Enable deploying chrome-tests-syncer independently (#4647)

### Motivation
Decouple the chrome-tests-syncer code releases from the rest of
clusterfuzz, in order to enable it to be deployed independently of the
clusterfuzz instances by the chrome fuzzing team .

### Changes
* Extended the butler deploy script to produce a dedicated deployment
package for "--release=chrome-tests-syncer" and to produce both the prod
and tests-syncer packages when releasing to prod.
* Changed chrome-tests-syncer Dockerfile/start.sh to pull from this
dedicated deployment package.
* Moved and edited the tests-syncer code to be executed as a k8s
cronjob.
* (side effect) Updated unit test to account for deploying the
chrome-tests-syncer manifest.

### Tests
* Deployed a chrome-tests-syncer release using this PR and produced the
dedicated artifact
(https://screenshot.googleplex.com/8W88DmYwFKPkSfZ.png).
* Built an updated chrome-tests-syncer image that pulls from this
artifact (https://screenshot.googleplex.com/8HqEvV7bnBggBPj.png).

* Created the chrome-tests-syncer cronjob (along with PR in
clusterfuzz-config: 2428107) pulling from this updated image, applied it
to the GKE and it ran successfully:

![image](https://github.com/user-attachments/assets/b32441f1-daae-4c12-9476-97583fad3982)

* Mocked a deployment with --release=prod (by printing the command
instead of actually uploading to gcs):
<img width="960" alt="Screenshot 2025-02-05 at 11 48 31"
src="https://github.com/user-attachments/assets/bd035374-07d1-453a-9db4-2ed0cd39d92f"
/>
 
Bug: b/389048599

butler.py

@@ -286,7 +286,10 @@ def main():
   parser_package.add_argument(
       '-p', '--platform', choices=['linux', 'macos', 'windows', 'all'])
   parser_package.add_argument(
-      '-r', '--release', choices=['prod', 'candidate'], default='prod')
+      '-r',
+      '--release',
+      choices=['prod', 'candidate', 'chrome-tests-syncer'],
+      default='prod')
 
   parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine')
   parser_deploy.add_argument(
@@ -303,7 +306,10 @@ def main():
   parser_deploy.add_argument(
       '--targets', nargs='*', default=['appengine', 'k8s', 'zips'])
   parser_deploy.add_argument(
-      '--release', '-r', choices=['prod', 'candidate'], default='prod')
+      '--release',
+      '-r',
+      choices=['prod', 'candidate', 'chrome-tests-syncer'],
+      default='prod')
 
   parser_run_server = subparsers.add_parser(
       'run_server', help='Run the local Clusterfuzz server.')

docker/chromium/tests-syncer/Dockerfile

@@ -13,11 +13,9 @@
 # limitations under the License.
 FROM gcr.io/clusterfuzz-images/base
 
-ENV RUN_CMD \
-    "python3.11 $ROOT_DIR/src/python/other-bots/chromium-tests-syncer/run.py"
+ENV RUN_CMD "python3.11 $ROOT_DIR/src/python/bot/startup/run_cron.py chrome_tests_syncer"
 ENV DISABLE_MOUNTS True
 ENV EXTRA_PATH "/data/depot_tools"
-ENV SYNC_INTERVAL 43200
 ENV TESTS_ARCHIVE_BUCKET "clusterfuzz-data"
 ENV TESTS_ARCHIVE_NAME "web_tests.zip"
 ENV TESTS_DIR /home/$USER/tests

docker/chromium/tests-syncer/start.sh

@@ -13,10 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-DEPLOYMENT_ZIP="linux-3.zip"
-if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
-    DEPLOYMENT_ZIP="linux-3-candidate.zip"
-fi
+DEPLOYMENT_ZIP="linux-3-chrome-tests-syncer.zip"
+
 export DEPLOYMENT_ZIP
 
 source /data/setup_common.sh

src/clusterfuzz/_internal/base/utils.py

@@ -261,17 +261,19 @@ def _get_manifest_release_suffix(release):
   suffix = ''
   if sys.version_info.major == 3:
     suffix += '.3'
-  if release == 'candidate':
-    suffix += '-candidate'
+  if release == 'prod':
+    return suffix
+  suffix += f'-{release}'
   return suffix
 
 
 def _get_deployment_zip_release_suffix(release):
   suffix = ''
   if sys.version_info.major == 3:
     suffix += '-3'
-  if release == 'candidate':
-    suffix += '-candidate'
+  if release == 'prod':
+    return suffix
+  suffix += f'-{release}'
   return suffix
 
 

src/python/other-bots/chromium-tests-syncer/run.py → src/clusterfuzz/_internal/cron/chrome_tests_syncer.py

@@ -24,16 +24,13 @@
 import re
 import subprocess
 import tarfile
-import time
 
 from clusterfuzz._internal.base import utils
 from clusterfuzz._internal.bot import testcase_manager
 from clusterfuzz._internal.bot.tasks import setup
 from clusterfuzz._internal.datastore import data_types
-from clusterfuzz._internal.datastore import ndb_init
 from clusterfuzz._internal.datastore import ndb_utils
 from clusterfuzz._internal.metrics import logs
-from clusterfuzz._internal.metrics import monitor
 from clusterfuzz._internal.metrics import monitoring_metrics
 from clusterfuzz._internal.system import archive
 from clusterfuzz._internal.system import environment
@@ -338,23 +335,9 @@ def main():
   tests_archive_name = environment.get_value('TESTS_ARCHIVE_NAME')
   tests_directory = environment.get_value('TESTS_DIR')
 
-  # Intervals are in seconds.
-  sync_interval = environment.get_value('SYNC_INTERVAL')
-  fail_wait = environment.get_value('FAIL_WAIT')
-
-  while True:
-    sleep_secs = sync_interval
-
-    try:
-      with monitor.wrap_with_monitoring(), ndb_init.context():
-        sync_tests(tests_archive_bucket, tests_archive_name, tests_directory)
-    except Exception as e:
-      logs.error(f'Failed to sync tests: {e}')
-      sleep_secs = fail_wait
-
-    logs.info(f'Sleeping for {sleep_secs} seconds.')
-    time.sleep(sleep_secs)
-
-
-if __name__ == '__main__':
-  main()
+  try:
+    sync_tests(tests_archive_bucket, tests_archive_name, tests_directory)
+    return True
+  except Exception as e:
+    logs.error(f'Failed to sync tests: {e}')
+    return False

src/clusterfuzz/_internal/tests/core/local/butler/deploy_test.py

@@ -53,6 +53,7 @@ def setUp(self):
     os.environ['ROOT_DIR'] = '.'
     self.mock.now.return_value = datetime.datetime(2017, 1, 3, 12, 1)
     self.manifest_target = 'clusterfuzz-source.manifest.3'
+    self.additional_manifest_target = 'clusterfuzz-source.manifest.3-chrome-tests-syncer'
 
   def _check_env_variables(self, yaml_paths):
     """Check that environment variables are written to yaml paths."""
@@ -182,6 +183,10 @@ def test_app(self):
         mock.call('gsutil cp src/appengine/resources/'
                   'clusterfuzz-source.manifest '
                   'gs://test-deployment-bucket/' + self.manifest_target),
+        mock.call('gsutil cp src/appengine/resources/'
+                  'clusterfuzz-source.manifest '
+                  'gs://test-deployment-bucket/' +
+                  self.additional_manifest_target),
         mock.call('python butler.py run setup --config-dir /config_dir '
                   '--non-dry-run'),
     ])
@@ -264,6 +269,10 @@ def test_app_retry(self):
         mock.call('gsutil cp src/appengine/resources/'
                   'clusterfuzz-source.manifest '
                   'gs://test-deployment-bucket/' + self.manifest_target),
+        mock.call('gsutil cp src/appengine/resources/'
+                  'clusterfuzz-source.manifest '
+                  'gs://test-deployment-bucket/' +
+                  self.additional_manifest_target),
         mock.call('python butler.py run setup --config-dir /config_dir '
                   '--non-dry-run'),
     ])

src/local/butler/constants.py

@@ -41,6 +41,9 @@
     ('linux', ('manylinux2014_x86_64')),
 ])
 
+# Additional required packages when deploying to prod.
+ADDITIONAL_RELEASES = ['chrome-tests-syncer']
+
 if sys.version_info.major == 3 and sys.version_info.minor == 7:
   ABIS = {'linux': 'cp37m', 'windows': 'cp37m', 'macos': 'cp37m'}
 elif sys.version_info.major == 3 and sys.version_info.minor == 8:

src/local/butler/deploy.py

@@ -117,11 +117,14 @@ def _deploy_app_prod(project,
       _deploy_zip(
           deployment_bucket, package_zip_path, test_deployment=test_deployment)
 
-    _deploy_manifest(
-        deployment_bucket,
-        constants.PACKAGE_TARGET_MANIFEST_PATH,
-        test_deployment=test_deployment,
-        release=release)
+    releases = [release]
+    releases += constants.ADDITIONAL_RELEASES if release == 'prod' else []
+    for rel in releases:
+      _deploy_manifest(
+          deployment_bucket,
+          constants.PACKAGE_TARGET_MANIFEST_PATH,
+          test_deployment=test_deployment,
+          release=rel)
 
 
 def _deploy_app_staging(project, yaml_paths):
@@ -563,9 +566,8 @@ def execute(args):
   package_zip_paths = []
   if deploy_zips:
     for platform_name in platforms:
-      package_zip_paths.append(
-          package.package(
-              revision, platform_name=platform_name, release=args.release))
+      package_zip_paths += package.package(
+          revision, platform_name=platform_name, release=args.release)
   else:
     # package.package calls these, so only set these up if we're not packaging,
     # since they can be fairly slow.

src/local/butler/package.py

@@ -15,6 +15,7 @@
 
 import os
 import re
+import shutil
 import sys
 import zipfile
 
@@ -135,7 +136,20 @@ def package(revision,
 
   print()
   print('%s is ready.' % target_zip_path)
-  return target_zip_path
+
+  targets_zip_paths = [target_zip_path]
+  if platform_name and release == 'prod':
+    # Copy prod package into additional releases.
+    for add_release in constants.ADDITIONAL_RELEASES:
+      add_target_zip_name = utils.get_platform_deployment_filename(
+          platform_name, release=add_release)
+      add_target_zip_path = os.path.join(target_zip_dir, add_target_zip_name)
+      _clear_zip(add_target_zip_path)
+      shutil.copy2(target_zip_path, add_target_zip_path)
+      print('\n%s is ready.' % add_target_zip_path)
+      targets_zip_paths.append(add_target_zip_path)
+
+  return targets_zip_paths
 
 
 def execute(args):