Update distribution build jenkins job with option for incremental bui…

…ld (#4356) Signed-off-by: Zelin Hao <[email protected]>
opensearch-project · Jan 24, 2024 · 982a6c7 · 982a6c7
1 parent 685f23a
commit 982a6c7
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 14 deletions.
diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile
@@ -91,6 +91,17 @@ pipeline {
             description: 'Continue building the distribution even if a one or more component fails',
             defaultValue: true
         )
+        booleanParam(
+            name: 'INCREMENTAL',
+            description: 'Whether to trigger incremental build. Defaults to false.',
+            defaultValue: false
+        )
+        string(
+            name: 'PREVIOUS_BUILD_ID',
+            description: 'The build ID used to download previous build artifacts. Defaults to latest.',
+            defaultValue: 'latest',
+            trim: true
+        )
     }
     stages {
         stage('verify-parameters') {
@@ -178,7 +189,9 @@ pipeline {
                                 platform: 'linux',
                                 architecture: 'x64',
                                 distribution: 'tar',
-                                continueOnError: params.CONTINUE_ON_ERROR
+                                continueOnError: params.CONTINUE_ON_ERROR,
+                                incremental: params.INCREMENTAL,
+                                previousBuildId: params.PREVIOUS_BUILD_ID
                             )
                             String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                             String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, "latest"), "builds", "opensearch", "manifest.yml"].join("/")
@@ -290,6 +303,8 @@ pipeline {
                                         architecture: 'x64',
                                         distribution: 'rpm',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -411,6 +426,8 @@ pipeline {
                                         architecture: 'x64',
                                         distribution: 'deb',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -524,6 +541,8 @@ pipeline {
                                         architecture: 'arm64',
                                         distribution: 'tar',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-arm64-tar-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -674,6 +693,8 @@ pipeline {
                                         architecture: 'arm64',
                                         distribution: 'rpm',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -799,6 +820,8 @@ pipeline {
                                         architecture: 'arm64',
                                         distribution: 'deb',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -907,7 +930,9 @@ pipeline {
                                 platform: 'windows',
                                 architecture: 'x64',
                                 distribution: "zip",
-                                continueOnError: params.CONTINUE_ON_ERROR
+                                continueOnError: params.CONTINUE_ON_ERROR,
+                                incremental: params.INCREMENTAL,
+                                previousBuildId: params.PREVIOUS_BUILD_ID
                             )
                             String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                             String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)

diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -84,6 +84,17 @@ pipeline {
             description: 'Continue building the distribution even if a one or more component fails',
             defaultValue: true
         )
+        booleanParam(
+            name: 'INCREMENTAL',
+            description: 'Whether to trigger incremental build. Defaults to false.',
+            defaultValue: false
+        )
+        string(
+            name: 'PREVIOUS_BUILD_ID',
+            description: 'The build ID used to download previous build artifacts. Defaults to latest.',
+            defaultValue: 'latest',
+            trim: true
+        )
     }
     stages {
         stage('verify-parameters') {
@@ -169,7 +180,9 @@ pipeline {
                                 platform: 'linux',
                                 architecture: 'x64',
                                 distribution: 'tar',
-                                continueOnError: params.CONTINUE_ON_ERROR
+                                continueOnError: params.CONTINUE_ON_ERROR,
+                                incremental: params.INCREMENTAL,
+                                previousBuildId: params.PREVIOUS_BUILD_ID
                             )
                             String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                             String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
@@ -262,6 +275,8 @@ pipeline {
                                         architecture: 'x64',
                                         distribution: 'rpm',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -376,6 +391,8 @@ pipeline {
                                         architecture: 'x64',
                                         distribution: 'deb',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -480,7 +497,9 @@ pipeline {
                                 platform: 'linux',
                                 architecture: 'arm64',
                                 distribution: 'tar',
-                                continueOnError: params.CONTINUE_ON_ERROR
+                                continueOnError: params.CONTINUE_ON_ERROR,
+                                incremental: params.INCREMENTAL,
+                                previousBuildId: params.PREVIOUS_BUILD_ID
                             )
                             String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                             String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
@@ -553,6 +572,8 @@ pipeline {
                                         architecture: 'arm64',
                                         distribution: 'rpm',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -667,6 +688,8 @@ pipeline {
                                         architecture: 'arm64',
                                         distribution: 'deb',
                                         continueOnError: params.CONTINUE_ON_ERROR,
+                                        incremental: params.INCREMENTAL,
+                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                         stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                     )
                                 }
@@ -771,7 +794,9 @@ pipeline {
                                 platform: 'windows',
                                 architecture: 'x64',
                                 distribution: 'zip',
-                                continueOnError: params.CONTINUE_ON_ERROR
+                                continueOnError: params.CONTINUE_ON_ERROR,
+                                incremental: params.INCREMENTAL,
+                                previousBuildId: params.PREVIOUS_BUILD_ID
                             )
                             String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                             String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)

diff --git a/manifests/2.12.0/opensearch-2.12.0.yml b/manifests/2.12.0/opensearch-2.12.0.yml
@@ -56,6 +56,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - job-scheduler
   - name: cross-cluster-replication
     repository: https://github.com/opensearch-project/cross-cluster-replication.git
     ref: 2.x
@@ -65,6 +67,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - common-utils
   - name: ml-commons
     repository: https://github.com/opensearch-project/ml-commons.git
     ref: 2.x
@@ -74,6 +78,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version: opensearch-ml-plugin
+    depends_on:
+      - common-utils
   - name: neural-search
     repository: https://github.com/opensearch-project/neural-search.git
     ref: 2.x
@@ -83,6 +89,9 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - ml-commons
+      - k-NN
   - name: notifications-core
     repository: https://github.com/opensearch-project/notifications.git
     ref: 2.x
@@ -93,6 +102,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version: opensearch-notifications-core
+    depends_on:
+      - common-utils
   - name: notifications
     repository: https://github.com/opensearch-project/notifications.git
     ref: 2.x
@@ -103,6 +114,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version: notifications
+    depends_on:
+      - common-utils
   - name: opensearch-observability
     repository: https://github.com/opensearch-project/observability.git
     ref: 2.x
@@ -112,6 +125,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - common-utils
   - name: opensearch-reports
     repository: https://github.com/opensearch-project/reporting.git
     ref: 2.x
@@ -121,6 +136,9 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - common-utils
+      - job-scheduler
   - name: sql
     repository: https://github.com/opensearch-project/sql.git
     ref: 2.x
@@ -130,6 +148,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version: opensearch-sql-plugin
+    depends_on:
+      - ml-commons
   - name: asynchronous-search
     repository: https://github.com/opensearch-project/asynchronous-search.git
     ref: 2.x
@@ -139,6 +159,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - common-utils
   - name: anomaly-detection
     repository: https://github.com/opensearch-project/anomaly-detection.git
     ref: 2.x
@@ -148,6 +170,9 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - common-utils
+      - job-scheduler
   - name: alerting
     repository: https://github.com/opensearch-project/alerting.git
     ref: 2.x
@@ -157,6 +182,8 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version: alerting
+    depends_on:
+      - common-utils
   - name: security-analytics
     repository: https://github.com/opensearch-project/security-analytics.git
     ref: 2.x
@@ -165,6 +192,8 @@ components:
       - windows
     checks:
       - gradle:properties:version
+    depends_on:
+      - common-utils
   - name: index-management
     repository: https://github.com/opensearch-project/index-management.git
     ref: 2.x
@@ -173,6 +202,9 @@ components:
       - windows
     checks:
       - gradle:properties:version
+    depends_on:
+      - common-utils
+      - job-scheduler
   - name: performance-analyzer
     repository: https://github.com/opensearch-project/performance-analyzer.git
     ref: 2.x
@@ -199,3 +231,5 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+    depends_on:
+      - common-utils
diff --git a/src/build_workflow/build_recorder.py b/src/build_workflow/build_recorder.py
@@ -59,6 +59,7 @@ def __init__(self, target: BuildTarget, build_manifest: BuildManifest = None) ->
 
             if build_manifest:
                 self.data = build_manifest.__to_dict__()
+                self.data["build"]["id"] = target.build_id
                 for component in build_manifest.components.select():
                     self.components_hash[component.name] = component.__to_dict__()
             else:

diff --git a/src/run_build.py b/src/run_build.py
@@ -9,6 +9,7 @@
 import logging
 import os
 import sys
+import uuid
 
 from build_workflow.build_args import BuildArgs
 from build_workflow.build_incremental import BuildIncremental
@@ -49,19 +50,20 @@ def main() -> int:
         buildIncremental = BuildIncremental(manifest, args.distribution)
         list_of_updated_plugins = buildIncremental.commits_diff(manifest)
         components = buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)
-        if not components:
-            logging.info("No commit difference found between any components. Skipping the build")
-            return 0
-
-        logging.info(f"Plugins for incremental build: {components}")
 
         build_manifest_path = os.path.join(args.distribution, "builds", manifest.build.filename, "manifest.yml")
         if not os.path.exists(build_manifest_path):
             logging.error(f"Previous build manifest missing at path: {build_manifest_path}")
+        build_manifest = BuildManifest.from_path(build_manifest_path)
 
-        logging.info(f"Build {components} incrementally.")
+        if not components:
+            logging.info("No commit difference found between any components. Skipping the build.")
+            build_manifest.build.id = os.getenv("BUILD_NUMBER") or uuid.uuid4().hex
+            build_manifest.to_file(build_manifest_path)
+            logging.info(f"Updating the build ID in the build manifest to {build_manifest.build.id}.")
+            return 0
 
-        build_manifest = BuildManifest.from_path(build_manifest_path)
+        logging.info(f"Plugins for incremental build: {components}")
 
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
         logging.info(f"Building in {work_dir.name}")

diff --git a/tests/test_run_build.py b/tests/test_run_build.py
@@ -259,16 +259,23 @@ def test_main_incremental(self, mock_build_incremental: MagicMock, mock_temp: Ma
         mock_build_incremental.return_value.commits_diff.assert_called()
         mock_build_incremental.return_value.rebuild_plugins.assert_called()
 
+    @patch.dict(os.environ, {"BUILD_NUMBER": "1234"})
     @patch("argparse._sys.argv", ["run_build.py", INPUT_MANIFEST_PATH, "--incremental", "-p", "linux"])
     @patch("run_build.BuildIncremental.commits_diff", return_value=MagicMock())
+    @patch("manifests.build_manifest.BuildManifest.from_path")
     @patch("run_build.BuildIncremental.rebuild_plugins", return_value=MagicMock())
     @patch("run_build.logging.info")
     def test_build_incremental_no_change(self, mock_logging_info: MagicMock,
-                                         mock_build_incremental: MagicMock, *mocks: Any) -> None:
+                                         mock_build_incremental: MagicMock, mock_build_manifest: MagicMock,
+                                         *mocks: Any) -> None:
         mock_build_incremental.return_value = []
+        mock_build_manifest.return_value = self.BUILD_MANIFEST
         main()
         mock_logging_info.assert_has_calls([
-            call("No commit difference found between any components. Skipping the build")
+            call("No commit difference found between any components. Skipping the build.")
+        ], any_order=True)
+        mock_logging_info.assert_has_calls([
+            call("Updating the build ID in the build manifest to 1234.")
         ], any_order=True)
 
     @patch("argparse._sys.argv", ["run_build.py", INPUT_MANIFEST_PATH, "--incremental"])

diff --git a/tests/tests_build_workflow/test_build_recorder.py b/tests/tests_build_workflow/test_build_recorder.py
@@ -261,6 +261,8 @@ def test_append_component_with_existing_manifest(self) -> None:
                          "e3c8902dea26fd20f56a6f144042b2623f652e3e")
         self.assertEqual(mock.build_manifest.components_hash.get("security").get("version"), "2.12.0.0")
 
+        self.assertEqual(mock.build_manifest.data.get("build").get("id"), "1")
+
         mock.record_component(
             "job-scheduler",
             MagicMock(