From 982a6c7c82d4ceb9216854180dc4889c6d60fdab Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 24 Jan 2024 13:44:29 -0800 Subject: [PATCH] Update distribution build jenkins job with option for incremental build (#4356) Signed-off-by: Zelin Hao --- .../distribution-build.jenkinsfile | 29 ++++++++++++++-- .../opensearch/distribution-build.jenkinsfile | 31 +++++++++++++++-- manifests/2.12.0/opensearch-2.12.0.yml | 34 +++++++++++++++++++ src/build_workflow/build_recorder.py | 1 + src/run_build.py | 16 +++++---- tests/test_run_build.py | 11 ++++-- .../test_build_recorder.py | 2 ++ 7 files changed, 110 insertions(+), 14 deletions(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 21fb0640d4..351d1fae1e 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -91,6 +91,17 @@ pipeline { description: 'Continue building the distribution even if a one or more component fails', defaultValue: true ) + booleanParam( + name: 'INCREMENTAL', + description: 'Whether to trigger incremental build. Defaults to false.', + defaultValue: false + ) + string( + name: 'PREVIOUS_BUILD_ID', + description: 'The build ID used to download previous build artifacts. Defaults to latest.', + defaultValue: 'latest', + trim: true + ) } stages { stage('verify-parameters') { @@ -178,7 +189,9 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'tar', - continueOnError: params.CONTINUE_ON_ERROR + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, "latest"), "builds", "opensearch", "manifest.yml"].join("/") @@ -290,6 +303,8 @@ pipeline { architecture: 'x64', distribution: 'rpm', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -411,6 +426,8 @@ pipeline { architecture: 'x64', distribution: 'deb', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -524,6 +541,8 @@ pipeline { architecture: 'arm64', distribution: 'tar', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-arm64-tar-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -674,6 +693,8 @@ pipeline { architecture: 'arm64', distribution: 'rpm', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -799,6 +820,8 @@ pipeline { architecture: 'arm64', distribution: 'deb', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -907,7 +930,9 @@ pipeline { platform: 'windows', architecture: 'x64', distribution: "zip", - continueOnError: params.CONTINUE_ON_ERROR + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 116dcb0bec..3a0ffa7c39 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -84,6 +84,17 @@ pipeline { description: 'Continue building the distribution even if a one or more component fails', defaultValue: true ) + booleanParam( + name: 'INCREMENTAL', + description: 'Whether to trigger incremental build. Defaults to false.', + defaultValue: false + ) + string( + name: 'PREVIOUS_BUILD_ID', + description: 'The build ID used to download previous build artifacts. Defaults to latest.', + defaultValue: 'latest', + trim: true + ) } stages { stage('verify-parameters') { @@ -169,7 +180,9 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'tar', - continueOnError: params.CONTINUE_ON_ERROR + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) @@ -262,6 +275,8 @@ pipeline { architecture: 'x64', distribution: 'rpm', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -376,6 +391,8 @@ pipeline { architecture: 'x64', distribution: 'deb', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -480,7 +497,9 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'tar', - continueOnError: params.CONTINUE_ON_ERROR + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) @@ -553,6 +572,8 @@ pipeline { architecture: 'arm64', distribution: 'rpm', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -667,6 +688,8 @@ pipeline { architecture: 'arm64', distribution: 'deb', continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID, stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -771,7 +794,9 @@ pipeline { platform: 'windows', architecture: 'x64', distribution: 'zip', - continueOnError: params.CONTINUE_ON_ERROR + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) diff --git a/manifests/2.12.0/opensearch-2.12.0.yml b/manifests/2.12.0/opensearch-2.12.0.yml index 94e8fb7cdf..08049e5716 100644 --- a/manifests/2.12.0/opensearch-2.12.0.yml +++ b/manifests/2.12.0/opensearch-2.12.0.yml @@ -56,6 +56,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - job-scheduler - name: cross-cluster-replication repository: https://github.com/opensearch-project/cross-cluster-replication.git ref: 2.x @@ -65,6 +67,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - common-utils - name: ml-commons repository: https://github.com/opensearch-project/ml-commons.git ref: 2.x @@ -74,6 +78,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-ml-plugin + depends_on: + - common-utils - name: neural-search repository: https://github.com/opensearch-project/neural-search.git ref: 2.x @@ -83,6 +89,9 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - ml-commons + - k-NN - name: notifications-core repository: https://github.com/opensearch-project/notifications.git ref: 2.x @@ -93,6 +102,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-notifications-core + depends_on: + - common-utils - name: notifications repository: https://github.com/opensearch-project/notifications.git ref: 2.x @@ -103,6 +114,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version: notifications + depends_on: + - common-utils - name: opensearch-observability repository: https://github.com/opensearch-project/observability.git ref: 2.x @@ -112,6 +125,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - common-utils - name: opensearch-reports repository: https://github.com/opensearch-project/reporting.git ref: 2.x @@ -121,6 +136,9 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - common-utils + - job-scheduler - name: sql repository: https://github.com/opensearch-project/sql.git ref: 2.x @@ -130,6 +148,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-sql-plugin + depends_on: + - ml-commons - name: asynchronous-search repository: https://github.com/opensearch-project/asynchronous-search.git ref: 2.x @@ -139,6 +159,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - common-utils - name: anomaly-detection repository: https://github.com/opensearch-project/anomaly-detection.git ref: 2.x @@ -148,6 +170,9 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - common-utils + - job-scheduler - name: alerting repository: https://github.com/opensearch-project/alerting.git ref: 2.x @@ -157,6 +182,8 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version: alerting + depends_on: + - common-utils - name: security-analytics repository: https://github.com/opensearch-project/security-analytics.git ref: 2.x @@ -165,6 +192,8 @@ components: - windows checks: - gradle:properties:version + depends_on: + - common-utils - name: index-management repository: https://github.com/opensearch-project/index-management.git ref: 2.x @@ -173,6 +202,9 @@ components: - windows checks: - gradle:properties:version + depends_on: + - common-utils + - job-scheduler - name: performance-analyzer repository: https://github.com/opensearch-project/performance-analyzer.git ref: 2.x @@ -199,3 +231,5 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + depends_on: + - common-utils diff --git a/src/build_workflow/build_recorder.py b/src/build_workflow/build_recorder.py index 4356fe4478..cb4cc38ef9 100644 --- a/src/build_workflow/build_recorder.py +++ b/src/build_workflow/build_recorder.py @@ -59,6 +59,7 @@ def __init__(self, target: BuildTarget, build_manifest: BuildManifest = None) -> if build_manifest: self.data = build_manifest.__to_dict__() + self.data["build"]["id"] = target.build_id for component in build_manifest.components.select(): self.components_hash[component.name] = component.__to_dict__() else: diff --git a/src/run_build.py b/src/run_build.py index cf4e590f21..9057183778 100755 --- a/src/run_build.py +++ b/src/run_build.py @@ -9,6 +9,7 @@ import logging import os import sys +import uuid from build_workflow.build_args import BuildArgs from build_workflow.build_incremental import BuildIncremental @@ -49,19 +50,20 @@ def main() -> int: buildIncremental = BuildIncremental(manifest, args.distribution) list_of_updated_plugins = buildIncremental.commits_diff(manifest) components = buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest) - if not components: - logging.info("No commit difference found between any components. Skipping the build") - return 0 - - logging.info(f"Plugins for incremental build: {components}") build_manifest_path = os.path.join(args.distribution, "builds", manifest.build.filename, "manifest.yml") if not os.path.exists(build_manifest_path): logging.error(f"Previous build manifest missing at path: {build_manifest_path}") + build_manifest = BuildManifest.from_path(build_manifest_path) - logging.info(f"Build {components} incrementally.") + if not components: + logging.info("No commit difference found between any components. Skipping the build.") + build_manifest.build.id = os.getenv("BUILD_NUMBER") or uuid.uuid4().hex + build_manifest.to_file(build_manifest_path) + logging.info(f"Updating the build ID in the build manifest to {build_manifest.build.id}.") + return 0 - build_manifest = BuildManifest.from_path(build_manifest_path) + logging.info(f"Plugins for incremental build: {components}") with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: logging.info(f"Building in {work_dir.name}") diff --git a/tests/test_run_build.py b/tests/test_run_build.py index 92443f9618..b058b01933 100644 --- a/tests/test_run_build.py +++ b/tests/test_run_build.py @@ -259,16 +259,23 @@ def test_main_incremental(self, mock_build_incremental: MagicMock, mock_temp: Ma mock_build_incremental.return_value.commits_diff.assert_called() mock_build_incremental.return_value.rebuild_plugins.assert_called() + @patch.dict(os.environ, {"BUILD_NUMBER": "1234"}) @patch("argparse._sys.argv", ["run_build.py", INPUT_MANIFEST_PATH, "--incremental", "-p", "linux"]) @patch("run_build.BuildIncremental.commits_diff", return_value=MagicMock()) + @patch("manifests.build_manifest.BuildManifest.from_path") @patch("run_build.BuildIncremental.rebuild_plugins", return_value=MagicMock()) @patch("run_build.logging.info") def test_build_incremental_no_change(self, mock_logging_info: MagicMock, - mock_build_incremental: MagicMock, *mocks: Any) -> None: + mock_build_incremental: MagicMock, mock_build_manifest: MagicMock, + *mocks: Any) -> None: mock_build_incremental.return_value = [] + mock_build_manifest.return_value = self.BUILD_MANIFEST main() mock_logging_info.assert_has_calls([ - call("No commit difference found between any components. Skipping the build") + call("No commit difference found between any components. Skipping the build.") + ], any_order=True) + mock_logging_info.assert_has_calls([ + call("Updating the build ID in the build manifest to 1234.") ], any_order=True) @patch("argparse._sys.argv", ["run_build.py", INPUT_MANIFEST_PATH, "--incremental"]) diff --git a/tests/tests_build_workflow/test_build_recorder.py b/tests/tests_build_workflow/test_build_recorder.py index e411a1ceb5..aac631bcd7 100644 --- a/tests/tests_build_workflow/test_build_recorder.py +++ b/tests/tests_build_workflow/test_build_recorder.py @@ -261,6 +261,8 @@ def test_append_component_with_existing_manifest(self) -> None: "e3c8902dea26fd20f56a6f144042b2623f652e3e") self.assertEqual(mock.build_manifest.components_hash.get("security").get("version"), "2.12.0.0") + self.assertEqual(mock.build_manifest.data.get("build").get("id"), "1") + mock.record_component( "job-scheduler", MagicMock(