From bac808f8f19731a0228dea87da07a96095176751 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Thu, 4 Jan 2024 15:07:21 -0800 Subject: [PATCH 1/9] Add Signed-off-by: Zelin Hao --- vars/buildManifest.groovy | 4 ++++ vars/retrievePreviousBuild.groovy | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 vars/retrievePreviousBuild.groovy diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index 5c13372d4..c40294748 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -18,6 +18,10 @@ @param args.continueOnError - Do not fail the distribution build on any plugin component failure. Defaults to null */ void call(Map args = [:]) { +// if (args.incremental == true) { +// retrievePreviousBuild() +// } + sh(([ './build.sh', args.inputManifest ?: "manifests/${INPUT_MANIFEST}", diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy new file mode 100644 index 000000000..39680479f --- /dev/null +++ b/vars/retrievePreviousBuild.groovy @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + /** Library to call build.sh script (builds OpenSearch and OpenSearch Dashboards distribution) + @param Map args = [:] args A map of the following parameters + @param args.inputManifest - Relative path to input manifest containing all the components to build. + @param args.distribution - Type of distribution to build. Defaults to null. + @param args.componentName - Name of the single component to build. Defaults to null and builds all the components in the manifest. + @param args.platform - Platform to build. Defaults to null. + @param args.architecture - Architecture to build. Defaults to null. + @param args.snapshot - Boolean value. Defaults to null. + @param args.lock - Generate a stable reference manifest. Defaults to null. + @param args.continueOnError - Do not fail the distribution build on any plugin component failure. Defaults to null + */ +void call(Map args = [:]) { + if (args.incremental == true) { + retrievePreviousBuild() + } + +} From 40310f04b380c60e873ebf67382eade6bd7d2e8d Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Fri, 12 Jan 2024 17:27:11 -0800 Subject: [PATCH 2/9] Add library to retrieve previous build artifacts Signed-off-by: Zelin Hao --- .github/workflows/version-increment.yml | 2 +- build.gradle | 2 +- src/jenkins/BuildManifest.groovy | 6 +- .../opensearch-dashboards-input-2.12.0.yml | 18 ++ tests/data/opensearch-input-2.12.0.yml | 205 ++++++++++++++++++ .../jenkins/TestRetrievePreviousBuild.groovy | 55 +++++ .../jobs/BuildManifest_Jenkinsfile.txt | 2 +- .../jobs/RetrievePreviousBuild_Jenkinsfile | 35 +++ .../RetrievePreviousBuild_Jenkinsfile.txt | 33 +++ .../RetrievePreviousBuildLibTester.groovy | 54 +++++ vars/buildManifest.groovy | 10 +- vars/retrievePreviousBuild.groovy | 62 +++++- 12 files changed, 467 insertions(+), 17 deletions(-) create mode 100644 tests/data/opensearch-dashboards-input-2.12.0.yml create mode 100644 tests/data/opensearch-input-2.12.0.yml create mode 100644 tests/jenkins/TestRetrievePreviousBuild.groovy create mode 100644 tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile create mode 100644 tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy diff --git a/.github/workflows/version-increment.yml b/.github/workflows/version-increment.yml index 738f57761..5830707a9 100644 --- a/.github/workflows/version-increment.yml +++ b/.github/workflows/version-increment.yml @@ -33,7 +33,7 @@ jobs: uses: peter-evans/create-pull-request@v4 with: token: ${{ steps.github_app_token.outputs.token }} - base: '5.x' + base: '6.x' committer: opensearch-ci-bot author: opensearch-ci-bot commit-message: | diff --git a/build.gradle b/build.gradle index 1a88de297..d7eea41fb 100644 --- a/build.gradle +++ b/build.gradle @@ -121,7 +121,7 @@ jacocoTestReport { } } -String version = '5.12.0' +String version = '6.0.0' task updateVersion { doLast { diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy index 221a51c13..cf300dcb2 100644 --- a/src/jenkins/BuildManifest.groovy +++ b/src/jenkins/BuildManifest.groovy @@ -116,7 +116,11 @@ class BuildManifest implements Serializable { public String getIndexFileRoot(String jobName) { return [ jobName, - this.build.version + this.build.version, + "index", + this.build.platform, + this.build.architecture, + this.build.distribution ].join("/") } diff --git a/tests/data/opensearch-dashboards-input-2.12.0.yml b/tests/data/opensearch-dashboards-input-2.12.0.yml new file mode 100644 index 000000000..e2084c046 --- /dev/null +++ b/tests/data/opensearch-dashboards-input-2.12.0.yml @@ -0,0 +1,18 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch Dashboards + version: 2.12.0 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-build-v1 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: 361df7741b699e67c03f68a8550a5b8edcb1c838 + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: a0be908272a67dec0d39211b432c9d415eee780c + - name: observabilityDashboards + repository: https://github.com/opensearch-project/dashboards-observability.git + ref: 42a0be2aedc4f28e0e37cde6936783767881d5e7 diff --git a/tests/data/opensearch-input-2.12.0.yml b/tests/data/opensearch-input-2.12.0.yml new file mode 100644 index 000000000..955b85ea5 --- /dev/null +++ b/tests/data/opensearch-input-2.12.0.yml @@ -0,0 +1,205 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 2.12.0 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 05c2befd7d01fab4aef4f0d3d6722d2da240b2c6 + checks: + - gradle:publish + - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: ace0d5a6bf4b489bbeaccad65f62f6ccf20df491 + checks: + - gradle:publish + - gradle:properties:version + platforms: + - linux + - windows + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: aaf09b0211df15dd74ff2756f2590c360b03486b + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: 7ccb10cf5339701217c4019932bd26bf304a93aa + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: geospatial + repository: https://github.com/opensearch-project/geospatial.git + ref: f48c9dabcd4d955e5d88b0670d519cd7d341581c + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - job-scheduler + - name: security + repository: https://github.com/opensearch-project/security.git + ref: ea0d4ff4775ea3d8793458393ccadbe0ec8c0ef0 + platforms: + - linux + - windows + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: ee577a1729dec1b0bb05ddc3b767a54e6e905401 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: 2ee80cde78c7099d5132af7308933724b32c1f0d + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-ml-plugin + platforms: + - linux + - windows + depends_on: + - common-utils + - name: neural-search + repository: https://github.com/opensearch-project/neural-search.git + ref: 5442f84398f8c65378ba71dc009d75550810fce5 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - ml-commons + - k-NN + - name: notifications-core + repository: https://github.com/opensearch-project/notifications.git + ref: cf64c411ce4f85ce67734feb8704467b2232d171 + working_directory: notifications + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-notifications-core + platforms: + - linux + - windows + depends_on: + - common-utils + - name: notifications + repository: https://github.com/opensearch-project/notifications.git + ref: cf64c411ce4f85ce67734feb8704467b2232d171 + working_directory: notifications + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: notifications + platforms: + - linux + - windows + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: 40b0260573d5ba2cd8aa371febddd5816bff4967 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: opensearch-reports + repository: https://github.com/opensearch-project/reporting.git + ref: e8e5025b9577038c6c160b6ea5a1610312b66180 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: e090584440cdf4832b4225241622570fc5a8c80d + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-sql-plugin + platforms: + - linux + - windows + depends_on: + - ml-commons + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: 3fd781eaee620895d10067baf9a368d970257455 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: 72673698e062b061e8c6734c4dd52d43208ffa3d + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: 944abf29a42ce71d8e5d62c61768f826d34b2a45 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: alerting + platforms: + - linux + - windows + depends_on: + - common-utils + - name: security-analytics + repository: https://github.com/opensearch-project/security-analytics.git + ref: 22fa49d9d02734e561be55993ff336eb713318b1 + checks: + - gradle:properties:version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: 11aa4469ae2b9fb3be714ae24ef52d9bfb127510 + checks: + - gradle:properties:version + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy new file mode 100644 index 000000000..e066641ad --- /dev/null +++ b/tests/jenkins/TestRetrievePreviousBuild.groovy @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package jenkins.tests + +import org.junit.* +import jenkins.tests.BuildPipelineTest +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat +import static org.hamcrest.core.IsNot.not + +class TestRetrievePreviousBuild extends BuildPipelineTest { + @Before + void setUp() { + super.setUp() + + binding.setVariable('JOB_NAME', 'dummy_job') + this.registerLibTester(new RetrievePreviousBuildLibTester('tests/data/opensearch-input-2.12.0.yml', 'linux', 'x64', 'tar', '123')) + helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod("s3Download", [Map]) + } + + @Test + void testRetrievePreviousBuild() { + super.testPipeline('tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile') + def shCommands = getCommands('sh', 'mkdir') + assertThat(shCommands, hasItems('mkdir -p tar && mv /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace')) + assertThat(shCommands, hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')) + + assertThat(shCommands, hasItems('mkdir -p zip && mv /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace')) + assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))) + + } + + def getCommands(method, text) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == method + }.collect { call -> + callArgsToString(call) + }.findAll { command -> + command.contains(text) + } + return shCommands + } +} + diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt index c1f3602f8..b922db96f 100644 --- a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt @@ -28,7 +28,7 @@ BuildManifest.getMinArtifact() BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz) BuildManifest.getIndexFileRoot(distribution-build-opensearch) - BuildManifest_Jenkinsfile.echo(distribution-build-opensearch/1.1.0) + BuildManifest_Jenkinsfile.echo(distribution-build-opensearch/1.1.0/index/linux/x64/tar) BuildManifest.getCommitId(OpenSearch) BuildManifest_Jenkinsfile.echo(b7334f49d530ffd1a3f7bd0e5832b9b2a9caa583) BuildManifest.getRepo(OpenSearch) diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile new file mode 100644 index 000000000..3fd6d55cc --- /dev/null +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('Test build manifest jenkins var') { + steps { + script { + retrievePreviousBuild( + inputManifest: "tests/data/opensearch-input-2.12.0.yml", + distribution: "tar", + architecture: "x64", + platform: "linux", + incremental: "123" + ) + retrievePreviousBuild( + inputManifest: "tests/data/opensearch-dashboards-input-2.12.0.yml", + distribution: "zip", + architecture: "x64", + platform: "windows", + incremental: "1234" + ) + } + } + } + } +} + diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt new file mode 100644 index 000000000..816a411d7 --- /dev/null +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt @@ -0,0 +1,33 @@ + RetrievePreviousBuild_Jenkinsfile.run() + RetrievePreviousBuild_Jenkinsfile.pipeline(groovy.lang.Closure) + RetrievePreviousBuild_Jenkinsfile.echo(Executing on agent [label:none]) + RetrievePreviousBuild_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure) + RetrievePreviousBuild_Jenkinsfile.script(groovy.lang.Closure) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, incremental=123}) + retrievePreviousBuild.legacySCM(groovy.lang.Closure) + retrievePreviousBuild.library({identifier=jenkins@jenkins_incremental, retriever=null}) + retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml}) + InputManifest.asBoolean() + retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/2.12.0/123/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) + downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true}) + retrievePreviousBuild.sh(mkdir -p tar && mv /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace) + retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.) + retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, incremental=1234}) + retrievePreviousBuild.legacySCM(groovy.lang.Closure) + retrievePreviousBuild.library({identifier=jenkins@jenkins_incremental, retriever=null}) + retrievePreviousBuild.readYaml({file=tests/data/opensearch-dashboards-input-2.12.0.yml}) + InputManifest.asBoolean() + retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/2.12.0/1234/windows/x64/zip/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) + downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}) + retrievePreviousBuild.sh(mkdir -p zip && mv /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace) diff --git a/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy new file mode 100644 index 000000000..6f016b40d --- /dev/null +++ b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.CoreMatchers.nullValue +import static org.hamcrest.MatcherAssert.assertThat + +class RetrievePreviousBuildLibTester extends LibFunctionTester { + private String inputManifestPath + private String distribution = 'tar' + private String platform = 'linux' + private String architecture = 'tar' + private String incremental = 'latest' + + public RetrievePreviousBuildLibTester(String inputManifestPath, String platform, String architecture, String distribution, String incremental){ + this.inputManifestPath = inputManifestPath + this.platform = platform + this.architecture = architecture + this.distribution = distribution + this.incremental = incremental + } + + @Override + void configure(helper, binding) { + } + + @Override + void parameterInvariantsAssertions(call) { + assertThat(call.args.inputManifest.first(), notNullValue()) + assertThat(call.args.platform.first(), notNullValue()) + assertThat(call.args.architecture.first(), notNullValue()) + assertThat(call.args.distribution.first(), notNullValue()) + assertThat(call.args.incremental.first(), notNullValue()) + } + + @Override + boolean expectedParametersMatcher(call) { + return call.args.inputManifest.first().equals(this.inputManifestPath) + && call.args.platform.first().equals(this.platform) + && call.args.architecture.first().equals(this.architecture) + && call.args.distribution.first().equals(this.distribution) + && call.args.incremental.first().equals(this.incremental) + } + + @Override + String libFunctionName() { + return 'retrievePreviousBuild' + } +} diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index c40294748..b66376f38 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -18,9 +18,12 @@ @param args.continueOnError - Do not fail the distribution build on any plugin component failure. Defaults to null */ void call(Map args = [:]) { -// if (args.incremental == true) { -// retrievePreviousBuild() -// } + boolean incremental_enabled = args.incremental != null && !args.incremental.isEmpty() && !args.incremental.equalsIgnoreCase("false") + + if (incremental_enabled) { + echo("Start retreiving library") + retrievePreviousBuild(args) + } sh(([ './build.sh', @@ -32,5 +35,6 @@ void call(Map args = [:]) { args.snapshot ? '--snapshot' : null, args.lock ? '--lock' : null, args.continueOnError ? '--continue-on-error' : null, + incremental_enabled ? '--incremental': null ] - null).join(' ')) } diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index 39680479f..5303e4f73 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -6,20 +6,62 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ - /** Library to call build.sh script (builds OpenSearch and OpenSearch Dashboards distribution) +/** Library to call build.sh script (builds OpenSearch and OpenSearch Dashboards distribution) @param Map args = [:] args A map of the following parameters @param args.inputManifest - Relative path to input manifest containing all the components to build. - @param args.distribution - Type of distribution to build. Defaults to null. - @param args.componentName - Name of the single component to build. Defaults to null and builds all the components in the manifest. - @param args.platform - Platform to build. Defaults to null. - @param args.architecture - Architecture to build. Defaults to null. - @param args.snapshot - Boolean value. Defaults to null. - @param args.lock - Generate a stable reference manifest. Defaults to null. - @param args.continueOnError - Do not fail the distribution build on any plugin component failure. Defaults to null + @param args.platform - Platform of previous build to retrieve. + @param args.architecture - Architecture of previous build to retrieve. + @param args.distribution - Distribution of previous build to retrieve. + @param args.incremental - String value contains the build number of previous build or the latest. */ void call(Map args = [:]) { - if (args.incremental == true) { - retrievePreviousBuild() + def lib = library(identifier: 'jenkins@jenkins_incremental', retriever: legacySCM(scm)) + def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) + + def DISTRIBUTION_JOB_NAME = "${JOB_NAME}" + def revision = inputManifestObj.build.version + + def DISTRIBUTION_PLATFORM = args.platform + def DISTRIBUTION_ARCHITECTURE = args.architecture + def distribution = args.distribution + def prefixPath = "${WORKSPACE}/download" + def DISTRIBUTION_BUILD_NUMBER + + if (args.incremental.equalsIgnoreCase("latest")) { + withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + downloadFromS3( + assumedRoleName: "opensearch-bundle", + roleAccountNumberCred: "jenkins-aws-account-public", + downloadPath: "${DISTRIBUTION_JOB_NAME}/${revision}/index.json", + bucketName: "${ARTIFACT_BUCKET_NAME}", + localPath: "${prefixPath}/index.json", + force: true, + ) + } + DISTRIBUTION_BUILD_NUMBER = sh( + script: "cat ${prefixPath}/index.json | jq -r \".latest\"", + returnStdout: true + ).trim() + } else { + DISTRIBUTION_BUILD_NUMBER = args.incremental + } + + def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" + + withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + downloadFromS3( + assumedRoleName: "opensearch-bundle", + roleAccountNumberCred: "jenkins-aws-account-public", + downloadPath: "${artifactPath}/", + bucketName: "${ARTIFACT_BUCKET_NAME}", + localPath: "${prefixPath}", + force: true, + ) + } + sh("mkdir -p ${distribution} && mv ${prefixPath}/${artifactPath} ${WORKSPACE}") + if (inputManifestObj.build.getFilename().equals("opensearch")) { + echo("Setting up Maven Local for OpenSearch build.") + sh("mkdir -p ~/.m2/repository/org/ && cp -r ${distribution}/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/") } } From c3010f6975e2baac1df627c9d4716dc5cd7f4899 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Tue, 16 Jan 2024 10:21:55 -0800 Subject: [PATCH 3/9] Update the format Signed-off-by: Zelin Hao --- tests/jenkins/TestRetrievePreviousBuild.groovy | 4 ++-- tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt | 8 ++++---- vars/buildManifest.groovy | 2 +- vars/retrievePreviousBuild.groovy | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy index e066641ad..20370bdb6 100644 --- a/tests/jenkins/TestRetrievePreviousBuild.groovy +++ b/tests/jenkins/TestRetrievePreviousBuild.groovy @@ -33,10 +33,10 @@ class TestRetrievePreviousBuild extends BuildPipelineTest { void testRetrievePreviousBuild() { super.testPipeline('tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile') def shCommands = getCommands('sh', 'mkdir') - assertThat(shCommands, hasItems('mkdir -p tar && mv /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace')) + assertThat(shCommands, hasItems('mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace')) assertThat(shCommands, hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')) - assertThat(shCommands, hasItems('mkdir -p zip && mv /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace')) + assertThat(shCommands, hasItems('mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace')) assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))) } diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt index 816a411d7..891e3c188 100644 --- a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt @@ -5,7 +5,7 @@ RetrievePreviousBuild_Jenkinsfile.script(groovy.lang.Closure) RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, incremental=123}) retrievePreviousBuild.legacySCM(groovy.lang.Closure) - retrievePreviousBuild.library({identifier=jenkins@jenkins_incremental, retriever=null}) + retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml}) InputManifest.asBoolean() retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) @@ -15,12 +15,12 @@ downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true}) - retrievePreviousBuild.sh(mkdir -p tar && mv /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace) + retrievePreviousBuild.sh(mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace) retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.) retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/) RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, incremental=1234}) retrievePreviousBuild.legacySCM(groovy.lang.Closure) - retrievePreviousBuild.library({identifier=jenkins@jenkins_incremental, retriever=null}) + retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) retrievePreviousBuild.readYaml({file=tests/data/opensearch-dashboards-input-2.12.0.yml}) InputManifest.asBoolean() retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) @@ -30,4 +30,4 @@ downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}) - retrievePreviousBuild.sh(mkdir -p zip && mv /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace) + retrievePreviousBuild.sh(mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace) diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index b66376f38..b8e53f1bd 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -35,6 +35,6 @@ void call(Map args = [:]) { args.snapshot ? '--snapshot' : null, args.lock ? '--lock' : null, args.continueOnError ? '--continue-on-error' : null, - incremental_enabled ? '--incremental': null + incremental_enabled ? '--incremental' : null ] - null).join(' ')) } diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index 5303e4f73..ec6c5860a 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -15,7 +15,7 @@ @param args.incremental - String value contains the build number of previous build or the latest. */ void call(Map args = [:]) { - def lib = library(identifier: 'jenkins@jenkins_incremental', retriever: legacySCM(scm)) + def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) def DISTRIBUTION_JOB_NAME = "${JOB_NAME}" @@ -58,7 +58,7 @@ void call(Map args = [:]) { force: true, ) } - sh("mkdir -p ${distribution} && mv ${prefixPath}/${artifactPath} ${WORKSPACE}") + sh("mkdir -p ${distribution} && mv -v ${prefixPath}/${artifactPath} ${WORKSPACE}") if (inputManifestObj.build.getFilename().equals("opensearch")) { echo("Setting up Maven Local for OpenSearch build.") sh("mkdir -p ~/.m2/repository/org/ && cp -r ${distribution}/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/") From a57dbf88b00e51d8029179d764400840b870c74e Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Tue, 16 Jan 2024 10:24:13 -0800 Subject: [PATCH 4/9] Update echo message Signed-off-by: Zelin Hao --- vars/buildManifest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index b8e53f1bd..2e93a5577 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ - /** Library to call build.sh script (builds OpenSearch and OpenSearch Dashboards distribution) + /** Library to download previous build artifacts used for incremental build. @param Map args = [:] args A map of the following parameters @param args.inputManifest - Relative path to input manifest containing all the components to build. @param args.distribution - Type of distribution to build. Defaults to null. @@ -21,7 +21,7 @@ void call(Map args = [:]) { boolean incremental_enabled = args.incremental != null && !args.incremental.isEmpty() && !args.incremental.equalsIgnoreCase("false") if (incremental_enabled) { - echo("Start retreiving library") + echo("Incremental build enabled! Retrieving previous build library") retrievePreviousBuild(args) } From a3fd33baf49d242ac8cfff18bde9a62990e7917c Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Tue, 16 Jan 2024 13:40:17 -0800 Subject: [PATCH 5/9] Update index.json using curl Signed-off-by: Zelin Hao --- vars/retrievePreviousBuild.groovy | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index ec6c5860a..831708390 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -28,20 +28,11 @@ void call(Map args = [:]) { def DISTRIBUTION_BUILD_NUMBER if (args.incremental.equalsIgnoreCase("latest")) { - withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { - downloadFromS3( - assumedRoleName: "opensearch-bundle", - roleAccountNumberCred: "jenkins-aws-account-public", - downloadPath: "${DISTRIBUTION_JOB_NAME}/${revision}/index.json", - bucketName: "${ARTIFACT_BUCKET_NAME}", - localPath: "${prefixPath}/index.json", - force: true, - ) - } DISTRIBUTION_BUILD_NUMBER = sh( - script: "cat ${prefixPath}/index.json | jq -r \".latest\"", + script: "curl -sL https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\"", returnStdout: true ).trim() + //Once we have new index.json, URL will be changed to: https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${revision}/index/${platform}/${architecture}/${distribution}/index.json } else { DISTRIBUTION_BUILD_NUMBER = args.incremental } From d4edbe94db301b292e56d2bef97c55d998da8558 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 17 Jan 2024 13:30:39 -0800 Subject: [PATCH 6/9] Update description Signed-off-by: Zelin Hao --- vars/buildManifest.groovy | 2 +- vars/retrievePreviousBuild.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index 2e93a5577..19317d34a 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ - /** Library to download previous build artifacts used for incremental build. + /** Library to call build.sh script (builds OpenSearch and OpenSearch Dashboards distribution) @param Map args = [:] args A map of the following parameters @param args.inputManifest - Relative path to input manifest containing all the components to build. @param args.distribution - Type of distribution to build. Defaults to null. diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index 831708390..4bdc97b15 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -/** Library to call build.sh script (builds OpenSearch and OpenSearch Dashboards distribution) +/** Library to download previous build artifacts used for incremental build. @param Map args = [:] args A map of the following parameters @param args.inputManifest - Relative path to input manifest containing all the components to build. @param args.platform - Platform of previous build to retrieve. From d9b1d369c8e53194fa2feb8e8ac1353d457eca42 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 17 Jan 2024 14:53:33 -0800 Subject: [PATCH 7/9] Make parameters more generic Signed-off-by: Zelin Hao --- .../jobs/RetrievePreviousBuild_Jenkinsfile | 4 ++-- .../RetrievePreviousBuild_Jenkinsfile.txt | 4 ++-- .../RetrievePreviousBuildLibTester.groovy | 10 +++++----- vars/buildManifest.groovy | 5 +++-- vars/retrievePreviousBuild.groovy | 19 ++++++++++--------- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile index 3fd6d55cc..a2d334119 100644 --- a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile @@ -18,14 +18,14 @@ pipeline { distribution: "tar", architecture: "x64", platform: "linux", - incremental: "123" + previousBuildId: "123" ) retrievePreviousBuild( inputManifest: "tests/data/opensearch-dashboards-input-2.12.0.yml", distribution: "zip", architecture: "x64", platform: "windows", - incremental: "1234" + previousBuildId: "1234" ) } } diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt index 891e3c188..4cd1d6e23 100644 --- a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt @@ -3,7 +3,7 @@ RetrievePreviousBuild_Jenkinsfile.echo(Executing on agent [label:none]) RetrievePreviousBuild_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure) RetrievePreviousBuild_Jenkinsfile.script(groovy.lang.Closure) - RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, incremental=123}) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, previousBuildId=123}) retrievePreviousBuild.legacySCM(groovy.lang.Closure) retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml}) @@ -18,7 +18,7 @@ retrievePreviousBuild.sh(mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace) retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.) retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/) - RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, incremental=1234}) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, previousBuildId=1234}) retrievePreviousBuild.legacySCM(groovy.lang.Closure) retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) retrievePreviousBuild.readYaml({file=tests/data/opensearch-dashboards-input-2.12.0.yml}) diff --git a/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy index 6f016b40d..ac3303e2e 100644 --- a/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy +++ b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy @@ -15,14 +15,14 @@ class RetrievePreviousBuildLibTester extends LibFunctionTester { private String distribution = 'tar' private String platform = 'linux' private String architecture = 'tar' - private String incremental = 'latest' + private String previousBuildId = 'latest' - public RetrievePreviousBuildLibTester(String inputManifestPath, String platform, String architecture, String distribution, String incremental){ + public RetrievePreviousBuildLibTester(String inputManifestPath, String platform, String architecture, String distribution, String previousBuildId){ this.inputManifestPath = inputManifestPath this.platform = platform this.architecture = architecture this.distribution = distribution - this.incremental = incremental + this.previousBuildId = previousBuildId } @Override @@ -35,7 +35,7 @@ class RetrievePreviousBuildLibTester extends LibFunctionTester { assertThat(call.args.platform.first(), notNullValue()) assertThat(call.args.architecture.first(), notNullValue()) assertThat(call.args.distribution.first(), notNullValue()) - assertThat(call.args.incremental.first(), notNullValue()) + assertThat(call.args.previousBuildId.first(), notNullValue()) } @Override @@ -44,7 +44,7 @@ class RetrievePreviousBuildLibTester extends LibFunctionTester { && call.args.platform.first().equals(this.platform) && call.args.architecture.first().equals(this.architecture) && call.args.distribution.first().equals(this.distribution) - && call.args.incremental.first().equals(this.incremental) + && call.args.previousBuildId.first().equals(this.previousBuildId) } @Override diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index 19317d34a..110e24391 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -16,12 +16,13 @@ @param args.snapshot - Boolean value. Defaults to null. @param args.lock - Generate a stable reference manifest. Defaults to null. @param args.continueOnError - Do not fail the distribution build on any plugin component failure. Defaults to null + @param args.incremental - Boolean value to enable incremental build. */ void call(Map args = [:]) { - boolean incremental_enabled = args.incremental != null && !args.incremental.isEmpty() && !args.incremental.equalsIgnoreCase("false") + boolean incremental_enabled = args.incremental != null && args.incremental if (incremental_enabled) { - echo("Incremental build enabled! Retrieving previous build library") + echo("Incremental build enabled! Retrieving previous build library.") retrievePreviousBuild(args) } diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index 4bdc97b15..b37728aff 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -6,20 +6,21 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -/** Library to download previous build artifacts used for incremental build. +/** Library to download previous build artifacts into the workspace. @param Map args = [:] args A map of the following parameters @param args.inputManifest - Relative path to input manifest containing all the components to build. + @param args.jobName - Relative path to input manifest containing all the components to build. @param args.platform - Platform of previous build to retrieve. @param args.architecture - Architecture of previous build to retrieve. @param args.distribution - Distribution of previous build to retrieve. - @param args.incremental - String value contains the build number of previous build or the latest. + @param args.previousBuildId - Build id of previous build for incremental build. Defaults to latest. */ void call(Map args = [:]) { def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) - def DISTRIBUTION_JOB_NAME = "${JOB_NAME}" - def revision = inputManifestObj.build.version + def DISTRIBUTION_JOB_NAME = args.jobName ? args.jobName : "${JOB_NAME}" + def version = inputManifestObj.build.version def DISTRIBUTION_PLATFORM = args.platform def DISTRIBUTION_ARCHITECTURE = args.architecture @@ -27,17 +28,17 @@ void call(Map args = [:]) { def prefixPath = "${WORKSPACE}/download" def DISTRIBUTION_BUILD_NUMBER - if (args.incremental.equalsIgnoreCase("latest")) { + if (args.previousBuildId.equalsIgnoreCase("latest")) { DISTRIBUTION_BUILD_NUMBER = sh( - script: "curl -sL https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\"", + script: "curl -sL https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"", returnStdout: true ).trim() - //Once we have new index.json, URL will be changed to: https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${revision}/index/${platform}/${architecture}/${distribution}/index.json + //Once we have new index.json, URL will be changed to: https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${version}/index/${platform}/${architecture}/${distribution}/index.json } else { - DISTRIBUTION_BUILD_NUMBER = args.incremental + DISTRIBUTION_BUILD_NUMBER = args.previousBuildId } - def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" + def artifactPath = "${DISTRIBUTION_JOB_NAME}/${version}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { downloadFromS3( From 6e106d9205780b703b7be0dd6d00eedef79e4fab Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 17 Jan 2024 16:33:34 -0800 Subject: [PATCH 8/9] Default previous build id Signed-off-by: Zelin Hao --- vars/retrievePreviousBuild.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index b37728aff..29572d8fd 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -19,23 +19,24 @@ void call(Map args = [:]) { def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) - def DISTRIBUTION_JOB_NAME = args.jobName ? args.jobName : "${JOB_NAME}" + def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}" def version = inputManifestObj.build.version def DISTRIBUTION_PLATFORM = args.platform def DISTRIBUTION_ARCHITECTURE = args.architecture def distribution = args.distribution def prefixPath = "${WORKSPACE}/download" + def previousBuildId = args.previousBuildId ?: "latest" def DISTRIBUTION_BUILD_NUMBER - if (args.previousBuildId.equalsIgnoreCase("latest")) { + if (previousBuildId.equalsIgnoreCase("latest")) { DISTRIBUTION_BUILD_NUMBER = sh( script: "curl -sL https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"", returnStdout: true ).trim() //Once we have new index.json, URL will be changed to: https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${version}/index/${platform}/${architecture}/${distribution}/index.json } else { - DISTRIBUTION_BUILD_NUMBER = args.previousBuildId + DISTRIBUTION_BUILD_NUMBER = previousBuildId } def artifactPath = "${DISTRIBUTION_JOB_NAME}/${version}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" From 74182be89d47fccf88b9d292a03173f124de718b Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 17 Jan 2024 16:45:47 -0800 Subject: [PATCH 9/9] Add assert for s3download Signed-off-by: Zelin Hao --- tests/jenkins/TestRetrievePreviousBuild.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy index 20370bdb6..4daeec7e2 100644 --- a/tests/jenkins/TestRetrievePreviousBuild.groovy +++ b/tests/jenkins/TestRetrievePreviousBuild.groovy @@ -39,6 +39,15 @@ class TestRetrievePreviousBuild extends BuildPipelineTest { assertThat(shCommands, hasItems('mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace')) assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))) + def s3DownloadCommands = getCommands('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands, hasItems( + "{file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true}".toString())) + + assertThat(s3DownloadCommands, hasItems( + "{file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}".toString())) } def getCommands(method, text) {