Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update distribution build jenkins job with option for incremental build #4356

Merged
merged 9 commits into from
Jan 24, 2024
Merged

Update distribution build jenkins job with option for incremental build #4356

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b83d53f
Update build job
zelinh Jan 4, 2024
eb7f9cf
Add incremental parameter to distribution build jobs
zelinh Jan 16, 2024
e2494df
Merge remote-tracking branch 'upstream/main' into jenkins_incremental
zelinh Jan 16, 2024
67f4e3c
Update description for parameters
zelinh Jan 18, 2024
ed5c31f
Add tests to make sure the build ID
zelinh Jan 19, 2024
8f1b17c
Resolve id discrepancy when no component changes
zelinh Jan 23, 2024
5412d2d
Remove unnecessary logging
zelinh Jan 24, 2024
2f5bef3
Merge remote-tracking branch 'upstream/main' into jenkins_incremental
zelinh Jan 24, 2024
8d2866d
Nit fix
zelinh Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions jenkins/opensearch-dashboards/distribution-build.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* compatible open source license.
*/

lib = library(identifier: 'jenkins@5.12.0', retriever: modernSCM([
lib = library(identifier: 'jenkins@6.1.0', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))
Expand Down Expand Up @@ -91,6 +91,17 @@ pipeline {
description: 'Continue building the distribution even if a one or more component fails',
defaultValue: true
)
booleanParam(
name: 'INCREMENTAL',
description: 'Whether to trigger incremental build. Defaults to false.',
defaultValue: false
)
string(
name: 'PREVIOUS_BUILD_ID',
description: 'The build ID used to download previous build artifacts. Defaults to latest.',
defaultValue: 'latest',
trim: true
)
}
stages {
stage('verify-parameters') {
Expand Down Expand Up @@ -178,7 +189,9 @@ pipeline {
platform: 'linux',
architecture: 'x64',
distribution: 'tar',
continueOnError: params.CONTINUE_ON_ERROR
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, "latest"), "builds", "opensearch", "manifest.yml"].join("/")
Expand Down Expand Up @@ -272,6 +285,8 @@ pipeline {
architecture: 'x64',
distribution: 'rpm',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -374,6 +389,8 @@ pipeline {
architecture: 'x64',
distribution: 'deb',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -468,6 +485,8 @@ pipeline {
architecture: 'arm64',
distribution: 'tar',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-arm64-tar-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -598,6 +617,8 @@ pipeline {
architecture: 'arm64',
distribution: 'rpm',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -703,6 +724,8 @@ pipeline {
architecture: 'arm64',
distribution: 'deb',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -792,7 +815,9 @@ pipeline {
platform: 'windows',
architecture: 'x64',
distribution: "zip",
continueOnError: params.CONTINUE_ON_ERROR
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
Expand Down Expand Up @@ -965,4 +990,4 @@ def markStageUnstableIfPluginsFailedToBuild() {
if (stageLogs.any{e -> e.contains('Failed plugins are')}) {
unstable('Some plugins failed to build. See the ./build.sh step for logs and more details')
}
}
}
37 changes: 31 additions & 6 deletions jenkins/opensearch/distribution-build.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* compatible open source license.
*/

lib = library(identifier: 'jenkins@5.12.0', retriever: modernSCM([
lib = library(identifier: 'jenkins@6.1.0', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))
Expand Down Expand Up @@ -84,6 +84,17 @@ pipeline {
description: 'Continue building the distribution even if a one or more component fails',
defaultValue: true
)
booleanParam(
name: 'INCREMENTAL',
description: 'Whether to trigger incremental build. Defaults to false.',
defaultValue: false
)
string(
name: 'PREVIOUS_BUILD_ID',
description: 'The build ID used to download previous build artifacts. Defaults to latest.',
defaultValue: 'latest',
trim: true
)
}
stages {
stage('verify-parameters') {
Expand Down Expand Up @@ -169,7 +180,9 @@ pipeline {
platform: 'linux',
architecture: 'x64',
distribution: 'tar',
continueOnError: params.CONTINUE_ON_ERROR
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
Expand Down Expand Up @@ -256,6 +269,8 @@ pipeline {
architecture: 'x64',
distribution: 'rpm',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand All @@ -266,7 +281,7 @@ pipeline {
if (params.CONTINUE_ON_ERROR) {
markStageUnstableIfPluginsFailedToBuild()
}
postCleanup()
postCleanup()
}
}
}
Expand Down Expand Up @@ -353,6 +368,8 @@ pipeline {
architecture: 'x64',
distribution: 'deb',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand All @@ -363,7 +380,7 @@ pipeline {
if (params.CONTINUE_ON_ERROR) {
markStageUnstableIfPluginsFailedToBuild()
}
postCleanup()
postCleanup()
}
}
}
Expand Down Expand Up @@ -437,7 +454,9 @@ pipeline {
platform: 'linux',
architecture: 'arm64',
distribution: 'tar',
continueOnError: params.CONTINUE_ON_ERROR
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
Expand Down Expand Up @@ -508,6 +527,8 @@ pipeline {
architecture: 'arm64',
distribution: 'rpm',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -605,6 +626,8 @@ pipeline {
architecture: 'arm64',
distribution: 'deb',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID,
stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}"
)
}
Expand Down Expand Up @@ -689,7 +712,9 @@ pipeline {
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
continueOnError: params.CONTINUE_ON_ERROR
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
Expand Down
34 changes: 34 additions & 0 deletions manifests/2.12.0/opensearch-2.12.0.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- job-scheduler
peterzhuamazon marked this conversation as resolved.
Show resolved Hide resolved
- name: cross-cluster-replication
repository: https://github.com/opensearch-project/cross-cluster-replication.git
ref: 2.x
Expand All @@ -65,6 +67,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- common-utils
- name: ml-commons
repository: https://github.com/opensearch-project/ml-commons.git
ref: 2.x
Expand All @@ -74,6 +78,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: opensearch-ml-plugin
depends_on:
- common-utils
- name: neural-search
repository: https://github.com/opensearch-project/neural-search.git
ref: 2.x
Expand All @@ -83,6 +89,9 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- ml-commons
- k-NN
- name: notifications-core
repository: https://github.com/opensearch-project/notifications.git
ref: 2.x
Expand All @@ -93,6 +102,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: opensearch-notifications-core
depends_on:
- common-utils
- name: notifications
repository: https://github.com/opensearch-project/notifications.git
ref: 2.x
Expand All @@ -103,6 +114,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: notifications
depends_on:
- common-utils
- name: opensearch-observability
repository: https://github.com/opensearch-project/observability.git
ref: 2.x
Expand All @@ -112,6 +125,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- common-utils
- name: opensearch-reports
repository: https://github.com/opensearch-project/reporting.git
ref: 2.x
Expand All @@ -121,6 +136,9 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- common-utils
- job-scheduler
- name: sql
repository: https://github.com/opensearch-project/sql.git
ref: 2.x
Expand All @@ -130,6 +148,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: opensearch-sql-plugin
depends_on:
- ml-commons
- name: asynchronous-search
repository: https://github.com/opensearch-project/asynchronous-search.git
ref: 2.x
Expand All @@ -139,6 +159,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- common-utils
- name: anomaly-detection
repository: https://github.com/opensearch-project/anomaly-detection.git
ref: 2.x
Expand All @@ -148,6 +170,9 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- common-utils
- job-scheduler
- name: alerting
repository: https://github.com/opensearch-project/alerting.git
ref: 2.x
Expand All @@ -157,6 +182,8 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: alerting
depends_on:
- common-utils
- name: security-analytics
repository: https://github.com/opensearch-project/security-analytics.git
ref: 2.x
Expand All @@ -165,6 +192,8 @@ components:
- windows
checks:
- gradle:properties:version
depends_on:
- common-utils
- name: index-management
repository: https://github.com/opensearch-project/index-management.git
ref: 2.x
Expand All @@ -173,6 +202,9 @@ components:
- windows
checks:
- gradle:properties:version
depends_on:
- common-utils
- job-scheduler
- name: performance-analyzer
repository: https://github.com/opensearch-project/performance-analyzer.git
ref: 2.x
Expand All @@ -199,3 +231,5 @@ components:
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
depends_on:
- common-utils
1 change: 1 addition & 0 deletions src/build_workflow/build_recorder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def __init__(self, target: BuildTarget, build_manifest: BuildManifest = None) ->

if build_manifest:
self.data = build_manifest.__to_dict__()
self.data["build"]["id"] = target.build_id
zelinh marked this conversation as resolved.
Show resolved Hide resolved
for component in build_manifest.components.select():
self.components_hash[component.name] = component.__to_dict__()
else:
Expand Down
16 changes: 9 additions & 7 deletions src/run_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import logging
import os
import sys
import uuid

from build_workflow.build_args import BuildArgs
from build_workflow.build_incremental import BuildIncremental
Expand Down Expand Up @@ -49,19 +50,20 @@ def main() -> int:
buildIncremental = BuildIncremental(manifest, args.distribution)
list_of_updated_plugins = buildIncremental.commits_diff(manifest)
components = buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)
if not components:
logging.info("No commit difference found between any components. Skipping the build")
return 0

logging.info(f"Plugins for incremental build: {components}")

build_manifest_path = os.path.join(args.distribution, "builds", manifest.build.filename, "manifest.yml")
if not os.path.exists(build_manifest_path):
logging.error(f"Previous build manifest missing at path: {build_manifest_path}")
build_manifest = BuildManifest.from_path(build_manifest_path)

logging.info(f"Build {components} incrementally.")
if not components:
logging.info("No commit difference found between any components. Skipping the build.")
build_manifest.build.id = os.getenv("BUILD_NUMBER") or uuid.uuid4().hex
build_manifest.to_file(build_manifest_path)
logging.info(f"Updating the build ID of build manifest to {build_manifest.build.id}.")
zelinh marked this conversation as resolved.
Show resolved Hide resolved
return 0

build_manifest = BuildManifest.from_path(build_manifest_path)
logging.info(f"Plugins for incremental build: {components}")

with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
logging.info(f"Building in {work_dir.name}")
Expand Down
Loading
Loading