[Backport to 6.x] Refactor retrievePreviousBuild and buildManifest li…

…brary (#396) (#404) Signed-off-by: Zelin Hao <zelinhao@amazon.com>
opensearch-project · Mar 29, 2024 · 2c93df9 · 2c93df9
1 parent 1bc08b1
commit 2c93df9
Show file tree

Hide file tree

Showing 14 changed files with 168 additions and 34 deletions.
diff --git a/tests/jenkins/TestbuildManifestVar.groovy b/tests/jenkins/TestbuildManifestVar.groovy
@@ -8,13 +8,25 @@
  */
 package jenkins.tests
 
-import org.junit.Test
+import org.junit.*
 import jenkins.tests.BuildPipelineTest
 import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
 import static org.hamcrest.CoreMatchers.hasItems
 import static org.hamcrest.MatcherAssert.assertThat
 
 class TestbuildManifestVar extends BuildPipelineTest {
+    @Before
+    void setUp() {
+        super.setUp()
+
+        binding.setVariable('JOB_NAME', 'dummy-build-job')
+        binding.setVariable('PUBLIC_ARTIFACT_URL', 'dummy-url')
+        helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
+        helper.registerAllowedMethod("s3Download", [Map])
+    }
 
     @Test
     void testbuildManifestWithSnapshotContinueOnError() {
@@ -40,6 +52,13 @@ class TestbuildManifestVar extends BuildPipelineTest {
         assertThat(shCommands, hasItems('./build.sh tests/data/opensearch-2.0.0.yml -d rpm --component common-utils --snapshot --lock'))
     }
 
+    @Test
+    void testbuildManifestWithIncrementalBuild() {
+        super.testPipeline('tests/jenkins/jobs/BuildShManifestIncremental_Jenkinsfile')
+        def shCommands = getCommands('sh', 'build.sh')
+        assertThat(shCommands, hasItems('./build.sh tests/data/opensearch-input-2.12.0.yml -d tar -p linux -a x64 --incremental'))
+    }
+
     def getCommands(method, text) {
         def shCommands = helper.callStack.findAll { call ->
             call.methodName == method

diff --git a/tests/jenkins/jobs/BuildArchive_Jenkinsfile.txt b/tests/jenkins/jobs/BuildArchive_Jenkinsfile.txt
@@ -7,6 +7,10 @@
                   buildArchive.legacySCM(groovy.lang.Closure)
                   buildArchive.library({identifier=jenkins@6.4.0, retriever=null})
                   buildArchive.buildManifest({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, distribution=tar, stashName=test-stash-tar})
+                     buildManifest.legacySCM(groovy.lang.Closure)
+                     buildManifest.library({identifier=jenkins@main, retriever=null})
+                     buildManifest.readYaml({file=tests/data/opensearch-1.3.0.yml})
+                     InputManifest.asBoolean()
                      buildManifest.sh(./build.sh tests/data/opensearch-1.3.0.yml -d tar -p linux -a x64)
                   buildArchive.echo(Stashing builds to assemble later with name: test-stash-tar)
                   buildArchive.stash({includes=tar/builds/**, name=test-stash-tar})

diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt
@@ -9,6 +9,10 @@
                   buildAssembleUpload.readYaml({file=tests/data/opensearch-1.3.0.yml})
                   InputManifest.asBoolean()
                   buildAssembleUpload.buildManifest({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, distribution=tar})
+                     buildManifest.legacySCM(groovy.lang.Closure)
+                     buildManifest.library({identifier=jenkins@main, retriever=null})
+                     buildManifest.readYaml({file=tests/data/opensearch-1.3.0.yml})
+                     InputManifest.asBoolean()
                      buildManifest.sh(./build.sh tests/data/opensearch-1.3.0.yml -d tar -p linux -a x64)
                   buildAssembleUpload.readYaml({file=tar/builds/opensearch/manifest.yml})
                   BuildManifest.asBoolean()

diff --git a/tests/jenkins/jobs/BuildShManifestIncremental_Jenkinsfile b/tests/jenkins/jobs/BuildShManifestIncremental_Jenkinsfile
@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('Test build manifest jenkins var') {
+            steps {
+                script {
+                    buildManifest(
+                        inputManifest: "tests/data/opensearch-input-2.12.0.yml",
+                        distribution: "tar",
+                        platform: "linux",
+                        architecture: "x64",
+                        incremental: true,
+                        previousBuildId: "latest"
+                    )
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/jenkins/jobs/BuildShManifestIncremental_Jenkinsfile.txt b/tests/jenkins/jobs/BuildShManifestIncremental_Jenkinsfile.txt
@@ -0,0 +1,35 @@
+   BuildShManifestIncremental_Jenkinsfile.run()
+      BuildShManifestIncremental_Jenkinsfile.pipeline(groovy.lang.Closure)
+         BuildShManifestIncremental_Jenkinsfile.echo(Executing on agent [label:none])
+         BuildShManifestIncremental_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure)
+            BuildShManifestIncremental_Jenkinsfile.script(groovy.lang.Closure)
+               BuildShManifestIncremental_Jenkinsfile.buildManifest({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, platform=linux, architecture=x64, incremental=true, previousBuildId=latest})
+                  buildManifest.legacySCM(groovy.lang.Closure)
+                  buildManifest.library({identifier=jenkins@main, retriever=null})
+                  buildManifest.readYaml({file=tests/data/opensearch-input-2.12.0.yml})
+                  InputManifest.asBoolean()
+                  buildManifest.sh({script=curl -sL dummy-url/dummy-build-job/2.12.0/index/linux/x64/tar/index.json | jq -r ".latest" > /dev/null 2>&1, returnStatus=true})
+                  buildManifest.sh({script=curl -sL dummy-url/dummy-build-job/2.12.0/index.json | jq -r ".latest" > /dev/null 2>&1, returnStatus=true})
+                  buildManifest.echo(Use new URL path for the latest index.)
+                  buildManifest.sh({script=curl -sL dummy-url/dummy-build-job/2.12.0/index/linux/x64/tar/index.json | jq -r ".latest", returnStdout=true})
+                  buildManifest.echo(Incremental build enabled! Retrieving previous build library.)
+                  buildManifest.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, platform=linux, architecture=x64, incremental=true, previousBuildId=latest, distributionBuildNumber=bbb
+ccc})
+                     retrievePreviousBuild.legacySCM(groovy.lang.Closure)
+                     retrievePreviousBuild.library({identifier=jenkins@6.4.0, retriever=null})
+                     retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml})
+                     InputManifest.asBoolean()
+                     retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                     retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                        retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy-build-job/2.12.0/bbb
+ccc/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true})
+                           downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER})
+                           downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
+                              downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
+                                 downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy-build-job/2.12.0/bbb
+ccc/linux/x64/tar/, force=true})
+                     retrievePreviousBuild.sh(rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy-build-job/2.12.0/bbb
+ccc/linux/x64/tar/* /tmp/workspace/tar)
+                     retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.)
+                     retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/)
+                  buildManifest.sh(./build.sh tests/data/opensearch-input-2.12.0.yml -d tar -p linux -a x64 --incremental)
diff --git a/tests/jenkins/jobs/BuildShManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildShManifest_Jenkinsfile.txt
@@ -4,8 +4,20 @@
          BuildShManifest_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure)
             BuildShManifest_Jenkinsfile.script(groovy.lang.Closure)
                BuildShManifest_Jenkinsfile.buildManifest({inputManifest=tests/data/opensearch-2.0.0.yml, distribution=tar, snapshot=true, continueOnError=true})
+                  buildManifest.legacySCM(groovy.lang.Closure)
+                  buildManifest.library({identifier=jenkins@main, retriever=null})
+                  buildManifest.readYaml({file=tests/data/opensearch-2.0.0.yml})
+                  InputManifest.asBoolean()
                   buildManifest.sh(./build.sh tests/data/opensearch-2.0.0.yml -d tar --snapshot --continue-on-error)
                BuildShManifest_Jenkinsfile.buildManifest({componentName=job-scheduler, inputManifest=tests/data/opensearch-2.0.0.yml, distribution=tar, snapshot=true})
+                  buildManifest.legacySCM(groovy.lang.Closure)
+                  buildManifest.library({identifier=jenkins@main, retriever=null})
+                  buildManifest.readYaml({file=tests/data/opensearch-2.0.0.yml})
+                  InputManifest.asBoolean()
                   buildManifest.sh(./build.sh tests/data/opensearch-2.0.0.yml -d tar --component job-scheduler --snapshot)
                BuildShManifest_Jenkinsfile.buildManifest({componentName=common-utils, inputManifest=tests/data/opensearch-2.0.0.yml, distribution=rpm, lock=true, snapshot=true})
+                  buildManifest.legacySCM(groovy.lang.Closure)
+                  buildManifest.library({identifier=jenkins@main, retriever=null})
+                  buildManifest.readYaml({file=tests/data/opensearch-2.0.0.yml})
+                  InputManifest.asBoolean()
                   buildManifest.sh(./build.sh tests/data/opensearch-2.0.0.yml -d rpm --component common-utils --snapshot --lock)
diff --git a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt
@@ -8,6 +8,10 @@
                   buildUploadManifestSHA.library({identifier=jenkins@6.4.0, retriever=null})
                   buildUploadManifestSHA.getManifestSHA({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
                      getManifestSHA.buildManifest({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true})
+                        buildManifest.legacySCM(groovy.lang.Closure)
+                        buildManifest.library({identifier=jenkins@main, retriever=null})
+                        buildManifest.readYaml({file=tests/data/opensearch-1.3.0.yml})
+                        InputManifest.asBoolean()
                         buildManifest.sh(./build.sh tests/data/opensearch-1.3.0.yml -p linux -a x64 --lock)
                      getManifestSHA.sha1(tests/data/opensearch-1.3.0.yml.lock)
                      getManifestSHA.echo(Manifest SHA: sha1)

diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt
@@ -5,6 +5,10 @@
             GetManifestSHA_Jenkinsfile.script(groovy.lang.Closure)
                GetManifestSHA_Jenkinsfile.getManifestSHA({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
                   getManifestSHA.buildManifest({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true})
+                     buildManifest.legacySCM(groovy.lang.Closure)
+                     buildManifest.library({identifier=jenkins@main, retriever=null})
+                     buildManifest.readYaml({file=tests/data/opensearch-1.3.0.yml})
+                     InputManifest.asBoolean()
                      buildManifest.sh(./build.sh tests/data/opensearch-1.3.0.yml -p linux -a x64 --lock)
                   getManifestSHA.sha1(tests/data/opensearch-1.3.0.yml.lock)
                   getManifestSHA.echo(Manifest SHA: sha1)

diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt
@@ -5,6 +5,10 @@
             GetManifestSHA_Jenkinsfile.script(groovy.lang.Closure)
                GetManifestSHA_Jenkinsfile.getManifestSHA({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
                   getManifestSHA.buildManifest({inputManifest=tests/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true})
+                     buildManifest.legacySCM(groovy.lang.Closure)
+                     buildManifest.library({identifier=jenkins@main, retriever=null})
+                     buildManifest.readYaml({file=tests/data/opensearch-1.3.0.yml})
+                     InputManifest.asBoolean()
                      buildManifest.sh(./build.sh tests/data/opensearch-1.3.0.yml -p linux -a x64 --lock)
                   getManifestSHA.sha1(tests/data/opensearch-1.3.0.yml.lock)
                   getManifestSHA.echo(Manifest SHA: sha1)

diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile
@@ -18,14 +18,14 @@ pipeline {
                         distribution: "tar",
                         architecture: "x64",
                         platform: "linux",
-                        previousBuildId: "123"
+                        distributionBuildNumber: "123"
                     )
                     retrievePreviousBuild(
                         inputManifest: "tests/data/opensearch-dashboards-input-2.12.0.yml",
                         distribution: "zip",
                         architecture: "x64",
                         platform: "windows",
-                        previousBuildId: "1234"
+                        distributionBuildNumber: "1234"
                     )
                 }
             }

diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt
@@ -3,7 +3,7 @@
          RetrievePreviousBuild_Jenkinsfile.echo(Executing on agent [label:none])
          RetrievePreviousBuild_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure)
             RetrievePreviousBuild_Jenkinsfile.script(groovy.lang.Closure)
-               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, previousBuildId=123})
+               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, distributionBuildNumber=123})
                   retrievePreviousBuild.legacySCM(groovy.lang.Closure)
                   retrievePreviousBuild.library({identifier=jenkins@6.4.0, retriever=null})
                   retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml})
@@ -18,7 +18,7 @@
                   retrievePreviousBuild.sh(rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar/* /tmp/workspace/tar)
                   retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.)
                   retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/)
-               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, previousBuildId=1234})
+               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, distributionBuildNumber=1234})
                   retrievePreviousBuild.legacySCM(groovy.lang.Closure)
                   retrievePreviousBuild.library({identifier=jenkins@6.4.0, retriever=null})
                   retrievePreviousBuild.readYaml({file=tests/data/opensearch-dashboards-input-2.12.0.yml})

diff --git a/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy
@@ -15,14 +15,14 @@ class RetrievePreviousBuildLibTester extends LibFunctionTester {
     private String distribution = 'tar'
     private String platform = 'linux'
     private String architecture = 'tar'
-    private String previousBuildId = 'latest'
+    private String distributionBuildNumber = 'latest'
 
-    public RetrievePreviousBuildLibTester(String inputManifestPath, String platform, String architecture, String distribution, String previousBuildId){
+    public RetrievePreviousBuildLibTester(String inputManifestPath, String platform, String architecture, String distribution, String distributionBuildNumber){
         this.inputManifestPath = inputManifestPath
         this.platform = platform
         this.architecture = architecture
         this.distribution = distribution
-        this.previousBuildId = previousBuildId
+        this.distributionBuildNumber = distributionBuildNumber
     }
 
     @Override
@@ -35,7 +35,7 @@ class RetrievePreviousBuildLibTester extends LibFunctionTester {
         assertThat(call.args.platform.first(), notNullValue())
         assertThat(call.args.architecture.first(), notNullValue())
         assertThat(call.args.distribution.first(), notNullValue())
-        assertThat(call.args.previousBuildId.first(), notNullValue())
+        assertThat(call.args.distributionBuildNumber.first(), notNullValue())
     }
 
     @Override
@@ -44,7 +44,7 @@ class RetrievePreviousBuildLibTester extends LibFunctionTester {
             && call.args.platform.first().equals(this.platform)
             && call.args.architecture.first().equals(this.architecture)
             && call.args.distribution.first().equals(this.distribution)
-            && call.args.previousBuildId.first().equals(this.previousBuildId)
+            && call.args.distributionBuildNumber.first().equals(this.distributionBuildNumber)
     }
 
     @Override

diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy
@@ -21,8 +21,50 @@
 void call(Map args = [:]) {
     boolean incremental_enabled = args.incremental != null && args.incremental
 
+    def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm))
+    def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest))
+
+    def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}"
+    def version = inputManifestObj.build.version
+
+    def DISTRIBUTION_PLATFORM = args.platform
+    def DISTRIBUTION_ARCHITECTURE = args.architecture
+    def distribution = args.distribution
+    def previousBuildId = args.previousBuildId ?: "latest"
+    def DISTRIBUTION_BUILD_NUMBER
+
+    if (incremental_enabled && previousBuildId.equalsIgnoreCase("latest")) {
+        def latestIndexStatus = sh (
+                script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1",
+                returnStatus: true
+        )
+        def latestIndexStatusOld = sh (
+                script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\" > /dev/null 2>&1",
+                returnStatus: true
+        )
+        if (latestIndexStatus == 0) {
+            echo("Use new URL path for the latest index.")
+            DISTRIBUTION_BUILD_NUMBER = sh(
+                    script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"",
+                    returnStdout: true
+            ).trim()
+        } else if (latestIndexStatusOld == 0) {
+            echo("Use old URL path for the latest index.")
+            DISTRIBUTION_BUILD_NUMBER = sh(
+                    script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"",
+                    returnStdout: true
+            ).trim()
+        } else {
+            echo("No latest build for ${version} is available. Building all components from the manifest.")
+            incremental_enabled = false
+        }
+    } else {
+        DISTRIBUTION_BUILD_NUMBER = previousBuildId
+    }
+
     if (incremental_enabled) {
         echo("Incremental build enabled! Retrieving previous build library.")
+        args.distributionBuildNumber = DISTRIBUTION_BUILD_NUMBER
         retrievePreviousBuild(args)
     }