[BACKPORT 6.x] add compare logic to opensearch build libraries (#478)

build.gradle

@@ -127,7 +127,7 @@ jacocoTestReport {
     }
 }
 
-String version = '6.8.0'
+String version = '6.8.1'
 
 task updateVersion {
     doLast {

tests/jenkins/TestRunCompareTestScript.groovy

@@ -0,0 +1,74 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.equalTo
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+
+class TestRunCompareTestScript extends BuildPipelineTest {
+    @Before
+    void setUp() {
+        this.registerLibTester(new RunCompareTestScriptLibTester(
+                'compare',
+                '12345',
+                '54321',
+                'csv',
+                'left',
+                '/results/final_results',
+                'true',
+                'compare-tester',
+                'tests/data/opensearch-1.3.0-bundle.yml'
+        ))
+        super.setUp()
+    }
+
+    @Test
+    public void testRunCompareTestScript_PipelineSingleNode() {
+        super.testPipeline("tests/jenkins/jobs/CompareTest_Jenkinsfile")
+    }
+
+    @Test 
+    void testRunCompareTestScript_verifyScriptExecutionSingleNode() {
+        runScript("tests/jenkins/jobs/CompareTest_Jenkinsfile")
+
+        def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
+            shCommand -> shCommand.contains('./test.sh')
+        }
+
+        assertThat(testScriptCommands.size(), equalTo(2))
+        assertThat(testScriptCommands, hasItem(
+                "set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester"
+        ))
+        assertThat(testScriptCommands, hasItem(
+                "set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester --results-format=csv --results-numbers-align=left --results-file=/results/final_results --show-in-results=true"
+        ))
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+                collect {
+                    call ->
+                        callArgsToString(call)
+                }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+
+}

tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt

@@ -6,7 +6,7 @@
                CompareBenchmarkRun_Jenkinsfile.getCompareBenchmarkIds({baselineClusterConfig=test-baseline-config, distributionVersion=3.0.0, workload=big5, pullRequestNumber=12345})
                   getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER})
                   getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD})
-                  getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-endpoint, variable=DATASTORE_ENDPOINT})
+                  getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-nlb-endpoint, variable=DATASTORE_ENDPOINT})
                   getCompareBenchmarkIds.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD, DATASTORE_ENDPOINT], groovy.lang.Closure)
                      getCompareBenchmarkIds.sh({script=
               curl -X POST "https://DATASTORE_ENDPOINT/benchmark-results-*/_search" -ku DATASTORE_USER:DATASTORE_PASSWORD -H 'Content-Type: application/json' -d '{

tests/jenkins/jobs/CompareTest_Jenkinsfile

@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('compare-test') {
+            parallel {
+                stage('test-with-all-args') {
+                    steps {
+                        script {
+                            runBenchmarkTestScript(command: COMMAND,
+                            baseline: BASELINE,
+                            contender: CONTENDER,
+                            results_format: RESULTS_FORMAT,
+                            results_numbers_align: RESULTS_NUMBERS_ALIGN,
+                            results_file: RESULTS_FILE,
+                            show_in_results: SHOW_IN_RESULTS,
+                            suffix: SUFFIX,
+                            bundleManifest: BUNDLE_MANIFEST
+                            )
+                        }
+                    }
+                }
+                stage('test-with-base-args') {
+                    steps {
+                        script {
+                            runBenchmarkTestScript(command: COMMAND,
+                            baseline: BASELINE,
+                            contender: CONTENDER,
+                            suffix: SUFFIX,
+                            bundleManifest: BUNDLE_MANIFEST
+                            )
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

tests/jenkins/jobs/CompareTest_Jenkinsfile.txt

@@ -0,0 +1,39 @@
+   CompareTest_Jenkinsfile.run()
+      CompareTest_Jenkinsfile.pipeline(groovy.lang.Closure)
+         CompareTest_Jenkinsfile.echo(Executing on agent [label:none])
+         CompareTest_Jenkinsfile.stage(test-with-all-args, groovy.lang.Closure)
+            CompareTest_Jenkinsfile.script(groovy.lang.Closure)
+               CompareTest_Jenkinsfile.runBenchmarkTestScript({command=compare, baseline=12345, contender=54321, results_format=csv, results_numbers_align=left, results_file=/results/final_results, show_in_results=true, suffix=compare-tester, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml})
+                  runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
+                  runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
+                  runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
+                  BuildManifest.asBoolean()
+                  runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true})
+                  runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER})
+                  runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD})
+                  runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
+                     runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
+                     runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
+                  runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester --results-format=csv --results-numbers-align=left --results-file=/results/final_results --show-in-results=true)
+         CompareTest_Jenkinsfile.stage(test-with-base-args, groovy.lang.Closure)
+            CompareTest_Jenkinsfile.script(groovy.lang.Closure)
+               CompareTest_Jenkinsfile.runBenchmarkTestScript({command=compare, baseline=12345, contender=54321, suffix=compare-tester, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml})
+                  runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
+                  runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
+                  runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
+                  BuildManifest.asBoolean()
+                  runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true})
+                  runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER})
+                  runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD})
+                  runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
+                     runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
+                     runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
+                  runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester)

tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy

@@ -41,10 +41,10 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{
     private String captureSegmentReplicationStat
     private String telemetryParams
 
-    public RunBenchmarkTestScriptLibTester(command, bundleManifest, distributionUrl, distributionVersion, insecure, workload, singleNode, minDistribution, use50PercentHeap,
-                                           enableRemoteStore, managerNodeCount, dataNodeCount, dataInstanceType, userTag, workloadParams,
-                                           testProcedure, excludeTasks, includeTasks,
-                                           additionalConfig, captureNodeStat, captureSegmentReplicationStat, telemetryParams){
+    public RunBenchmarkTestScriptLibTester(command, bundleManifest, distributionUrl, distributionVersion, insecure, workload, 
+                                           singleNode, minDistribution, use50PercentHeap, enableRemoteStore, managerNodeCount, 
+                                           dataNodeCount, dataInstanceType, userTag, workloadParams, testProcedure, excludeTasks, 
+                                           includeTasks, additionalConfig, captureNodeStat, captureSegmentReplicationStat, telemetryParams){
         this.command = command
         this.bundleManifest = bundleManifest
         this.distributionUrl = distributionUrl

tests/jenkins/lib-testers/RunCompareTestScriptLibTest.groovy

@@ -0,0 +1,82 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+import static org.hamcrest.CoreMatchers.notNullValue
+import static org.hamcrest.MatcherAssert.assertThat
+
+class RunCompareTestScriptLibTester extends LibFunctionTester{
+
+    private String command
+    private String baseline
+    private String contender
+    private String results_format
+    private String results_numbers_align
+    private String results_file
+    private String show_in_results
+    private String suffix
+    private String bundleManifest
+
+    public RunCompareTestScriptLibTester(command, baseline, contender, results_format,
+                                        results_numbers_align, results_file, show_in_results,
+                                        suffix, bundleManifest){
+        this.command = command
+        this.baseline = baseline
+        this.contender = contender
+        this.results_format = results_format
+        this.results_numbers_align = results_numbers_align
+        this.results_file = results_file
+        this.show_in_results = show_in_results
+        this.suffix = suffix
+        this.bundleManifest = bundleManifest
+    }
+
+
+    @Override
+    String libFunctionName() {
+        return 'runBenchmarkTestScript'
+    }
+
+    @Override
+    void parameterInvariantsAssertions(Object call) {}
+
+    @Override
+    boolean expectedParametersMatcher(Object call) {
+        if (!this.bundleManifest.isEmpty()) {
+            return call.args.bundleManifest.first().toString().equals(this.bundleManifest)
+        }
+        return call.args.distributionUrl.first().toString().equals(this.distributionUrl)
+    }
+
+    @Override
+    void configure(Object helper, Object binding) {
+        binding.setVariable('COMMAND', command)
+        binding.setVariable('SUFFIX', suffix)
+        binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config')
+        binding.setVariable('BASELINE', baseline)
+        binding.setVariable('CONTENDER', contender)
+        binding.setVariable('RESULTS_FORMAT', results_format)
+        binding.setVariable('RESULTS_NUMBERS_ALIGN', results_numbers_align)
+        binding.setVariable('RESULTS_FILE', results_file)
+        binding.setVariable('SHOW_IN_RESULTS', show_in_results)
+        binding.setVariable('BUNDLE_MANIFEST', bundleManifest)
+        helper.registerAllowedMethod("withAWS", [Map, Closure], {
+            args,
+            closure ->
+                closure.delegate = delegate
+                return helper.callClosure(closure)
+        })
+        helper.registerAllowedMethod("s3Download", [Map])
+        helper.registerAllowedMethod("withAWS", [Map, Closure], {
+            args,
+            closure ->
+                closure.delegate = delegate
+                return helper.callClosure(closure)
+        })
+        helper.registerAllowedMethod("withCredentials", [Map])
+    }
+}

vars/getCompareBenchmarkIds.groovy

@@ -31,7 +31,7 @@ Map<String, String> call(Map args = [:]) {
 String getBaselineTestExecutionId(baselineClusterConfig, distributionVersion, workload) {
     withCredentials([string(credentialsId: 'benchmark-metrics-datastore-user', variable: 'DATASTORE_USER'),
                      string(credentialsId: 'benchmark-metrics-datastore-password', variable: 'DATASTORE_PASSWORD'),
-                     string(credentialsId: 'benchmark-metrics-datastore-endpoint', variable: 'DATASTORE_ENDPOINT')]) {
+                     string(credentialsId: 'benchmark-metrics-datastore-nlb-endpoint', variable: 'DATASTORE_ENDPOINT')]) {
         def curlCommand = """
               curl -X POST "https://${DATASTORE_ENDPOINT}/benchmark-results-*/_search" -ku ${DATASTORE_USER}:${DATASTORE_PASSWORD} -H 'Content-Type: application/json' -d '{
               "size": 1,