From 8c79a56f25a0bf209452c9d6a37691a41b4e5835 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 30 Jul 2024 13:19:55 -0700 Subject: [PATCH] Download manifest bug Signed-off-by: Prudhvi Godithi --- tests/jenkins/TestRunBenchmarkTestScript.groovy | 2 +- tests/jenkins/TestRunPerfTestScript.groovy | 2 +- tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt | 8 ++++++-- tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt | 8 ++++++-- tests/jenkins/jobs/PerfTest_Jenkinsfile.txt | 8 ++++++-- vars/downloadBuildManifest.groovy | 5 +++-- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/jenkins/TestRunBenchmarkTestScript.groovy b/tests/jenkins/TestRunBenchmarkTestScript.groovy index 2857cce00..3e6386140 100644 --- a/tests/jenkins/TestRunBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScript.groovy @@ -61,7 +61,7 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest { assertThat(curlCommands.size(), equalTo(2)) assertThat(curlCommands, hasItem( - "curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString() + "curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString() )) def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index b06c36861..d3bf89383 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -47,7 +47,7 @@ class TestRunPerfTestScript extends BuildPipelineTest { assertThat(curlCommands.size(), equalTo(2)) assertThat(curlCommands, hasItem( - "curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString() + "curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString() )) def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt index 3572db6ed..aa05034e8 100644 --- a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt @@ -6,7 +6,9 @@ BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@main, retriever=null}) - downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml")) + downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) @@ -34,7 +36,9 @@ BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@main, retriever=null}) - downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml")) + downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt index 9fc0b24ef..bed6dfd76 100644 --- a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt @@ -6,7 +6,9 @@ BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@main, retriever=null}) - downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml")) + downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) @@ -34,7 +36,9 @@ BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@main, retriever=null}) - downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml")) + downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) diff --git a/tests/jenkins/jobs/PerfTest_Jenkinsfile.txt b/tests/jenkins/jobs/PerfTest_Jenkinsfile.txt index a56af91c0..142d5468a 100644 --- a/tests/jenkins/jobs/PerfTest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PerfTest_Jenkinsfile.txt @@ -6,7 +6,9 @@ PerfTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@main, retriever=null}) - downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml")) + downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() PerfTest_Jenkinsfile.runPerfTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1, stackNameSuffix=307}) @@ -32,7 +34,9 @@ PerfTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@main, retriever=null}) - downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml")) + downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() PerfTest_Jenkinsfile.runPerfTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1, stackNameSuffix=307}) diff --git a/vars/downloadBuildManifest.groovy b/vars/downloadBuildManifest.groovy index d0d21b974..19b58d34b 100644 --- a/vars/downloadBuildManifest.groovy +++ b/vars/downloadBuildManifest.groovy @@ -8,8 +8,9 @@ */ def call(Map args = [:]) { def lib = library(identifier: "jenkins@main", retriever: legacySCM(scm)) - - sh "curl -sSL ${args.url} --output ${args.path}" + sh "mkdir -p \$(dirname \"${args.path}\")" + sh "touch ${args.path}" + sh "curl -sSL --retry 5 ${args.url} --output ${args.path}" def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: args.path)) return buildManifestObj }