From 69106da1edfb4c58acba04a5a2c621e47e68dae8 Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Sun, 29 Sep 2024 14:33:51 -0400 Subject: [PATCH 01/10] Add el8 to build yaml --- etc/scipipe/build_matrix.yaml | 6 +++--- pipelines/release/run_rebuild.groovy | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index efb07b8f..a12b5dff 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -78,9 +78,9 @@ scipipe-lsstsw-ci_imsim: canonical: products: &canonical_products lsst_distrib lsst_ci lsst_middleware lsstsw_config: - <<: *el7-conda - label: snowflake-0 - display_name: centos-7 + - <<: *el7-conda + label: snowflake-0 + - <<: *linux-9-arm workspace: snowflake/release # # eups distrib tarball configuration -- used by diff --git a/pipelines/release/run_rebuild.groovy b/pipelines/release/run_rebuild.groovy index 5a6799c7..6f0cc7f9 100644 --- a/pipelines/release/run_rebuild.groovy +++ b/pipelines/release/run_rebuild.groovy @@ -52,7 +52,7 @@ notify.wrap { splenvRef = params.SPLENV_REF } - def slug = util.lsstswConfigSlug(lsstswConfig) + // def slug = util.lsstswConfigSlug(lsstswConfig) def run = { ws(canonical.workspace) { @@ -61,10 +61,10 @@ notify.wrap { def buildParams = [ EUPS_PKGROOT: "${cwd}/distrib", GIT_SSH_COMMAND: 'ssh -o StrictHostKeyChecking=no', - K8S_DIND_LIMITS_CPU: "4", + // K8S_DIND_LIMITS_CPU: "4", LSST_BUILD_DOCS: buildDocs, LSST_COMPILER: lsstswConfig.compiler, - LSST_JUNIT_PREFIX: slug, + // LSST_JUNIT_PREFIX: slug, LSST_PREP_ONLY: prepOnly, LSST_PRODUCTS: products, LSST_PYTHON_VERSION: lsstswConfig.python, @@ -91,9 +91,10 @@ notify.wrap { } stage('build') { - util.insideDockerWrap( - image: lsstswConfig.image, - pull: true, + util.lsstswBuildMatrix(lsstswConfig, buildParams, true + // util.insideDockerWrap( + // image: lsstswConfig.image, + // pull: true, ) { // only setup sshagent if we are going to push if (versiondbPush) { @@ -143,6 +144,7 @@ notify.wrap { } // stage('push docs') } // ws } // run +util = load 'pipelines/lib/util.groovy' util.nodeWrap(lsstswConfig.label) { timeout(time: timelimit, unit: 'HOURS') { From 1c9f504d3251394087f8e2ca0bf11b5928eafdc1 Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Tue, 15 Oct 2024 14:25:02 -0400 Subject: [PATCH 02/10] Update run_rebuild to build as a matrix. --- etc/scipipe/build_matrix.yaml | 2 +- pipelines/lib/util.groovy | 16 ++-- pipelines/release/run_publish.groovy | 2 +- pipelines/release/run_rebuild.groovy | 113 +++++---------------------- 4 files changed, 29 insertions(+), 104 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index a12b5dff..33b8d2a0 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -157,5 +157,5 @@ newinstall: github_repo: lsst/lsst git_ref: main eups: - base_url: https://eups.lsst.codes/stack + base_url: https://eups.lsst.codes/stack_test s3_bucket: eups.lsst.codes diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index adcb30af..adfce629 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -626,7 +626,7 @@ def void requireEnvVars(List rev) { // note that `env` isn't a map and #get doesn't work as expected rev.each { it -> if (env."${it}" == null) { - error "${it} envirnoment variable is required" + error "${it} environment variable is required" } } } @@ -691,7 +691,8 @@ def void getManifest(String rebuildId, String filename) { step([$class: 'CopyArtifact', projectName: buildJob, - filter: manifest_artifact, + filter: "**/lsstsw/build/manifext.txt", + target: manifest_artifact, selector: [ $class: 'SpecificBuildSelector', buildNumber: rebuildId // wants a string @@ -1385,20 +1386,20 @@ def String runRebuild(Map p) { parameters: jobParameters, wait: true, ) - nodeTiny { - manifestArtifact = 'lsstsw/build/manifest.txt' - + def manifestArtifact = 'lsstsw/build/manifest.txt' step([$class: 'CopyArtifact', projectName: useP.job, - filter: manifestArtifact, + filter: "**/lsstsw/build/manifest.txt", + //target: manifestArtifact, + target: 'buildmanifest', selector: [ $class: 'SpecificBuildSelector', buildNumber: result.id, ], ]) + def manifestId = parseManifestId(readFile('buildmanifest/linux-9-x86/lsstsw/build/manifest.txt')) - def manifestId = parseManifestId(readFile(manifestArtifact)) echo "parsed manifest id: ${manifestId}" return manifestId } // nodeTiny @@ -2300,4 +2301,5 @@ def void nodeWrap(String label, Closure run) { } } + return this; diff --git a/pipelines/release/run_publish.groovy b/pipelines/release/run_publish.groovy index 32ee032a..856dfc03 100644 --- a/pipelines/release/run_publish.groovy +++ b/pipelines/release/run_publish.groovy @@ -33,7 +33,7 @@ notify.wrap { Boolean pushS3 = (! params.NO_PUSH?.toBoolean()) def canonical = scipipe.canonical - def lsstswConfig = canonical.lsstsw_config + def lsstswConfig = canonical.lsstsw_config[0] def splenvRef = lsstswConfig.splenv_ref if (params.SPLENV_REF) { diff --git a/pipelines/release/run_rebuild.groovy b/pipelines/release/run_rebuild.groovy index 6f0cc7f9..6aaef282 100644 --- a/pipelines/release/run_rebuild.groovy +++ b/pipelines/release/run_rebuild.groovy @@ -1,6 +1,6 @@ properties([ copyArtifactPermission('/release/*'), -]); +]) node('jenkins-manager') { dir('jenkins-dm-jobs') { @@ -45,110 +45,33 @@ notify.wrap { } def canonical = scipipe.canonical - def lsstswConfig = canonical.lsstsw_config + def lsstswConfigs = canonical.lsstsw_config + def cwd = '/j/' + canonical.workspace - def splenvRef = lsstswConfig.splenv_ref - if (params.SPLENV_REF) { - splenvRef = params.SPLENV_REF - } - - // def slug = util.lsstswConfigSlug(lsstswConfig) - - def run = { - ws(canonical.workspace) { - def cwd = pwd() - - def buildParams = [ + def buildParams = [ EUPS_PKGROOT: "${cwd}/distrib", GIT_SSH_COMMAND: 'ssh -o StrictHostKeyChecking=no', - // K8S_DIND_LIMITS_CPU: "4", + K8S_DIND_LIMITS_CPU: "4", LSST_BUILD_DOCS: buildDocs, - LSST_COMPILER: lsstswConfig.compiler, - // LSST_JUNIT_PREFIX: slug, LSST_PREP_ONLY: prepOnly, LSST_PRODUCTS: products, - LSST_PYTHON_VERSION: lsstswConfig.python, - LSST_SPLENV_REF: splenvRef, LSST_REFS: refs, - VERSIONDB_PUSH: versiondbPush, - VERSIONDB_REPO: versiondbRepo, + // VERSIONDB_PUSH: versiondbPush, + // VERSIONDB_REPO: versiondbRepo, ] + + // override conda env ref from build_matrix.yaml + if (params.SPLENV_REF) { + buildParams['LSST_SPLENV_REF'] = params.SPLENV_REF + } - def runJW = { - // note that util.jenkinsWrapper() clones the ci-scripts repo, which is - // used by the push docs stage - try { - util.jenkinsWrapper(buildParams) - } finally { - util.jenkinsWrapperPost(null, prepOnly) - } - } - - def withVersiondbCredentials = { closure -> - sshagent (credentials: ['github-jenkins-versiondb']) { - closure() - } - } - - stage('build') { - util.lsstswBuildMatrix(lsstswConfig, buildParams, true - // util.insideDockerWrap( - // image: lsstswConfig.image, - // pull: true, - ) { - // only setup sshagent if we are going to push - if (versiondbPush) { - withVersiondbCredentials(runJW) - } else { - runJW() - } - } // util.insideDockerWrap - } // stage('build') - - stage('push docs') { - if (buildDocs) { - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'aws-doxygen-push', - usernameVariable: 'AWS_ACCESS_KEY_ID', - passwordVariable: 'AWS_SECRET_ACCESS_KEY' - ], - [ - $class: 'StringBinding', - credentialsId: 'doxygen-push-bucket', - variable: 'DOXYGEN_S3_BUCKET' - ]]) { - withEnv([ - "EUPS_PKGROOT=${cwd}/distrib", - "HOME=${cwd}/home", - ]) { - // the current iteration of the awscli container is alpine based - // and doesn't work with util.insideDockerWrap. However, the aws - // cli seems to work OK without trying to lookup the username. - docker.image(util.defaultAwscliImage()).inside { - // alpine does not include bash by default - util.posixSh ''' - # provides DOC_PUSH_PATH - . ./ci-scripts/settings.cfg.sh - - aws s3 cp \ - --only-show-errors \ - --recursive \ - "${DOC_PUSH_PATH}/" \ - "s3://${DOXYGEN_S3_BUCKET}/stack/doxygen/" - ''' - } // util.insideDockerWrap - } // withEnv - } // withCredentials - } - } // stage('push docs') - } // ws - } // run -util = load 'pipelines/lib/util.groovy' + if (lsstswConfigs == null) { + error "invalid value for BUILD_CONFIG: ${BUILD_CONFIG}" + } - util.nodeWrap(lsstswConfig.label) { - timeout(time: timelimit, unit: 'HOURS') { - run() + timeout(time: 12, unit: 'HOURS') { + stage('build') { + util.lsstswBuildMatrix(lsstswConfigs, buildParams ) } } } // notify.wrap From 36cb525f23cbafad3f5d52191dcf6591cf170766 Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Mon, 2 Dec 2024 13:48:23 -0500 Subject: [PATCH 03/10] Updated run publish to run in a matrix config --- jobs/run_publish.groovy | 2 +- pipelines/release/run_publish.groovy | 126 ++++++++++++++++++++++++--- 2 files changed, 117 insertions(+), 11 deletions(-) diff --git a/jobs/run_publish.groovy b/jobs/run_publish.groovy index 3968f07e..797387e6 100644 --- a/jobs/run_publish.groovy +++ b/jobs/run_publish.groovy @@ -18,6 +18,6 @@ p.pipeline().with { stringParam('SPLENV_REF', scipipe.template.splenv_ref, 'conda env ref') stringParam('RUBINENV_VER', scipipe.template.splenv_ref, 'conda env ref') // enable for debugging only - // booleanParam('NO_PUSH', true, 'Skip s3 push.') + booleanParam('NO_PUSH', true, 'Skip s3 push.') } } diff --git a/pipelines/release/run_publish.groovy b/pipelines/release/run_publish.groovy index 856dfc03..b8d10a2b 100644 --- a/pipelines/release/run_publish.groovy +++ b/pipelines/release/run_publish.groovy @@ -33,9 +33,9 @@ notify.wrap { Boolean pushS3 = (! params.NO_PUSH?.toBoolean()) def canonical = scipipe.canonical - def lsstswConfig = canonical.lsstsw_config[0] + def lsstswConfigs = canonical.lsstsw_config - def splenvRef = lsstswConfig.splenv_ref + def splenvRef = lsstswConfigs.splenv_ref if (params.SPLENV_REF) { splenvRef = params.SPLENV_REF } @@ -43,12 +43,15 @@ notify.wrap { if (params.RUBINENV_VER) { rubinEnvVer = params.RUBINENV_VER } - - def slug = util.lsstswConfigSlug(lsstswConfig) - - def run = { - ws(canonical.workspace) { + def matrix = [:] + lsstswConfigs.each{ lsstswConfig -> + def slug = util.lsstswConfigSlug(lsstswConfig) + matrix[slug] ={ + def run = { + def workingDir = canonical.workspace + "/${lsstswConfig.display_name}" + ws(workingDir) { def cwd = pwd() + println(cwd) def pkgroot = "${cwd}/distrib" stage('create packages') { @@ -83,6 +86,8 @@ notify.wrap { pull: true, ) { util.bash ''' + pwd + ls -a ARGS=() ARGS+=('-b' "$MANIFEST_ID") ARGS+=('-t' "$EUPS_TAG") @@ -98,7 +103,7 @@ notify.wrap { # this can be retrived using the -b option. # (note: bin/setup.sh is now deprecated) source ./lsstsw/bin/envconfig -n "lsst-scipipe-$LSST_SPLENV_REF" - + cat conda/envs/lsst-scipipe-*/share/eups/ups_db/global.tags publish "${ARGS[@]}" ''' } @@ -142,10 +147,111 @@ notify.wrap { } // stage('push packages') } // ws } // run - - util.nodeWrap(lsstswConfig.label) { + util.nodeWrap(lsstswConfig.label) { timeout(time: timelimit, unit: 'HOURS') { run() } } + } + parallel matrix + } } // notify.wrap + // def slug = util.lsstswConfigSlug(lsstswConfig) + + // def run = { + // ws(canonical.workspace) { + // def cwd = pwd() + // def pkgroot = "${cwd}/distrib" + + // stage('create packages') { + // dir('lsstsw') { + // util.cloneLsstsw() + // } + + // def tagDir = "${pkgroot}/tags" + + // // remove any pre-existing eups tags to prevent them from being + // // [re]published + // // the src pkgroot has tags under ./tags/ + // dir(tagDir) { + // deleteDir() + // } + + // def env = [ + // "HOME=${cwd}/home", + // "EUPS_PKGROOT=${pkgroot}", + // "EUPS_USERDATA=${cwd}/home/.eups_userdata", + // "EUPSPKG_SOURCE=${eupspkgSource}", + // "LSST_SPLENV_REF=${splenvRef}", + // "RUBINENV_VER=${rubinEnvVer}", + // "MANIFEST_ID=${manifestId}", + // "EUPS_TAG=${eupsTag}", + // "PRODUCTS=${products}", + // ] + + // withEnv(env) { + // util.insideDockerWrap( + // image: lsstswConfig.image, + // pull: true, + // ) { + // util.bash ''' + // ARGS=() + // ARGS+=('-b' "$MANIFEST_ID") + // ARGS+=('-t' "$EUPS_TAG") + // # enable debug output + // ARGS+=('-d') + // # split whitespace separated EUPS products into separate array + // # elements by not quoting + // ARGS+=($PRODUCTS) + + // export EUPSPKG_SOURCE="$EUPSPKG_SOURCE" + + // # setting up the same environment used in the previous build step + // # this can be retrived using the -b option. + // # (note: bin/setup.sh is now deprecated) + // source ./lsstsw/bin/envconfig -n "lsst-scipipe-$LSST_SPLENV_REF" + + // publish "${ARGS[@]}" + // ''' + // } + // } // util.insideDockerWrap + // } // stage('publish') + + // stage('push packages') { + // if (pushS3) { + // withCredentials([[ + // $class: 'UsernamePasswordMultiBinding', + // credentialsId: 'aws-eups-push', + // usernameVariable: 'AWS_ACCESS_KEY_ID', + // passwordVariable: 'AWS_SECRET_ACCESS_KEY' + // ], + // [ + // $class: 'StringBinding', + // credentialsId: 'eups-push-bucket', + // variable: 'EUPS_S3_BUCKET' + // ]]) { + // def env = [ + // "EUPS_PKGROOT=${pkgroot}", + // "HOME=${cwd}/home", + // "EUPS_S3_OBJECT_PREFIX=stack/src/" + // ] + // withEnv(env) { + // docker.image(util.defaultAwscliImage()).inside { + // // alpine does not include bash by default + // util.posixSh ''' + // aws s3 cp \ + // --only-show-errors \ + // --recursive \ + // "${EUPS_PKGROOT}/" \ + // "s3://${EUPS_S3_BUCKET}/${EUPS_S3_OBJECT_PREFIX}" + // ''' + // } // .inside + // } // withEnv + // } // withCredentials + // } else { + // echo "skipping s3 push." + // } + // } // stage('push packages') + // } // ws + // } // run + From 933288b43d20fc6da681283f7eb42adcae794d64 Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Tue, 3 Dec 2024 21:09:05 -0500 Subject: [PATCH 04/10] Move rebuild back to run_rebuild --- etc/scipipe/build_matrix.yaml | 2 + pipelines/lib/util.groovy | 27 +++--- pipelines/release/nightly_release.groovy | 7 +- pipelines/release/run_publish.groovy | 109 +--------------------- pipelines/release/run_rebuild.groovy | 114 ++++++++++++++++++++--- 5 files changed, 123 insertions(+), 136 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index 33b8d2a0..5cb3afb4 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -81,6 +81,8 @@ canonical: - <<: *el7-conda label: snowflake-0 - <<: *linux-9-arm + label: arm64 # need to move to snowflake-aarch64 + splenv_ref: *splenv_ref workspace: snowflake/release # # eups distrib tarball configuration -- used by diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index adfce629..0c434991 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -297,7 +297,8 @@ def void runPublish(Map p) { 'PRODUCTS', ]) useP.parameters = [ - TIMEOUT: '1' // should be string + TIMEOUT: '1', // should be string + NO_PUSH: true ] + p.parameters def jobParameters = [ @@ -686,21 +687,19 @@ def void createDirs(List eds) { * @param filename String Output filename. */ def void getManifest(String rebuildId, String filename) { - def manifest_artifact = 'lsstsw/build/manifest.txt' def buildJob = 'release/run-rebuild' - step([$class: 'CopyArtifact', + step([$class: 'CopyArtifact', projectName: buildJob, - filter: "**/lsstsw/build/manifext.txt", - target: manifest_artifact, - selector: [ - $class: 'SpecificBuildSelector', + filter: "**/lsstsw/build/manifest.txt", + target: 'buildmanifest', + selector: [ + $class: 'SpecificBuildSelector', buildNumber: rebuildId // wants a string - ], - ]) - - def manifest = readFile manifest_artifact - writeFile(file: filename, text: manifest) + ], + ]) + def manifest = readFile('buildmanifest/linux-9-x86/lsstsw/build/manifest.txt') + writeFile(file: filename, text: manifest) } // getManifest /** @@ -1387,7 +1386,9 @@ def String runRebuild(Map p) { wait: true, ) nodeTiny { - def manifestArtifact = 'lsstsw/build/manifest.txt' + + println("This is the output") + step([$class: 'CopyArtifact', projectName: useP.job, filter: "**/lsstsw/build/manifest.txt", diff --git a/pipelines/release/nightly_release.groovy b/pipelines/release/nightly_release.groovy index c1a61ca2..017c43ff 100644 --- a/pipelines/release/nightly_release.groovy +++ b/pipelines/release/nightly_release.groovy @@ -31,7 +31,7 @@ notify.wrap { def manifestId = null def stackResults = null - def lsstswConfig = scipipe.canonical.lsstsw_config + def lsstswConfigs = scipipe.canonical.lsstsw_config def run = { stage('format nightly tag') { @@ -46,7 +46,7 @@ notify.wrap { manifestId = util.runRebuild( parameters: [ PRODUCTS: products, - BUILD_DOCS: true, + BUILD_DOCS: false, ], ) } // retry @@ -64,6 +64,7 @@ notify.wrap { MANIFEST_ID: manifestId, EUPS_TAG: tagName, PRODUCTS: products, + BUILD_DOCS: false, ], ) } // retry @@ -134,7 +135,7 @@ notify.wrap { EUPS_TAG: eupsTag, DOCKER_TAGS: extraDockerTags, MANIFEST_ID: manifestId, - LSST_COMPILER: lsstswConfig.compiler, + LSST_COMPILER: lsstswConfig.compiler[0], ], ) } // retry diff --git a/pipelines/release/run_publish.groovy b/pipelines/release/run_publish.groovy index b8d10a2b..23c2b7c8 100644 --- a/pipelines/release/run_publish.groovy +++ b/pipelines/release/run_publish.groovy @@ -51,7 +51,6 @@ notify.wrap { def workingDir = canonical.workspace + "/${lsstswConfig.display_name}" ws(workingDir) { def cwd = pwd() - println(cwd) def pkgroot = "${cwd}/distrib" stage('create packages') { @@ -86,8 +85,6 @@ notify.wrap { pull: true, ) { util.bash ''' - pwd - ls -a ARGS=() ARGS+=('-b' "$MANIFEST_ID") ARGS+=('-t' "$EUPS_TAG") @@ -103,7 +100,6 @@ notify.wrap { # this can be retrived using the -b option. # (note: bin/setup.sh is now deprecated) source ./lsstsw/bin/envconfig -n "lsst-scipipe-$LSST_SPLENV_REF" - cat conda/envs/lsst-scipipe-*/share/eups/ups_db/global.tags publish "${ARGS[@]}" ''' } @@ -111,7 +107,7 @@ notify.wrap { } // stage('publish') stage('push packages') { - if (pushS3) { + if (false) { withCredentials([[ $class: 'UsernamePasswordMultiBinding', credentialsId: 'aws-eups-push', @@ -153,105 +149,6 @@ notify.wrap { } } } - parallel matrix } -} // notify.wrap - // def slug = util.lsstswConfigSlug(lsstswConfig) - - // def run = { - // ws(canonical.workspace) { - // def cwd = pwd() - // def pkgroot = "${cwd}/distrib" - - // stage('create packages') { - // dir('lsstsw') { - // util.cloneLsstsw() - // } - - // def tagDir = "${pkgroot}/tags" - - // // remove any pre-existing eups tags to prevent them from being - // // [re]published - // // the src pkgroot has tags under ./tags/ - // dir(tagDir) { - // deleteDir() - // } - - // def env = [ - // "HOME=${cwd}/home", - // "EUPS_PKGROOT=${pkgroot}", - // "EUPS_USERDATA=${cwd}/home/.eups_userdata", - // "EUPSPKG_SOURCE=${eupspkgSource}", - // "LSST_SPLENV_REF=${splenvRef}", - // "RUBINENV_VER=${rubinEnvVer}", - // "MANIFEST_ID=${manifestId}", - // "EUPS_TAG=${eupsTag}", - // "PRODUCTS=${products}", - // ] - - // withEnv(env) { - // util.insideDockerWrap( - // image: lsstswConfig.image, - // pull: true, - // ) { - // util.bash ''' - // ARGS=() - // ARGS+=('-b' "$MANIFEST_ID") - // ARGS+=('-t' "$EUPS_TAG") - // # enable debug output - // ARGS+=('-d') - // # split whitespace separated EUPS products into separate array - // # elements by not quoting - // ARGS+=($PRODUCTS) - - // export EUPSPKG_SOURCE="$EUPSPKG_SOURCE" - - // # setting up the same environment used in the previous build step - // # this can be retrived using the -b option. - // # (note: bin/setup.sh is now deprecated) - // source ./lsstsw/bin/envconfig -n "lsst-scipipe-$LSST_SPLENV_REF" - - // publish "${ARGS[@]}" - // ''' - // } - // } // util.insideDockerWrap - // } // stage('publish') - - // stage('push packages') { - // if (pushS3) { - // withCredentials([[ - // $class: 'UsernamePasswordMultiBinding', - // credentialsId: 'aws-eups-push', - // usernameVariable: 'AWS_ACCESS_KEY_ID', - // passwordVariable: 'AWS_SECRET_ACCESS_KEY' - // ], - // [ - // $class: 'StringBinding', - // credentialsId: 'eups-push-bucket', - // variable: 'EUPS_S3_BUCKET' - // ]]) { - // def env = [ - // "EUPS_PKGROOT=${pkgroot}", - // "HOME=${cwd}/home", - // "EUPS_S3_OBJECT_PREFIX=stack/src/" - // ] - // withEnv(env) { - // docker.image(util.defaultAwscliImage()).inside { - // // alpine does not include bash by default - // util.posixSh ''' - // aws s3 cp \ - // --only-show-errors \ - // --recursive \ - // "${EUPS_PKGROOT}/" \ - // "s3://${EUPS_S3_BUCKET}/${EUPS_S3_OBJECT_PREFIX}" - // ''' - // } // .inside - // } // withEnv - // } // withCredentials - // } else { - // echo "skipping s3 push." - // } - // } // stage('push packages') - // } // ws - // } // run - + parallel matrix +} // notify.wrap \ No newline at end of file diff --git a/pipelines/release/run_rebuild.groovy b/pipelines/release/run_rebuild.groovy index 6aaef282..5c785584 100644 --- a/pipelines/release/run_rebuild.groovy +++ b/pipelines/release/run_rebuild.groovy @@ -1,6 +1,6 @@ properties([ copyArtifactPermission('/release/*'), -]) +]); node('jenkins-manager') { dir('jenkins-dm-jobs') { @@ -46,32 +46,118 @@ notify.wrap { def canonical = scipipe.canonical def lsstswConfigs = canonical.lsstsw_config - def cwd = '/j/' + canonical.workspace + def splenvRef = lsstswConfigs.splenv_ref + println("This is the env ref: " + splenvRef) + if (params.SPLENV_REF) { + splenvRef = params.SPLENV_REF + } + + def matrix = [:] + lsstswConfigs.each{ lsstswConfig -> + def slug = util.lsstswConfigSlug(lsstswConfig) + matrix[slug] ={ + def run = { + def workingDir = canonical.workspace + "/${lsstswConfig.display_name}" + ws(workingDir) { + def cwd = pwd() + splenvRef = lsstswConfig.splenv_ref + if (params.SPLENV_REF) { + splenvRef = params.SPLENV_REF + } - def buildParams = [ + def buildParams = [ EUPS_PKGROOT: "${cwd}/distrib", GIT_SSH_COMMAND: 'ssh -o StrictHostKeyChecking=no', K8S_DIND_LIMITS_CPU: "4", LSST_BUILD_DOCS: buildDocs, + LSST_COMPILER: lsstswConfig.compiler, + LSST_JUNIT_PREFIX: slug, LSST_PREP_ONLY: prepOnly, LSST_PRODUCTS: products, + LSST_PYTHON_VERSION: lsstswConfig.python, + LSST_SPLENV_REF: splenvRef, LSST_REFS: refs, // VERSIONDB_PUSH: versiondbPush, // VERSIONDB_REPO: versiondbRepo, ] - - // override conda env ref from build_matrix.yaml - if (params.SPLENV_REF) { - buildParams['LSST_SPLENV_REF'] = params.SPLENV_REF - } - if (lsstswConfigs == null) { - error "invalid value for BUILD_CONFIG: ${BUILD_CONFIG}" - } + def runJW = { + // note that util.jenkinsWrapper() clones the ci-scripts repo, which is + // used by the push docs stage + try { + util.jenkinsWrapper(buildParams) + } finally { + util.jenkinsWrapperPost(null, prepOnly) + } + } - timeout(time: 12, unit: 'HOURS') { - stage('build') { - util.lsstswBuildMatrix(lsstswConfigs, buildParams ) + def withVersiondbCredentials = { closure -> + sshagent (credentials: ['github-jenkins-versiondb']) { + closure() + } + } + + stage('build') { + util.insideDockerWrap( + image: lsstswConfig.image, + pull: true, + ) { + // only setup sshagent if we are going to push + if (versiondbPush) { + withVersiondbCredentials(runJW) + } else { + runJW() + } + } // util.insideDockerWrap + } // stage('build') + + stage('push docs') { + if (buildDocs) { + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'aws-doxygen-push', + usernameVariable: 'AWS_ACCESS_KEY_ID', + passwordVariable: 'AWS_SECRET_ACCESS_KEY' + ], + [ + $class: 'StringBinding', + credentialsId: 'doxygen-push-bucket', + variable: 'DOXYGEN_S3_BUCKET' + ]]) { + withEnv([ + "EUPS_PKGROOT=${cwd}/distrib", + "HOME=${cwd}/home", + ]) { + // the current iteration of the awscli container is alpine based + // and doesn't work with util.insideDockerWrap. However, the aws + // cli seems to work OK without trying to lookup the username. + docker.image(util.defaultAwscliImage()).inside { + // alpine does not include bash by default + util.posixSh ''' + # provides DOC_PUSH_PATH + . ./ci-scripts/settings.cfg.sh + + aws s3 cp \ + --only-show-errors \ + --recursive \ + "${DOC_PUSH_PATH}/" \ + "s3://${DOXYGEN_S3_BUCKET}/stack/doxygen/" + ''' + } // util.insideDockerWrap + } // withEnv + } // withCredentials + } + } // stage('push docs') + } // ws + } // run + + util.nodeWrap(lsstswConfig.label) { + timeout(time: timelimit, unit: 'HOURS') { + run() + } } } +} +parallel matrix + } // notify.wrap From f8a474f4f24b9d78a05c8151428d2f5a24f4fdca Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Thu, 19 Dec 2024 16:08:39 -0500 Subject: [PATCH 05/10] Switch newInstall repo --- etc/scipipe/build_matrix.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index 5cb3afb4..5903ac1d 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -157,7 +157,7 @@ newinstall: repo: ghcr.io/lsst-dm/docker-newinstall tag: 9-latest github_repo: lsst/lsst - git_ref: main + git_ref: tickets/DM-48154 eups: base_url: https://eups.lsst.codes/stack_test s3_bucket: eups.lsst.codes From c4026c7685a75f6c45fa1fce5e1cc7a80e0c9a62 Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Thu, 19 Dec 2024 16:08:55 -0500 Subject: [PATCH 06/10] Adjustments for testing in prod --- pipelines/lib/util.groovy | 5 +- pipelines/release/nightly_release.groovy | 200 ++++++++++++----------- 2 files changed, 103 insertions(+), 102 deletions(-) diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index 0c434991..589d7bb9 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -698,7 +698,7 @@ def void getManifest(String rebuildId, String filename) { buildNumber: rebuildId // wants a string ], ]) - def manifest = readFile('buildmanifest/linux-9-x86/lsstsw/build/manifest.txt') + def manifest = readFile('buildmanifest/lsstsw/build/manifest.txt') writeFile(file: filename, text: manifest) } // getManifest @@ -1387,8 +1387,6 @@ def String runRebuild(Map p) { ) nodeTiny { - println("This is the output") - step([$class: 'CopyArtifact', projectName: useP.job, filter: "**/lsstsw/build/manifest.txt", @@ -1400,6 +1398,7 @@ def String runRebuild(Map p) { ], ]) def manifestId = parseManifestId(readFile('buildmanifest/linux-9-x86/lsstsw/build/manifest.txt')) + echo sh(returnStdout: true, script: 'env|sort') echo "parsed manifest id: ${manifestId}" return manifestId diff --git a/pipelines/release/nightly_release.groovy b/pipelines/release/nightly_release.groovy index 017c43ff..ef2fdfa4 100644 --- a/pipelines/release/nightly_release.groovy +++ b/pipelines/release/nightly_release.groovy @@ -55,7 +55,9 @@ notify.wrap { stage('eups publish') { def pub = [:] - [eupsTag, 'd_latest'].each { tagName -> + [eupsTag, + //'d_latest' + ].each { tagName -> pub[tagName] = { retry(retries) { util.runPublish( @@ -77,39 +79,39 @@ notify.wrap { util.waitForS3() // NOOP / DRY_RUN - stage('git tag eups products') { - retry(retries) { - util.nodeWrap('docker') { - // needs eups distrib tag to be sync'd from s3 -> k8s volume - util.githubTagRelease( - options: [ - '--dry-run': true, - '--org': scipipe.release_tag_org, - '--manifest': manifestId, - '--eups-tag': eupsTag, - ], - args: [gitTag], - ) - } // util.nodeWrap - } // retry - } // stage + // stage('git tag eups products') { + // retry(retries) { + // util.nodeWrap('docker') { + // // needs eups distrib tag to be sync'd from s3 -> k8s volume + // util.githubTagRelease( + // options: [ + // '--dry-run': true, + // '--org': scipipe.release_tag_org, + // '--manifest': manifestId, + // '--eups-tag': eupsTag, + // ], + // args: [gitTag], + // ) + // } // util.nodeWrap + // } // retry + // } // stage // add aux repo tags *after* tagging eups product repos so as to avoid a // trainwreck if an aux repo has been pulled into the build (without // first being removed from the aux team). - stage('git tag auxilliaries') { - retry(retries) { - util.nodeWrap('docker') { - util.githubTagTeams( - options: [ - '--dry-run': true, - '--org': scipipe.release_tag_org, - '--tag': gitTag, - ], - ) - } // util.nodeWrap - } // retry - } // stage + // stage('git tag auxilliaries') { + // retry(retries) { + // util.nodeWrap('docker') { + // util.githubTagTeams( + // options: [ + // '--dry-run': true, + // '--org': scipipe.release_tag_org, + // '--tag': gitTag, + // ], + // ) + // } // util.nodeWrap + // } // retry + // } // stage stage('build eups tarballs') { util.buildTarballMatrix( @@ -119,13 +121,13 @@ notify.wrap { EUPS_TAG: eupsTag, SMOKE: true, RUN_SCONS_CHECK: true, - PUBLISH: true, + PUBLISH: false, ], retries: retries, ) } // stage - util.waitForS3() + // util.waitForS3() stage('build stack image') { retry(retries) { @@ -135,7 +137,7 @@ notify.wrap { EUPS_TAG: eupsTag, DOCKER_TAGS: extraDockerTags, MANIFEST_ID: manifestId, - LSST_COMPILER: lsstswConfig.compiler[0], + LSST_COMPILER: lsstswConfigs.compiler[0], ], ) } // retry @@ -143,72 +145,72 @@ notify.wrap { def triggerMe = [:] - triggerMe['build Science Platform Notebook Aspect Lab image'] = { - retry(retries) { - // based on lsstsqre/stack image - build( - job: 'sqre/infra/build-sciplatlab', - parameters: [ - string(name: 'TAG', value: eupsTag), - ], - wait: false, - ) - } // retry - } - - triggerMe['verify_drp_metrics'] = { - retry(1) { - // based on lsstsqre/stack image - build( - job: 'sqre/verify_drp_metrics', - parameters: [ - string(name: 'DOCKER_IMAGE', value: stackResults.image), - booleanParam( - name: 'NO_PUSH', - value: scipipe.release.step.verify_drp_metrics.no_push, - ), - booleanParam(name: 'WIPEOUT', value: false), - string(name: 'GIT_REF', value: 'main'), - ], - wait: false, - ) - } // retry - } - - triggerMe['doc build'] = { - retry(retries) { - build( - job: 'sqre/infra/documenteer', - parameters: [ - string(name: 'EUPS_TAG', value: eupsTag), - string(name: 'LTD_SLUG', value: eupsTag), - string(name: 'RELEASE_IMAGE', value: stackResults.image), - booleanParam( - name: 'PUBLISH', - value: scipipe.release.step.documenteer.publish, - ), - ], - wait: false, - ) - } // retry - } - - triggerMe['ap_verify'] = { - retry(retries) { - build( - job: 'scipipe/ap_verify', - parameters: [ - string(name: 'DOCKER_IMAGE', value: stackResults.image), - booleanParam( - name: 'NO_PUSH', - value: scipipe.release.step.ap_verify.no_push, - ), - booleanParam(name: 'WIPEOUT', value: false), - ], - wait: false, - ) - } // retry - } + // triggerMe['build Science Platform Notebook Aspect Lab image'] = { + // retry(retries) { + // // based on lsstsqre/stack image + // build( + // job: 'sqre/infra/build-sciplatlab', + // parameters: [ + // string(name: 'TAG', value: eupsTag), + // ], + // wait: false, + // ) + // } // retry + // } + + // triggerMe['verify_drp_metrics'] = { + // retry(1) { + // // based on lsstsqre/stack image + // build( + // job: 'sqre/verify_drp_metrics', + // parameters: [ + // string(name: 'DOCKER_IMAGE', value: stackResults.image), + // booleanParam( + // name: 'NO_PUSH', + // value: scipipe.release.step.verify_drp_metrics.no_push, + // ), + // booleanParam(name: 'WIPEOUT', value: false), + // string(name: 'GIT_REF', value: 'main'), + // ], + // wait: false, + // ) + // } // retry + // } + + // triggerMe['doc build'] = { + // retry(retries) { + // build( + // job: 'sqre/infra/documenteer', + // parameters: [ + // string(name: 'EUPS_TAG', value: eupsTag), + // string(name: 'LTD_SLUG', value: eupsTag), + // string(name: 'RELEASE_IMAGE', value: stackResults.image), + // booleanParam( + // name: 'PUBLISH', + // value: scipipe.release.step.documenteer.publish, + // ), + // ], + // wait: false, + // ) + // } // retry + // } + + // triggerMe['ap_verify'] = { + // retry(retries) { + // build( + // job: 'scipipe/ap_verify', + // parameters: [ + // string(name: 'DOCKER_IMAGE', value: stackResults.image), + // booleanParam( + // name: 'NO_PUSH', + // value: scipipe.release.step.ap_verify.no_push, + // ), + // booleanParam(name: 'WIPEOUT', value: false), + // ], + // wait: false, + // ) + // } // retry + // } stage('triggered jobs') { parallel triggerMe From 3ea491fc4ca49c9a45280652e1b8b80e0ba49d9e Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Sat, 21 Dec 2024 01:22:57 -0500 Subject: [PATCH 07/10] Rework artifact copy --- pipelines/lib/util.groovy | 4 ++-- pipelines/release/run_rebuild.groovy | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index 589d7bb9..2c4314b3 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -1397,8 +1397,8 @@ def String runRebuild(Map p) { buildNumber: result.id, ], ]) - def manifestId = parseManifestId(readFile('buildmanifest/linux-9-x86/lsstsw/build/manifest.txt')) - echo sh(returnStdout: true, script: 'env|sort') + def manifestId = parseManifestId(readFile('buildmanifest/lsstsw/build/manifest.txt')) + // echo sh(returnStdout: true, script: 'env|sort') echo "parsed manifest id: ${manifestId}" return manifestId diff --git a/pipelines/release/run_rebuild.groovy b/pipelines/release/run_rebuild.groovy index 5c785584..696184d2 100644 --- a/pipelines/release/run_rebuild.groovy +++ b/pipelines/release/run_rebuild.groovy @@ -46,11 +46,6 @@ notify.wrap { def canonical = scipipe.canonical def lsstswConfigs = canonical.lsstsw_config - def splenvRef = lsstswConfigs.splenv_ref - println("This is the env ref: " + splenvRef) - if (params.SPLENV_REF) { - splenvRef = params.SPLENV_REF - } def matrix = [:] lsstswConfigs.each{ lsstswConfig -> From c51835d187d97d405aab1a879654168245ee1d4b Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Mon, 23 Dec 2024 16:47:03 -0500 Subject: [PATCH 08/10] Add arm to build stack docker --- etc/scipipe/build_matrix.yaml | 2 +- pipelines/release/docker/build_stack.groovy | 16 ++++++++++-- pipelines/release/nightly_release.groovy | 28 ++++++++++----------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index 5903ac1d..b1de602d 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -81,7 +81,7 @@ canonical: - <<: *el7-conda label: snowflake-0 - <<: *linux-9-arm - label: arm64 # need to move to snowflake-aarch64 + label: snowflake-arch-0 splenv_ref: *splenv_ref workspace: snowflake/release # diff --git a/pipelines/release/docker/build_stack.groovy b/pipelines/release/docker/build_stack.groovy index d9e43c54..fc446fed 100644 --- a/pipelines/release/docker/build_stack.groovy +++ b/pipelines/release/docker/build_stack.groovy @@ -36,6 +36,9 @@ notify.wrap { String manifestId = params.MANIFEST_ID ?: '' String lsstCompiler = params.LSST_COMPILER ?: '' + + def canonical = scipipe.canonical + def lsstswConfigs = canonical.lsstsw_config def release = scipipe.scipipe_release def dockerfile = release.dockerfile def dockerRegistry = release.docker_registry @@ -49,9 +52,14 @@ notify.wrap { def timestamp = util.epochMilliToUtc(currentBuild.startTimeInMillis) def shebangtronUrl = util.shebangtronUrl() + def matrix = [:] + lsstswConfigs.each{ lsstswConfig -> + def slug = util.lsstswConfigSlug(lsstswConfig) + matrix[slug] ={ + def newinstallImage = newinstall.docker_registry.repo def newinstallTagBase = newinstall.docker_registry.tag - def splenvRef = scipipe.canonical.lsstsw_config.splenv_ref + def splenvRef = lsstswConfig.splenv_ref if (params.SPLENV_REF) { splenvRef = params.SPLENV_REF } @@ -139,7 +147,7 @@ notify.wrap { } // push } // run - util.nodeWrap('docker') { + util.nodeWrap(lsstswConfig.label) { try { timeout(time: timelimit, unit: 'HOURS') { run() @@ -164,4 +172,8 @@ notify.wrap { } // stage } // try } // util.nodeWrap + } +} +parallel matrix + } // notify.wrap diff --git a/pipelines/release/nightly_release.groovy b/pipelines/release/nightly_release.groovy index ef2fdfa4..0352d65b 100644 --- a/pipelines/release/nightly_release.groovy +++ b/pipelines/release/nightly_release.groovy @@ -76,7 +76,7 @@ notify.wrap { parallel pub } // stage - util.waitForS3() + // util.waitForS3() // NOOP / DRY_RUN // stage('git tag eups products') { @@ -113,19 +113,19 @@ notify.wrap { // } // retry // } // stage - stage('build eups tarballs') { - util.buildTarballMatrix( - tarballConfigs: scipipe.tarball.build_config, - parameters: [ - PRODUCTS: tarballProducts, - EUPS_TAG: eupsTag, - SMOKE: true, - RUN_SCONS_CHECK: true, - PUBLISH: false, - ], - retries: retries, - ) - } // stage + // stage('build eups tarballs') { + // util.buildTarballMatrix( + // tarballConfigs: scipipe.tarball.build_config, + // parameters: [ + // PRODUCTS: tarballProducts, + // EUPS_TAG: eupsTag, + // SMOKE: true, + // RUN_SCONS_CHECK: true, + // PUBLISH: false, + // ], + // retries: retries, + // ) + // } // stage // util.waitForS3() From 9243e0f27b9a38dd2698c58a42b34042631e074c Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Sun, 29 Dec 2024 14:53:54 -0500 Subject: [PATCH 09/10] Add arm to tarball --- etc/scipipe/build_matrix.yaml | 14 ++++++++++-- jobs/tarball.groovy | 2 +- pipelines/release/nightly_release.groovy | 28 ++++++++++++------------ pipelines/release/tarball.groovy | 2 +- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index b1de602d..d370d789 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -105,6 +105,16 @@ tarball: osfamily: osx timelimit: 8 allow_fail: true + - <<: *tarball_defaults + <<: *linux-9-arm + platform: el8-arm + osfamily: redhat + - <<: *tarball_defaults + <<: *macarm64-conda + platform: 14-arm + osfamily: osx + timelimit: 8 + allow_fail: true # # X-release pattern pipelines # @@ -129,7 +139,7 @@ versiondb: release_tag_org: lsst lsstsw: github_repo: lsst/lsstsw - git_ref: main + git_ref: tickets/DM-46554 ciscripts: github_repo: lsst-sqre/ci-scripts git_ref: main @@ -157,7 +167,7 @@ newinstall: repo: ghcr.io/lsst-dm/docker-newinstall tag: 9-latest github_repo: lsst/lsst - git_ref: tickets/DM-48154 + git_ref: tickets/DM-46554 eups: base_url: https://eups.lsst.codes/stack_test s3_bucket: eups.lsst.codes diff --git a/jobs/tarball.groovy b/jobs/tarball.groovy index 033f28f2..566d6319 100644 --- a/jobs/tarball.groovy +++ b/jobs/tarball.groovy @@ -20,7 +20,7 @@ p.pipeline().with { booleanParam('WIPEOUT', false, 'Completely wipe out workspace(s) before starting build.') stringParam('TIMEOUT', '23', 'build timeout in hours') stringParam('IMAGE', null, 'Pipeline base docker image (e.g. docker.io/lsstdm/scipipe-base:8 or empty for macOS)') - choiceParam('LABEL', ['docker', 'arm64', 'osx-13', 'osx-14'], 'Jenkins build agent label') + choiceParam('LABEL', ['docker', 'arm64', 'osx-13', 'osx-14', 'mini'], 'Jenkins build agent label') stringParam('COMPILER', 'conda-system', 'compiler version string') choiceParam('PYTHON_VERSION', ['3'], 'Python major version') stringParam('MINIVER', scipipe.template.tarball_defaults.miniver, 'Miniconda installer version') diff --git a/pipelines/release/nightly_release.groovy b/pipelines/release/nightly_release.groovy index 0352d65b..a4b23b0e 100644 --- a/pipelines/release/nightly_release.groovy +++ b/pipelines/release/nightly_release.groovy @@ -113,21 +113,21 @@ notify.wrap { // } // retry // } // stage - // stage('build eups tarballs') { - // util.buildTarballMatrix( - // tarballConfigs: scipipe.tarball.build_config, - // parameters: [ - // PRODUCTS: tarballProducts, - // EUPS_TAG: eupsTag, - // SMOKE: true, - // RUN_SCONS_CHECK: true, - // PUBLISH: false, - // ], - // retries: retries, - // ) - // } // stage + stage('build eups tarballs') { + util.buildTarballMatrix( + tarballConfigs: scipipe.tarball.build_config, + parameters: [ + PRODUCTS: tarballProducts, + EUPS_TAG: eupsTag, + SMOKE: true, + RUN_SCONS_CHECK: true, + PUBLISH: false, + ], + retries: retries, + ) + } // stage - // util.waitForS3() + util.waitForS3() stage('build stack image') { retry(retries) { diff --git a/pipelines/release/tarball.groovy b/pipelines/release/tarball.groovy index 7cf6694e..4431fc5a 100644 --- a/pipelines/release/tarball.groovy +++ b/pipelines/release/tarball.groovy @@ -142,7 +142,7 @@ def void linuxTarballs( } // util.withEupsEnv } // run() - util.nodeWrap('docker') { + util.nodeWrap(label) { timeout(time: timelimit, unit: 'HOURS') { run() } From f038402b41cf60d74a01462aba3862d0167b6432 Mon Sep 17 00:00:00 2001 From: Ross Ceballo Date: Tue, 31 Dec 2024 13:13:07 -0500 Subject: [PATCH 10/10] Run like prod --- etc/scipipe/build_matrix.yaml | 1 - jobs/run_publish.groovy | 2 +- pipelines/lib/util.groovy | 3 +- pipelines/release/nightly_release.groovy | 62 ++++++++++++------------ 4 files changed, 33 insertions(+), 35 deletions(-) diff --git a/etc/scipipe/build_matrix.yaml b/etc/scipipe/build_matrix.yaml index d370d789..02dad050 100644 --- a/etc/scipipe/build_matrix.yaml +++ b/etc/scipipe/build_matrix.yaml @@ -82,7 +82,6 @@ canonical: label: snowflake-0 - <<: *linux-9-arm label: snowflake-arch-0 - splenv_ref: *splenv_ref workspace: snowflake/release # # eups distrib tarball configuration -- used by diff --git a/jobs/run_publish.groovy b/jobs/run_publish.groovy index 797387e6..3968f07e 100644 --- a/jobs/run_publish.groovy +++ b/jobs/run_publish.groovy @@ -18,6 +18,6 @@ p.pipeline().with { stringParam('SPLENV_REF', scipipe.template.splenv_ref, 'conda env ref') stringParam('RUBINENV_VER', scipipe.template.splenv_ref, 'conda env ref') // enable for debugging only - booleanParam('NO_PUSH', true, 'Skip s3 push.') + // booleanParam('NO_PUSH', true, 'Skip s3 push.') } } diff --git a/pipelines/lib/util.groovy b/pipelines/lib/util.groovy index 2c4314b3..cc3936b0 100644 --- a/pipelines/lib/util.groovy +++ b/pipelines/lib/util.groovy @@ -297,8 +297,7 @@ def void runPublish(Map p) { 'PRODUCTS', ]) useP.parameters = [ - TIMEOUT: '1', // should be string - NO_PUSH: true + TIMEOUT: '1' // should be string ] + p.parameters def jobParameters = [ diff --git a/pipelines/release/nightly_release.groovy b/pipelines/release/nightly_release.groovy index a4b23b0e..cb9e3a8b 100644 --- a/pipelines/release/nightly_release.groovy +++ b/pipelines/release/nightly_release.groovy @@ -76,42 +76,42 @@ notify.wrap { parallel pub } // stage - // util.waitForS3() + util.waitForS3() // NOOP / DRY_RUN - // stage('git tag eups products') { - // retry(retries) { - // util.nodeWrap('docker') { - // // needs eups distrib tag to be sync'd from s3 -> k8s volume - // util.githubTagRelease( - // options: [ - // '--dry-run': true, - // '--org': scipipe.release_tag_org, - // '--manifest': manifestId, - // '--eups-tag': eupsTag, - // ], - // args: [gitTag], - // ) - // } // util.nodeWrap - // } // retry - // } // stage + stage('git tag eups products') { + retry(retries) { + util.nodeWrap('docker') { + // needs eups distrib tag to be sync'd from s3 -> k8s volume + util.githubTagRelease( + options: [ + '--dry-run': true, + '--org': scipipe.release_tag_org, + '--manifest': manifestId, + '--eups-tag': eupsTag, + ], + args: [gitTag], + ) + } // util.nodeWrap + } // retry + } // stage // add aux repo tags *after* tagging eups product repos so as to avoid a // trainwreck if an aux repo has been pulled into the build (without // first being removed from the aux team). - // stage('git tag auxilliaries') { - // retry(retries) { - // util.nodeWrap('docker') { - // util.githubTagTeams( - // options: [ - // '--dry-run': true, - // '--org': scipipe.release_tag_org, - // '--tag': gitTag, - // ], - // ) - // } // util.nodeWrap - // } // retry - // } // stage + stage('git tag auxilliaries') { + retry(retries) { + util.nodeWrap('docker') { + util.githubTagTeams( + options: [ + '--dry-run': true, + '--org': scipipe.release_tag_org, + '--tag': gitTag, + ], + ) + } // util.nodeWrap + } // retry + } // stage stage('build eups tarballs') { util.buildTarballMatrix( @@ -121,7 +121,7 @@ notify.wrap { EUPS_TAG: eupsTag, SMOKE: true, RUN_SCONS_CHECK: true, - PUBLISH: false, + PUBLISH: true, ], retries: retries, )