From 4e0b5939a3cc9c18323f1c9605cd724fb01b95e5 Mon Sep 17 00:00:00 2001 From: kirrg001 Date: Thu, 2 May 2024 17:39:41 +0200 Subject: [PATCH] ci: added skip-cache option --- .tekton/listeners/esm.yaml | 2 + .tekton/listeners/main.yaml | 4 +- .tekton/listeners/node-x-manual.yaml | 2 + .tekton/listeners/node-x.yaml | 2 + .tekton/listeners/pr.yaml | 2 + .tekton/pipeline/currency-bot-pipeline.yaml | 2 + .tekton/pipeline/default-pipeline.yaml | 6 +- .tekton/pipeline/trigger-template.yaml | 7 +- .tekton/tasks/install-npm-dependencies.yaml | 148 +++++++++--------- .../templates/default-pipeline.yaml.template | 6 +- 10 files changed, 107 insertions(+), 74 deletions(-) diff --git a/.tekton/listeners/esm.yaml b/.tekton/listeners/esm.yaml index edaa04aee8..75485f88ae 100644 --- a/.tekton/listeners/esm.yaml +++ b/.tekton/listeners/esm.yaml @@ -32,6 +32,8 @@ spec: value: "true" - name: context value: "node-$(params.node-version)-esm" + - name: skip-cache + value: $(params.skip-cache) --- diff --git a/.tekton/listeners/main.yaml b/.tekton/listeners/main.yaml index f551122dae..beb04ba6a3 100644 --- a/.tekton/listeners/main.yaml +++ b/.tekton/listeners/main.yaml @@ -26,7 +26,9 @@ spec: - name: pl-name value: $(event.after) - name: pipeline-ref - value: "default-pipeline" + value: "default-pipeline" + - name: skip-cache + value: $(params.skip-cache) --- diff --git a/.tekton/listeners/node-x-manual.yaml b/.tekton/listeners/node-x-manual.yaml index fec9add67a..b492114566 100644 --- a/.tekton/listeners/node-x-manual.yaml +++ b/.tekton/listeners/node-x-manual.yaml @@ -30,6 +30,8 @@ spec: value: "default-pipeline" - name: esm value: $(params.esm) + - name: skip-cache + value: $(params.skip-cache) --- diff --git a/.tekton/listeners/node-x.yaml b/.tekton/listeners/node-x.yaml index f79510f2f5..d73003111a 100644 --- a/.tekton/listeners/node-x.yaml +++ b/.tekton/listeners/node-x.yaml @@ -27,6 +27,8 @@ spec: value: "nightly-$(params.node-version)" - name: pipeline-ref value: "default-pipeline" + - name: skip-cache + value: $(params.skip-cache) --- diff --git a/.tekton/listeners/pr.yaml b/.tekton/listeners/pr.yaml index 3f06d62911..352c57eac7 100644 --- a/.tekton/listeners/pr.yaml +++ b/.tekton/listeners/pr.yaml @@ -35,6 +35,8 @@ spec: value: $(event.pull_request.number) - name: pipeline-ref value: "default-pipeline" + - name: skip-cache + value: $(params.skip-cache) --- diff --git a/.tekton/pipeline/currency-bot-pipeline.yaml b/.tekton/pipeline/currency-bot-pipeline.yaml index 10e32469e8..9c33302a93 100644 --- a/.tekton/pipeline/currency-bot-pipeline.yaml +++ b/.tekton/pipeline/currency-bot-pipeline.yaml @@ -104,6 +104,8 @@ spec: value: $(params.node-version) - name: npm-version value: $(params.npm-version) + - name: skip-cache + value: "false" workspaces: - name: output workspace: artifacts diff --git a/.tekton/pipeline/default-pipeline.yaml b/.tekton/pipeline/default-pipeline.yaml index fe17f1bbd0..0fae1524ea 100644 --- a/.tekton/pipeline/default-pipeline.yaml +++ b/.tekton/pipeline/default-pipeline.yaml @@ -38,6 +38,8 @@ spec: value: $(params.coverage) - name: context value: $(params.context) + - name: skip-cache + value: $(params.skip-cache) workspaces: - name: artifacts tasks: @@ -124,7 +126,9 @@ spec: - name: node-version value: $(params.node-version) - name: npm-version - value: $(params.npm-version) + value: $(params.npm-version) + - name: skip-cache + value: $(params.skip-cache) workspaces: - name: output workspace: artifacts diff --git a/.tekton/pipeline/trigger-template.yaml b/.tekton/pipeline/trigger-template.yaml index a35134af87..73856af467 100644 --- a/.tekton/pipeline/trigger-template.yaml +++ b/.tekton/pipeline/trigger-template.yaml @@ -43,6 +43,9 @@ spec: - name: coverage default: "false" value: $(params.coverage) + - name: skip-cache + default: "false" + value: $(params.skip-cache) - name: context default: "node-$(params.node-version)" value: $(params.context) @@ -116,4 +119,6 @@ spec: - name: coverage value: $(params.coverage) - name: context - value: $(params.context) \ No newline at end of file + value: $(params.context) + - name: skip-cache + value: $(params.skip-cache) \ No newline at end of file diff --git a/.tekton/tasks/install-npm-dependencies.yaml b/.tekton/tasks/install-npm-dependencies.yaml index 77c72a5ad2..a964b47d79 100644 --- a/.tekton/tasks/install-npm-dependencies.yaml +++ b/.tekton/tasks/install-npm-dependencies.yaml @@ -13,7 +13,9 @@ spec: - name: node-version value: $(params.node-version) - name: npm-version - value: $(params.npm-version) + value: $(params.npm-version) + - name: skip-cache + value: $(params.skip-cache) workspaces: - name: output mountPath: /artifacts @@ -34,71 +36,29 @@ spec: if [ -n "$(params.npm-version)" ]; then npm install npm@$(params.npm-version) -g - fi + fi - echo "Restoring node_modules cache..." - curl -fsSL https://clis.cloud.ibm.com/install/linux | sh - ibmcloud plugin install cloud-object-storage -f -r 'IBM Cloud' - ibmcloud login -a https://cloud.ibm.com -r eu-de --apikey $API_KEY - - apt-get update -y - apt-get install zstd -y - - checksum=$(sha256sum package-lock.json | awk '{print $1}') - ibmcloud cos download --bucket npm-cache --key node-modules-$(params.target-branch)-$(params.node-version)-$checksum ./node-modules.tar.zst - - # Check the exit code of the download command - if [ $? -eq 0 ]; then - npmInstall=false - echo "Download successful." - - zstd -d node-modules.tar.zst -o node-modules.tar - tar -xf node-modules.tar - - # Iterate over packages/*/ directories - for package_dir in packages/*; do - if [ -d "$ARTIFACTS_PATH/$package_dir" ]; then - cd $ARTIFACTS_PATH/$package_dir - package_name=$(basename "$package_dir") - - # Download the cache for the specific package - pkg_name="node-modules-$(params.target-branch)-$(params.node-version)-$package_name-$checksum" - - echo "Downloading cache for $pkg_name" - - ibmcloud cos download --bucket npm-cache --key $pkg_name $pkg_name.tar.zst - - if [ $? -eq 0 ]; then - # Decompress and extract the cache - echo "Download successful." - - zstd -d $pkg_name.tar.zst -o "node-modules.tar" - tar -xf "node-modules.tar" - else - npmInstall=true - fi - fi - done - - cd $ARTIFACTS_PATH - - echo "npm install: $npmInstall" - - if [ "$npmInstall" == true ]; then - npm install + if [ "$(params.skip-cache)" == "true" ]; then + echo "Skipping npm cache..." + echo "Installing npm dependencies..." + npm install --loglevel verbose + else + echo "Restoring node_modules cache..." + curl -fsSL https://clis.cloud.ibm.com/install/linux | sh + ibmcloud plugin install cloud-object-storage -f -r 'IBM Cloud' + ibmcloud login -a https://cloud.ibm.com -r eu-de --apikey $API_KEY - rm -rf node-modules.tar.zst - rm -rf node-modules.tar - fi + apt-get update -y + apt-get install zstd -y - echo "Restored node_modules cache..." - else - ibmcloud cos download --bucket npm-cache --key node-modules-main-$(params.node-version)-$checksum ./node-modules.tar.zst + checksum=$(sha256sum package-lock.json | awk '{print $1}') + ibmcloud cos download --bucket npm-cache --key node-modules-$(params.target-branch)-$(params.node-version)-$checksum ./node-modules.tar.zst + # Check the exit code of the download command if [ $? -eq 0 ]; then - npmInstall=false - echo "Download successful. Restoring node_modules cache..." - + npmInstall=false + echo "Download successful." + zstd -d node-modules.tar.zst -o node-modules.tar tar -xf node-modules.tar @@ -106,37 +66,85 @@ spec: for package_dir in packages/*; do if [ -d "$ARTIFACTS_PATH/$package_dir" ]; then cd $ARTIFACTS_PATH/$package_dir - package_name=$(basename "$package_dir") # Download the cache for the specific package pkg_name="node-modules-$(params.target-branch)-$(params.node-version)-$package_name-$checksum" - echo "Downloading cache for $pkg_name" - ibmcloud cos download --bucket npm-cache --key $pkg_name $pkg_name.tar.zst + echo "Downloading cache for $pkg_name" + + ibmcloud cos download --bucket npm-cache --key $pkg_name $pkg_name.tar.zst + if [ $? -eq 0 ]; then # Decompress and extract the cache echo "Download successful." + zstd -d $pkg_name.tar.zst -o "node-modules.tar" tar -xf "node-modules.tar" else npmInstall=true - fi + fi fi done cd $ARTIFACTS_PATH - rm -rf node-modules.tar.zst - rm -rf node-modules.tar + echo "npm install: $npmInstall" if [ "$npmInstall" == true ]; then npm install --loglevel verbose + + rm -rf node-modules.tar.zst + rm -rf node-modules.tar fi echo "Restored node_modules cache..." else - echo "Download from main was not successful. Installing node_modules..." - npm install --loglevel verbose + ibmcloud cos download --bucket npm-cache --key node-modules-main-$(params.node-version)-$checksum ./node-modules.tar.zst + + if [ $? -eq 0 ]; then + npmInstall=false + echo "Download successful. Restoring node_modules cache..." + + zstd -d node-modules.tar.zst -o node-modules.tar + tar -xf node-modules.tar + + # Iterate over packages/*/ directories + for package_dir in packages/*; do + if [ -d "$ARTIFACTS_PATH/$package_dir" ]; then + cd $ARTIFACTS_PATH/$package_dir + + package_name=$(basename "$package_dir") + + # Download the cache for the specific package + pkg_name="node-modules-$(params.target-branch)-$(params.node-version)-$package_name-$checksum" + echo "Downloading cache for $pkg_name" + ibmcloud cos download --bucket npm-cache --key $pkg_name $pkg_name.tar.zst + + if [ $? -eq 0 ]; then + # Decompress and extract the cache + echo "Download successful." + zstd -d $pkg_name.tar.zst -o "node-modules.tar" + tar -xf "node-modules.tar" + else + npmInstall=true + fi + fi + done + + cd $ARTIFACTS_PATH + + rm -rf node-modules.tar.zst + rm -rf node-modules.tar + + if [ "$npmInstall" == true ]; then + npm install --loglevel verbose + fi + + echo "Restored node_modules cache..." + else + echo "Download from main was not successful. Installing node_modules..." + npm install --loglevel verbose + fi fi - fi \ No newline at end of file + fi diff --git a/.tekton/templates/default-pipeline.yaml.template b/.tekton/templates/default-pipeline.yaml.template index c65564ad4d..9bbb9dbaea 100644 --- a/.tekton/templates/default-pipeline.yaml.template +++ b/.tekton/templates/default-pipeline.yaml.template @@ -38,6 +38,8 @@ spec: value: $(params.coverage) - name: context value: $(params.context) + - name: skip-cache + value: $(params.skip-cache) workspaces: - name: artifacts tasks: @@ -124,7 +126,9 @@ spec: - name: node-version value: $(params.node-version) - name: npm-version - value: $(params.npm-version) + value: $(params.npm-version) + - name: skip-cache + value: $(params.skip-cache) workspaces: - name: output workspace: artifacts