From f65150066ba8501bc98592eed328757a0f7e0b41 Mon Sep 17 00:00:00 2001 From: Hassan Date: Wed, 20 Nov 2024 12:08:56 -0600 Subject: [PATCH] chore: update all jobs to use python 3.9 or greater. --- dataeng/resources/dbt-docs.sh | 8 ++++---- dataeng/resources/dbt-run.sh | 8 ++++---- dataeng/resources/dbtsource-freshness.sh | 8 ++++---- dataeng/resources/prefect-flows-deployment.sh | 4 ++-- .../retirement-partner-report-cleanup.sh | 2 +- .../resources/retirement-partner-reporter.sh | 2 +- .../resources/setup-exporter-email-optin.sh | 2 +- dataeng/resources/setup-exporter.sh | 2 +- .../resources/user-retirement-bulk-status.sh | 2 +- .../warehouse-transforms-ci-manual.sh | 8 ++++---- .../warehouse-transforms-ci-master-merges.sh | 18 +++++++++--------- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/dataeng/resources/dbt-docs.sh b/dataeng/resources/dbt-docs.sh index 1a14273ef..3c34707b1 100644 --- a/dataeng/resources/dbt-docs.sh +++ b/dataeng/resources/dbt-docs.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -ex -# Creating python 3.8 virtual environment to run dbt warehouse-transform job -PYTHON38_VENV="py38_venv" -virtualenv --python=python3.8 --clear "${PYTHON38_VENV}" -source "${PYTHON38_VENV}/bin/activate" +# Creating python 3.11 virtual environment to run dbt warehouse-transform job +PYTHON311_VENV="py311_venv" +virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" +source "${PYTHON311_VENV}/bin/activate" # Setup cd $WORKSPACE/warehouse-transforms diff --git a/dataeng/resources/dbt-run.sh b/dataeng/resources/dbt-run.sh index 18aa6a152..b664c2c60 100644 --- a/dataeng/resources/dbt-run.sh +++ b/dataeng/resources/dbt-run.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -ex -# Creating python 3.8 virtual environment to run dbt warehouse-transform job -PYTHON38_VENV="py38_venv" -virtualenv --python=python3.8 --clear "${PYTHON38_VENV}" -source "${PYTHON38_VENV}/bin/activate" +# Creating python 3.11 virtual environment to run dbt warehouse-transform job +PYTHON311_VENV="py311_venv" +virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" +source "${PYTHON311_VENV}/bin/activate" # Setup to run dbt commands cd $WORKSPACE/warehouse-transforms diff --git a/dataeng/resources/dbtsource-freshness.sh b/dataeng/resources/dbtsource-freshness.sh index bb1b08540..0a0dd4369 100644 --- a/dataeng/resources/dbtsource-freshness.sh +++ b/dataeng/resources/dbtsource-freshness.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -ex -# Creating python 3.8 virtual environment to run dbt warehouse-transform job -PYTHON38_VENV="py38_venv" -virtualenv --python=python3.8 --clear "${PYTHON38_VENV}" -source "${PYTHON38_VENV}/bin/activate" +# Creating python 3.11 virtual environment to run dbt warehouse-transform job +PYTHON311_VENV="py311_venv" +virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" +source "${PYTHON311_VENV}/bin/activate" # Setup cd $WORKSPACE/warehouse-transforms diff --git a/dataeng/resources/prefect-flows-deployment.sh b/dataeng/resources/prefect-flows-deployment.sh index bc49bb1b7..879d12dcc 100644 --- a/dataeng/resources/prefect-flows-deployment.sh +++ b/dataeng/resources/prefect-flows-deployment.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -ex -# Creating python3.8 virtual env +# Creating python3.9 virtual env PYTHON_VENV="python_venv" -virtualenv --python=python3.8 --clear "${PYTHON_VENV}" +virtualenv --python=python3.9 --clear "${PYTHON_VENV}" source "${PYTHON_VENV}/bin/activate" # Removing prefix 'prefect-flows-deployment-' diff --git a/dataeng/resources/retirement-partner-report-cleanup.sh b/dataeng/resources/retirement-partner-report-cleanup.sh index 4859fdfb4..9f9939df2 100644 --- a/dataeng/resources/retirement-partner-report-cleanup.sh +++ b/dataeng/resources/retirement-partner-report-cleanup.sh @@ -6,7 +6,7 @@ set -ex # setting on the jenkins worker, it would be safest to keep the builds from # clobbering each other's virtualenvs. VENV="venv-${BUILD_NUMBER}" -virtualenv --python=python3.8 --clear "${VENV}" +virtualenv --python=python3.9 --clear "${VENV}" source "${VENV}/bin/activate" #Fetch secrets from AWS diff --git a/dataeng/resources/retirement-partner-reporter.sh b/dataeng/resources/retirement-partner-reporter.sh index 8465256a4..be9e6fcca 100644 --- a/dataeng/resources/retirement-partner-reporter.sh +++ b/dataeng/resources/retirement-partner-reporter.sh @@ -12,7 +12,7 @@ env # setting on the jenkins worker, it would be safest to keep the builds from # clobbering each other's virtualenvs. VENV="venv-${BUILD_NUMBER}" -virtualenv --python=python3.8 --clear "${VENV}" +virtualenv --python=python3.9 --clear "${VENV}" source "${VENV}/bin/activate" # Make sure that when we try to write unicode to the console, it diff --git a/dataeng/resources/setup-exporter-email-optin.sh b/dataeng/resources/setup-exporter-email-optin.sh index f8dd3f5a6..af769373c 100644 --- a/dataeng/resources/setup-exporter-email-optin.sh +++ b/dataeng/resources/setup-exporter-email-optin.sh @@ -6,7 +6,7 @@ mkdir -p /var/lib/jenkins/tmp/analytics-exporter/course-data # Create and activate a virtualenv in shell script EXPORTER_VENV="exporter_venv" -virtualenv --python=python3.8 --clear "${EXPORTER_VENV}" +virtualenv --python=python3.9 --clear "${EXPORTER_VENV}" source "${EXPORTER_VENV}/bin/activate" # Install requirements into this (exporter) virtual environment diff --git a/dataeng/resources/setup-exporter.sh b/dataeng/resources/setup-exporter.sh index b5813e5e7..2cddb89ee 100644 --- a/dataeng/resources/setup-exporter.sh +++ b/dataeng/resources/setup-exporter.sh @@ -6,7 +6,7 @@ mkdir -p /var/lib/jenkins/tmp/analytics-exporter/course-data # Create and activate a virtualenv in shell script EXPORTER_VENV="exporter_venv" -virtualenv --python=python3.8 --clear "${EXPORTER_VENV}" +virtualenv --python=python3.9 --clear "${EXPORTER_VENV}" source "${EXPORTER_VENV}/bin/activate" # Install requirements into this (exporter) virtual environment diff --git a/dataeng/resources/user-retirement-bulk-status.sh b/dataeng/resources/user-retirement-bulk-status.sh index f4c1883fb..4a8cdf644 100644 --- a/dataeng/resources/user-retirement-bulk-status.sh +++ b/dataeng/resources/user-retirement-bulk-status.sh @@ -12,7 +12,7 @@ env # setting on the jenkins worker, it would be safest to keep the builds from # clobbering each other's virtualenvs. VENV="venv-${BUILD_NUMBER}" -virtualenv --python=python3.8 --clear "${VENV}" +virtualenv --python=python3.9 --clear "${VENV}" source "${VENV}/bin/activate" # Make sure that when we try to write unicode to the console, it diff --git a/dataeng/resources/warehouse-transforms-ci-manual.sh b/dataeng/resources/warehouse-transforms-ci-manual.sh index cf9f48ac7..d1187b697 100644 --- a/dataeng/resources/warehouse-transforms-ci-manual.sh +++ b/dataeng/resources/warehouse-transforms-ci-manual.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -ex -# Creating python 3.8 virtual environment to run dbt warehouse-transform job -PYTHON38_VENV="py38_venv" -virtualenv --python=python3.8 --clear "${PYTHON38_VENV}" -source "${PYTHON38_VENV}/bin/activate" +# Creating python 3.11 virtual environment to run dbt warehouse-transform job +PYTHON311_VENV="py311_venv" +virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" +source "${PYTHON311_VENV}/bin/activate" # Specifying GITHUB_PR_ID and WAREHOUSE_TRANSFORMS_BRANCH is a must if [[ "$GITHUB_PR_ID" == "" || "$WAREHOUSE_TRANSFORMS_BRANCH" == "" ]] diff --git a/dataeng/resources/warehouse-transforms-ci-master-merges.sh b/dataeng/resources/warehouse-transforms-ci-master-merges.sh index 8193c2b3e..6ddbdd837 100644 --- a/dataeng/resources/warehouse-transforms-ci-master-merges.sh +++ b/dataeng/resources/warehouse-transforms-ci-master-merges.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -ex -# Creating python 3.8 virtual environment to run dbt warehouse-transform job -PYTHON38_VENV="py38_venv" -virtualenv --python=python3.8 --clear "${PYTHON38_VENV}" -source "${PYTHON38_VENV}/bin/activate" +# Creating python 3.11 virtual environment to run dbt warehouse-transform job +PYTHON311_VENV="py311_venv" +virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" +source "${PYTHON311_VENV}/bin/activate" # Setup to run python script to create snowflake schema cd $WORKSPACE/analytics-tools/snowflake @@ -44,24 +44,24 @@ then # Schema_Name will be the Github Commit ID e.g. 1724 prefixed with 'merged' and sufixed with project name e.g. 1724_reporting export CI_SCHEMA_NAME=merged_${COMMIT_ID}_reporting # Schema is dynamically created against each run. - # profiles.yml contains the name of Schema which is used to create output models when dbt runs. - python create_ci_schema.py --key_path $KEY_PATH --passphrase_path $PASSPHRASE_PATH --automation_user $USER --account $ACCOUNT --db_name $DB_NAME --schema_name $CI_SCHEMA_NAME + # profiles.yml contains the name of Schema which is used to create output models when dbt runs. + python create_ci_schema.py --key_path $KEY_PATH --passphrase_path $PASSPHRASE_PATH --automation_user $USER --account $ACCOUNT --db_name $DB_NAME --schema_name $CI_SCHEMA_NAME # create_ci_schema python script not just create schema but also drops the schema if it exists already, and the reason for doing so is if dbt model changes tables that are # created in seed job it will fail, so dropping those tables or deleting the whole schema is important to avoid such failure. We noticed while create_ci_schema being running # the dbt commands below starts running as they were using different sessions (warehouse and users), in order to complete the drop and create operation before running dbt adding sleep sleep 10s DBT_PROJECT_PATH='reporting' # Full dbt run on merges to master CI (Might decide to run Slim CI in future) - DBT_RUN_OPTIONS='' + DBT_RUN_OPTIONS='' DBT_RUN_EXCLUDE='' ## Add excluded models here if any # Full dbt test on merges to master CI (Might decide to run Slim CI in future) DBT_TEST_OPTIONS='' - DBT_TEST_EXCLUDE='' + DBT_TEST_EXCLUDE='' source $WORKSPACE/jenkins-job-dsl/dataeng/resources/warehouse-transforms-ci-dbt.sh cd $WORKSPACE/analytics-tools/snowflake - python remove_ci_schema.py --key_path $KEY_PATH --passphrase_path $PASSPHRASE_PATH --automation_user $USER --account $ACCOUNT --db_name $DB_NAME --schema_name $CI_SCHEMA_NAME + python remove_ci_schema.py --key_path $KEY_PATH --passphrase_path $PASSPHRASE_PATH --automation_user $USER --account $ACCOUNT --db_name $DB_NAME --schema_name $CI_SCHEMA_NAME fi