From 28b95f2a9ad94874fafba5b4f1c3ded963ad89d1 Mon Sep 17 00:00:00 2001 From: Yuriy Natarov Date: Mon, 16 Oct 2023 20:18:11 +0300 Subject: [PATCH 1/2] Remove unused 2.6.3 --- versions/2.6.3/CHANGELOG.md | 32 -------------------------------- versions/2.6.3/VERSION | 1 - versions/2.6.3/requirements.txt | 14 -------------- 3 files changed, 47 deletions(-) delete mode 100644 versions/2.6.3/CHANGELOG.md delete mode 100644 versions/2.6.3/VERSION delete mode 100644 versions/2.6.3/requirements.txt diff --git a/versions/2.6.3/CHANGELOG.md b/versions/2.6.3/CHANGELOG.md deleted file mode 100644 index d8f9435..0000000 --- a/versions/2.6.3/CHANGELOG.md +++ /dev/null @@ -1,32 +0,0 @@ -# Changelog - -DoubleCloud Airflow Base Image 2.6.3-3, 21.09.23 ----------------------------------------- - -### Fixed -TEST: Updated workflow step - -DoubleCloud Airflow Base Image 2.6.3-2, 21.09.23 ----------------------------------------- - -### Fixed -Test: Add requests - -DoubleCloud Airflow Base Image 2.6.3-1, 21.09.23 ----------------------------------------- - -### New -Added extra Python packages: -- apache-airflow-providers-amazon==8.1.0 -- apache-airflow-providers-celery==3.2.0 -- apache-airflow-providers-cncf-kubernetes==7.0.0 -- apache-airflow-providers-elasticsearch==4.5.0 -- apache-airflow-providers-google==10.1.1 -- apache-airflow-providers-http==4.4.1 -- apache-airflow-providers-microsoft-azure==6.1.1 -- apache-airflow-providers-postgres==5.5.0 -- apache-airflow-providers-redis==3.2.0 -- apache-airflow-providers-snowflake==4.1.0 -- astro-sdk-python==1.6.1 -- astronomer-cosmos==0.7.5 -- openlineage-airflow==0.28.0 diff --git a/versions/2.6.3/VERSION b/versions/2.6.3/VERSION deleted file mode 100644 index 00750ed..0000000 --- a/versions/2.6.3/VERSION +++ /dev/null @@ -1 +0,0 @@ -3 diff --git a/versions/2.6.3/requirements.txt b/versions/2.6.3/requirements.txt deleted file mode 100644 index 8c2ed34..0000000 --- a/versions/2.6.3/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -apache-airflow-providers-amazon==8.1.0 -apache-airflow-providers-celery==3.2.0 -apache-airflow-providers-cncf-kubernetes==7.0.0 -apache-airflow-providers-elasticsearch==4.5.0 -apache-airflow-providers-google==10.1.1 -apache-airflow-providers-http==4.4.1 -apache-airflow-providers-microsoft-azure==6.1.1 -apache-airflow-providers-postgres==5.5.0 -apache-airflow-providers-redis==3.2.0 -apache-airflow-providers-snowflake==4.1.0 -astro-sdk-python==1.6.1 -astronomer-cosmos==0.7.5 -openlineage-airflow==0.28.0 -requests==2.31.0 From 34cfcee938f9b774f4bf3ac3ba368498b134861b Mon Sep 17 00:00:00 2001 From: Yuriy Natarov Date: Mon, 16 Oct 2023 20:54:03 +0300 Subject: [PATCH 2/2] Upgraded and synced packages --- .../workflows/manual-build-and-publish.yaml | 18 +++++--- Dockerfile | 2 +- versions/2.5.3/requirements.txt | 42 +++++++++++++------ versions/2.7.1/requirements.txt | 42 +++++++++++++------ 4 files changed, 72 insertions(+), 32 deletions(-) diff --git a/.github/workflows/manual-build-and-publish.yaml b/.github/workflows/manual-build-and-publish.yaml index 9fb7b8a..2ccade2 100644 --- a/.github/workflows/manual-build-and-publish.yaml +++ b/.github/workflows/manual-build-and-publish.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: version: - description: Version tag of base Airflow image + description: Version of base Airflow image required: true type: string changelogNew: @@ -46,28 +46,36 @@ jobs: - name: Update changelog run: python versions/update_changelog.py --af-version ${{ github.event.inputs.version }} --build-version ${{ env.BUILD_VERSION }} --changelog-new '${{ github.event.inputs.changelogNew }}' --changelog-fixes '${{ github.event.inputs.changelogFixes }}' - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # @v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Configure image metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # @v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=semver,pattern={{version}},value=v${{ github.event.inputs.version }} type=semver,pattern={{version}},value=v${{ github.event.inputs.version }}-${{ env.BUILD_VERSION }} + - name: Get python version from requirements.txt + run: | + PY_VER=$(grep python_version versions/${{ github.event.inputs.version }}/requirements.txt | cut -d '=' -f 2) + if [ -z $PY_VER ]; then + echo "Please specify python version in this image requirements.txt in format # python_version=VER" + exit 1 + fi + echo "AIRFLOW_TAG=${{ github.event.inputs.version }}-python${PY_VER}" >> $GITHUB_ENV - name: Build and push image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # @v5 with: context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - AIRFLOW_TAG=${{ github.event.inputs.version }} + AIRFLOW_TAG=${{ env.AIRFLOW_TAG }} - name: Commit changes run: | git config user.name "GitHub Actions" diff --git a/Dockerfile b/Dockerfile index 3ed22f8..bbbcc87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ ARG AIRFLOW_TAG FROM apache/airflow:${AIRFLOW_TAG} COPY versions/${AIRFLOW_VERSION}/requirements.txt /requirements.txt -RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" -r /requirements.txt +RUN pip install --no-cache-dir -r /requirements.txt diff --git a/versions/2.5.3/requirements.txt b/versions/2.5.3/requirements.txt index 8c2ed34..d2c4fd0 100644 --- a/versions/2.5.3/requirements.txt +++ b/versions/2.5.3/requirements.txt @@ -1,14 +1,30 @@ -apache-airflow-providers-amazon==8.1.0 -apache-airflow-providers-celery==3.2.0 -apache-airflow-providers-cncf-kubernetes==7.0.0 -apache-airflow-providers-elasticsearch==4.5.0 -apache-airflow-providers-google==10.1.1 -apache-airflow-providers-http==4.4.1 -apache-airflow-providers-microsoft-azure==6.1.1 -apache-airflow-providers-postgres==5.5.0 -apache-airflow-providers-redis==3.2.0 -apache-airflow-providers-snowflake==4.1.0 -astro-sdk-python==1.6.1 -astronomer-cosmos==0.7.5 -openlineage-airflow==0.28.0 +# python_version=3.9 +# apache-airflow version MUST be equal to airflow version +apache-airflow[async,azure,amazon,elasticsearch,google,password,cncf.kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv]==2.5.3 +apache-airflow-providers-amazon==8.6.0 +apache-airflow-providers-apache-hive==6.2.0 +apache-airflow-providers-apache-livy==3.6.0 +apache-airflow-providers-celery==3.3.3 +apache-airflow-providers-cncf-kubernetes==7.5.0 +apache-airflow-providers-common-sql==1.7.1 +apache-airflow-providers-databricks==4.6.0 +apache-airflow-providers-datadog==3.4.0 +apache-airflow-providers-dbt-cloud==3.4.0 +apache-airflow-providers-elasticsearch==5.0.1 +apache-airflow-providers-ftp==3.5.1 +apache-airflow-providers-google==10.7.0 +apache-airflow-providers-http==4.5.1 +apache-airflow-providers-imap==3.3.1 +apache-airflow-providers-microsoft-azure==6.3.0 +apache-airflow-providers-microsoft-mssql==3.5.0 +apache-airflow-providers-postgres==5.6.0 +apache-airflow-providers-redis==3.3.1 +apache-airflow-providers-sftp==4.7.0 +apache-airflow-providers-snowflake==5.0.1 +apache-airflow-providers-sqlite==3.4.3 +apache-airflow-providers-ssh==3.8.0 +astro-sdk-python==1.7.0 +astronomer-cosmos==1.2.0 +astronomer-providers[all]==1.17.3 +openlineage-airflow==1.1.0 requests==2.31.0 diff --git a/versions/2.7.1/requirements.txt b/versions/2.7.1/requirements.txt index 35d0d77..b445002 100644 --- a/versions/2.7.1/requirements.txt +++ b/versions/2.7.1/requirements.txt @@ -1,14 +1,30 @@ -apache-airflow-providers-amazon==8.1.0 -apache-airflow-providers-celery==3.2.0 -apache-airflow-providers-cncf-kubernetes==7.0.0 -apache-airflow-providers-elasticsearch==4.5.0 - apache-airflow-providers-google==10.1.1 -apache-airflow-providers-http==4.4.1 -apache-airflow-providers-microsoft-azure==6.1.1 -apache-airflow-providers-postgres==5.5.0 -apache-airflow-providers-redis==3.2.0 -apache-airflow-providers-snowflake==4.1.0 -astro-sdk-python==1.6.1 -astronomer-cosmos==0.7.5 -openlineage-airflow==0.28.0 +# python_version=3.11 +# apache-airflow version MUST be equal to airflow version +apache-airflow[async,azure,amazon,elasticsearch,google,password,cncf.kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv]==2.7.1 +apache-airflow-providers-amazon==8.6.0 +apache-airflow-providers-apache-hive==6.2.0 +apache-airflow-providers-apache-livy==3.6.0 +apache-airflow-providers-celery==3.3.3 +apache-airflow-providers-cncf-kubernetes==7.5.0 +apache-airflow-providers-common-sql==1.7.1 +apache-airflow-providers-databricks==4.6.0 +apache-airflow-providers-datadog==3.4.0 +apache-airflow-providers-dbt-cloud==3.4.0 +apache-airflow-providers-elasticsearch==5.0.1 +apache-airflow-providers-ftp==3.5.1 +apache-airflow-providers-google==10.7.0 +apache-airflow-providers-http==4.5.1 +apache-airflow-providers-imap==3.3.1 +apache-airflow-providers-microsoft-azure==6.3.0 +apache-airflow-providers-microsoft-mssql==3.5.0 +apache-airflow-providers-postgres==5.6.0 +apache-airflow-providers-redis==3.3.1 +apache-airflow-providers-sftp==4.7.0 +apache-airflow-providers-snowflake==5.0.1 +apache-airflow-providers-sqlite==3.4.3 +apache-airflow-providers-ssh==3.8.0 +astro-sdk-python==1.7.0 +astronomer-cosmos==1.2.0 +astronomer-providers[all]==1.17.3 +openlineage-airflow==1.1.0 requests==2.31.0