PROC-66: Align providers and extensions with astronomer images

.github/workflows/manual-build-and-publish.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: Version tag of base Airflow image
+        description: Version of base Airflow image
         required: true
         type: string
       changelogNew:
@@ -46,28 +46,36 @@ jobs:
       - name: Update changelog
         run: python versions/update_changelog.py --af-version ${{ github.event.inputs.version }} --build-version ${{ env.BUILD_VERSION }} --changelog-new '${{ github.event.inputs.changelogNew }}' --changelog-fixes '${{ github.event.inputs.changelogFixes }}'
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # @v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Configure image metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # @v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=semver,pattern={{version}},value=v${{ github.event.inputs.version }}
             type=semver,pattern={{version}},value=v${{ github.event.inputs.version }}-${{ env.BUILD_VERSION }}
+      - name: Get python version from requirements.txt
+        run: |
+          PY_VER=$(grep python_version versions/${{ github.event.inputs.version }}/requirements.txt | cut -d '=' -f 2)
+          if [ -z $PY_VER ]; then
+            echo "Please specify python version in this image requirements.txt in format # python_version=VER"
+            exit 1
+          fi
+          echo "AIRFLOW_TAG=${{ github.event.inputs.version }}-python${PY_VER}" >> $GITHUB_ENV
       - name: Build and push image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # @v5
         with:
           context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
-            AIRFLOW_TAG=${{ github.event.inputs.version }}
+            AIRFLOW_TAG=${{ env.AIRFLOW_TAG }}
       - name: Commit changes
         run: |
           git config user.name "GitHub Actions"

Dockerfile

@@ -1,4 +1,4 @@
 ARG AIRFLOW_TAG
 FROM apache/airflow:${AIRFLOW_TAG}
 COPY versions/${AIRFLOW_VERSION}/requirements.txt /requirements.txt
-RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" -r /requirements.txt
+RUN pip install --no-cache-dir -r /requirements.txt

versions/2.5.3/requirements.txt

@@ -1,14 +1,30 @@
-apache-airflow-providers-amazon==8.1.0
-apache-airflow-providers-celery==3.2.0
-apache-airflow-providers-cncf-kubernetes==7.0.0
-apache-airflow-providers-elasticsearch==4.5.0
-apache-airflow-providers-google==10.1.1
-apache-airflow-providers-http==4.4.1
-apache-airflow-providers-microsoft-azure==6.1.1
-apache-airflow-providers-postgres==5.5.0
-apache-airflow-providers-redis==3.2.0
-apache-airflow-providers-snowflake==4.1.0
-astro-sdk-python==1.6.1
-astronomer-cosmos==0.7.5
-openlineage-airflow==0.28.0
+# python_version=3.9
+# apache-airflow version MUST be equal to airflow version
+apache-airflow[async,azure,amazon,elasticsearch,google,password,cncf.kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv]==2.5.3
+apache-airflow-providers-amazon==8.6.0
+apache-airflow-providers-apache-hive==6.2.0
+apache-airflow-providers-apache-livy==3.6.0
+apache-airflow-providers-celery==3.3.3
+apache-airflow-providers-cncf-kubernetes==7.5.0
+apache-airflow-providers-common-sql==1.7.1
+apache-airflow-providers-databricks==4.6.0
+apache-airflow-providers-datadog==3.4.0
+apache-airflow-providers-dbt-cloud==3.4.0
+apache-airflow-providers-elasticsearch==5.0.1
+apache-airflow-providers-ftp==3.5.1
+apache-airflow-providers-google==10.7.0
+apache-airflow-providers-http==4.5.1
+apache-airflow-providers-imap==3.3.1
+apache-airflow-providers-microsoft-azure==6.3.0
+apache-airflow-providers-microsoft-mssql==3.5.0
+apache-airflow-providers-postgres==5.6.0
+apache-airflow-providers-redis==3.3.1
+apache-airflow-providers-sftp==4.7.0
+apache-airflow-providers-snowflake==5.0.1
+apache-airflow-providers-sqlite==3.4.3
+apache-airflow-providers-ssh==3.8.0
+astro-sdk-python==1.7.0
+astronomer-cosmos==1.2.0
+astronomer-providers[all]==1.17.3
+openlineage-airflow==1.1.0
 requests==2.31.0

versions/2.7.1/requirements.txt

@@ -1,14 +1,30 @@
-apache-airflow-providers-amazon==8.1.0
-apache-airflow-providers-celery==3.2.0
-apache-airflow-providers-cncf-kubernetes==7.0.0
-apache-airflow-providers-elasticsearch==4.5.0
-  apache-airflow-providers-google==10.1.1
-apache-airflow-providers-http==4.4.1
-apache-airflow-providers-microsoft-azure==6.1.1
-apache-airflow-providers-postgres==5.5.0
-apache-airflow-providers-redis==3.2.0
-apache-airflow-providers-snowflake==4.1.0
-astro-sdk-python==1.6.1
-astronomer-cosmos==0.7.5
-openlineage-airflow==0.28.0
+# python_version=3.11
+# apache-airflow version MUST be equal to airflow version
+apache-airflow[async,azure,amazon,elasticsearch,google,password,cncf.kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv]==2.7.1
+apache-airflow-providers-amazon==8.6.0
+apache-airflow-providers-apache-hive==6.2.0
+apache-airflow-providers-apache-livy==3.6.0
+apache-airflow-providers-celery==3.3.3
+apache-airflow-providers-cncf-kubernetes==7.5.0
+apache-airflow-providers-common-sql==1.7.1
+apache-airflow-providers-databricks==4.6.0
+apache-airflow-providers-datadog==3.4.0
+apache-airflow-providers-dbt-cloud==3.4.0
+apache-airflow-providers-elasticsearch==5.0.1
+apache-airflow-providers-ftp==3.5.1
+apache-airflow-providers-google==10.7.0
+apache-airflow-providers-http==4.5.1
+apache-airflow-providers-imap==3.3.1
+apache-airflow-providers-microsoft-azure==6.3.0
+apache-airflow-providers-microsoft-mssql==3.5.0
+apache-airflow-providers-postgres==5.6.0
+apache-airflow-providers-redis==3.3.1
+apache-airflow-providers-sftp==4.7.0
+apache-airflow-providers-snowflake==5.0.1
+apache-airflow-providers-sqlite==3.4.3
+apache-airflow-providers-ssh==3.8.0
+astro-sdk-python==1.7.0
+astronomer-cosmos==1.2.0
+astronomer-providers[all]==1.17.3
+openlineage-airflow==1.1.0
 requests==2.31.0