From 8282419f81658b1bd29f552b1d1bb7794a674aa1 Mon Sep 17 00:00:00 2001 From: kukushking Date: Wed, 10 Jul 2024 20:13:31 +0100 Subject: [PATCH] update mwaa to 2.9.2 & update requirements/constraints (#177) * update mwaa constraints Signed-off-by: Anton Kukushkin * changelog Signed-off-by: Anton Kukushkin * update mwaa to 2.9.2 Signed-off-by: Anton Kukushkin * update mwaa constraints to align with 2.9.2 Signed-off-by: Anton Kukushkin * update mwaa version in the workflow Signed-off-by: Anton Kukushkin * reconcile dependencies Signed-off-by: Anton Kukushkin * changelog Signed-off-by: Anton Kukushkin --------- Signed-off-by: Anton Kukushkin --- .github/workflows/mwaa-requirements.check.yml | 2 +- CHANGELOG.md | 2 + .../requirements/constraint-requirements.txt | 847 +++++++++--------- data/mwaa/requirements/requirements.txt | 8 +- manifests/mwaa-ml-training/mwaa-modules.yaml | 2 +- 5 files changed, 446 insertions(+), 415 deletions(-) diff --git a/.github/workflows/mwaa-requirements.check.yml b/.github/workflows/mwaa-requirements.check.yml index 2e8b794d..6472a816 100644 --- a/.github/workflows/mwaa-requirements.check.yml +++ b/.github/workflows/mwaa-requirements.check.yml @@ -20,7 +20,7 @@ jobs: name: Install Requirements runs-on: ubuntu-latest env: - AIRFLOW_VERSION: 2.8.1 + AIRFLOW_VERSION: 2.9.2 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index b2f797ed..a647b44e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - various typo fixes - various clean-ups to the SageMaker Service Catalog templates - fix opensearch removal policy +- update MWAA to 2.9.2 +- update mwaa constraints ## v1.2.0 diff --git a/data/mwaa/requirements/constraint-requirements.txt b/data/mwaa/requirements/constraint-requirements.txt index 1c4ce96c..66c7dd83 100644 --- a/data/mwaa/requirements/constraint-requirements.txt +++ b/data/mwaa/requirements/constraint-requirements.txt @@ -1,7 +1,7 @@ # -# This constraints file was automatically generated on 2024-01-16T07:06:10.475440 -# via "eager-upgrade" mechanism of PIP. For the "v2-8-test" branch of Airflow. +# This constraints file was automatically generated on 2024-06-06T07:19:46.079179 +# via "eager-upgrade" mechanism of PIP. For the "v2-9-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. # @@ -31,590 +31,612 @@ # pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" # Authlib==1.3.1 -Babel==2.14.0 +Babel==2.15.0 ConfigUpdater==3.2 Deprecated==1.2.14 -Flask-AppBuilder==4.3.10 +Events==0.5 +Flask-AppBuilder==4.4.1 Flask-Babel==2.0.0 Flask-Bcrypt==1.0.1 -Flask-Caching==2.1.0 +Flask-Caching==2.3.0 Flask-JWT-Extended==4.6.0 -Flask-Limiter==3.5.0 +Flask-Limiter==3.7.0 Flask-Login==0.6.3 Flask-SQLAlchemy==2.5.1 Flask-Session==0.5.0 Flask-WTF==1.2.1 Flask==2.2.5 -GitPython==3.1.41 +GitPython==3.1.43 JPype1==1.5.0 JayDeBeApi==1.2.3 Jinja2==3.1.4 Js2Py==0.74 -Mako==1.3.0 -Markdown==3.5.2 -MarkupSafe==2.1.3 -PyGithub==2.1.1 +Mako==1.3.5 +Markdown==3.6 +MarkupSafe==2.1.5 +PyAthena==3.8.3 +PyGithub==2.3.0 PyHive==0.7.0 PyJWT==2.8.0 PyNaCl==1.5.0 PyYAML==6.0.1 -Pygments==2.17.2 +Pygments==2.18.0 SQLAlchemy-JSONField==1.0.2 -SQLAlchemy-Utils==0.41.1 -SQLAlchemy==1.4.51 +SQLAlchemy-Utils==0.41.2 +SQLAlchemy==1.4.52 SecretStorage==3.3.3 Sphinx==5.3.0 WTForms==3.1.2 Werkzeug==2.2.3 adal==1.2.7 -adlfs==2023.12.0 -aiobotocore==2.11.2 +adlfs==2024.4.1 +aiobotocore==2.13.0 aiofiles==23.2.1 -aiohttp==3.9.4 +aiohttp==3.9.5 aioitertools==0.11.0 aioresponses==0.7.6 aiosignal==1.3.1 alabaster==0.7.16 alembic==1.13.1 -alibabacloud-adb20211201==1.2.6 -alibabacloud-tea==0.3.5 -alibabacloud_credentials==0.3.2 +alibabacloud-adb20211201==1.3.5 +alibabacloud-tea==0.3.6 +alibabacloud_credentials==0.3.3 alibabacloud_endpoint_util==0.0.3 alibabacloud_gateway_spi==0.0.1 alibabacloud_openapi_util==0.2.2 -alibabacloud_tea_openapi==0.3.8 -alibabacloud_tea_util==0.3.11 +alibabacloud_tea_openapi==0.3.9 +alibabacloud_tea_util==0.3.12 alibabacloud_tea_xml==0.0.2 -aliyun-python-sdk-core==2.14.0 -aliyun-python-sdk-kms==2.16.2 +aliyun-python-sdk-core==2.15.1 +aliyun-python-sdk-kms==2.16.3 amqp==5.2.0 analytics-python==1.2.9 -annotated-types==0.6.0 +annotated-types==0.7.0 +ansicolors==1.1.8 anyascii==0.3.2 -anyio==4.2.0 -apache-airflow-providers-airbyte==3.5.1 -apache-airflow-providers-alibaba==2.7.1 -apache-airflow-providers-amazon==8.16.0 -apache-airflow-providers-apache-beam==5.5.0 -apache-airflow-providers-apache-cassandra==3.4.1 -apache-airflow-providers-apache-drill==2.6.0 -apache-airflow-providers-apache-druid==3.7.0 -apache-airflow-providers-apache-flink==1.3.0 -apache-airflow-providers-apache-hdfs==4.3.2 -apache-airflow-providers-apache-hive==6.4.1 -apache-airflow-providers-apache-impala==1.3.0 -apache-airflow-providers-apache-kafka==1.3.1 -apache-airflow-providers-apache-kylin==3.5.0 -apache-airflow-providers-apache-livy==3.7.1 -apache-airflow-providers-apache-pig==4.3.0 -apache-airflow-providers-apache-pinot==4.3.0 -apache-airflow-providers-apache-spark==4.7.0 -apache-airflow-providers-apprise==1.2.1 -apache-airflow-providers-arangodb==2.4.1 -apache-airflow-providers-asana==2.4.1 -apache-airflow-providers-atlassian-jira==2.5.0 -apache-airflow-providers-celery==3.5.1 -apache-airflow-providers-cloudant==3.4.1 -apache-airflow-providers-cncf-kubernetes==7.13.0 -apache-airflow-providers-cohere==1.1.1 -apache-airflow-providers-common-io==1.2.0 -apache-airflow-providers-common-sql==1.10.0 -apache-airflow-providers-databricks==6.0.0 -apache-airflow-providers-datadog==3.5.1 -apache-airflow-providers-dbt-cloud==3.5.1 -apache-airflow-providers-dingding==3.4.0 -apache-airflow-providers-discord==3.5.0 -apache-airflow-providers-docker==3.9.1 -apache-airflow-providers-elasticsearch==5.3.1 -apache-airflow-providers-exasol==4.4.1 -apache-airflow-providers-facebook==3.4.0 -apache-airflow-providers-ftp==3.7.0 -apache-airflow-providers-github==2.5.1 -apache-airflow-providers-google==10.13.1 -apache-airflow-providers-grpc==3.4.1 -apache-airflow-providers-hashicorp==3.6.1 -apache-airflow-providers-http==4.8.0 -apache-airflow-providers-imap==3.5.0 -apache-airflow-providers-influxdb==2.4.0 -apache-airflow-providers-jdbc==4.2.1 -apache-airflow-providers-jenkins==3.5.1 -apache-airflow-providers-microsoft-azure==8.5.1 -apache-airflow-providers-microsoft-mssql==3.6.0 -apache-airflow-providers-microsoft-psrp==2.5.0 -apache-airflow-providers-microsoft-winrm==3.4.0 -apache-airflow-providers-mongo==4.0.0 -apache-airflow-providers-mysql==5.5.1 -apache-airflow-providers-neo4j==3.5.0 -apache-airflow-providers-odbc==4.4.0 -apache-airflow-providers-openai==1.1.0 -apache-airflow-providers-openfaas==3.4.0 -apache-airflow-providers-openlineage==1.4.0 -apache-airflow-providers-opensearch==1.1.1 -apache-airflow-providers-opsgenie==5.5.0 -apache-airflow-providers-oracle==3.9.1 -apache-airflow-providers-pagerduty==3.6.0 -apache-airflow-providers-papermill==3.6.0 -apache-airflow-providers-pgvector==1.1.0 -apache-airflow-providers-pinecone==1.1.1 -apache-airflow-providers-postgres==5.10.0 -apache-airflow-providers-presto==5.4.0 -apache-airflow-providers-redis==3.6.0 -apache-airflow-providers-salesforce==5.6.1 -apache-airflow-providers-samba==4.5.0 -apache-airflow-providers-segment==3.4.0 -apache-airflow-providers-sendgrid==3.4.0 -apache-airflow-providers-sftp==4.8.1 -apache-airflow-providers-singularity==3.4.0 -apache-airflow-providers-slack==8.5.1 -apache-airflow-providers-smtp==1.5.0 -apache-airflow-providers-snowflake==5.2.1 -apache-airflow-providers-sqlite==3.7.0 -apache-airflow-providers-ssh==3.10.0 -apache-airflow-providers-tableau==4.4.0 -apache-airflow-providers-tabular==1.4.1 -apache-airflow-providers-telegram==4.3.0 -apache-airflow-providers-trino==5.6.0 -apache-airflow-providers-vertica==3.7.0 -apache-airflow-providers-weaviate==1.3.0 -apache-airflow-providers-yandex==3.7.1 -apache-airflow-providers-zendesk==4.6.0 -apache-beam==2.53.0 -apispec==6.4.0 -apprise==1.7.1 -argcomplete==3.2.1 -asana==3.2.2 -asgiref==3.7.2 +anyio==4.4.0 +apache-airflow-providers-airbyte==3.8.1 +apache-airflow-providers-alibaba==2.8.1 +apache-airflow-providers-amazon==8.24.0 +apache-airflow-providers-apache-beam==5.7.1 +apache-airflow-providers-apache-cassandra==3.5.1 +apache-airflow-providers-apache-drill==2.7.1 +apache-airflow-providers-apache-druid==3.10.1 +apache-airflow-providers-apache-flink==1.4.1 +apache-airflow-providers-apache-hdfs==4.4.1 +apache-airflow-providers-apache-hive==8.1.1 +apache-airflow-providers-apache-iceberg==1.0.0 +apache-airflow-providers-apache-impala==1.4.1 +apache-airflow-providers-apache-kafka==1.4.1 +apache-airflow-providers-apache-kylin==3.6.1 +apache-airflow-providers-apache-livy==3.8.1 +apache-airflow-providers-apache-pig==4.4.1 +apache-airflow-providers-apache-pinot==4.4.1 +apache-airflow-providers-apache-spark==4.8.1 +apache-airflow-providers-apprise==1.3.1 +apache-airflow-providers-arangodb==2.5.1 +apache-airflow-providers-asana==2.5.1 +apache-airflow-providers-atlassian-jira==2.6.1 +apache-airflow-providers-celery==3.7.2 +apache-airflow-providers-cloudant==3.5.1 +apache-airflow-providers-cncf-kubernetes==8.3.1 +apache-airflow-providers-cohere==1.2.1 +apache-airflow-providers-common-io==1.3.2 +apache-airflow-providers-common-sql==1.14.0 +apache-airflow-providers-databricks==6.5.0 +apache-airflow-providers-datadog==3.6.1 +apache-airflow-providers-dbt-cloud==3.8.1 +apache-airflow-providers-dingding==3.5.1 +apache-airflow-providers-discord==3.7.1 +apache-airflow-providers-docker==3.12.0 +apache-airflow-providers-elasticsearch==5.4.1 +apache-airflow-providers-exasol==4.5.1 +apache-airflow-providers-fab==1.1.1 +apache-airflow-providers-facebook==3.5.1 +apache-airflow-providers-ftp==3.9.1 +apache-airflow-providers-github==2.6.1 +apache-airflow-providers-google==10.20.0 +apache-airflow-providers-grpc==3.5.1 +apache-airflow-providers-hashicorp==3.7.1 +apache-airflow-providers-http==4.11.1 +apache-airflow-providers-imap==3.6.1 +apache-airflow-providers-influxdb==2.5.1 +apache-airflow-providers-jdbc==4.3.1 +apache-airflow-providers-jenkins==3.6.1 +apache-airflow-providers-microsoft-azure==10.1.1 +apache-airflow-providers-microsoft-mssql==3.7.1 +apache-airflow-providers-microsoft-psrp==2.7.1 +apache-airflow-providers-microsoft-winrm==3.5.1 +apache-airflow-providers-mongo==4.1.1 +apache-airflow-providers-mysql==5.6.1 +apache-airflow-providers-neo4j==3.6.1 +apache-airflow-providers-odbc==4.6.1 +apache-airflow-providers-openai==1.2.1 +apache-airflow-providers-openfaas==3.5.1 +apache-airflow-providers-openlineage==1.8.0 +apache-airflow-providers-opensearch==1.2.1 +apache-airflow-providers-opsgenie==5.6.1 +apache-airflow-providers-oracle==3.10.1 +apache-airflow-providers-pagerduty==3.7.1 +apache-airflow-providers-papermill==3.7.1 +apache-airflow-providers-pgvector==1.2.1 +apache-airflow-providers-pinecone==2.0.0 +apache-airflow-providers-postgres==5.11.1 +apache-airflow-providers-presto==5.5.1 +apache-airflow-providers-qdrant==1.1.1 +apache-airflow-providers-redis==3.7.1 +apache-airflow-providers-salesforce==5.7.1 +apache-airflow-providers-samba==4.7.1 +apache-airflow-providers-segment==3.5.1 +apache-airflow-providers-sendgrid==3.5.1 +apache-airflow-providers-sftp==4.10.1 +apache-airflow-providers-singularity==3.5.1 +apache-airflow-providers-slack==8.7.1 +apache-airflow-providers-smtp==1.7.1 +apache-airflow-providers-snowflake==5.5.1 +apache-airflow-providers-sqlite==3.8.1 +apache-airflow-providers-ssh==3.11.1 +apache-airflow-providers-tableau==4.5.1 +apache-airflow-providers-tabular==1.5.1 +apache-airflow-providers-telegram==4.5.1 +apache-airflow-providers-teradata==2.2.0 +apache-airflow-providers-trino==5.7.1 +apache-airflow-providers-vertica==3.8.1 +apache-airflow-providers-weaviate==1.4.1 +apache-airflow-providers-yandex==3.11.1 +apache-airflow-providers-zendesk==4.7.1 +apache-beam==2.56.0 +apispec==6.6.1 +apprise==1.8.0 +argcomplete==3.3.0 +asana==3.2.3 +asgiref==3.8.1 asn1crypto==1.5.1 astroid==2.15.8 asttokens==2.4.1 +asyncssh==2.14.2 atlasclient==1.0.0 -atlassian-python-api==3.41.5 +atlassian-python-api==3.41.13 attrs==23.2.0 -aws-sam-translator==1.83.0 -aws-xray-sdk==2.12.1 -azure-batch==14.1.0 +aws-sam-translator==1.89.0 +aws-xray-sdk==2.14.0 +azure-batch==14.2.0 azure-common==1.1.28 -azure-core==1.29.6 -azure-cosmos==4.5.1 +azure-core==1.30.1 +azure-cosmos==4.7.0 azure-datalake-store==0.0.53 -azure-identity==1.16.1 -azure-keyvault-secrets==4.7.0 -azure-kusto-data==4.3.1 +azure-identity==1.16.0 +azure-keyvault-secrets==4.8.0 +azure-kusto-data==4.4.1 azure-mgmt-containerinstance==10.1.0 azure-mgmt-containerregistry==10.3.0 azure-mgmt-core==1.4.0 -azure-mgmt-cosmosdb==9.4.0 -azure-mgmt-datafactory==4.0.0 +azure-mgmt-cosmosdb==9.5.0 +azure-mgmt-datafactory==7.1.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==23.0.1 +azure-mgmt-resource==23.1.1 azure-mgmt-storage==21.1.0 azure-nspkg==3.0.2 -azure-servicebus==7.11.4 -azure-storage-blob==12.19.0 -azure-storage-file-datalake==12.14.0 -azure-storage-file-share==12.15.0 -azure-synapse-artifacts==0.18.0 +azure-servicebus==7.12.2 +azure-storage-blob==12.20.0 +azure-storage-file-datalake==12.15.0 +azure-storage-file-share==12.16.0 +azure-synapse-artifacts==0.19.0 azure-synapse-spark==0.7.0 backoff==2.2.1 -bcrypt==4.1.2 -beautifulsoup4==4.12.2 +backports.tarfile==1.2.0 +bcrypt==4.1.3 +beautifulsoup4==4.12.3 billiard==4.2.0 bitarray==2.9.2 -black==24.3.0 -blinker==1.7.0 -boto3==1.33.13 -botocore==1.33.13 +black==24.4.2 +blinker==1.8.2 +boto3==1.34.106 +botocore==1.34.106 cachelib==0.9.0 -cachetools==5.3.2 -cassandra-driver==3.29.0 +cachetools==5.3.3 +cassandra-driver==3.29.1 cattrs==23.2.3 -celery==5.3.6 -certifi==2024.7.4 +celery==5.4.0 +certifi==2024.6.2 cffi==1.16.0 cfgv==3.4.0 -cfn-lint==0.83.8 -cgroupspy==0.2.2 +cfn-lint==0.87.4 +cgroupspy==0.2.3 chardet==5.2.0 charset-normalizer==3.3.2 checksumdir==1.2.0 ciso8601==2.3.1 -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 click-plugins==1.1.1 click-repl==0.3.0 click==8.1.7 clickclick==20.10.2 cloudant==2.15.0 cloudpickle==2.2.1 -cohere==4.44 +cohere==4.57 colorama==0.4.6 colorlog==4.8.0 -comm==0.2.1 -confluent-kafka==2.3.0 +comm==0.2.2 +confluent-kafka==2.4.0 connexion==2.14.2 -coverage==7.4.0 +coverage==7.5.3 crcmod==1.7 -cron-descriptor==1.4.0 -croniter==2.0.1 +cron-descriptor==1.4.3 +croniter==2.0.5 cryptography==41.0.7 curlify==2.2.1 -databricks-sql-connector==2.9.3 -datadog==0.47.0 +databricks-sql-connector==2.9.6 +datadog==0.49.1 db-dtypes==1.2.0 -debugpy==1.8.0 +debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -deltalake==0.15.1 +deltalake==0.17.4 +diagrams==0.23.4 dill==0.3.8 distlib==0.3.8 distro==1.9.0 dnspython==2.6.1 -docker==7.0.0 +docker==7.1.0 docopt==0.6.2 -docutils==0.20.1 -duckdb==0.9.2 -ecdsa==0.18.0 -editables==0.5 -elastic-transport==8.11.0 -elasticsearch==8.11.1 -email-validator==1.3.1 +docstring_parser==0.16 +docutils==0.16 +duckdb==1.0.0 +elastic-transport==8.13.1 +elasticsearch==8.13.2 +email_validator==2.1.1 entrypoints==0.4 -eralchemy2==1.3.8 +eralchemy2==1.4.1 et-xmlfile==1.1.0 -eventlet==0.35.2 -execnet==2.0.2 +eventlet==0.36.1 +execnet==2.1.1 executing==2.0.1 -facebook-business==18.0.5 -fastavro==1.9.3 +facebook_business==19.0.3 +fastavro==1.9.4 fasteners==0.19 fastjsonschema==2.19.1 -filelock==3.13.1 +filelock==3.14.0 flower==2.0.1 frozenlist==1.4.1 fsspec==2023.12.2 -future==0.18.3 +future==1.0.0 gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==7.0.0 -gcloud-aio-storage==9.0.0 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.2.0 gcsfs==2023.12.2.post1 geomet==0.2.1.post1 -gevent==23.9.1 +gevent==24.2.1 gitdb==4.0.11 -google-ads==22.1.0 -google-analytics-admin==0.22.2 -google-api-core==2.15.0 -google-api-python-client==2.113.0 +google-ads==24.1.0 +google-analytics-admin==0.22.7 +google-api-core==2.19.0 +google-api-python-client==2.132.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.0 -google-auth==2.26.2 -google-cloud-aiplatform==1.39.0 -google-cloud-appengine-logging==1.4.0 +google-auth==2.29.0 +google-cloud-aiplatform==1.54.0 +google-cloud-appengine-logging==1.4.3 google-cloud-audit-log==0.2.5 -google-cloud-automl==2.12.0 -google-cloud-batch==0.17.7 -google-cloud-bigquery-datatransfer==3.13.0 -google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.16.0 -google-cloud-bigtable==2.22.0 -google-cloud-build==3.22.0 -google-cloud-compute==1.15.0 -google-cloud-container==2.37.0 +google-cloud-automl==2.13.3 +google-cloud-batch==0.17.21 +google-cloud-bigquery-datatransfer==3.15.3 +google-cloud-bigquery==3.20.1 +google-cloud-bigtable==2.23.1 +google-cloud-build==3.24.0 +google-cloud-compute==1.19.0 +google-cloud-container==2.46.0 google-cloud-core==2.4.1 -google-cloud-datacatalog==3.17.2 -google-cloud-dataflow-client==0.8.6 -google-cloud-dataform==0.5.5 -google-cloud-dataplex==1.11.0 -google-cloud-dataproc-metastore==1.14.0 -google-cloud-dataproc==5.8.0 -google-cloud-dlp==3.14.0 -google-cloud-kms==2.20.0 -google-cloud-language==2.12.0 -google-cloud-logging==3.9.0 -google-cloud-memcache==1.8.0 -google-cloud-monitoring==2.18.0 -google-cloud-orchestration-airflow==1.10.0 -google-cloud-os-login==2.13.0 -google-cloud-pubsub==2.19.0 -google-cloud-redis==2.14.0 -google-cloud-resource-manager==1.11.0 -google-cloud-run==0.10.1 -google-cloud-secret-manager==2.17.0 -google-cloud-spanner==3.41.0 -google-cloud-speech==2.23.0 -google-cloud-storage-transfer==1.10.0 -google-cloud-storage==2.14.0 -google-cloud-tasks==2.15.0 -google-cloud-texttospeech==2.15.1 -google-cloud-translate==3.14.0 -google-cloud-videointelligence==2.12.0 -google-cloud-vision==3.5.0 -google-cloud-workflows==1.13.0 +google-cloud-datacatalog==3.19.0 +google-cloud-dataflow-client==0.8.10 +google-cloud-dataform==0.5.9 +google-cloud-dataplex==2.0.0 +google-cloud-dataproc-metastore==1.15.3 +google-cloud-dataproc==5.9.3 +google-cloud-dlp==3.18.0 +google-cloud-kms==2.23.0 +google-cloud-language==2.13.3 +google-cloud-logging==3.10.0 +google-cloud-memcache==1.9.3 +google-cloud-monitoring==2.21.0 +google-cloud-orchestration-airflow==1.12.1 +google-cloud-os-login==2.14.3 +google-cloud-pubsub==2.21.2 +google-cloud-redis==2.15.3 +google-cloud-resource-manager==1.12.3 +google-cloud-run==0.10.5 +google-cloud-secret-manager==2.20.0 +google-cloud-spanner==3.47.0 +google-cloud-speech==2.26.0 +google-cloud-storage-transfer==1.11.3 +google-cloud-storage==2.16.0 +google-cloud-tasks==2.16.3 +google-cloud-texttospeech==2.16.3 +google-cloud-translate==3.15.3 +google-cloud-videointelligence==2.13.3 +google-cloud-vision==3.7.2 +google-cloud-workflows==1.14.3 google-crc32c==1.5.0 -google-re2==1.1 +google-re2==1.1.20240601 google-resumable-media==2.7.0 -googleapis-common-protos==1.62.0 +googleapis-common-protos==1.63.1 graphql-core==3.2.3 -graphviz==0.20.1 +graphviz==0.20.3 greenlet==3.0.3 grpc-google-iam-v1==0.13.0 +grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 -grpcio-status==1.60.0 -grpcio==1.60.0 +grpcio-status==1.62.2 +grpcio-tools==1.62.2 +grpcio==1.64.1 gssapi==1.8.3 gunicorn==22.0.0 h11==0.14.0 -hatch==1.9.1 -hatchling==1.21.0 +h2==4.1.0 +hatch==1.12.0 +hatchling==1.24.2 hdfs==2.7.3 hmsclient==0.1.1 -httpcore==0.16.3 +hpack==4.0.0 +httpcore==0.18.0 httplib2==0.22.0 -httpx==0.23.3 +httpx==0.25.0 humanize==4.9.0 -hvac==2.1.0 +hvac==2.2.0 +hyperframe==6.0.1 hyperlink==21.0.0 icdiff==2.0.7 -identify==2.5.33 +identify==2.5.36 idna==3.7 ijson==3.2.3 imagesize==1.4.1 importlib-metadata==6.11.0 -importlib-resources==6.1.1 +importlib_resources==6.4.0 impyla==0.19.0 incremental==22.10.0 inflection==0.5.1 -influxdb-client==1.39.0 +influxdb-client==1.43.0 iniconfig==2.0.0 ipdb==0.13.13 -ipykernel==6.28.0 -ipython==8.20.0 +ipykernel==6.29.4 +ipython==8.25.0 isodate==0.6.1 -itsdangerous==2.1.2 -jaraco.classes==3.3.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 jedi==0.19.1 jeepney==0.8.0 jmespath==0.10.0 +joserfc==0.11.1 jschema-to-python==1.2.3 json-merge-patch==0.2 jsondiff==2.0.0 jsonpatch==1.33 jsonpath-ng==1.6.1 -jsonpickle==3.0.2 +jsonpickle==3.0.4 jsonpointer==2.4 jsonschema-path==0.3.2 jsonschema-specifications==2023.12.1 -jsonschema==4.20.0 +jsonschema==4.22.0 junit-xml==1.9 -jupyter_client==8.6.0 -jupyter_core==5.7.1 -keyring==24.3.0 -kombu==5.3.5 +jupyter_client==8.6.2 +jupyter_core==5.7.2 +keyring==25.2.1 +kombu==5.3.7 krb5==0.5.1 -kubernetes-asyncio==24.2.3 -kubernetes==23.6.0 +kubernetes==29.0.0 +kubernetes_asyncio==29.0.0 kylinpy==2.8.4 lazy-object-proxy==1.10.0 ldap3==2.9.1 -limits==3.7.0 -linkify-it-py==2.0.2 +limits==3.12.0 +linkify-it-py==2.0.3 lockfile==0.12.2 loguru==0.7.2 -looker-sdk==23.20.1 -lxml==5.1.0 +looker-sdk==24.8.0 +lxml==5.2.2 lz4==4.3.3 markdown-it-py==3.0.0 -marshmallow-oneofschema==3.0.1 -marshmallow-sqlalchemy==0.26.1 -marshmallow==3.20.2 -matplotlib-inline==0.1.6 -mdit-py-plugins==0.4.0 +marshmallow-oneofschema==3.1.1 +marshmallow-sqlalchemy==0.28.2 +marshmallow==3.21.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.1 mdurl==0.1.2 +methodtools==0.4.7 +microsoft-kiota-abstractions==1.3.3 +microsoft-kiota-authentication-azure==1.0.0 +microsoft-kiota-http==1.3.1 mmhash3==3.0.1 mongomock==4.1.2 more-itertools==10.2.0 -moto==4.2.13 +moto==5.0.9 mpmath==1.3.0 msal-extensions==1.1.0 msal==1.28.0 +msgraph-core==1.0.0 msrest==0.7.1 -msrestazure==0.6.4 +msrestazure==0.6.4.post1 multi_key_dict==2.0.3 -multidict==6.0.4 +multidict==6.0.5 mypy-boto3-appflow==1.34.0 -mypy-boto3-rds==1.34.6 +mypy-boto3-rds==1.34.116 mypy-boto3-redshift-data==1.34.0 -mypy-boto3-s3==1.34.14 +mypy-boto3-s3==1.34.120 mypy-extensions==1.0.0 -mypy==1.2.0 -mysql-connector-python==8.0.29 -mysqlclient==2.2.1 -nbclient==0.9.0 -nbformat==5.9.2 -neo4j==5.16.0 -nest-asyncio==1.5.9 -networkx==3.2.1 -nh3==0.2.15 -nodeenv==1.8.0 -numpy==1.24.4 +mypy==1.9.0 +mysql-connector-python==8.4.0 +mysqlclient==2.2.4 +nbclient==0.10.0 +nbformat==5.10.4 +neo4j==5.20.0 +nest-asyncio==1.6.0 +networkx==3.3 +nh3==0.2.17 +nodeenv==1.9.1 +numpy==1.26.4 oauthlib==3.2.2 -objsize==0.6.1 -openai==1.7.2 +objsize==0.7.0 +openai==1.31.1 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 -openlineage-integration-common==1.7.0 -openlineage-python==1.7.0 -openlineage_sql==1.7.0 -openpyxl==3.1.2 -opensearch-py==2.4.2 -opentelemetry-api==1.22.0 -opentelemetry-exporter-otlp-proto-common==1.22.0 -opentelemetry-exporter-otlp-proto-grpc==1.22.0 -opentelemetry-exporter-otlp-proto-http==1.22.0 -opentelemetry-exporter-otlp==1.22.0 -opentelemetry-exporter-prometheus==0.43b0 -opentelemetry-proto==1.22.0 -opentelemetry-sdk==1.22.0 -opentelemetry-semantic-conventions==0.43b0 +openlineage-integration-common==1.16.0 +openlineage-python==1.16.0 +openlineage_sql==1.16.0 +openpyxl==3.1.3 +opensearch-py==2.6.0 +opentelemetry-api==1.25.0 +opentelemetry-exporter-otlp-proto-common==1.25.0 +opentelemetry-exporter-otlp-proto-grpc==1.25.0 +opentelemetry-exporter-otlp-proto-http==1.25.0 +opentelemetry-exporter-otlp==1.25.0 +opentelemetry-exporter-prometheus==0.46b0 +opentelemetry-proto==1.25.0 +opentelemetry-sdk==1.25.0 +opentelemetry-semantic-conventions==0.46b0 opsgenie-sdk==2.1.5 -oracledb==2.0.1 +oracledb==2.2.1 ordered-set==4.1.0 -orjson==3.9.15 -oss2==2.18.4 -packaging==23.2 -pandas-gbq==0.20.0 -pandas-stubs==2.0.2.230605 +orjson==3.10.3 +oss2==2.18.5 +packaging==24.0 +pandas-gbq==0.23.0 +pandas-stubs==2.2.2.240603 pandas==2.1.4 -papermill==2.5.0 +papermill==2.6.0 paramiko==3.4.0 -parso==0.8.3 +parso==0.8.4 pathable==0.4.3 pathspec==0.12.1 pbr==6.0.0 pdpyras==5.2.0 pendulum==3.0.0 pexpect==4.9.0 -pgvector==0.2.4 -pinecone-client==2.2.4 -pinotdb==5.1.2 -pipdeptree==2.13.2 -pipx==1.4.2 -pkginfo==1.9.6 -platformdirs==3.11.0 -pluggy==1.3.0 +pgvector==0.2.5 +pinecone-client==4.1.1 +pinecone-plugin-interface==0.0.7 +pinotdb==5.2.0 +pipdeptree==2.22.0 +pipx==1.6.0 +pkginfo==1.11.0 +platformdirs==4.2.2 +pluggy==1.5.0 ply==3.11 plyvel==1.5.1 portalocker==2.8.2 pprintpp==0.4.0 -pre-commit==3.6.0 +pre-commit==3.7.1 presto-python-client==0.8.4 prison==0.2.1 -prometheus-client==0.19.0 -prompt-toolkit==3.0.43 +prometheus_client==0.20.0 +prompt_toolkit==3.0.46 proto-plus==1.23.0 -protobuf==4.25.2 -psutil==5.9.7 +protobuf==4.25.3 +psutil==5.9.8 psycopg2-binary==2.9.9 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 -py-partiql-parser==0.5.0 +py-partiql-parser==0.5.5 py4j==0.10.9.7 -pyOpenSSL==23.3.0 +pyOpenSSL==24.1.0 pyarrow-hotfix==0.6 pyarrow==14.0.2 pyasn1-modules==0.3.0 pyasn1==0.5.1 -pycountry==23.12.11 -pycparser==2.21 +pycountry==24.6.1 +pycparser==2.22 pycryptodome==3.20.0 -pydantic==2.5.3 -pydantic_core==2.14.6 +pydantic==2.7.3 +pydantic_core==2.18.4 pydata-google-auth==1.8.2 pydot==1.4.2 -pydruid==0.6.6 +pydruid==0.6.9 pyenchant==3.2.2 pyexasol==0.25.2 -pygraphviz==1.12 -pyiceberg==0.5.1 +pygraphviz==1.13 +pyiceberg==0.6.1 pyjsparser==2.7.1 pykerberos==1.2.4 -pymongo==4.6.3 -pymssql==2.2.11 -pyodbc==5.0.1 -pyparsing==3.1.1 +pymongo==4.7.3 +pymssql==2.3.0 +pyodbc==5.1.0 +pyparsing==3.1.2 pypsrp==0.8.1 -pyspark==3.5.0 +pyspark==3.5.1 pyspnego==0.10.2 -pytest-asyncio==0.23.3 -pytest-cov==4.1.0 -pytest-httpx==0.21.3 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-custom-exit-code==0.3.0 pytest-icdiff==0.9 pytest-instafail==0.5.0 -pytest-mock==3.12.0 -pytest-rerunfailures==13.0 +pytest-mock==3.14.0 +pytest-rerunfailures==14.0 pytest-timeouts==1.2.1 -pytest-xdist==3.5.0 +pytest-xdist==3.6.1 pytest==7.4.4 -python-arango==7.9.0 +python-arango==8.0.0 python-daemon==3.0.1 -python-dateutil==2.8.2 -python-dotenv==1.0.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 python-http-client==3.3.7 python-jenkins==1.8.2 -python-jose==3.3.0 python-ldap==3.4.4 -python-nvd3==0.15.0 -python-slugify==8.0.1 +python-nvd3==0.16.0 +python-slugify==8.0.4 python-telegram-bot==20.2 python3-saml==1.16.0 -pytz==2023.3.post1 +pytz==2024.1 pywinrm==0.4.3 -pyzmq==25.1.2 +pyzmq==26.0.3 +qdrant-client==1.9.1 reactivex==4.0.4 -readme-renderer==42.0 -redis==4.6.0 -redshift-connector==2.0.918 -referencing==0.32.1 -regex==2023.12.25 -requests-file==1.5.1 -requests-kerberos==0.14.0 -requests-mock==1.11.0 +readme_renderer==43.0 +redis==5.0.4 +redshift-connector==2.1.1 +referencing==0.31.1 +regex==2024.5.15 +requests-file==2.1.0 +requests-kerberos==0.15.0 +requests-mock==1.12.1 requests-ntlm==1.2.0 requests-oauthlib==1.3.1 requests-toolbelt==1.0.0 -requests==2.32.2 -responses==0.24.1 +requests==2.31.0 +responses==0.25.0 restructuredtext_lint==1.4.0 rfc3339-validator==0.1.4 rfc3986==1.5.0 -rich-argparse==1.4.0 -rich-click==1.7.3 -rich==13.7.0 -rpds-py==0.17.1 +rich-argparse==1.5.0 +rich-click==1.8.2 +rich==13.7.1 +rpds-py==0.18.1 rsa==4.9 -ruff==0.1.11 +ruff==0.3.3 s3fs==2023.12.2 -s3transfer==0.8.2 +s3transfer==0.10.1 sarif-om==1.0.4 -scramp==1.4.4 +scramp==1.4.5 scrapbook==0.5.0 semver==3.0.2 sendgrid==6.11.0 sentinels==1.0.0 -sentry-sdk==1.39.2 +sentry-sdk==2.4.0 setproctitle==1.3.3 -shapely==2.0.2 +shapely==2.0.4 shellingham==1.5.4 -simple-salesforce==1.12.5 +simple-salesforce==1.12.6 six==1.16.0 -slack_sdk==3.26.2 -smbprotocol==1.12.0 +slack_sdk==3.27.2 +smbprotocol==1.13.0 smmap==5.0.1 -sniffio==1.3.0 +sniffio==1.3.1 snowballstemmer==2.2.0 -snowflake-connector-python==3.6.0 -snowflake-sqlalchemy==1.5.1 +snowflake-connector-python==3.10.1 +snowflake-sqlalchemy==1.5.3 sortedcontainers==2.4.0 soupsieve==2.5 sphinx-airflow-theme==0.0.12 @@ -623,7 +645,7 @@ sphinx-autoapi==2.1.1 sphinx-copybutton==0.5.2 sphinx-jinja==2.0.2 sphinx-rtd-theme==2.0.0 -sphinx_design==0.5.0 +sphinx_design==0.6.0 sphinxcontrib-applehelp==1.0.8 sphinxcontrib-devhelp==1.0.6 sphinxcontrib-htmlhelp==2.0.5 @@ -635,81 +657,88 @@ sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-spelling==8.0.0 spython==0.3.13 -sqlalchemy-bigquery==1.9.0 +sqlalchemy-bigquery==1.11.0 sqlalchemy-redshift==0.8.14 -sqlalchemy-spanner==1.6.2 +sqlalchemy-spanner==1.7.0 sqlalchemy_drill==1.1.4 sqlparse==0.5.0 -sshpubkeys==3.3.1 sshtunnel==0.4.0 stack-data==0.6.3 starkbank-ecdsa==2.2.0 statsd==4.0.1 +std-uritemplate==0.0.57 strictyaml==1.7.3 -sympy==1.12 -tableauserverclient==0.29 +sympy==1.12.1 +tableauserverclient==0.19.0 tabulate==0.9.0 -tenacity==8.2.3 +tenacity==8.3.0 +teradatasql==20.0.0.12 +teradatasqlalchemy==20.0.0.1 termcolor==2.4.0 text-unidecode==1.3 thrift-sasl==0.4.3 thrift==0.16.0 -time-machine==2.13.0 +time-machine==2.14.1 tomli_w==1.0.0 -tomlkit==0.12.3 -tornado==6.4.1 +tomlkit==0.12.5 +tornado==6.4 towncrier==23.11.0 -tqdm==4.66.3 -traitlets==5.14.1 -trino==0.327.0 -trove-classifiers==2024.1.8 -twine==4.0.2 -types-Deprecated==1.2.9.20240106 -types-Markdown==3.5.0.20240106 -types-PyMySQL==1.1.0.1 -types-PyYAML==6.0.12.12 -types-aiofiles==23.2.0.20240106 +tqdm==4.66.4 +traitlets==5.14.3 +trino==0.328.0 +trove-classifiers==2024.5.22 +twine==5.1.0 +typed-ast==1.5.5 +types-Deprecated==1.2.9.20240311 +types-Markdown==3.6.0.20240316 +types-PyMySQL==1.1.0.20240524 +types-PyYAML==6.0.12.20240311 +types-aiofiles==23.2.0.20240403 types-certifi==2021.10.8.3 -types-croniter==2.0.0.20240106 -types-docutils==0.20.0.20240106 -types-paramiko==3.4.0.20240106 -types-protobuf==4.24.0.20240106 -types-pyOpenSSL==23.3.0.20240106 -types-python-dateutil==2.8.19.20240106 -types-python-slugify==8.0.0.3 -types-pytz==2023.3.1.1 -types-redis==4.6.0.20240106 -types-requests==2.31.0.20240106 -types-setuptools==69.0.0.20240115 +types-cffi==1.16.0.20240331 +types-croniter==2.0.0.20240423 +types-docutils==0.21.0.20240423 +types-paramiko==3.4.0.20240423 +types-protobuf==5.26.0.20240422 +types-pyOpenSSL==24.1.0.20240425 +types-python-dateutil==2.9.0.20240316 +types-python-slugify==8.0.2.20240310 +types-pytz==2024.1.0.20240417 +types-redis==4.6.0.20240425 +types-requests==2.32.0.20240602 +types-setuptools==70.0.0.20240524 types-tabulate==0.9.0.20240106 types-termcolor==1.1.6.2 -types-toml==0.10.8.7 -typing_extensions==4.9.0 -tzdata==2023.4 +types-toml==0.10.8.20240310 +typing_extensions==4.12.1 +tzdata==2024.1 tzlocal==5.2 -uc-micro-py==1.0.2 +uc-micro-py==1.0.3 unicodecsv==0.14.1 -universal-pathlib==0.1.4 +universal_pathlib==0.2.2 uritemplate==4.1.1 -urllib3==2.0.7 -userpath==1.9.1 -validators==0.22.0 +urllib3==2.2.2 +userpath==1.9.2 +uv==0.2.2 +validators==0.28.3 vertica-python==1.3.8 vine==5.1.0 -virtualenv==20.25.0 -watchtower==3.0.1 +virtualenv==20.26.2 +watchtower==3.2.0 wcwidth==0.2.13 -weaviate-client==3.26.1 -websocket-client==1.7.0 +weaviate-client==3.26.2 +websocket-client==1.8.0 +wirerope==0.4.7 wrapt==1.16.0 xmlsec==1.3.13 xmltodict==0.13.0 -yamllint==1.33.0 -yandexcloud==0.253.0 +yamllint==1.35.1 +yandex-query-client==0.1.4 +yandexcloud==0.291.0 yarl==1.9.4 zeep==4.2.1 -zenpy==2.0.45 -zipp==3.17.0 +zenpy==2.0.49 +zipp==3.19.2 zope.event==5.0 -zope.interface==6.1 +zope.interface==6.4.post2 zstandard==0.22.0 diff --git a/data/mwaa/requirements/requirements.txt b/data/mwaa/requirements/requirements.txt index 293ca543..dfe845a9 100644 --- a/data/mwaa/requirements/requirements.txt +++ b/data/mwaa/requirements/requirements.txt @@ -1,14 +1,14 @@ --constraint https://raw.githubusercontent.com/awslabs/aiops-modules/main/data/mwaa/requirements/constraint-requirements.txt -aiobotocore==2.11.2 +aiobotocore==2.13.0 aioitertools==0.11.0 cloudpickle==2.2.1 contextlib2==21.6.0 dill==0.3.8 -docker==7.0.0 +docker==7.1.0 google-pasta==0.2.0 multiprocess==0.70.16 -numpy==1.24.4 +numpy==1.26.4 pandas==2.1.4 pathos==0.2.5 pox==0.3.4 @@ -19,4 +19,4 @@ schema==0.7.5 scipy==1.13.0 smdebug-rulesconfig==1.0.1 tblib==3.0.0 -tqdm==4.66.3 +tqdm==4.66.4 diff --git a/manifests/mwaa-ml-training/mwaa-modules.yaml b/manifests/mwaa-ml-training/mwaa-modules.yaml index 3d73d014..955b8274 100644 --- a/manifests/mwaa-ml-training/mwaa-modules.yaml +++ b/manifests/mwaa-ml-training/mwaa-modules.yaml @@ -17,6 +17,6 @@ parameters: name: networking key: PrivateSubnetIds - name: airflow-version - value: "2.8.1" + value: "2.9.2" - name: custom-requirements-path value: data/mwaa/requirements/requirements.txt \ No newline at end of file