diff --git a/.github/workflows/main-publish.yml b/.github/workflows/main-publish.yml index 57847ecbb..7a962cfca 100644 --- a/.github/workflows/main-publish.yml +++ b/.github/workflows/main-publish.yml @@ -28,7 +28,7 @@ jobs: with: python-version: 3.9 - name: Install Python dependencies for publish - run: pip install requests toml + run: pip install boto3 requests toml - name: Publish env: PYPI_USER: ${{ secrets.PYPI_USER }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f073f4a79..b21230d13 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,16 @@ dcicutils Change Log ---------- +7.10.0 +====== + +* Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for + S3 or SQS boto3 client/resource creation using the ``LOCALSTACK_S3_URL`` or ``LOCALSTACK_SQS_URL`` + environment variables to specify that these services should use a locally running ersatz + instance of S3 or SQS via localstack (https://localstack.cloud). +* Comment and typo cleanup in ``publish_to_pypi.py``. +* Updates PyYAML to ^6.0.1 because Mac M1 (with Python 3.9) only likes 5.3.1 (not 5.4.1) or 6+. + 7.9.0 ===== @@ -26,6 +36,7 @@ Change Log for conceptual compatibility with ``get_schemas``. * Minor tweaks to ``dump_results_to_json`` for style reasons, and repairs to its overly complex and error-prone unit test. + * Fix to license_utils.py for localstack-ext. 7.8.0 @@ -70,7 +81,6 @@ Change Log 7.6.0 ===== - * In ``creds_utils``: * Support for ``SMaHTKeyManager`` diff --git a/dcicutils/__init__.py b/dcicutils/__init__.py index e69de29bb..1842081dc 100644 --- a/dcicutils/__init__.py +++ b/dcicutils/__init__.py @@ -0,0 +1 @@ +from . import boto_monkey_patching # noqa diff --git a/dcicutils/boto_monkey_patching.py b/dcicutils/boto_monkey_patching.py new file mode 100644 index 000000000..d239ec2c9 --- /dev/null +++ b/dcicutils/boto_monkey_patching.py @@ -0,0 +1,68 @@ +# Module to monkey patch the boto3 client and resource functions to use a custom endpoint-url. +# Originally introduced June 2023 for overriding certain boto3 services (e.g. s3, sqs) to use the +# localstack utility, which provides a way to run some AWS services locally, for testing purposes. +# Currently only supported for S3 and SQS. To use this set the environment variables LOCALSTACK_S3_URL +# and/or LOCALSTACK_SQS_URL to the localstack URL, for example, http://localhost:4566. +# Reference: https://localstack.cloud + +import boto3 +import os +from typing import Optional + +LOCALSTACK_S3_URL_ENVIRON_NAME = "LOCALSTACK_S3_URL" +LOCALSTACK_SQS_URL_ENVIRON_NAME = "LOCALSTACK_SQS_URL" + + +_boto_client_original = boto3.client +_boto_resource_original = boto3.resource +_boto_service_overrides_supported = [ + {"service": "s3", "env": LOCALSTACK_S3_URL_ENVIRON_NAME}, + {"service": "sqs", "env": LOCALSTACK_SQS_URL_ENVIRON_NAME} +] + +# This will entirely disable this feature; for troubleshooting only. +_boto_monkey_patching_disabled = False + +# For import only in test_boto_monkey_patching; +# the list of AWS services for which we support this monkey patching facility (e.g. ["s3", "sqs"]). +_boto_monkey_patching_services = [item["service"] for item in _boto_service_overrides_supported] + + +# For import only in test_boto_monkey_patching. +def _boto_monkey_patching_endpoint_url_environ_name(service: str) -> Optional[str]: + """ + For the given AWS service name (e.g. "s3" or "sqs") returns environment variable name which + needs to be set in order to use a different (e.g. localstack version of the) endpoint URL + when creating a boto3 client or resource. E.g. given "s3" this will return "LOCALSTACK_S3_URL". + """ + for item in _boto_service_overrides_supported: + if item["service"] == service: + return item["env"] + return None + + +def _setup_monkey_patching_kwargs(*args, **kwargs) -> dict: + if not _boto_monkey_patching_disabled: + endpoint_url = kwargs.get("endpoint_url") + if not endpoint_url: + for service_override in _boto_service_overrides_supported: + if service_override["service"] in args: + endpoint_url = os.environ.get(service_override["env"]) + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + break + return kwargs + + +def _monkey_patched_boto_client(*args, **kwargs): + kwargs = _setup_monkey_patching_kwargs(*args, **kwargs) + return _boto_client_original(*args, **kwargs) + + +def _monkey_patched_boto_resource(*args, **kwargs): + kwargs = _setup_monkey_patching_kwargs(*args, **kwargs) + return _boto_resource_original(*args, **kwargs) + + +boto3.client = _monkey_patched_boto_client +boto3.resource = _monkey_patched_boto_resource diff --git a/dcicutils/captured_output.py b/dcicutils/captured_output.py new file mode 100644 index 000000000..e5db67c42 --- /dev/null +++ b/dcicutils/captured_output.py @@ -0,0 +1,69 @@ +from collections import namedtuple +from contextlib import contextmanager +import io +import sys +from typing import Optional + +_real_stdout = sys.stdout +_real_stderr = sys.stderr + +@contextmanager +def captured_output(capture: bool = True): + """ + Context manager to capture any/all output to stdout or stderr, and not actually output it to stdout + or stderr. Yields and object with a get_captured_output() method to get the output captured thus far, + and another uncaptured_print() method to actually print the given output to stdout, even though output + to stdout is being captured. Can be useful, for example, in creating command-line scripts which invoke + code which outputs a lot of info, warning, error, etc to stdout or stderr, and we want to suprress that + output; but with the yielded uncaptured_print() method output specific to the script can actually be + output (to stdout); and/or can also optionally output any/all captured output, e.g. for debugging or + troubleshooting purposes. Disable this capture, without having to restructure your code WRT the usage + of the with-clause with this context manager, pass False as an argument to this context manager. + """ + + original_stdout = _real_stdout + original_stderr = _real_stderr + captured_output = io.StringIO() + + def set_original_output() -> None: + sys.stdout = original_stdout + sys.stderr = original_stderr + + def set_captured_output() -> None: + if capture: + sys.stdout = captured_output + sys.stderr = captured_output + + def uncaptured_print(*args, **kwargs) -> None: + set_original_output() + print(*args, **kwargs) + set_captured_output() + + def uncaptured_input(message: str) -> str: + set_original_output() + value = input(message) + set_captured_output() + return value + + def get_captured_output() -> Optional[str]: + return captured_output.getvalue() if capture else None + + try: + set_captured_output() + Result = namedtuple("Result", ["get_captured_output", "uncaptured_print", "uncaptured_input"]) + yield Result(get_captured_output, uncaptured_print, uncaptured_input) + finally: + set_original_output() + + +@contextmanager +def uncaptured_output(): + original_stdout = sys.stdout + original_stderr = sys.stderr + sys.stdout = _real_stdout + sys.stderr = _real_stderr + try: + yield + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr diff --git a/dcicutils/license_utils.py b/dcicutils/license_utils.py index 855fa5c80..be472df6a 100644 --- a/dcicutils/license_utils.py +++ b/dcicutils/license_utils.py @@ -874,13 +874,18 @@ class C4InfrastructureLicenseChecker(LicenseChecker): ], + 'Other/Proprietary License': [ + # This is known to be offered under Apache-2.0 license. + # Ref: https://github.com/localstack/localstack/blob/master/LICENSE.txt + 'localstack-ext' + ], + 'UNLICENSED': [ # The udn-browser library is our own and has been observed to sometimes show up in some contexts # as UNLICENSED, when really it's MIT. # Ref: https://github.com/dbmi-bgm/udn-browser/blob/main/LICENSE 'udn-browser', ], - } diff --git a/dcicutils/scripts/publish_to_pypi.py b/dcicutils/scripts/publish_to_pypi.py index a454782b2..1bcefe9e1 100644 --- a/dcicutils/scripts/publish_to_pypi.py +++ b/dcicutils/scripts/publish_to_pypi.py @@ -5,12 +5,13 @@ # 1. The git repo MUST NOT contain unstaged changes. # 2. The git repo MUST NOT contain staged but uncommitted changes. # 3. The git repo MUST NOT contain committed but unpushed changes. -# 4. The git repo package directories MUST NOT contain untracked files, +# 4. The git repo MUST be tagged (its most recent commit must be for a tag). +# 5. The git repo package directories MUST NOT contain untracked files, # OR if they do contain untracked files then you must confirm this is OK. -# 5. The version being published must NOT have already been published. +# 6. The version being published must NOT have already been published. # # ASSUMES you have these credentials environment variables correctly set for PyPi publishing; -# although a --username and --password are also supported to set these via command-line. +# although a --username and --password are alternatively supported to set these via command-line. # # - PYPI_USER # - PYPI_PASSWORD @@ -22,19 +23,15 @@ # option to skip this confimation, however it is only allowed when running in the # context of GitHub actions - it checks for the GITHUB_ACTIONS environment variable. # -# Prints warning if PY -# -# FYI: This was created late April 2023 after a junk file containing development -# logging output containing passwords was accidentally published to PyPi; -# item #4 above specifically addresses/prevents this. Perhaps better -# would be if publishing only happened via GitHub actions. +# FYI: This was created late April 2023 after a junk file containing development logging +# output containing passwords was accidentally published to PyPi; item #5 above specifically +# addresses/prevents this. Perhaps better would be if publishing only happened via GitHub actions. import argparse import os import requests import subprocess import toml - from typing import Tuple, Union @@ -111,9 +108,9 @@ def publish_package(pypi_username: str = None, pypi_password: str = None, force_ if pypi_username != PYPI_API_TOKEN_USERNAME: if not force_allow_username: # Just in case someone really really needs this we will allow for now: --force-allow-username - ERROR_PRINT(f"Publishing with username/pasword is no longer allowed; must use API token instead.") + ERROR_PRINT(f"Publishing with username/password is no longer allowed; must use API token instead.") return False - WARNING_PRINT(f"Publishing with username/pasword is NOT recommmended; use API token instead;" + WARNING_PRINT(f"Publishing with username/password is NOT recommmended; use API token instead;" f"only allowing because you said: --force-allow-username") poetry_publish_command = [ "poetry", "publish", diff --git a/docs/source/dcicutils.rst b/docs/source/dcicutils.rst index f15307d0e..82955f19e 100644 --- a/docs/source/dcicutils.rst +++ b/docs/source/dcicutils.rst @@ -23,6 +23,13 @@ beanstalk_utils :members: +boto_monkey_patching +^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: dcicutils.boto_monkey_patching + :members: + + codebuild_utils ^^^^^^^^^^^^^^^ diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 52914b886..9015f395f 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -30,3 +30,7 @@ Central metadata functions The most useful utilities functions for most users are the metadata functions, which generally are used to access, create, or edit object metadata on the Fourfront portal. Since this utilities module is a pip-installable Python package, they can be leveraged as an API to the portal in your scripts. All of these functions are contained within ``dcicutils.ff_utils.py``. See example usage of these functions `here <./examples.html#metadata>`_ + +Local development notes +^^^^^^^^^^^^^^^^^^^^^^^ +For local debugging and development, there is some support for the ``localstack`` package (https://localstack.cloud/), or something like it, to use a local ersatz version of certain AWS services, namely, currently, for S3 and SQS, whereby any boto3 based calls which use these services will optionally use the local ersatz version. To take advantage of this simply set the environment variables ``LOCALSTACK_S3_URL`` and/or ``LOCALSTACK_SQS_URL`` environment variables when running your process, whatever that may be, e.g. ``export LOCALSTACK_S3_URL=http://localhost:4566``. To install ``localstack`` use ``pip install localstack``, and to start it use ``localstack``. FYI this support is implemented via "monkey patching" hooked on these environment variables (see the ``boto_monkey_patching.py`` module for details). diff --git a/poetry.lock b/poetry.lock index d7e77523c..31102b044 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "async-timeout" @@ -400,6 +400,18 @@ files = [ [package.dependencies] typing-extensions = {version = "*", markers = "python_version < \"3.8\""} +[[package]] +name = "cachetools" +version = "5.0.0" +description = "Extensible memoizing collections and decorators" +category = "dev" +optional = false +python-versions = "~=3.7" +files = [ + {file = "cachetools-5.0.0-py3-none-any.whl", hash = "sha256:8fecd4203a38af17928be7b90689d8083603073622229ca7077b72d8e5a976e4"}, + {file = "cachetools-5.0.0.tar.gz", hash = "sha256:486471dfa8799eb7ec503a8059e263db000cdda20075ce5e48903087f79d5fd6"}, +] + [[package]] name = "certifi" version = "2023.7.22" @@ -574,6 +586,22 @@ files = [ {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, ] +[[package]] +name = "click" +version = "8.1.6" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + [[package]] name = "colorama" version = "0.4.6" @@ -708,6 +736,54 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "dill" +version = "0.3.2" +description = "serialize all of python" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*" +files = [ + {file = "dill-0.3.2.zip", hash = "sha256:6e12da0d8e49c220e8d6e97ee8882002e624f1160289ce85ec2cc0a5246b3a2e"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "dnslib" +version = "0.9.23" +description = "Simple library to encode/decode DNS wire-format packets" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "dnslib-0.9.23-py2-none-any.whl", hash = "sha256:9eb851ac721eea51834d43795478ac9b48272c61ba97cc4a160668b50aff39ec"}, + {file = "dnslib-0.9.23-py3-none-any.whl", hash = "sha256:46137e8ef6ef52b24a16d47e0786a99dd103ab1e71eea616f21371accbccc557"}, + {file = "dnslib-0.9.23.tar.gz", hash = "sha256:310196d3e38ce2051b61eebbd2f1d08fcc934fa3360f22031864d16efe8bca77"}, +] + +[[package]] +name = "dnspython" +version = "2.3.0" +description = "DNS toolkit" +category = "dev" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {file = "dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] + +[package.extras] +curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] +dnssec = ["cryptography (>=2.6,<40.0)"] +doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.11.0)"] +doq = ["aioquic (>=0.9.20)"] +idna = ["idna (>=2.1,<4.0)"] +trio = ["trio (>=0.14,<0.23)"] +wmi = ["wmi (>=1.5.1,<2.0.0)"] + [[package]] name = "docker" version = "4.4.4" @@ -730,6 +806,25 @@ websocket-client = ">=0.32.0" ssh = ["paramiko (>=2.4.2)"] tls = ["cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] +[[package]] +name = "ecdsa" +version = "0.18.0" +description = "ECDSA cryptographic signature library (pure python)" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "ecdsa-0.18.0-py2.py3-none-any.whl", hash = "sha256:80600258e7ed2f16b9aa1d7c295bd70194109ad5a30fdee0eaeefef1d4c559dd"}, + {file = "ecdsa-0.18.0.tar.gz", hash = "sha256:190348041559e21b22a1d65cee485282ca11a6f81d503fddb84d5017e9ed1e49"}, +] + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +gmpy = ["gmpy"] +gmpy2 = ["gmpy2"] + [[package]] name = "elasticsearch" version = "7.13.4" @@ -884,6 +979,117 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "localstack" +version = "2.2.0" +description = "LocalStack - A fully functional local Cloud stack" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "localstack-2.2.0.tar.gz", hash = "sha256:3b3ed3d4e13fff791ebcc8d9e58617b7ba62b87ae3a6925645050134b1643e2d"}, +] + +[package.dependencies] +localstack-core = "*" +localstack-ext = "2.2.0" + +[package.extras] +full = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.2.0)"] +runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.2.0)"] + +[[package]] +name = "localstack-core" +version = "2.2.0" +description = "The core library and runtime of LocalStack" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "localstack-core-2.2.0.tar.gz", hash = "sha256:cc46a5e585f4265e9273439335ef1fa2f4b9163f157145d298b62d388f75fb60"}, + {file = "localstack_core-2.2.0-py3-none-any.whl", hash = "sha256:34b41b22da4134089ab9bdec71049d5a7cb45ba58d30e62545285b54cd4b3629"}, +] + +[package.dependencies] +cachetools = ">=5.0.0,<5.1.0" +click = ">=7.0" +cryptography = "*" +dill = "0.3.2" +dnspython = ">=1.16.0" +importlib-metadata = {version = "<5.0", markers = "python_version < \"3.8\""} +plux = ">=1.3.1" +psutil = ">=5.4.8,<6.0.0" +python-dotenv = ">=0.19.1" +pyyaml = ">=5.1" +requests = ">=2.20.0" +rich = ">=12.3.0" +semver = ">=2.10" +stevedore = ">=3.4.0" +tailer = ">=0.4.1" +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["Cython", "autoflake", "black (==22.3.0)", "coveralls (>=3.3.1)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "isort (==5.12.0)", "networkx (>=2.8.4)", "pandoc", "pre-commit (==2.13.0)", "pypandoc", "pyproject-flake8 (>=6.0.0.post1)", "rstr (>=3.2.0)"] +full = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.12.0)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.31.2)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (>=0.14.4)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.13.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] +runtime = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.12.0)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.31.2)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (>=0.14.4)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.13.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] +test = ["coverage[toml] (>=5.5)", "deepdiff (>=5.5.0)", "jsonpath-ng (>=1.5.3)", "pytest (==6.2.4)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)"] +typehint = ["boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]"] + +[[package]] +name = "localstack-ext" +version = "2.2.0" +description = "Extensions for LocalStack" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "localstack-ext-2.2.0.tar.gz", hash = "sha256:04b2b8d5345a60db6979e0de199865be7be44305a8d2e181d7d837e30691a859"}, +] + +[package.dependencies] +dill = ">=0.3.2" +dnslib = ">=0.9.10" +dnspython = ">=1.16.0" +localstack-core = "2.2.0" +plux = ">=1.3.1" +pyaes = ">=1.6.0" +python-dateutil = ">=2.8" +python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"]} +requests = ">=2.20.0" +tabulate = "*" +windows-curses = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +package = ["python-minifier"] +runtime = ["Js2Py (>=0.71)", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "amazon.ion (>=0.9.3,<0.10)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "dirtyjson (==1.0.7)", "docker (>=6.0.1)", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "kafka-python", "kubernetes (==21.7.0)", "localstack-core[runtime] (==2.2.0)", "localstack-plugin-persistence (>=0.1.3)", "moto-ext[all]", "paho-mqtt (>=1.5)", "parquet (>=1.3.1)", "parse (==1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.1.0)", "presto-python-client (>=0.7.0)", "pyftpdlib (>=1.5.6)", "pyhive[hive] (>=0.6.1)", "pyion2json (>=0.0.2)", "pymysql", "pyqldb (>=3.2,<4.0)", "python-snappy (>=0.6)", "readerwriterlock (>=1.0.7)", "rsa (>=4.0)", "sql-metadata (>=2.6.0)", "srp-ext (==1.0.7.1)", "testing.common.database (>=1.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] +test = ["Js2Py (>=0.71)", "PyAthena[pandas]", "aiohttp", "aws_xray_sdk (>=2.4.2)", "awsiotsdk", "awswrangler (>=2.19.0)", "black (==22.3.0)", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "gremlinpython", "isort (==5.12.0)", "jws (>=0.1.3)", "localstack-core[test] (==2.2.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "packaging (<22)", "paramiko (>=2.11.0,<2.12.0)", "portalocker", "pre-commit (==2.13.0)", "pyarrow", "pykafka", "pymongo", "pymssql", "pyproject-flake8 (==6.0.0.post1)", "pytest (==6.2.4)", "pytest-dependency (==0.5.1)", "pytest-httpserver (>=1.0.1)", "pytest-instafail (>=0.4.2)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)", "python-terraform", "redis (==4.5.4)", "redshift_connector", "stomp.py (==8.0.1)"] + +[[package]] +name = "markdown-it-py" +version = "2.2.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, + {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" +typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "mccabe" version = "0.7.0" @@ -896,6 +1102,18 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "mirakuru" version = "2.5.1" @@ -948,6 +1166,18 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "pbr" +version = "5.11.1" +description = "Python Build Reasonableness" +category = "dev" +optional = false +python-versions = ">=2.6" +files = [ + {file = "pbr-5.11.1-py2.py3-none-any.whl", hash = "sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b"}, + {file = "pbr-5.11.1.tar.gz", hash = "sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3"}, +] + [[package]] name = "pip-licenses" version = "3.5.5" @@ -985,6 +1215,24 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "plux" +version = "1.4.0" +description = "A dynamic code loading framework for building pluggable Python distributions" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "plux-1.4.0-py3-none-any.whl", hash = "sha256:9ee34e9a339a2f512db69fc2592eaabedbccd6280102b06a74c32ec5268b1916"}, + {file = "plux-1.4.0.tar.gz", hash = "sha256:4a432e2bd0fd50eb165781fd2a885e2169492f1f699429afda8130a3efdd4590"}, +] + +[package.dependencies] +stevedore = ">=3.4" + +[package.extras] +dev = ["black (==22.3.0)", "isort (==5.9.1)", "pytest (==6.2.4)"] + [[package]] name = "port-for" version = "0.7.1" @@ -1035,6 +1283,29 @@ files = [ {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, ] +[[package]] +name = "pyaes" +version = "1.6.1" +description = "Pure-Python Implementation of the AES block-cipher and common modes of operation" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pyaes-1.6.1.tar.gz", hash = "sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + [[package]] name = "pycodestyle" version = "2.9.1" @@ -1071,6 +1342,21 @@ files = [ {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, ] +[[package]] +name = "pygments" +version = "2.16.1" +description = "Pygments is a syntax highlighting package written in Python." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pyjwt" version = "2.8.0" @@ -1224,6 +1510,44 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "0.21.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-dotenv-0.21.1.tar.gz", hash = "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49"}, + {file = "python_dotenv-0.21.1-py3-none-any.whl", hash = "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-jose" +version = "3.3.0" +description = "JOSE implementation in Python" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "python-jose-3.3.0.tar.gz", hash = "sha256:55779b5e6ad599c6336191246e95eb2293a9ddebd555f796a65f838f07e5d78a"}, + {file = "python_jose-3.3.0-py2.py3-none-any.whl", hash = "sha256:9b1376b023f8b298536eedd47ae1089bcdb848f1535ab30555cd92002d78923a"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryptography\""} +ecdsa = "!=0.15" +pyasn1 = "*" +rsa = "*" + +[package.extras] +cryptography = ["cryptography (>=3.4.0)"] +pycrypto = ["pyasn1", "pycrypto (>=2.6.0,<2.7.0)"] +pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] + [[package]] name = "pytz" version = "2023.3" @@ -1260,41 +1584,52 @@ files = [ [[package]] name = "pyyaml" -version = "5.4.1" +version = "6.0.1" description = "YAML parser and emitter for Python" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.6" files = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] [[package]] @@ -1355,6 +1690,41 @@ files = [ [package.extras] idna2008 = ["idna"] +[[package]] +name = "rich" +version = "13.5.2" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "dev" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, + {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +category = "dev" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + [[package]] name = "s3transfer" version = "0.6.1" @@ -1373,6 +1743,18 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "semver" +version = "3.0.1" +description = "Python helper for Semantic Versioning (https://semver.org)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, + {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1409,6 +1791,22 @@ files = [ {file = "soupsieve-2.4.1.tar.gz", hash = "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"}, ] +[[package]] +name = "stevedore" +version = "3.5.2" +description = "Manage dynamic plugins for Python applications" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "stevedore-3.5.2-py3-none-any.whl", hash = "sha256:fa2630e3d0ad3e22d4914aff2501445815b9a4467a6edc49387c667a38faf5bf"}, + {file = "stevedore-3.5.2.tar.gz", hash = "sha256:cf99f41fc0d5a4f185ca4d3d42b03be9011b0a1ec1a4ea1a282be1b4b306dcc2"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} +pbr = ">=2.0.0,<2.1.0 || >2.1.0" + [[package]] name = "structlog" version = "19.2.0" @@ -1430,6 +1828,32 @@ dev = ["coverage", "freezegun (>=0.2.8)", "pre-commit", "pretend", "pytest (>=3. docs = ["sphinx", "twisted"] tests = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest (>=3.3.0)", "python-rapidjson", "simplejson"] +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "tailer" +version = "0.4.1" +description = "Python tail is a simple implementation of GNU tail and head." +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "tailer-0.4.1.tar.gz", hash = "sha256:78d60f23a1b8a2d32f400b3c8c06b01142ac7841b75d8a1efcb33515877ba531"}, +] + [[package]] name = "toml" version = "0.10.2" @@ -1575,6 +1999,28 @@ WebOb = ">=1.2" docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.8)"] tests = ["PasteDeploy", "WSGIProxy2", "coverage", "mock", "nose (<1.3.0)", "pyquery"] +[[package]] +name = "windows-curses" +version = "2.3.1" +description = "Support for the standard curses module on Windows" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "windows_curses-2.3.1-cp310-cp310-win32.whl", hash = "sha256:2644f4547ae5124ce5129b66faa59ee0995b7b7205ed5e3920f6ecfef2e46275"}, + {file = "windows_curses-2.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b541520895649c0835771775034a2b4edf36da3c3d9381c5022b5b4f9a5014e"}, + {file = "windows_curses-2.3.1-cp311-cp311-win32.whl", hash = "sha256:25e7ff3d77aed6c747456b06fbc1528d67fc59d1ef3be9ca244774e65e6bdbb2"}, + {file = "windows_curses-2.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:395656bfe88d6f60cb18604605d423e0f2d1c3a8f550507dca5877a9d0b3a0f3"}, + {file = "windows_curses-2.3.1-cp36-cp36m-win32.whl", hash = "sha256:6ea8e1c4536fee248ee3f88e5010871df749932b7e829e2f012e5d23bd2fe31d"}, + {file = "windows_curses-2.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59856b41676c4b3eb527eb6b1478803d4dc92413b2e63aea762407807ffcd3ac"}, + {file = "windows_curses-2.3.1-cp37-cp37m-win32.whl", hash = "sha256:9cd0ba6efde23930736eff45a0aa0af6fd82e60b4787a46157ef4956d2c52b06"}, + {file = "windows_curses-2.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f9a7fcd03934e40238f9bbeddae51e3fdc442f28bca50afccdc521245ed39439"}, + {file = "windows_curses-2.3.1-cp38-cp38-win32.whl", hash = "sha256:5c55ebafdb402cfa927174a03d651cd1b1e76d6e6cf71818f9d3378636c00e74"}, + {file = "windows_curses-2.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:a551aaa09d6ec28f64ade8e85fd0c52880c8e9114729a79c34803104e49bed71"}, + {file = "windows_curses-2.3.1-cp39-cp39-win32.whl", hash = "sha256:aab7e28133bf81769cddf8b3c3c8ab89e76cd43effd371c6370e918b6dfccf1b"}, + {file = "windows_curses-2.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:85675de4ae7058348140daae83a8a7b81147a84ef9ab699307b3168f9490292f"}, +] + [[package]] name = "zipp" version = "3.15.0" @@ -1594,4 +2040,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.10" -content-hash = "b8d6612bb28cfb9da79306a82b2ac35a20678e1f62ef86c93b8af3c3d1ed798e" +content-hash = "5d652586eef7475ad10cf40bb197feb7b14b77736f2f6431ddca25cf29decd8c" diff --git a/pyproject.toml b/pyproject.toml index 70f90b624..76b630289 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.9.0" +version = "7.9.0.1b2" # TODO: To become 7.10.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" @@ -46,7 +46,7 @@ aws-requests-auth = ">=0.4.2,<1" docker = "^4.4.4" gitpython = "^3.1.2" pytz = ">=2020.4" -PyYAML = ">=5.1,<5.5" +PyYAML = "^6.0.1" requests = "^2.21.0" rfc3986 = "^1.4.0" structlog = "^19.2.0" @@ -70,6 +70,7 @@ coverage = ">=7.2.3" # coveralls = ">=3.3.1" flake8 = ">=3.9.2" flaky = ">=3.7.0" +localstack = "^2.1.0" pip-licenses = "^3.5.5" pytest = ">=4.5.0" pytest-cov = ">=2.7.1" diff --git a/test/conftest.py b/test/conftest.py index 3b7de844e..abac63e67 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,11 @@ import os + +# Disable any AWS endpoint-url (ala boto_monkey_patching/localstack) environment variables overrides for testing. +if "LOCALSTACK_S3_URL" in os.environ: + os.environ.pop("LOCALSTACK_S3_URL") +if "LOCALSTACK_SQS_URL" in os.environ: + os.environ.pop("LOCALSTACK_SQS_URL") + import pytest import requests diff --git a/test/test_boto_monkey_patching.py b/test/test_boto_monkey_patching.py new file mode 100644 index 000000000..8b43010a2 --- /dev/null +++ b/test/test_boto_monkey_patching.py @@ -0,0 +1,69 @@ +import boto3 +import os +from typing import Optional +from dcicutils.misc_utils import override_environ + +# Reach into the implementation details of the boto_monkey_patching module +# to get list of boto3 services for which we support monkey patching, and to +# get the environment variable name used to do the associated endpoint-url override. +from dcicutils.boto_monkey_patching import _boto_monkey_patching_services +from dcicutils.boto_monkey_patching import _boto_monkey_patching_endpoint_url_environ_name + + +_override_endpoint_url = "http://localhost:4566" + + +def _is_default_aws_endpoint_url(endpoint_url: str, service: str): + return (endpoint_url == f"https://{service}.amazonaws.com" or + endpoint_url == f"https://{service}.{os.environ.get('AWS_DEFAULT_REGION')}.amazonaws.com") + + +def _environ_overrides(service: str, endpoint_url: Optional[str]) -> dict: + return { + _boto_monkey_patching_endpoint_url_environ_name(service): endpoint_url, + "AWS_DEFAULT_REGION": "us-east-1" + } + + +def _test_boto_monkey_patching_client_without_overriding(service: str): + with override_environ(**_environ_overrides(service, None)): + s3_client = boto3.client(service) + assert _is_default_aws_endpoint_url(s3_client.meta.endpoint_url, service) + + +def _test_boto_monkey_patching_resource_without_overriding(service: str): + with override_environ(**_environ_overrides(service, None)): + s3_resource = boto3.resource(service) + assert _is_default_aws_endpoint_url(s3_resource.meta.client._endpoint.host, service) + + +def _test_boto_monkey_patching_client_with_overriding(service: str): + with override_environ(**_environ_overrides(service, _override_endpoint_url)): + s3_client = boto3.client(service) + assert s3_client.meta.endpoint_url == _override_endpoint_url + + +def _test_boto_monkey_patching_resource_with_overriding(service: str): + with override_environ(**_environ_overrides(service, _override_endpoint_url)): + s3_resource = boto3.resource(service) + assert s3_resource.meta.client._endpoint.host == _override_endpoint_url + + +def test_boto_monkey_patching_client_without_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_client_without_overriding(service) + + +def test_boto_monkey_patching_resource_without_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_resource_without_overriding(service) + + +def test_boto_monkey_patching_client_with_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_client_with_overriding(service) + + +def test_boto_monkey_patching_resource_with_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_resource_with_overriding(service) diff --git a/test/test_s3_utils.py b/test/test_s3_utils.py index 3cfd05c7e..0e98959cd 100644 --- a/test/test_s3_utils.py +++ b/test/test_s3_utils.py @@ -409,7 +409,7 @@ def test_s3utils_get_google_key(): keys = s3u.get_google_key() assert isinstance(keys, dict) assert keys['type'] == 'service_account' - assert keys["project_id"] == "fourdn-fourfront" + assert keys["project_id"] == "fourfront-396315" # yes, this is a magic constant for dict_key in ['private_key_id', 'private_key', 'client_email', 'client_id', 'auth_uri', 'client_x509_cert_url']: assert keys[dict_key]