S3 localstack ize

.github/workflows/main-publish.yml

@@ -28,7 +28,7 @@ jobs:
         with:
           python-version: 3.9
       - name: Install Python dependencies for publish
-        run: pip install requests toml
+        run: pip install boto3 requests toml
       - name: Publish
         env:
           PYPI_USER: ${{ secrets.PYPI_USER }}

CHANGELOG.rst

@@ -6,6 +6,14 @@ dcicutils
 Change Log
 ----------
 
+7.7.2
+=====
+
+* Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for
+  for S3 or SQS boto3 client/resource creation using the LOCALSTACK_S3_URL or LOCALSTACK_SQS_URL
+  environment variables to specify that these services should use a locally running ersatz
+  instance of S3 or SQS via localstack.
+
 
 7.7.1
 =====
@@ -34,7 +42,6 @@ Change Log
 
 7.6.0
 =====
-
 * In ``creds_utils``:
 
   * Support for ``SMaHTKeyManager``

dcicutils/__init__.py

@@ -0,0 +1 @@
+from . import boto_monkey_patching  # noqa

dcicutils/boto_monkey_patching.py

@@ -0,0 +1,68 @@
+# Module to monkey patch the boto3 client and resource functions to use a custom endpoint-url.
+# Originally introduced June 2023 for overriding certain boto3 services (e.g. s3, sqs) to use the
+# localstack utility, which provides a way to run some AWS services locally, for testing purposes.
+# Currently only supported for S3 and SQS. To use this set the environment variables LOCALSTACK_S3_URL
+# and/or LOCALSTACK_SQS_URL to the localstack URL, for example, http://localhost:4566.
+# Reference: https://localstack.cloud
+
+import boto3
+import os
+from typing import Optional
+
+LOCALSTACK_S3_URL_ENVIRON_NAME = "LOCALSTACK_S3_URL"
+LOCALSTACK_SQS_URL_ENVIRON_NAME = "LOCALSTACK_SQS_URL"
+
+
+_boto_client_original = boto3.client
+_boto_resource_original = boto3.resource
+_boto_service_overrides_supported = [
+    {"service": "s3", "env": LOCALSTACK_S3_URL_ENVIRON_NAME},
+    {"service": "sqs", "env": LOCALSTACK_SQS_URL_ENVIRON_NAME}
+]
+
+# This will entirely disable this feature; for troubleshooting only.
+_boto_monkey_patching_disabled = False
+
+# For import only in test_boto_monkey_patching;
+# the list of AWS services for which we support this monkey patching facility (e.g. ["s3", "sqs"]).
+_boto_monkey_patching_services = [item["service"] for item in _boto_service_overrides_supported]
+
+
+# For import only in test_boto_monkey_patching.
+def _boto_monkey_patching_endpoint_url_environ_name(service: str) -> Optional[str]:
+    """
+    For the given AWS service name (e.g. "s3" or "sqs") returns environment variable name which
+    needs to be set in order to use a different (e.g. localstack version of the) endpoint URL
+    when creating a boto3 client or resource. E.g. given "s3" this will return "LOCALSTACK_S3_URL".
+    """
+    for item in _boto_service_overrides_supported:
+        if item["service"] == service:
+            return item["env"]
+    return None
+
+
+def _setup_monkey_patching_kwargs(*args, **kwargs) -> dict:
+    if not _boto_monkey_patching_disabled:
+        endpoint_url = kwargs.get("endpoint_url")
+        if not endpoint_url:
+            for service_override in _boto_service_overrides_supported:
+                if service_override["service"] in args:
+                    endpoint_url = os.environ.get(service_override["env"])
+                    if endpoint_url:
+                        kwargs["endpoint_url"] = endpoint_url
+                    break
+    return kwargs
+
+
+def _monkey_patched_boto_client(*args, **kwargs):
+    kwargs = _setup_monkey_patching_kwargs(*args, **kwargs)
+    return _boto_client_original(*args, **kwargs)
+
+
+def _monkey_patched_boto_resource(*args, **kwargs):
+    kwargs = _setup_monkey_patching_kwargs(*args, **kwargs)
+    return _boto_resource_original(*args, **kwargs)
+
+
+boto3.client = _monkey_patched_boto_client
+boto3.resource = _monkey_patched_boto_resource

dcicutils/license_utils.py

@@ -862,6 +862,10 @@ class C4InfrastructureLicenseChecker(LicenseChecker):
             'typed-function',  # LICENSE at https://www.npmjs.com/package/typed-function?activeTab=code
 
         ],
+
+        'Other/Proprietary License': [
+            'localstack-ext'
+        ]
     }
 
 

docs/source/dcicutils.rst

@@ -23,6 +23,13 @@ beanstalk_utils
    :members:
 
 
+boto_monkey_patching
+^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: dcicutils.boto_monkey_patching
+   :members:
+
+
 codebuild_utils
 ^^^^^^^^^^^^^^^
 

docs/source/getting_started.rst

@@ -30,3 +30,7 @@ Central metadata functions
 The most useful utilities functions for most users are the metadata functions, which generally are used to access, create, or edit object metadata on the Fourfront portal. Since this utilities module is a pip-installable Python package, they can be leveraged as an API to the portal in your scripts. All of these functions are contained within  ``dcicutils.ff_utils.py``.
 
 See example usage of these functions `here <./examples.html#metadata>`_
+
+Local development notes
+^^^^^^^^^^^^^^^^^^^^^^^
+For local debugging and development, there is some support for the ``localstack`` package (https://localstack.cloud/), or something like it, to use a local ersatz version of certain AWS services, namely, currently, for S3 and SQS, whereby any boto3 based calls which use these services will optionally use the local ersatz version. To take advantage of this simply set the environment variables ``LOCALSTACK_S3_URL`` and/or ``LOCALSTACK_SQS_URL`` environment variables when running your process, whatever that may be, e.g. ``export LOCALSTACK_S3_URL=http://localhost:4566``. To install ``localstack`` use ``pip install localstack``, and to start it use ``localstack``. FYI this support is implemented via "monkey patching" hooked on these environment variables (see the ``boto_monkey_patching.py`` module for details).