Made cachetools optional for libssm and libsm

* Both libssm and libsm modules will use cachetools if it
  is found in the PYTHONPATH or sys.modules. Else fetch
  from the remote.
* Added unit tests
* Bumped rc3 release

Resolves #2 #4 #25

docker-compose.yml

@@ -1,5 +1,5 @@
 # NOTE: if we update the container image version in here, also do update GHA caches step in prbuild.yml
-version: '3.1'
+version: '3.8'
 
 services:
 
@@ -8,3 +8,7 @@ services:
     container_name: libumccr_localstack
     ports:
       - "4566:4566"
+    volumes:
+      - "./init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh"
+    environment:
+      - DEBUG=1

init-aws.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+awslocal s3 mb s3://my-bucket
+awslocal sqs create-queue --region ap-southeast-2 --queue-name my-queue
+awslocal ssm put-parameter --region ap-southeast-2 --name my-param --type String --value 'Hello'
+awslocal ssm put-parameter --region ap-southeast-2 --name my-param-secure --type SecureString --value 'Sello'  # pragma: allow-secret
+awslocal secretsmanager create-secret --region ap-southeast-2 --name MyTestSecret --secret-string "HealTheWorld"  # pragma: allow-secret
+awslocal secretsmanager create-secret --region ap-southeast-2 --name MyTestSecretJson --secret-string "{\"user\":\"diegor\",\"password\":\"EXAMPLE-PASSWORD\"}"  # pragma: allow-secret

libumccr/aws/libsm.py

@@ -6,12 +6,24 @@
 import base64
 import logging
 
-from cachetools.func import lru_cache
-
 from libumccr.aws import sm_client
+from libumccr.utils import load_package_if_found
 
 logger = logging.getLogger(__name__)
 
+if load_package_if_found("cachetools"):
+    logger.info(f"cachetools found, using LRU cache")
+    from cachetools.func import lru_cache
+else:
+    def lru_cache(maxsize):
+        logger.info(f"cachetools not found, skipping LRU cache with maxsize={maxsize}")
+
+        def wrapper(func):
+            func.cache_clear = lambda: None
+            return func
+
+        return wrapper
+
 
 @lru_cache(maxsize=64)
 def get_secret(secret_name):

libumccr/aws/libssm.py

@@ -5,12 +5,24 @@
 """
 import logging
 
-from cachetools.func import lru_cache
-
 from libumccr.aws import ssm_client
+from libumccr.utils import load_package_if_found
 
 logger = logging.getLogger(__name__)
 
+if load_package_if_found("cachetools"):
+    logger.info(f"cachetools found, using LRU cache")
+    from cachetools.func import lru_cache
+else:
+    def lru_cache(maxsize):
+        logger.info(f"cachetools not found, skipping LRU cache with maxsize={maxsize}")
+
+        def wrapper(func):
+            func.cache_clear = lambda: None
+            return func
+
+        return wrapper
+
 
 @lru_cache(maxsize=64)
 def get_secret(key) -> str:

setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name="libumccr",
-    version="0.4.0rc2",
+    version="0.4.0rc3",
     author="UMCCR and Contributors",
     author_email="services@umccr.org",
     description="UMCCR Reusable Python modules",
@@ -48,13 +48,15 @@
             "tox",
             "nose2",
             "awscli-local",
+            "cachetools",
         ],
         "libgdrive": [
             "requests",
             "pandas",
             "gspread",
             "gspread-pandas",
             "google-auth",
+            "cachetools",
         ],
         "aws": [
             "boto3",

tests/aws/test_libsm.py

@@ -1,11 +1,79 @@
+import logging
+import sys
+import uuid
 from unittest import TestCase, skip
+from unittest.mock import patch
 
-from libumccr.aws import libsm
+from mockito import when
+
+from libumccr import aws
+
+logger = logging.getLogger(__name__)
 
 
 class LibSmUnitTests(TestCase):
-    # TODO https://github.com/umccr/libumccr/issues/2
-    pass
+
+    def setUp(self):
+        from libumccr.aws import libsm
+        mock_sm = aws.client(
+            'secretsmanager',
+            endpoint_url='http://localhost:4566',
+            region_name='ap-southeast-2',
+            aws_access_key_id=str(uuid.uuid4()),
+            aws_secret_access_key=str(uuid.uuid4()),
+            aws_session_token=f"{uuid.uuid4()}_{uuid.uuid4()}"
+        )
+        when(aws).sm_client(...).thenReturn(mock_sm)
+        when(libsm).sm_client(...).thenReturn(mock_sm)
+
+    def test_cache_clear(self):
+        """
+        python -m unittest tests.aws.test_libsm.LibSmUnitTests.test_cache_clear
+        """
+        from libumccr.aws import libsm
+        # logger.info(dir(libsm.get_secret))
+        # logger.info(type(libsm.get_secret))
+        libsm.get_secret.cache_clear()
+        self.assertTrue(hasattr(libsm.get_secret, "cache_clear"))
+
+    @patch.dict(sys.modules, {"cachetools": None})
+    @patch("libumccr.utils.load_package_if_found")
+    def test_cache_clear_cachetools_not_found(self, load_package_if_found):
+        """
+        python -m unittest tests.aws.test_libsm.LibSmUnitTests.test_cache_clear_cachetools_not_found
+        """
+        load_package_if_found.return_value = False
+
+        from libumccr.aws import libsm
+        libsm.get_secret.cache_clear()
+
+        self.assertIsNone(sys.modules['cachetools'])
+        self.assertTrue(hasattr(libsm.get_secret, "cache_clear"))
+
+    @patch.dict(sys.modules, {"cachetools": None})
+    @patch("libumccr.utils.load_package_if_found")
+    def test_get_secret_no_cache(self, load_package_if_found):
+        """
+        python -m unittest tests.aws.test_libsm.LibSmUnitTests.test_get_secret_no_cache
+        """
+        load_package_if_found.return_value = False
+
+        from libumccr.aws import libsm
+        libsm.get_secret.cache_clear()
+        value = libsm.get_secret(secret_name='MyTestSecret')
+        logger.info(value)
+        self.assertEqual(value, 'HealTheWorld')
+        self.assertIsNone(sys.modules['cachetools'])
+
+    def test_get_secret(self):
+        """
+        python -m unittest tests.aws.test_libsm.LibSmUnitTests.test_get_secret
+        """
+
+        from libumccr.aws import libsm
+        value = libsm.get_secret(secret_name='MyTestSecret')
+        logger.info(value)
+        self.assertEqual(value, 'HealTheWorld')
 
 
 class LibSmIntegrationTests(TestCase):
@@ -15,11 +83,11 @@ def test_get_secret(self):
         """
         python -m unittest tests.aws.test_libsm.LibSmIntegrationTests.test_get_secret
         """
-
+        from libumccr.aws import libsm
         lookup_name = "IcaSecretsPortal"
 
         secret = libsm.get_secret(secret_name=lookup_name)
 
         self.assertIsNotNone(secret)
         self.assertIsInstance(secret, str)
-        # print(secret)
+        logger.info(secret)

tests/aws/test_libssm.py

@@ -1,11 +1,89 @@
+import logging
+import sys
+import uuid
 from unittest import TestCase, skip
+from unittest.mock import patch
 
-from libumccr.aws import libssm
+from mockito import when
+
+from libumccr import aws
+
+logger = logging.getLogger(__name__)
 
 
 class LibSsmUnitTests(TestCase):
-    # TODO https://github.com/umccr/libumccr/issues/4
-    pass
+
+    def setUp(self):
+        from libumccr.aws import libssm
+        mock_ssm = aws.client(
+            'ssm',
+            endpoint_url='http://localhost:4566',
+            region_name='ap-southeast-2',
+            aws_access_key_id=str(uuid.uuid4()),
+            aws_secret_access_key=str(uuid.uuid4()),
+            aws_session_token=f"{uuid.uuid4()}_{uuid.uuid4()}"
+        )
+        when(aws).ssm_client(...).thenReturn(mock_ssm)
+        when(libssm).ssm_client(...).thenReturn(mock_ssm)
+
+    def test_cache_clear(self):
+        """
+        python -m unittest tests.aws.test_libssm.LibSsmUnitTests.test_cache_clear
+        """
+        from libumccr.aws import libssm
+        # logger.info(dir(libssm.get_secret))
+        # logger.info(type(libssm.get_secret))
+        libssm.get_secret.cache_clear()
+        self.assertTrue(hasattr(libssm.get_secret, "cache_clear"))
+
+    @patch.dict(sys.modules, {"cachetools": None})
+    @patch("libumccr.utils.load_package_if_found")
+    def test_cache_clear_cachetools_not_found(self, load_package_if_found):
+        """
+        python -m unittest tests.aws.test_libssm.LibSsmUnitTests.test_cache_clear_cachetools_not_found
+        """
+        load_package_if_found.return_value = False
+
+        from libumccr.aws import libssm
+        libssm.get_secret.cache_clear()
+
+        self.assertIsNone(sys.modules['cachetools'])
+        self.assertTrue(hasattr(libssm.get_secret, "cache_clear"))
+
+    @patch.dict(sys.modules, {"cachetools": None})
+    @patch("libumccr.utils.load_package_if_found")
+    def test_get_secret_no_cache(self, load_package_if_found):
+        """
+        python -m unittest tests.aws.test_libssm.LibSsmUnitTests.test_get_secret_no_cache
+        """
+        load_package_if_found.return_value = False
+
+        from libumccr.aws import libssm
+        libssm.get_secret.cache_clear()
+        value = libssm.get_secret(key='my-param-secure')
+        logger.info(value)
+        self.assertEqual(value, 'Sello')
+        self.assertIsNone(sys.modules['cachetools'])
+
+    def test_get_secret(self):
+        """
+        python -m unittest tests.aws.test_libssm.LibSsmUnitTests.test_get_secret
+        """
+
+        from libumccr.aws import libssm
+        value = libssm.get_secret(key='my-param-secure')
+        logger.info(value)
+        self.assertEqual(value, 'Sello')
+
+    def test_get_ssm_param(self):
+        """
+        python -m unittest tests.aws.test_libssm.LibSsmUnitTests.test_get_ssm_param
+        """
+
+        from libumccr.aws import libssm
+        value = libssm.get_ssm_param(name='my-param')
+        logger.info(value)
+        self.assertEqual(value, 'Hello')
 
 
 class LibSsmIntegrationTests(TestCase):
@@ -15,11 +93,11 @@ def test_get_secret(self):
         """
         python -m unittest tests.aws.test_libssm.LibSsmIntegrationTests.test_get_secret
         """
-
+        from libumccr.aws import libssm
         key = "/iap/jwt-token"
 
         value = libssm.get_secret(key=key)
 
         self.assertIsNotNone(value)
         self.assertIsInstance(value, str)
-        # print(value)
+        logger.info(value)