From e13010bc513a2100a013552709bd420c6419a613 Mon Sep 17 00:00:00 2001 From: Victor San Kho Lin Date: Wed, 16 Oct 2024 19:44:21 +1100 Subject: [PATCH 1/2] Removed localstack dependency from toolchain * Replaced unittest case mock with botocore Stubber and Mockito * Removed also Docker compose stack as no longer need * Simplified GitHub Action CI setup --- .github/workflows/prbuild.yml | 23 ---------- Makefile | 12 ------ README.md | 2 - compose.yml | 10 ----- init-aws.sh | 7 --- tests/aws/__init__.py | 36 +--------------- tests/aws/test_libeb.py | 81 +++++++++++++++++++++++++++++------ tests/aws/test_libsm.py | 21 +++++---- tests/aws/test_libssm.py | 22 +++++----- 9 files changed, 91 insertions(+), 123 deletions(-) delete mode 100644 compose.yml delete mode 100755 init-aws.sh diff --git a/.github/workflows/prbuild.yml b/.github/workflows/prbuild.yml index 6a3245c..9ec8286 100644 --- a/.github/workflows/prbuild.yml +++ b/.github/workflows/prbuild.yml @@ -33,25 +33,6 @@ jobs: with: extra_args: --only-verified - - name: Restore Localstack Image Cache if it exists - id: cache-docker-localstack - uses: actions/cache@v4 - with: - path: ci/cache/docker/localstack - key: cache-docker-localstack-3.8 - - - name: Update Localstack Image Cache if cache miss - if: steps.cache-docker-localstack.outputs.cache-hit != 'true' - run: docker pull public.ecr.aws/localstack/localstack:3.8 && mkdir -p ci/cache/docker/localstack && docker image save public.ecr.aws/localstack/localstack:3.8 --output ./ci/cache/docker/localstack/localstack-3.8.tar - - - name: Use Localstack Image Cache if cache hit - if: steps.cache-docker-localstack.outputs.cache-hit == 'true' - run: docker image load --input ./ci/cache/docker/localstack/localstack-3.8.tar - - - name: Set up compose stack - run: | - docker compose -f compose.yml up -d - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -68,7 +49,3 @@ jobs: - name: Run test suite run: | make test - - - name: Tear down compose stack - run: | - docker compose -f compose.yml down diff --git a/Makefile b/Makefile index 10c3ab3..5aa10bd 100644 --- a/Makefile +++ b/Makefile @@ -18,18 +18,6 @@ deep: scan baseline: @detect-secrets scan --exclude-files '^(yarn.lock|.yarn/|.local/|openapi/)' > .secrets.baseline -up: - @docker compose up -d - -down: - @docker compose down - -stop: - @docker compose down - -ps: - @docker compose ps - pytest: @py.test --no-cov tests/ diff --git a/README.md b/README.md index ac297d6..f46d35e 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,6 @@ _This arrangement is sometime also known as deferring feature flag toggle at run git clone https://github.com/umccr/libumccr.git cd libumccr conda activate libumccr -make up -make ps make install all make check make test diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 13071f3..0000000 --- a/compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -services: - localstack: - image: public.ecr.aws/localstack/localstack:3.8 - container_name: libumccr_localstack - ports: - - "4566:4566" - volumes: - - "./init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh" - environment: - - DEBUG=1 diff --git a/init-aws.sh b/init-aws.sh deleted file mode 100755 index 96e157a..0000000 --- a/init-aws.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -awslocal s3 mb s3://my-bucket -awslocal sqs create-queue --region ap-southeast-2 --queue-name my-queue -awslocal ssm put-parameter --region ap-southeast-2 --name my-param --type String --value 'Hello' -awslocal ssm put-parameter --region ap-southeast-2 --name my-param-secure --type SecureString --value 'Sello' # pragma: allow-secret -awslocal secretsmanager create-secret --region ap-southeast-2 --name MyTestSecret --secret-string "HealTheWorld" # pragma: allow-secret -awslocal secretsmanager create-secret --region ap-southeast-2 --name MyTestSecretJson --secret-string "{\"user\":\"diegor\",\"password\":\"EXAMPLE-PASSWORD\"}" # pragma: allow-secret diff --git a/tests/aws/__init__.py b/tests/aws/__init__.py index 0c18942..cd26534 100644 --- a/tests/aws/__init__.py +++ b/tests/aws/__init__.py @@ -1,11 +1,7 @@ import logging -import uuid from unittest import TestCase -from mockito import when, unstub - -from libumccr import aws -from libumccr.aws import libsqs, libeb +from mockito import unstub logger = logging.getLogger() logger.setLevel(logging.INFO) @@ -14,35 +10,7 @@ class AWSTestCase(TestCase): def setUp(self) -> None: - super().setUp() - - mock_sqs = aws.client( - 'sqs', - endpoint_url='http://localhost:4566', - region_name='us-east-1', - aws_access_key_id=str(uuid.uuid4()), - aws_secret_access_key=str(uuid.uuid4()), - aws_session_token=f"{uuid.uuid4()}_{uuid.uuid4()}" - ) - when(aws).sqs_client(...).thenReturn(mock_sqs) - when(libsqs).sqs_client(...).thenReturn(mock_sqs) - - mock_eb = aws.client( - 'events', - endpoint_url='http://localhost:4566', - region_name='us-east-1', - aws_access_key_id=str(uuid.uuid4()), - aws_secret_access_key=str(uuid.uuid4()), - aws_session_token=f"{uuid.uuid4()}_{uuid.uuid4()}" - ) - when(aws).eb_client(...).thenReturn(mock_eb) - when(libeb).eb_client(...).thenReturn(mock_eb) + super(AWSTestCase, self).setUp() def tearDown(self) -> None: unstub() - - def verify_local(self): - queue_urls = libsqs.sqs_client().list_queues()['QueueUrls'] - logger.info(f"SQS_QUEUE_URLS={queue_urls}") - self.assertIn('4566', queue_urls[0]) - logger.info(f"-" * 32) diff --git a/tests/aws/test_libeb.py b/tests/aws/test_libeb.py index d09f07a..1aea660 100644 --- a/tests/aws/test_libeb.py +++ b/tests/aws/test_libeb.py @@ -1,11 +1,24 @@ from datetime import datetime +import boto3 +from botocore.stub import Stubber +from libumccr import aws +from mockito import when + from libumccr.aws import libeb from tests.aws import AWSTestCase, logger class LibEBUnitTests(AWSTestCase): + def setUp(self): + super(LibEBUnitTests, self).setUp() + mock_eb_client = boto3.client('events') + self.stubber = Stubber(mock_eb_client) + self.stubber.activate() + when(aws).eb_client(...).thenReturn(mock_eb_client) + when(libeb).eb_client(...).thenReturn(mock_eb_client) + def test_eb_client(self): """ python -m unittest tests.aws.test_libeb.LibEBUnitTests.test_eb_client @@ -19,6 +32,20 @@ def test_get_event_buses(self): """ python -m unittest tests.aws.test_libeb.LibEBUnitTests.test_get_event_buses """ + self.stubber.add_response('list_event_buses', { + 'EventBuses': [ + { + 'Name': 'string', + 'Arn': 'string', + 'Description': 'string', + 'Policy': 'string', + 'CreationTime': datetime(2015, 1, 1), + 'LastModifiedTime': datetime(2015, 1, 1) + }, + ], + 'NextToken': 'string' + }) + resp = libeb.get_event_buses() logger.info(resp) self.assertIsNotNone(resp) @@ -27,18 +54,16 @@ def test_emit_events(self): """ python -m unittest tests.aws.test_libeb.LibEBUnitTests.test_emit_events """ - # when(BaseClient)._make_api_call(...).thenReturn( - # { - # "FailedEntryCount": 123, - # "Entries": [ - # { - # "EventId": "string", - # "ErrorCode": "string", - # "ErrorMessage": "string" - # }, - # ] - # } - # ) + self.stubber.add_response('put_events', { + 'FailedEntryCount': 123, + 'Entries': [ + { + 'EventId': 'string', + 'ErrorCode': 'string', + 'ErrorMessage': 'string' + }, + ] + }) resp = libeb.emit_events([ { @@ -61,6 +86,17 @@ def test_emit_event(self): """ python -m unittest tests.aws.test_libeb.LibEBUnitTests.test_emit_event """ + self.stubber.add_response('put_events', { + 'FailedEntryCount': 123, + 'Entries': [ + { + 'EventId': 'string', + 'ErrorCode': 'string', + 'ErrorMessage': 'string' + }, + ] + }) + resp = libeb.emit_event( { "Time": datetime(2015, 1, 1), @@ -82,6 +118,27 @@ def test_dispatch_events(self): """ python -m unittest tests.aws.test_libeb.LibEBUnitTests.test_dispatch_events """ + self.stubber.add_response('put_events', { + 'FailedEntryCount': 123, + 'Entries': [ + { + 'EventId': 'string', + 'ErrorCode': 'string', + 'ErrorMessage': 'string' + }, + ] + }) + self.stubber.add_response('put_events', { + 'FailedEntryCount': 123, + 'Entries': [ + { + 'EventId': 'string', + 'ErrorCode': 'string', + 'ErrorMessage': 'string' + }, + ] + }) + mock_entries = [] for n in range(20): mock_entries.append( diff --git a/tests/aws/test_libsm.py b/tests/aws/test_libsm.py index 3c8df36..78ea23b 100644 --- a/tests/aws/test_libsm.py +++ b/tests/aws/test_libsm.py @@ -1,9 +1,10 @@ import logging import sys -import uuid from unittest import TestCase, skip from unittest.mock import patch +import boto3 +from botocore.stub import Stubber from mockito import when from libumccr import aws @@ -15,16 +16,14 @@ class LibSmUnitTests(TestCase): def setUp(self): from libumccr.aws import libsm - mock_sm = aws.client( - 'secretsmanager', - endpoint_url='http://localhost:4566', - region_name='ap-southeast-2', - aws_access_key_id=str(uuid.uuid4()), - aws_secret_access_key=str(uuid.uuid4()), - aws_session_token=f"{uuid.uuid4()}_{uuid.uuid4()}" - ) - when(aws).sm_client(...).thenReturn(mock_sm) - when(libsm).sm_client(...).thenReturn(mock_sm) + mock_sm_client = boto3.client('secretsmanager') + stubber = Stubber(mock_sm_client) + stubber.add_response('get_secret_value', { + 'SecretString': 'HealTheWorld', # pragma: allowlist secret + }) + stubber.activate() + when(aws).sm_client(...).thenReturn(mock_sm_client) + when(libsm).sm_client(...).thenReturn(mock_sm_client) def test_cache_clear(self): """ diff --git a/tests/aws/test_libssm.py b/tests/aws/test_libssm.py index df08abc..707c785 100644 --- a/tests/aws/test_libssm.py +++ b/tests/aws/test_libssm.py @@ -1,9 +1,10 @@ import logging import sys -import uuid from unittest import TestCase, skip from unittest.mock import patch +import boto3 +from botocore.stub import Stubber from mockito import when from libumccr import aws @@ -15,16 +16,13 @@ class LibSsmUnitTests(TestCase): def setUp(self): from libumccr.aws import libssm - mock_ssm = aws.client( - 'ssm', - endpoint_url='http://localhost:4566', - region_name='ap-southeast-2', - aws_access_key_id=str(uuid.uuid4()), - aws_secret_access_key=str(uuid.uuid4()), - aws_session_token=f"{uuid.uuid4()}_{uuid.uuid4()}" - ) - when(aws).ssm_client(...).thenReturn(mock_ssm) - when(libssm).ssm_client(...).thenReturn(mock_ssm) + mock_ssm_client = boto3.client('ssm') + stubber = Stubber(mock_ssm_client) + stubber.add_response('get_parameter', {'Parameter': {'Value': 'Sello'}}) + stubber.activate() + when(aws).ssm_client(...).thenReturn(mock_ssm_client) + when(libssm).ssm_client(...).thenReturn(mock_ssm_client) + super(LibSsmUnitTests, self).setUp() def test_cache_clear(self): """ @@ -83,7 +81,7 @@ def test_get_ssm_param(self): from libumccr.aws import libssm value = libssm.get_ssm_param(name='my-param') logger.info(value) - self.assertEqual(value, 'Hello') + self.assertEqual(value, 'Sello') class LibSsmIntegrationTests(TestCase): From aee41a437830c928b33bccde66dec5cf9f7f38e1 Mon Sep 17 00:00:00 2001 From: Victor San Kho Lin Date: Wed, 16 Oct 2024 19:53:31 +1100 Subject: [PATCH 2/2] Fixed aws region name for stubber client --- tests/aws/test_libeb.py | 2 +- tests/aws/test_libsm.py | 2 +- tests/aws/test_libssm.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/aws/test_libeb.py b/tests/aws/test_libeb.py index 1aea660..3a28513 100644 --- a/tests/aws/test_libeb.py +++ b/tests/aws/test_libeb.py @@ -13,7 +13,7 @@ class LibEBUnitTests(AWSTestCase): def setUp(self): super(LibEBUnitTests, self).setUp() - mock_eb_client = boto3.client('events') + mock_eb_client = boto3.client('events', region_name='us-east-1') self.stubber = Stubber(mock_eb_client) self.stubber.activate() when(aws).eb_client(...).thenReturn(mock_eb_client) diff --git a/tests/aws/test_libsm.py b/tests/aws/test_libsm.py index 78ea23b..5e7ea1d 100644 --- a/tests/aws/test_libsm.py +++ b/tests/aws/test_libsm.py @@ -16,7 +16,7 @@ class LibSmUnitTests(TestCase): def setUp(self): from libumccr.aws import libsm - mock_sm_client = boto3.client('secretsmanager') + mock_sm_client = boto3.client('secretsmanager', region_name='us-east-1') stubber = Stubber(mock_sm_client) stubber.add_response('get_secret_value', { 'SecretString': 'HealTheWorld', # pragma: allowlist secret diff --git a/tests/aws/test_libssm.py b/tests/aws/test_libssm.py index 707c785..1e6a5df 100644 --- a/tests/aws/test_libssm.py +++ b/tests/aws/test_libssm.py @@ -16,7 +16,7 @@ class LibSsmUnitTests(TestCase): def setUp(self): from libumccr.aws import libssm - mock_ssm_client = boto3.client('ssm') + mock_ssm_client = boto3.client('ssm', region_name='us-east-1') stubber = Stubber(mock_ssm_client) stubber.add_response('get_parameter', {'Parameter': {'Value': 'Sello'}}) stubber.activate()