From 9b61bee8d43f2428442d3155e5318dda1305cf40 Mon Sep 17 00:00:00 2001 From: Evan Parish <104009494+EvanParish@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:37:10 -0500 Subject: [PATCH] #1466 - Pinpoint Callback Lambda Speedup --- .talismanrc | 52 ++++++++++--------- ci/docker-compose-test.yml | 5 ++ .../pinpoint_callback_lambda.py | 50 +++++++++++++++--- .../test_pinpoint_callback_lambda.py | 35 +++++++++++++ 4 files changed, 111 insertions(+), 31 deletions(-) create mode 100644 tests/lambda_functions/pinpoint_callback/test_pinpoint_callback_lambda.py diff --git a/.talismanrc b/.talismanrc index cfcf39cf69..9603c69b61 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,26 +1,30 @@ fileignoreconfig: - - filename: README.md - checksum: b2cbbb8508af49abccae8b35b317f7ce09215f3508430ed31440add78f450e5a - - filename: app/celery/contact_information_tasks.py - checksum: 80d0acf88bafb1358583016b9e143f4523ef1160d6eacdc9754ca68859b90eae - - filename: app/notifications/process_notifications.py - checksum: ae4e31c6eb56d91ec80ae09d13baf4558cf461c65f08893b93fee43f036a17a7 - - filename: app/service/rest.py - checksum: b42aefd1ae0e6ea76e75db4cf14d425facd0941943b17f7ba2e41f850ad1ec23 - - filename: app/template/rest.py - checksum: 1e5bdac8bc694d50f8f656dec127dd036b7b1b5b6156e3282d3411956c71ba0b - - filename: lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py - checksum: 7bd4900e14b1fa789bbb2568b8a8d7a400e3c8350ba32fb44cc0b5b66a2df037 - - filename: lambda_functions/ses_callback/ses_callback_lambda.py - checksum: b20c36921290a9609f158784e2a3278c36190887e6054ea548004a67675fd79b - - filename: poetry.lock - checksum: 34d12acdf749363555c31add4e7e7afa9e2a27afd792bd98c85f331b87bd7112 - - filename: scripts/trigger_task.py - checksum: 0e9d244dbe285de23fc84bb643407963dacf7d25a3358373f01f6272fb217778 - - filename: tests/app/celery/test_process_ga4_measurement_task.py - checksum: d33a6911258922f4bd3d149c90c2ee16c021a8e59e462594e4b1cd972902d689 - - filename: tests/app/conftest.py - checksum: a80aa727586db82ed1b50bdb81ddfe1379e649a9dfc1ece2c36047486b41b83d - - filename: tests/app/notifications/test_process_notifications_for_profile_v3.py - checksum: 4e15e63d349635131173ffdd7aebcd547621db08de877ef926d3a41fde72d065 +- filename: README.md + checksum: b2cbbb8508af49abccae8b35b317f7ce09215f3508430ed31440add78f450e5a +- filename: app/celery/contact_information_tasks.py + checksum: 80d0acf88bafb1358583016b9e143f4523ef1160d6eacdc9754ca68859b90eae +- filename: app/notifications/process_notifications.py + checksum: ae4e31c6eb56d91ec80ae09d13baf4558cf461c65f08893b93fee43f036a17a7 +- filename: app/service/rest.py + checksum: b42aefd1ae0e6ea76e75db4cf14d425facd0941943b17f7ba2e41f850ad1ec23 +- filename: app/template/rest.py + checksum: 1e5bdac8bc694d50f8f656dec127dd036b7b1b5b6156e3282d3411956c71ba0b +- filename: app/va/va_profile/va_profile_client.py + checksum: 6f4a0f7b8bb1fee23ae53cbcc8de6f23ca51a341a087e4910fde07ab599e5fde +- filename: ci/docker-compose-test.yml + checksum: e3efec2749e8c19e60f5bfc68eafabe24eba647530a482ceccfc4e0e62cff424 +- filename: lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py + checksum: 7bd4900e14b1fa789bbb2568b8a8d7a400e3c8350ba32fb44cc0b5b66a2df037 +- filename: lambda_functions/ses_callback/ses_callback_lambda.py + checksum: b20c36921290a9609f158784e2a3278c36190887e6054ea548004a67675fd79b +- filename: poetry.lock + checksum: 34d12acdf749363555c31add4e7e7afa9e2a27afd792bd98c85f331b87bd7112 +- filename: scripts/trigger_task.py + checksum: 0e9d244dbe285de23fc84bb643407963dacf7d25a3358373f01f6272fb217778 +- filename: tests/app/celery/test_process_ga4_measurement_task.py + checksum: d33a6911258922f4bd3d149c90c2ee16c021a8e59e462594e4b1cd972902d689 +- filename: tests/app/conftest.py + checksum: a80aa727586db82ed1b50bdb81ddfe1379e649a9dfc1ece2c36047486b41b83d +- filename: tests/app/notifications/test_process_notifications_for_profile_v3.py + checksum: 4e15e63d349635131173ffdd7aebcd547621db08de877ef926d3a41fde72d065 version: "1.0" diff --git a/ci/docker-compose-test.yml b/ci/docker-compose-test.yml index 34b95e58da..b8f1f040e6 100644 --- a/ci/docker-compose-test.yml +++ b/ci/docker-compose-test.yml @@ -8,6 +8,11 @@ services: POETRY_ARGS: --with static_tools,test FLASK_DEBUG: 1 TEST_FOLDER: tests + environment: + - AWS_ACCESS_KEY_ID=your_access_key_id + - AWS_DEFAULT_REGION=us-gov-west-1 + - AWS_SECRET_ACCESS_KEY=your_secret_access_key + - VAEC_AWS_ACCESS_KEY_ID=your_vaec_aws_access_key_id image: ¬ify_image test-notification-api volumes: ¬ify_volumes - "../:/app" diff --git a/lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py b/lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py index 0f216909b7..af14b8a299 100644 --- a/lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py +++ b/lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py @@ -1,19 +1,49 @@ import base64 import json -import boto3 +import logging import os import uuid +import boto3 +from botocore.exceptions import ClientError + +# constants +AWS_REGION = 'us-gov-west-1' ROUTING_KEY = 'delivery-status-result-tasks' +# environment variables +LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO') + +# set up logger +logger = logging.getLogger('PinpointCallbackLambda') + +try: + logger.setLevel(LOG_LEVEL) +except ValueError: + logger.setLevel('INFO') + logger.warning('Invalid log level specified. Defaulting to INFO.') + +# set up sqs resource +queue_name = f"{os.getenv('NOTIFICATION_QUEUE_PREFIX')}{ROUTING_KEY}" +try: + sqs = boto3.resource('sqs', region_name=AWS_REGION) + queue = sqs.get_queue_by_name(QueueName=queue_name) +except ClientError as e: + logger.critical( + 'ClientError, failed to create SQS resource or could not get sqs queue "%s". Exception: %s', queue_name, e + ) + raise +except Exception as e: + logger.critical( + 'Exception, failed to create SQS resource or could not get sqs queue "%s". Exception: %s', queue_name, e + ) + raise + def lambda_handler( event, context, ): - sqs = boto3.resource('sqs') - queue = sqs.get_queue_by_name(QueueName=f"{os.getenv('NOTIFICATION_QUEUE_PREFIX')}{ROUTING_KEY}") - for record in event['Records']: task = { 'task': 'process-pinpoint-result', @@ -44,7 +74,13 @@ def lambda_handler( 'delivery_tag': str(uuid.uuid4()), }, } - msg = base64.b64encode(bytes(json.dumps(envelope), 'utf-8')).decode('utf-8') - queue.send_message(MessageBody=msg) - return {'statusCode': 200} + msg = base64.b64encode(bytes(json.dumps(envelope), 'utf-8')).decode('utf-8') + try: + queue.send_message(MessageBody=msg) + except ClientError as e: + logger.critical('ClientError, failed to send message to SQS queue "%s". Exception: %s', queue_name, e) + raise + except Exception as e: + logger.critical('Exception, failed to send message to SQS queue "%s". Exception: %s', queue_name, e) + raise diff --git a/tests/lambda_functions/pinpoint_callback/test_pinpoint_callback_lambda.py b/tests/lambda_functions/pinpoint_callback/test_pinpoint_callback_lambda.py new file mode 100644 index 0000000000..c4fc6ca39c --- /dev/null +++ b/tests/lambda_functions/pinpoint_callback/test_pinpoint_callback_lambda.py @@ -0,0 +1,35 @@ +import boto3 +import pytest +from botocore.exceptions import ClientError +from moto import mock_aws + + +@mock_aws +def create_sqs_queue(): + sqs = boto3.resource('sqs', region_name='us-gov-west-1') + sqs.create_queue(QueueName='vanotify-delivery-status-result-tasks') + + +@pytest.mark.serial +@mock_aws +def test_pinpoint_callback_lambda_raises_client_error(): + # marked serial because this test has to run before the next one so the function is not already in memory + with pytest.raises(ClientError): + # this is imported here because importing it at the top of the file causes the exception to be thrown there + from lambda_functions.pinpoint_callback.pinpoint_callback_lambda import lambda_handler + + lambda_handler({}, {}) + + +@mock_aws +def test_pinpoint_callback_lambda_handler_success(): + # there has to be a better way to do this, right? + create_sqs_queue() + # this is imported here because the queue must be created before the lambda is imported + from lambda_functions.pinpoint_callback.pinpoint_callback_lambda import lambda_handler + + # test data + event = {'Records': [{'kinesis': {'data': 'test-data'}}]} + context = {} + + assert lambda_handler(event, context) is None