#1466 - Pinpoint Callback Lambda Speedup

.talismanrc

@@ -1,26 +1,30 @@
 fileignoreconfig:
-  - filename: README.md
-    checksum: b2cbbb8508af49abccae8b35b317f7ce09215f3508430ed31440add78f450e5a
-  - filename: app/celery/contact_information_tasks.py
-    checksum: 80d0acf88bafb1358583016b9e143f4523ef1160d6eacdc9754ca68859b90eae
-  - filename: app/notifications/process_notifications.py
-    checksum: ae4e31c6eb56d91ec80ae09d13baf4558cf461c65f08893b93fee43f036a17a7
-  - filename: app/service/rest.py
-    checksum: b42aefd1ae0e6ea76e75db4cf14d425facd0941943b17f7ba2e41f850ad1ec23
-  - filename: app/template/rest.py
-    checksum: 1e5bdac8bc694d50f8f656dec127dd036b7b1b5b6156e3282d3411956c71ba0b
-  - filename: lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py
-    checksum: 7bd4900e14b1fa789bbb2568b8a8d7a400e3c8350ba32fb44cc0b5b66a2df037
-  - filename: lambda_functions/ses_callback/ses_callback_lambda.py
-    checksum: b20c36921290a9609f158784e2a3278c36190887e6054ea548004a67675fd79b
-  - filename: poetry.lock
-    checksum: 34d12acdf749363555c31add4e7e7afa9e2a27afd792bd98c85f331b87bd7112
-  - filename: scripts/trigger_task.py
-    checksum: 0e9d244dbe285de23fc84bb643407963dacf7d25a3358373f01f6272fb217778
-  - filename: tests/app/celery/test_process_ga4_measurement_task.py
-    checksum: d33a6911258922f4bd3d149c90c2ee16c021a8e59e462594e4b1cd972902d689
-  - filename: tests/app/conftest.py
-    checksum: a80aa727586db82ed1b50bdb81ddfe1379e649a9dfc1ece2c36047486b41b83d
-  - filename: tests/app/notifications/test_process_notifications_for_profile_v3.py
-    checksum: 4e15e63d349635131173ffdd7aebcd547621db08de877ef926d3a41fde72d065
+- filename: README.md
+  checksum: b2cbbb8508af49abccae8b35b317f7ce09215f3508430ed31440add78f450e5a
+- filename: app/celery/contact_information_tasks.py
+  checksum: 80d0acf88bafb1358583016b9e143f4523ef1160d6eacdc9754ca68859b90eae
+- filename: app/notifications/process_notifications.py
+  checksum: ae4e31c6eb56d91ec80ae09d13baf4558cf461c65f08893b93fee43f036a17a7
+- filename: app/service/rest.py
+  checksum: b42aefd1ae0e6ea76e75db4cf14d425facd0941943b17f7ba2e41f850ad1ec23
+- filename: app/template/rest.py
+  checksum: 1e5bdac8bc694d50f8f656dec127dd036b7b1b5b6156e3282d3411956c71ba0b
+- filename: app/va/va_profile/va_profile_client.py
+  checksum: 6f4a0f7b8bb1fee23ae53cbcc8de6f23ca51a341a087e4910fde07ab599e5fde
+- filename: ci/docker-compose-test.yml
+  checksum: e3efec2749e8c19e60f5bfc68eafabe24eba647530a482ceccfc4e0e62cff424
+- filename: lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py
+  checksum: 7bd4900e14b1fa789bbb2568b8a8d7a400e3c8350ba32fb44cc0b5b66a2df037
+- filename: lambda_functions/ses_callback/ses_callback_lambda.py
+  checksum: b20c36921290a9609f158784e2a3278c36190887e6054ea548004a67675fd79b
+- filename: poetry.lock
+  checksum: 34d12acdf749363555c31add4e7e7afa9e2a27afd792bd98c85f331b87bd7112
+- filename: scripts/trigger_task.py
+  checksum: 0e9d244dbe285de23fc84bb643407963dacf7d25a3358373f01f6272fb217778
+- filename: tests/app/celery/test_process_ga4_measurement_task.py
+  checksum: d33a6911258922f4bd3d149c90c2ee16c021a8e59e462594e4b1cd972902d689
+- filename: tests/app/conftest.py
+  checksum: a80aa727586db82ed1b50bdb81ddfe1379e649a9dfc1ece2c36047486b41b83d
+- filename: tests/app/notifications/test_process_notifications_for_profile_v3.py
+  checksum: 4e15e63d349635131173ffdd7aebcd547621db08de877ef926d3a41fde72d065
 version: "1.0"

ci/docker-compose-test.yml

@@ -8,6 +8,11 @@ services:
         POETRY_ARGS: --with static_tools,test
         FLASK_DEBUG: 1
         TEST_FOLDER: tests
+    environment:
+      - AWS_ACCESS_KEY_ID=your_access_key_id
+      - AWS_DEFAULT_REGION=us-gov-west-1
+      - AWS_SECRET_ACCESS_KEY=your_secret_access_key
+      - VAEC_AWS_ACCESS_KEY_ID=your_vaec_aws_access_key_id
     image: &notify_image test-notification-api
     volumes: &notify_volumes
       - "../:/app"

lambda_functions/pinpoint_callback/pinpoint_callback_lambda.py

@@ -1,19 +1,49 @@
 import base64
 import json
-import boto3
+import logging
 import os
 import uuid
 
+import boto3
+from botocore.exceptions import ClientError
+
+# constants
+AWS_REGION = 'us-gov-west-1'
 ROUTING_KEY = 'delivery-status-result-tasks'
 
+# environment variables
+LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
+
+# set up logger
+logger = logging.getLogger('PinpointCallbackLambda')
+
+try:
+    logger.setLevel(LOG_LEVEL)
+except ValueError:
+    logger.setLevel('INFO')
+    logger.warning('Invalid log level specified. Defaulting to INFO.')
+
+# set up sqs resource
+queue_name = f"{os.getenv('NOTIFICATION_QUEUE_PREFIX')}{ROUTING_KEY}"
+try:
+    sqs = boto3.resource('sqs', region_name=AWS_REGION)
+    queue = sqs.get_queue_by_name(QueueName=queue_name)
+except ClientError as e:
+    logger.critical(
+        'ClientError, failed to create SQS resource or could not get sqs queue "%s". Exception: %s', queue_name, e
+    )
+    raise
+except Exception as e:
+    logger.critical(
+        'Exception, failed to create SQS resource or could not get sqs queue "%s". Exception: %s', queue_name, e
+    )
+    raise
+
 
 def lambda_handler(
     event,
     context,
 ):
-    sqs = boto3.resource('sqs')
-    queue = sqs.get_queue_by_name(QueueName=f"{os.getenv('NOTIFICATION_QUEUE_PREFIX')}{ROUTING_KEY}")
-
     for record in event['Records']:
         task = {
             'task': 'process-pinpoint-result',
@@ -44,7 +74,13 @@ def lambda_handler(
                 'delivery_tag': str(uuid.uuid4()),
             },
         }
-        msg = base64.b64encode(bytes(json.dumps(envelope), 'utf-8')).decode('utf-8')
-        queue.send_message(MessageBody=msg)
 
-    return {'statusCode': 200}
+        msg = base64.b64encode(bytes(json.dumps(envelope), 'utf-8')).decode('utf-8')
+        try:
+            queue.send_message(MessageBody=msg)
+        except ClientError as e:
+            logger.critical('ClientError, failed to send message to SQS queue "%s". Exception: %s', queue_name, e)
+            raise
+        except Exception as e:
+            logger.critical('Exception, failed to send message to SQS queue "%s". Exception: %s', queue_name, e)
+            raise

tests/lambda_functions/pinpoint_callback/test_pinpoint_callback_lambda.py

@@ -0,0 +1,35 @@
+import boto3
+import pytest
+from botocore.exceptions import ClientError
+from moto import mock_aws
+
+
+@mock_aws
+def create_sqs_queue():
+    sqs = boto3.resource('sqs', region_name='us-gov-west-1')
+    sqs.create_queue(QueueName='vanotify-delivery-status-result-tasks')
+
+
+@pytest.mark.serial
+@mock_aws
+def test_pinpoint_callback_lambda_raises_client_error():
+    # marked serial because this test has to run before the next one so the function is not already in memory
+    with pytest.raises(ClientError):
+        # this is imported here because importing it at the top of the file causes the exception to be thrown there
+        from lambda_functions.pinpoint_callback.pinpoint_callback_lambda import lambda_handler
+
+        lambda_handler({}, {})
+
+
+@mock_aws
+def test_pinpoint_callback_lambda_handler_success():
+    # there has to be a better way to do this, right?
+    create_sqs_queue()
+    # this is imported here because the queue must be created before the lambda is imported
+    from lambda_functions.pinpoint_callback.pinpoint_callback_lambda import lambda_handler
+
+    # test data
+    event = {'Records': [{'kinesis': {'data': 'test-data'}}]}
+    context = {}
+
+    assert lambda_handler(event, context) is None