From 6dbd5a4674b2df38e02a18fa4424dcd20eca3b5c Mon Sep 17 00:00:00 2001 From: David Kalbfleisch <1.21e9W@protonmail.com> Date: Tue, 25 Oct 2022 13:54:14 -0400 Subject: [PATCH] Lambda functions should read the database URL from SSM Parameter Store. Closes #902. (#904) --- .talismanrc | 2 +- .../va_profile_opt_in_out_lambda.py | 50 ++++++++++------- .../va_profile_remove_old_opt_outs_lambda.py | 56 +++++++++++++------ 3 files changed, 71 insertions(+), 37 deletions(-) diff --git a/.talismanrc b/.talismanrc index e3eb34dda5..503214b799 100644 --- a/.talismanrc +++ b/.talismanrc @@ -324,6 +324,6 @@ fileignoreconfig: - filename: migrations/env.py checksum: 2dd4f14a2d88a892182c9f3b32a5ff058691066982e71a72a6aa5a0856db1cd6 - filename: lambda_functions/va_profile/va_profile_opt_in_out_lambda.py - checksum: 07277537b6c68606f9672e2b7a3096d403a00612118e3d865d2ec31494868a85 + checksum: 71a7d63165dbd748d3195b68ad6c5f9ff476c3a3ac18339e67b2a69c6e503a3c - filename: tests/lambda_functions/va_profile/test_va_profile_integration.py checksum: 689a27fc1319888d900055c52d460b23b6c2b1dea97f4ec078a0e22ff631ae01 diff --git a/lambda_functions/va_profile/va_profile_opt_in_out_lambda.py b/lambda_functions/va_profile/va_profile_opt_in_out_lambda.py index 7b8bf2c712..61e1ba3224 100644 --- a/lambda_functions/va_profile/va_profile_opt_in_out_lambda.py +++ b/lambda_functions/va_profile/va_profile_opt_in_out_lambda.py @@ -41,21 +41,16 @@ logger = logging.getLogger("VAProfileOptInOut") logger.setLevel(logging.DEBUG) -CERTIFICATE_ARN = os.getenv("CERTIFICATE_ARN") -OPT_IN_OUT_QUERY = """SELECT va_profile_opt_in_out(%s, %s, %s, %s, %s);""" +ALB_CERTIFICATE_ARN = os.getenv("ALB_CERTIFICATE_ARN") +ALB_PRIVATE_KEY_PATH = os.getenv("ALB_PRIVATE_KEY_PATH") NOTIFY_ENVIRONMENT = os.getenv("NOTIFY_ENVIRONMENT") -ALB_PRIVATE_KEY_PATH = os.getenv("PRIVATE_KEY_PATH") -# TODO - Make this an SSM call. -SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI") +OPT_IN_OUT_QUERY = """SELECT va_profile_opt_in_out(%s, %s, %s, %s, %s);""" VA_PROFILE_DOMAIN = os.getenv("VA_PROFILE_DOMAIN") VA_PROFILE_PATH_BASE = "/communication-hub/communication/v1/status/changelog/" if NOTIFY_ENVIRONMENT is None: - # Without this value, this code cannot know the path to the required - # SSM Parameter Store values and other resources. - sys.exit("NOTIFY_ENVIRONMENT is not set.") - + sys.exit("NOTIFY_ENVIRONMENT is not set. Check the Lambda console.") if NOTIFY_ENVIRONMENT == "test": jwt_certificate_path = "tests/lambda_functions/va_profile/cert.pem" @@ -77,18 +72,36 @@ # access to VA Profile's private key. This variable with be populated later if needed. integration_testing_public_cert = None -# TODO - make SSM call; delete this block -if SQLALCHEMY_DATABASE_URI is None: - logger.error("SQLALCHEMY_DATABASE_URI is not set.") - sys.exit("Couldn't connect to the database.") +# Get the database URI. +if NOTIFY_ENVIRONMENT == "test": + sqlalchemy_database_uri = os.getenv("SQLALCHEMY_DATABASE_URI") +else: + # This is an AWS deployment environment. + database_uri_path = os.getenv("DATABASE_URI_PATH") + if database_uri_path is None: + # Without this value, this code cannot know the path to the required + # SSM Parameter Store resource. + sys.exit("DATABASE_URI_PATH is not set. Check the Lambda console.") + + logger.debug("Getting the database URI from SSM Parameter Store . . .") + ssm_client = boto3.client("ssm") + ssm_response: dict = ssm_client.get_parameter( + Name=database_uri_path, + WithDecryption=True + ) + logger.debug(". . . Retrieved the database URI from SSM Parameter Store.") + sqlalchemy_database_uri = ssm_response.get("Parameter", {}).get("Value") + +if sqlalchemy_database_uri is None: + sys.exit("Can't get the database URI.") # Making PUT requests requires presenting client certificates for mTLS. These are used programmatically via ssl.SSLContext. # The certificates are not necessary for testing, wherein the PUT request is mocked. ssl_context = None -if CERTIFICATE_ARN is None: - logger.error("CERTIFICATE_ARN is not set.") +if ALB_CERTIFICATE_ARN is None: + logger.error("ALB_CERTIFICATE_ARN is not set.") elif ALB_PRIVATE_KEY_PATH is None: logger.error("ALB_PRIVATE_KEY_PATH is not set.") elif NOTIFY_ENVIRONMENT != "test": @@ -96,11 +109,10 @@ # Get the client certificates from AWS ACM. logger.debug("Making a request to ACM . . .") acm_client = boto3.client("acm") - acm_response: dict = acm_client.get_certificate(CertificateArn=CERTIFICATE_ARN) + acm_response: dict = acm_client.get_certificate(CertificateArn=ALB_CERTIFICATE_ARN) logger.debug(". . . Finished the request to ACM.") # Get the private key from SSM Parameter Store. - # TODO - Get the database URI too. logger.debug("Getting the ALB private key from SSM Parameter Store . . .") ssm_client = boto3.client("ssm") ssm_response: dict = ssm_client.get_parameter( @@ -117,7 +129,7 @@ ssl_context = ssl.create_default_context(cadata=acm_response["CertificateChain"]) ssl_context.load_cert_chain(f.name) - except (OSError, ClientError, ssl.SSLError, ValidationError) as e: + except (OSError, ClientError, ssl.SSLError, ValidationError, KeyError) as e: logger.exception(e) if isinstance(e, ssl.SSLError): logger.error("The reason is: %s", e.reason) @@ -142,7 +154,7 @@ def make_database_connection(worker_id): try: logger.debug("Connecting to the database . . .") - connection = psycopg2.connect(SQLALCHEMY_DATABASE_URI + ('' if worker_id is None else f"_{worker_id}")) + connection = psycopg2.connect(sqlalchemy_database_uri + ('' if worker_id is None else f"_{worker_id}")) logger.debug(". . . Connected to the database.") except psycopg2.Warning as e: logger.warning(e) diff --git a/lambda_functions/va_profile_remove_old_opt_outs/va_profile_remove_old_opt_outs_lambda.py b/lambda_functions/va_profile_remove_old_opt_outs/va_profile_remove_old_opt_outs_lambda.py index 91a871be35..42ccd9e5ba 100644 --- a/lambda_functions/va_profile_remove_old_opt_outs/va_profile_remove_old_opt_outs_lambda.py +++ b/lambda_functions/va_profile_remove_old_opt_outs/va_profile_remove_old_opt_outs_lambda.py @@ -3,18 +3,36 @@ import psycopg2 import sys -# Set globals -REMOVE_OPTED_OUT_RECORDS_QUERY = """SELECT va_profile_remove_old_opt_outs();""" -SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI") -logger = logging.getLogger('va_profile_remove_old_opt_outs') +logger = logging.getLogger("va_profile_remove_old_opt_outs") logger.setLevel(logging.INFO) -# Verify environment is setup correctly -if SQLALCHEMY_DATABASE_URI is None: - logger.error("The database URI is not set.") - sys.exit("Couldn't connect to the database.") -else: - logger.info('Execution environment prepared...') +REMOVE_OPTED_OUT_RECORDS_QUERY = """SELECT va_profile_remove_old_opt_outs();""" + +# Get the database URI. The environment variable SQLALCHEMY_DATABASE_URI is +# set during unit testing. +sqlalchemy_database_uri = os.getenv("SQLALCHEMY_DATABASE_URI") + +if sqlalchemy_database_uri is None: + # This should be an AWS deployment environment. SQLALCHEMY_DATABASE_URI + # is not set in that case. + + database_uri_path = os.getenv("DATABASE_URI_PATH") + if database_uri_path is None: + # Without this value, this code cannot know the path to the required + # SSM Parameter Store resource. + sys.exit("DATABASE_URI_PATH is not set. Check the Lambda console.") + + logger.debug("Getting the database URI from SSM Parameter Store . . .") + ssm_client = boto3.client("ssm") + ssm_response: dict = ssm_client.get_parameter( + Name=database_uri_path, + WithDecryption=True + ) + logger.debug(". . . Retrieved the database URI from SSM Parameter Store.") + sqlalchemy_database_uri = ssm_response.get("Parameter", {}).get("Value") + +if sqlalchemy_database_uri is None: + sys.exit("Can't get the database URI.") def va_profile_remove_old_opt_outs_handler(event=None, context=None, worker_id=None): @@ -22,19 +40,21 @@ def va_profile_remove_old_opt_outs_handler(event=None, context=None, worker_id=N This function deletes any va_profile cache records that are opted out and greater than 24 hours old. """ + + logger.info("Removing old opt-outs . . .") connection = None # https://www.psycopg.org/docs/module.html#exceptions try: - logger.info('Connecting to database...') - connection = psycopg2.connect(SQLALCHEMY_DATABASE_URI + ('' if worker_id is None else f"_{worker_id}")) + logger.info("Connecting to the database...") + connection = psycopg2.connect(sqlalchemy_database_uri + ('' if worker_id is None else f"_{worker_id}")) + logger.info(". . . Connected to the database.") with connection.cursor() as c: - logger.info('Executing database function...') + logger.info("Executing the stored function") c.execute(REMOVE_OPTED_OUT_RECORDS_QUERY) - logger.info('Committing to database...') + logger.info("Committing the database transaction.") connection.commit() - logger.info('Completed commit...') except psycopg2.Warning as e: logger.warning(e) except psycopg2.Error as e: @@ -44,6 +64,8 @@ def va_profile_remove_old_opt_outs_handler(event=None, context=None, worker_id=N except Exception as e: logger.exception(e) finally: - if connection: + if connection is not None: connection.close() - logger.info('Connection closed...') + logger.info("Connection closed.") + + logger.info(". . . Finished removing old opt-outs.")