From 1c956ad4a2a4092060a769dc6054974d1f402541 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Thu, 8 Aug 2024 08:37:36 -0700 Subject: [PATCH 1/2] config --- infrastructure/mwaa_environment_variables.tpl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infrastructure/mwaa_environment_variables.tpl b/infrastructure/mwaa_environment_variables.tpl index bf65474a..f7743a39 100644 --- a/infrastructure/mwaa_environment_variables.tpl +++ b/infrastructure/mwaa_environment_variables.tpl @@ -14,5 +14,9 @@ "COGNITO_APP_SECRET": "${cognito_app_secret}", "STAC_INGESTOR_API_URL": "${stac_ingestor_api_url}", "STAC_URL": "${stac_url}", - "VECTOR_SECRET_NAME": "${vector_secret_name}" + "VECTOR_SECRET_NAME": "${vector_secret_name}", + "VECTOR_ECS_CONF":{ + "VECTOR_SECURITY_GROUP": ["${vector_security_group}"], + "VECTOR_SUBNETS": ["${vector_subnet_1}", "${vector_subnet_2}"] + } } From 1c290fc50ca3603ccff1ab838deac4d85d24cedd Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Thu, 8 Aug 2024 08:50:35 -0700 Subject: [PATCH 2/2] we need this anyhow --- dags/veda_data_pipeline/veda_process_vector_pipeline.py | 5 +++-- infrastructure/main.tf | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dags/veda_data_pipeline/veda_process_vector_pipeline.py b/dags/veda_data_pipeline/veda_process_vector_pipeline.py index da6524c4..be71c200 100644 --- a/dags/veda_data_pipeline/veda_process_vector_pipeline.py +++ b/dags/veda_data_pipeline/veda_process_vector_pipeline.py @@ -56,6 +56,7 @@ mwaa_stack_conf = Variable.get( "MWAA_STACK_CONF", default_var={}, deserialize_json=True ) + vector_ecs_conf = Variable.get("VECTOR_ECS_CONF", deserialize_json=True) ingest_vector = EcsRunTaskOperator( task_id="ingest_vector", @@ -96,8 +97,8 @@ }, network_configuration={ "awsvpcConfiguration": { - "securityGroups": mwaa_stack_conf.get("SECURITYGROUPS"), - "subnets": mwaa_stack_conf.get("SUBNETS"), + "securityGroups": vector_ecs_conf.get("VECTOR_SECURITY_GROUP"), + "subnets": vector_ecs_conf.get("VECTOR_SUBNETS"), }, }, awslogs_group=mwaa_stack_conf.get("LOG_GROUP_NAME"), diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 0523519b..bd99c9dd 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -101,6 +101,10 @@ resource "local_file" "mwaa_variables" { stac_ingestor_api_url = var.stac_ingestor_api_url stac_url = var.stac_url vector_secret_name = var.vector_secret_name + vector_subnet_1 = data.aws_subnets.private.ids[0] + vector_subnet_2 = data.aws_subnets.private.ids[1] + vector_security_group = aws_security_group.vector_sg.id + vector_vpc = var.vector_vpc }) filename = "/tmp/mwaa_vars.json" }