refactor: configure the Glue jobs with a map (#45)

ricardolsmendes · Dec 28, 2023 · 45bdd02 · 45bdd02
1 parent d9f6fa9
commit 45bdd02
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 34 deletions.
diff --git a/infrastructure/modules/glue/etl.tf b/infrastructure/modules/glue/etl.tf
@@ -1,39 +1,16 @@
-resource "aws_glue_job" "bronze_us_legislators" {
-  name              = "glue-ci-cd-bronze-us-legislators-${var.environment}"
-  description       = "Sample Bronze-layer job for the AWS Glue CI/CD Blueprint."
-  role_arn          = aws_iam_role.glue_service.arn
-  glue_version      = "4.0"
-  worker_type       = "G.1X"
-  number_of_workers = 2
-  execution_class   = "FLEX"
-  default_arguments = {
-    "--source-bucket-name" = "awsglue-datasets",
-    "--source-file-path"   = "examples/us-legislators/all/persons.json",
-    "--target-bucket-name" = var.data_bucket_id,
-    "--target-file-path"   = "bronze/awsglue-datasets/us-legislators/all-persons.json",
-  }
-  tags = local.default_tags
-  command {
-    script_location = "s3://${aws_s3_bucket.glue["scripts"].id}/s3_to_s3_job.py"
-  }
-}
+resource "aws_glue_job" "us_legislators" {
+  for_each = local.us_legislators_jobs_map
 
-resource "aws_glue_job" "silver_us_legislators" {
-  name              = "glue-ci-cd-silver-us-legislators-${var.environment}"
-  description       = "Sample Silver-layer job for the AWS Glue CI/CD Blueprint."
+  name              = each.value.name
+  description       = each.value.description
   role_arn          = aws_iam_role.glue_service.arn
   glue_version      = "4.0"
   worker_type       = "G.1X"
   number_of_workers = 2
   execution_class   = "FLEX"
-  default_arguments = {
-    "--source-bucket-name" = var.data_bucket_id,
-    "--source-file-path"   = "bronze/awsglue-datasets/us-legislators/all-persons.json",
-    "--target-bucket-name" = var.data_bucket_id,
-    "--target-table-path"  = "silver/us-legislators",
-  }
-  tags = local.default_tags
+  default_arguments = each.value.default_arguments
+  tags              = local.default_tags
   command {
-    script_location = "s3://${aws_s3_bucket.glue["scripts"].id}/s3_json_to_parquet_job.py"
+    script_location = each.value.script_location
   }
 }
diff --git a/infrastructure/modules/glue/locals.tf b/infrastructure/modules/glue/locals.tf
@@ -13,4 +13,29 @@ locals {
       name = "${var.glue_scripts_bucket_name}-${var.environment}"
     }
   }
+
+  us_legislators_jobs_map = {
+    bronze = {
+      name        = "glue-ci-cd-bronze-us-legislators-${var.environment}"
+      description = "Sample Bronze-layer job for the AWS Glue CI/CD Blueprint."
+      default_arguments = {
+        "--source-bucket-name" = "awsglue-datasets",
+        "--source-file-path"   = "examples/us-legislators/all/persons.json",
+        "--target-bucket-name" = var.data_bucket_id,
+        "--target-file-path"   = "bronze/awsglue-datasets/us-legislators/all-persons.json",
+      },
+      script_location = "s3://${aws_s3_bucket.glue["scripts"].id}/s3_to_s3_job.py"
+    },
+    silver = {
+      name        = "glue-ci-cd-silver-us-legislators-${var.environment}"
+      description = "Sample Silver-layer job for the AWS Glue CI/CD Blueprint."
+      default_arguments = {
+        "--source-bucket-name" = var.data_bucket_id,
+        "--source-file-path"   = "bronze/awsglue-datasets/us-legislators/all-persons.json",
+        "--target-bucket-name" = var.data_bucket_id,
+        "--target-table-path"  = "silver/us-legislators",
+      },
+      script_location = "s3://${aws_s3_bucket.glue["scripts"].id}/s3_json_to_parquet_job.py"
+    }
+  }
 }
diff --git a/infrastructure/modules/glue/orchestration.tf b/infrastructure/modules/glue/orchestration.tf
@@ -10,7 +10,7 @@ resource "aws_glue_trigger" "bronze_us_legislators" {
   workflow_name = aws_glue_workflow.us_legislators.name
   tags          = local.default_tags
   actions {
-    job_name = aws_glue_job.bronze_us_legislators.name
+    job_name = aws_glue_job.us_legislators["bronze"].name
   }
 }
 
@@ -21,12 +21,12 @@ resource "aws_glue_trigger" "silver_us_legislators" {
   tags          = local.default_tags
   predicate {
     conditions {
-      job_name = aws_glue_job.bronze_us_legislators.name
+      job_name = aws_glue_job.us_legislators["bronze"].name
       state    = "SUCCEEDED"
     }
   }
   actions {
-    job_name = aws_glue_job.silver_us_legislators.name
+    job_name = aws_glue_job.us_legislators["silver"].name
   }
 }
 
@@ -37,7 +37,7 @@ resource "aws_glue_trigger" "silver_us_legislators_crawler" {
   tags          = local.default_tags
   predicate {
     conditions {
-      job_name = aws_glue_job.silver_us_legislators.name
+      job_name = aws_glue_job.us_legislators["silver"].name
       state    = "SUCCEEDED"
     }
   }