From 359b2c4c346e637a43a297afeb7a2957a404b161 Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Fri, 11 Nov 2022 20:31:11 +0200 Subject: [PATCH 1/7] feat: added README.md --- README.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1ee54ff..61ce60b 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,58 @@ -# Google <> Terraform module -Terraform module for creation Google <> +# Google Composer Environments Terraform module +Terraform module for creation Google Composer Environments ## Usage +## Requirements +| Name | Version | +| ------------------------------------------------------------------------- | --------- | +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.24.0 | + +## Providers + +| Name | Version | +| ---------------------------------------------------------------- | --------- | +| [google](#requirement\_google) | >= 4.24.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| --------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| [google_composer_environment.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------- | :------: | +| [region](#input\_region) | The location or cloud resources region for the environment | `string` | n/a | yes | +| [product\_base\_name](#input\_product\_base\_name) | Cloud resources base name (used to create services) | `string` | n/a | yes | +| [env](#input\_env) | Variable to mark the environment of the resource (used to create services) | `string` | n/a | yes | +| [network](#input\_network) | The Compute Engine network to be used for machine communications | `string` | n/a | yes | +| [subnet](#input\_subnet) | The Compute Engine subnetwork to be used for machine communications | `string` | n/a | yes | +| [sa-email](#input\_sa-email) | The Google Cloud Platform Service Account to be used by the node VMs | `string` | n/a | yes | +| [image\_version](#input\_image\_version) | The version of the software running in the environment | `string` | "composer-2.0.22-airflow-2.2.5" | no | +| [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database | `string` | "ENVIRONMENT_SIZE_SMALL" | no | +| [env](#input\_env) | If true, access to the public endpoint of the GKE cluster is denied | `bool` | true | no | + +## Outputs + +| Name | Description | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------ | +| [airflow\_url](#output\_airflow\_url) | The URI of the Apache Airflow Web UI hosted within the environment | +| [dags\_bucket](#output\_dags\_bucket) | The Cloud Storage prefix of the DAGs for the environment | ## License -Apache 2 Licensed. For more information please see [LICENSE](https://github.com/data-platform-hq/terraform-azurerm<>/tree/master/LICENSE) +Apache 2 Licensed. For more information please see [LICENSE](https://github.com/data-platform-hq/terraform-google-composer-env/blob/main/LICENSE) + + + + From 7ed969ee2b70a2f17c471012cf92b588fc103728 Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Tue, 29 Nov 2022 11:03:05 +0200 Subject: [PATCH 2/7] feat: added terraforms files --- README.md | 26 +++++++++++--------------- main.tf | 27 +++++++++++++++++++++++++++ outputs.tf | 9 +++++++++ variables.tf | 47 +++++++++++++++++++++++++++++++++++++++++++++++ versions.tf | 9 +++++++++ 5 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 main.tf create mode 100644 outputs.tf create mode 100644 variables.tf create mode 100644 versions.tf diff --git a/README.md b/README.md index 61ce60b..52712b5 100644 --- a/README.md +++ b/README.md @@ -29,17 +29,17 @@ No modules. ## Inputs -| Name | Description | Type | Default | Required | -| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------- | :------: | -| [region](#input\_region) | The location or cloud resources region for the environment | `string` | n/a | yes | -| [product\_base\_name](#input\_product\_base\_name) | Cloud resources base name (used to create services) | `string` | n/a | yes | -| [env](#input\_env) | Variable to mark the environment of the resource (used to create services) | `string` | n/a | yes | -| [network](#input\_network) | The Compute Engine network to be used for machine communications | `string` | n/a | yes | -| [subnet](#input\_subnet) | The Compute Engine subnetwork to be used for machine communications | `string` | n/a | yes | -| [sa-email](#input\_sa-email) | The Google Cloud Platform Service Account to be used by the node VMs | `string` | n/a | yes | -| [image\_version](#input\_image\_version) | The version of the software running in the environment | `string` | "composer-2.0.22-airflow-2.2.5" | no | -| [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database | `string` | "ENVIRONMENT_SIZE_SMALL" | no | -| [env](#input\_env) | If true, access to the public endpoint of the GKE cluster is denied | `bool` | true | no | +| Name | Description | Type | Default | Required | +| ----------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------- | :------: | +| [region](#input\_region) | The location or cloud resources region for the environment | `string` | n/a | yes | +| [product\_base\_name](#input\_product\_base\_name) | Cloud resources base name (used to create services) | `string` | n/a | yes | +| [env](#input\_env) | Variable to mark the environment of the resource (used to create services) | `string` | n/a | yes | +| [network](#input\_network) | The Compute Engine network to be used for machine communications | `string` | n/a | yes | +| [subnet](#input\_subnet) | The Compute Engine subnetwork to be used for machine communications | `string` | n/a | yes | +| [sa-email](#input\_sa-email) | The Google Cloud Platform Service Account to be used by the node VMs | `string` | n/a | yes | +| [image\_version](#input\_image\_version) | The version of the software running in the environment | `string` | "composer-2.0.22-airflow-2.2.5" | no | +| [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database | `string` | "ENVIRONMENT_SIZE_SMALL" | no | +| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | If true, access to the public endpoint of the GKE cluster is denied | `bool` | true | no | ## Outputs @@ -52,7 +52,3 @@ No modules. ## License Apache 2 Licensed. For more information please see [LICENSE](https://github.com/data-platform-hq/terraform-google-composer-env/blob/main/LICENSE) - - - - diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..8326c2d --- /dev/null +++ b/main.tf @@ -0,0 +1,27 @@ + +resource "google_composer_environment" "main" { + name = "${var.env}-${var.product_base_name}-composer" + config { + + software_config { + image_version = var.image_version + } + + environment_size = var.environment_size + + node_config { + network = var.network + subnetwork = var.subnet + service_account = var.sa-email + } + private_environment_config { + enable_private_endpoint = var.enable_private_endpoint + } + } + + labels = { + "iacda-gcp-pbn" = var.product_base_name + "iacda-gcp-env" = var.env + "iacda-gcp-res" = "composer" + } +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..a1f898a --- /dev/null +++ b/outputs.tf @@ -0,0 +1,9 @@ +output "airflow_url" { + description = " The URI of the Apache Airflow Web UI hosted within this environment." + value = google_composer_environment.main.config.0.airflow_uri +} + +output "dags_bucket" { + description = "The Cloud Storage prefix of the DAGs for this environment" + value = google_composer_environment.main.config.0.dag_gcs_prefix +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..d3eb9f3 --- /dev/null +++ b/variables.tf @@ -0,0 +1,47 @@ +variable "region" { + description = "The location or cloud resources region for the environment" + type = string +} + +variable "product_base_name" { + description = "Cloud resources base name (used to create services)" + type = string +} + +variable "env" { + description = "Variable to mark the environment of the resource (used to create services)." + type = string +} + +variable "network" { + description = "The Compute Engine network to be used for machine communications" + type = string +} + +variable "subnet" { + description = "The Compute Engine subnetwork to be used for machine communications" + type = string +} + +variable "sa-email" { + description = "The Google Cloud Platform Service Account to be used by the node VMs" + type = string +} + +variable "image_version" { + description = "The version of the software running in the environment" + type = string + default = "composer-2.0.22-airflow-2.2.5" +} + +variable "environment_size" { + description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database" + type = string + default = "ENVIRONMENT_SIZE_SMALL" +} + +variable "enable_private_endpoint" { + description = "If true, access to the public endpoint of the GKE cluster is denied" + type = bool + default = true +} diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..d272246 --- /dev/null +++ b/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = "4.24.0" + } + } +} From 0f6c13196bd9272607900b3dd3b86f1569542ff2 Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Tue, 6 Dec 2022 22:33:34 +0200 Subject: [PATCH 3/7] refactored code according to the review --- README.md | 9 +++++---- main.tf | 6 +----- outputs.tf | 6 +++--- variables.tf | 22 ++++++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 52712b5..9d89919 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,14 @@ No modules. | [image\_version](#input\_image\_version) | The version of the software running in the environment | `string` | "composer-2.0.22-airflow-2.2.5" | no | | [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database | `string` | "ENVIRONMENT_SIZE_SMALL" | no | | [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | If true, access to the public endpoint of the GKE cluster is denied | `bool` | true | no | +| [labels](#input\_labels) | The labels associated with this dataset. You can use these to organize and group your datasets | `bool` | true | no | ## Outputs -| Name | Description | -| ----------------------------------------------------------------------- | ------------------------------------------------------------------ | -| [airflow\_url](#output\_airflow\_url) | The URI of the Apache Airflow Web UI hosted within the environment | -| [dags\_bucket](#output\_dags\_bucket) | The Cloud Storage prefix of the DAGs for the environment | +| Name | Description | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------- | +| [airflow\_url](#output\_airflow\_url) | The URI of the Apache Airflow Web UI hosted within the environment. | +| [dags\_bucket](#output\_dags\_bucket) | The Cloud Storage prefix of the DAGs for the environment. | ## License diff --git a/main.tf b/main.tf index 8326c2d..5980cc3 100644 --- a/main.tf +++ b/main.tf @@ -19,9 +19,5 @@ resource "google_composer_environment" "main" { } } - labels = { - "iacda-gcp-pbn" = var.product_base_name - "iacda-gcp-env" = var.env - "iacda-gcp-res" = "composer" - } + labels = var.labels } diff --git a/outputs.tf b/outputs.tf index a1f898a..403dbb6 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,9 +1,9 @@ output "airflow_url" { description = " The URI of the Apache Airflow Web UI hosted within this environment." - value = google_composer_environment.main.config.0.airflow_uri + value = google_composer_environment.main.config[0].airflow_uri } output "dags_bucket" { - description = "The Cloud Storage prefix of the DAGs for this environment" - value = google_composer_environment.main.config.0.dag_gcs_prefix + description = "The Cloud Storage prefix of the DAGs for this environment." + value = google_composer_environment.main.config[0].dag_gcs_prefix } diff --git a/variables.tf b/variables.tf index d3eb9f3..2f24222 100644 --- a/variables.tf +++ b/variables.tf @@ -1,10 +1,10 @@ variable "region" { - description = "The location or cloud resources region for the environment" + description = "The location or cloud resources region for the environment." type = string } variable "product_base_name" { - description = "Cloud resources base name (used to create services)" + description = "Cloud resources base name (used to create services)." type = string } @@ -14,34 +14,40 @@ variable "env" { } variable "network" { - description = "The Compute Engine network to be used for machine communications" + description = "The Compute Engine network to be used for machine communications." type = string } variable "subnet" { - description = "The Compute Engine subnetwork to be used for machine communications" + description = "The Compute Engine subnetwork to be used for machine communications." type = string } variable "sa-email" { - description = "The Google Cloud Platform Service Account to be used by the node VMs" + description = "The Google Cloud Platform Service Account to be used by the node VMs." type = string } variable "image_version" { - description = "The version of the software running in the environment" + description = "The version of the software running in the environment." type = string default = "composer-2.0.22-airflow-2.2.5" } variable "environment_size" { - description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database" + description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database." type = string default = "ENVIRONMENT_SIZE_SMALL" } variable "enable_private_endpoint" { - description = "If true, access to the public endpoint of the GKE cluster is denied" + description = "If true, access to the public endpoint of the GKE cluster is denied." type = bool default = true } + +variable "labels" { + description = "The labels associated with this dataset. You can use these to organize and group your datasets." + type = map(string) + default = {} +} From 25a03996d2c0065ee31a15574939bc9e21d10e01 Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Tue, 6 Dec 2022 22:44:59 +0200 Subject: [PATCH 4/7] fixed code according to github checks --- README.md | 3 +-- main.tf | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9d89919..6dd8a49 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,11 @@ No modules. | Name | Description | Type | Default | Required | | ----------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------- | :------: | -| [region](#input\_region) | The location or cloud resources region for the environment | `string` | n/a | yes | | [product\_base\_name](#input\_product\_base\_name) | Cloud resources base name (used to create services) | `string` | n/a | yes | | [env](#input\_env) | Variable to mark the environment of the resource (used to create services) | `string` | n/a | yes | | [network](#input\_network) | The Compute Engine network to be used for machine communications | `string` | n/a | yes | | [subnet](#input\_subnet) | The Compute Engine subnetwork to be used for machine communications | `string` | n/a | yes | -| [sa-email](#input\_sa-email) | The Google Cloud Platform Service Account to be used by the node VMs | `string` | n/a | yes | +| [sa_email](#input\_sa_email) | The Google Cloud Platform Service Account to be used by the node VMs | `string` | n/a | yes | | [image\_version](#input\_image\_version) | The version of the software running in the environment | `string` | "composer-2.0.22-airflow-2.2.5" | no | | [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database | `string` | "ENVIRONMENT_SIZE_SMALL" | no | | [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | If true, access to the public endpoint of the GKE cluster is denied | `bool` | true | no | diff --git a/main.tf b/main.tf index 5980cc3..5667e81 100644 --- a/main.tf +++ b/main.tf @@ -12,7 +12,7 @@ resource "google_composer_environment" "main" { node_config { network = var.network subnetwork = var.subnet - service_account = var.sa-email + service_account = var.sa_email } private_environment_config { enable_private_endpoint = var.enable_private_endpoint diff --git a/variables.tf b/variables.tf index 2f24222..85df1ee 100644 --- a/variables.tf +++ b/variables.tf @@ -23,7 +23,7 @@ variable "subnet" { type = string } -variable "sa-email" { +variable "sa_email" { description = "The Google Cloud Platform Service Account to be used by the node VMs." type = string } From eb5af61bbadc58388db7e44b39e9c915b0a8973c Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Tue, 6 Dec 2022 22:46:28 +0200 Subject: [PATCH 5/7] fixed code according to github checks --- variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/variables.tf b/variables.tf index 85df1ee..77fd7f1 100644 --- a/variables.tf +++ b/variables.tf @@ -1,8 +1,3 @@ -variable "region" { - description = "The location or cloud resources region for the environment." - type = string -} - variable "product_base_name" { description = "Cloud resources base name (used to create services)." type = string From 724604d1c1ae3d9ad4c3dff1b2d0bfa6549db28a Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Wed, 21 Dec 2022 22:05:25 +0200 Subject: [PATCH 6/7] fix: deleted empty line in min.tf --- main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.tf b/main.tf index 5667e81..a12817e 100644 --- a/main.tf +++ b/main.tf @@ -1,4 +1,3 @@ - resource "google_composer_environment" "main" { name = "${var.env}-${var.product_base_name}-composer" config { From a02006b62d40509c9a5156acc6eb23ec15a449e1 Mon Sep 17 00:00:00 2001 From: Ivan Smahlenko Date: Wed, 4 Jan 2023 01:05:50 +0200 Subject: [PATCH 7/7] fix: removed empty lines in main.tf --- main.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/main.tf b/main.tf index a12817e..3ac8a8e 100644 --- a/main.tf +++ b/main.tf @@ -1,13 +1,10 @@ resource "google_composer_environment" "main" { name = "${var.env}-${var.product_base_name}-composer" config { - software_config { image_version = var.image_version } - environment_size = var.environment_size - node_config { network = var.network subnetwork = var.subnet