diff --git a/README.md b/README.md index 6dd8a49..43d071d 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ Terraform module for creation Google Composer Environments ## Requirements | Name | Version | -| ------------------------------------------------------------------------- | --------- | +|---------------------------------------------------------------------------|-----------| | [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.24.0 | +| [google](#requirement\_google) | >= 4.69.1 | ## Providers | Name | Version | -| ---------------------------------------------------------------- | --------- | -| [google](#requirement\_google) | >= 4.24.0 | +|------------------------------------------------------------------|-----------| +| [google](#requirement\_google) | >= 4.69.1 | ## Modules @@ -24,29 +24,39 @@ No modules. ## Resources | Name | Type | -| --------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| [google_composer_environment.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment) | resource | +|-----------------------------------------------------------------------------------------------------------------------------------------|----------| +| [google_composer_environment.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment) | resource | ## Inputs -| Name | Description | Type | Default | Required | -| ----------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------- | :------: | -| [product\_base\_name](#input\_product\_base\_name) | Cloud resources base name (used to create services) | `string` | n/a | yes | -| [env](#input\_env) | Variable to mark the environment of the resource (used to create services) | `string` | n/a | yes | -| [network](#input\_network) | The Compute Engine network to be used for machine communications | `string` | n/a | yes | -| [subnet](#input\_subnet) | The Compute Engine subnetwork to be used for machine communications | `string` | n/a | yes | -| [sa_email](#input\_sa_email) | The Google Cloud Platform Service Account to be used by the node VMs | `string` | n/a | yes | -| [image\_version](#input\_image\_version) | The version of the software running in the environment | `string` | "composer-2.0.22-airflow-2.2.5" | no | -| [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database | `string` | "ENVIRONMENT_SIZE_SMALL" | no | -| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | If true, access to the public endpoint of the GKE cluster is denied | `bool` | true | no | -| [labels](#input\_labels) | The labels associated with this dataset. You can use these to organize and group your datasets | `bool` | true | no | +| Name | Description | Type | Default | Required | +|---------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-----------------------------------|:--------:| +| [product\_base\_name](#input\_product\_base\_name) | Cloud resources base name (used to create services) | `string` | n/a | yes | +| [project](#input\_project) | ID of the project in which the resources should be created | `string` | null | yes | +| [region](#input\_region) | The location or cloud resources region for the environment | `string` | null | yes | +| [env](#input\_env) | Variable to mark the environment of the resource (used to create services) | `string` | n/a | yes | +| [suffix](#input\_suffix) | Suffix for resource names | `string` | "" | no | +| [node\_config](#input\_node\_config) | An optional object containing Google Cloud Composer node configuration settings. You can set network, subnetwork, service_account, tags, ip_allocation_policy, and enable_ip_masq_agent parameters. All parameters are optional and can be individually skipped | `object` | {} | no | +| [software\_config](#input\_software\_config) | An optional object containing Google Cloud Composer software configuration settings. You can set airflow_config_overrides, pypi_packages, env_variables, image_version, and cloud_data_lineage_integration parameters. All parameters are optional and can be individually skipped | `object` | null | no | +| [private\_environment\_config](#input\_private\_environment\_config) | This variable allow to configure your Google Cloud project networking for Private IP environments | `object` | {enable_private_endpoint = false} | yes | +| [kms\_key\_name](#input\_kms\_key\_name) | Customer-managed Encryption Key available through Google's Key Management Service.
It must be the fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. Cannot be updated | `string` | null | no | +| [maintenance\_window](#input\_maintenance\_window) | The configuration settings for Cloud Composer maintenance windows | `object` | null | no | +| [recovery\_config](#input\_recovery\_config) | The configuration settings for recovery | `object` | null | no | +| [scheduler](#input\_scheduler) | Configuration for resources used by Airflow schedulers | `object` | null | no | +| [web\_server](#input\_web\_server) | Configuration for resources used by Airflow web server | `object` | null | no | +| [worker](#input\_worker) | Configuration for resources used by Airflow workers | `object` | null | no | +| [environment\_size](#input\_environment\_size) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Possible values are ENVIRONMENT_SIZE_SMALL, ENVIRONMENT_SIZE_MEDIUM, and ENVIRONMENT_SIZE_LARGE | `string` | null | no | +| [resilience\_mode](#input\_resilience\_mode) | The resilience mode states whether high resilience is enabled for the environment or not. Value for resilience mode is HIGH_RESILIENCE. If unspecified, defaults to standard resilience | `string` | null | no | +| [master\_authorized\_networks\_config](#input\_master\_authorized\_networks\_config) | Configuration options for the master authorized networks feature. Enabled master authorized networks will disallow all external traffic to access Kubernetes master through HTTPS except traffic from the given CIDR blocks, Google Compute Engine Public IPs and Google Prod IPs | `object` | null | no | +| [labels](#input\_labels) | The labels associated with this dataset. You can use these to organize and group your datasets | `map(string)` | {} | no | ## Outputs | Name | Description | -| ----------------------------------------------------------------------- | ------------------------------------------------------------------- | +|-------------------------------------------------------------------------|---------------------------------------------------------------------| | [airflow\_url](#output\_airflow\_url) | The URI of the Apache Airflow Web UI hosted within the environment. | | [dags\_bucket](#output\_dags\_bucket) | The Cloud Storage prefix of the DAGs for the environment. | + ## License diff --git a/main.tf b/main.tf index 3ac8a8e..2324558 100644 --- a/main.tf +++ b/main.tf @@ -1,17 +1,122 @@ -resource "google_composer_environment" "main" { - name = "${var.env}-${var.product_base_name}-composer" +locals { + suffix = length(var.suffix) == 0 ? "" : "-${var.suffix}" +} + +resource "google_composer_environment" "this" { + name = "${var.product_base_name}-${var.env}${local.suffix}-test-composer" + region = var.region + project = var.project config { - software_config { - image_version = var.image_version + dynamic "node_config" { + for_each = var.node_config != null ? [var.node_config] : [] + content { + network = lookup(var.node_config, "network", null) + subnetwork = lookup(var.node_config, "subnetwork", null) + service_account = lookup(var.node_config, "service_account", null) + tags = lookup(var.node_config, "tags", null) + ip_allocation_policy = lookup(var.node_config, "ip_allocation_policy", null) + enable_ip_masq_agent = lookup(var.node_config, "enable_ip_masq_agent", null) + } + } + + dynamic "software_config" { + for_each = var.software_config != null ? [var.software_config] : [] + content { + airflow_config_overrides = lookup(var.software_config, "airflow_config_overrides", null) + pypi_packages = lookup(var.software_config, "pypi_packages", null) + env_variables = lookup(var.software_config, "env_variables", null) + image_version = lookup(var.software_config, "image_version", null) + } } - environment_size = var.environment_size - node_config { - network = var.network - subnetwork = var.subnet - service_account = var.sa_email + + dynamic "private_environment_config" { + for_each = var.private_environment_config != null ? [var.private_environment_config] : [] + content { + enable_private_endpoint = lookup(var.private_environment_config, "enable_private_endpoint", null) + master_ipv4_cidr_block = lookup(var.private_environment_config, "master_ipv4_cidr_block", null) + cloud_sql_ipv4_cidr_block = lookup(var.private_environment_config, "cloud_sql_ipv4_cidr_block", null) + cloud_composer_network_ipv4_cidr_block = lookup(var.private_environment_config, "cloud_composer_network_ipv4_cidr_block", null) + enable_privately_used_public_ips = lookup(var.private_environment_config, "enable_privately_used_public_ips", null) + cloud_composer_connection_subnetwork = lookup(var.private_environment_config, "cloud_composer_connection_subnetwork", null) + } + } + + dynamic "encryption_config" { + for_each = var.kms_key_name != null ? [var.kms_key_name] : [] + content { + kms_key_name = var.kms_key_name + } } - private_environment_config { - enable_private_endpoint = var.enable_private_endpoint + + dynamic "maintenance_window" { + for_each = var.maintenance_window != null ? [var.maintenance_window] : [] + content { + start_time = lookup(var.maintenance_window, "start_time", null) + end_time = lookup(var.maintenance_window, "end_time", null) + recurrence = lookup(var.maintenance_window, "recurrence", null) + } + } + + dynamic "recovery_config" { + for_each = var.recovery_config != null ? [var.recovery_config] : [] + content { + scheduled_snapshots_config { + enabled = lookup(var.recovery_config, "enabled", null) + snapshot_location = lookup(var.recovery_config, "snapshot_location", null) + snapshot_creation_schedule = lookup(var.recovery_config, "snapshot_creation_schedule", null) + time_zone = lookup(var.recovery_config, "time_zone", null) + } + } + } + + dynamic "workloads_config" { + for_each = anytrue([var.scheduler != null, var.web_server != null, ]) ? flatten([var.scheduler, var.web_server]) : [] + content { + dynamic "scheduler" { + for_each = var.scheduler != null ? [var.scheduler] : [] + content { + cpu = lookup(var.scheduler, "cpu", null) + memory_gb = lookup(var.scheduler, "memory_gb", null) + storage_gb = lookup(var.scheduler, "storage_gb", null) + count = lookup(var.scheduler, "count", null) + } + } + dynamic "web_server" { + for_each = var.web_server != null ? [var.web_server] : [] + content { + cpu = lookup(var.web_server, "cpu", null) + memory_gb = lookup(var.web_server, "memory_gb", null) + storage_gb = lookup(var.web_server, "storage_gb", null) + } + } + dynamic "worker" { + for_each = var.worker != null ? [var.worker] : [] + content { + cpu = lookup(var.worker, "cpu", null) + memory_gb = lookup(var.worker, "memory_gb", null) + storage_gb = lookup(var.worker, "storage_gb", null) + min_count = lookup(var.worker, "min_count", null) + max_count = lookup(var.worker, "max_count", null) + } + } + } + } + + environment_size = var.environment_size != null ? var.environment_size : null + resilience_mode = var.resilience_mode != null ? var.resilience_mode : null + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config != null ? [var.master_authorized_networks_config] : [] + content { + enabled = lookup(var.master_authorized_networks_config, "enabled", false) + dynamic "cidr_blocks" { + for_each = var.master_authorized_networks_config.cidr_blocks + content { + display_name = lookup(cidr_blocks.value, "display_name", null) + cidr_block = lookup(cidr_blocks.value, "cidr_block", null) + } + } + } } } diff --git a/outputs.tf b/outputs.tf index 403dbb6..8f812ee 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,9 +1,9 @@ output "airflow_url" { description = " The URI of the Apache Airflow Web UI hosted within this environment." - value = google_composer_environment.main.config[0].airflow_uri + value = google_composer_environment.this.config[0].airflow_uri } output "dags_bucket" { description = "The Cloud Storage prefix of the DAGs for this environment." - value = google_composer_environment.main.config[0].dag_gcs_prefix + value = google_composer_environment.this.config[0].dag_gcs_prefix } diff --git a/variables.tf b/variables.tf index 77fd7f1..34702d3 100644 --- a/variables.tf +++ b/variables.tf @@ -3,42 +3,174 @@ variable "product_base_name" { type = string } -variable "env" { - description = "Variable to mark the environment of the resource (used to create services)." +variable "project" { + description = "ID of the project in which the resources should be created." type = string + default = null } -variable "network" { - description = "The Compute Engine network to be used for machine communications." +variable "region" { + description = "The location or cloud resources region for the environment." type = string + default = null } -variable "subnet" { - description = "The Compute Engine subnetwork to be used for machine communications." +variable "env" { + description = "Variable to mark the environment of the resource (used to create services)." type = string } -variable "sa_email" { - description = "The Google Cloud Platform Service Account to be used by the node VMs." +variable "suffix" { + description = "Suffix for resource names" type = string + default = "" +} + +variable "node_config" { + description = <<-EOT + An optional object containing Google Cloud Composer node configuration settings. You can set network, subnetwork, service_account, tags, ip_allocation_policy, and enable_ip_masq_agent parameters. All parameters are optional and can be individually skipped. + EOT + type = object({ + network = optional(string) + subnetwork = optional(string) + service_account = optional(string) + tags = optional(list(string)) + ip_allocation_policy = optional(list(object({ + cluster_secondary_range_name = optional(string) + services_secondary_range_name = optional(string) + cluster_ipv4_cidr_block = optional(string) + services_ipv4_cidr_block = optional(string) + use_ip_aliases = bool + }))) + enable_ip_masq_agent = optional(bool) + }) + default = {} } -variable "image_version" { - description = "The version of the software running in the environment." +variable "software_config" { + description = <<-EOT + An optional object containing Google Cloud Composer software configuration settings. You can set airflow_config_overrides, pypi_packages, env_variables, image_version, and cloud_data_lineage_integration parameters. All parameters are optional and can be individually skipped. + EOT + type = object({ + airflow_config_overrides = optional(map(string)) + pypi_packages = optional(map(string)) + env_variables = optional(map(string)) + image_version = string + }) + default = null +} + +variable "private_environment_config" { + description = "This variable allow to configure your Google Cloud project networking for Private IP environments" + type = object({ + enable_private_endpoint = bool + master_ipv4_cidr_block = optional(string) + cloud_sql_ipv4_cidr_block = optional(string) + cloud_composer_network_ipv4_cidr_block = optional(string) + enable_privately_used_public_ips = optional(bool) + cloud_composer_connection_subnetwork = optional(string) + }) + default = { + enable_private_endpoint = false + } +} + +variable "kms_key_name" { + description = <<-EOT +Customer-managed Encryption Key available through Google's Key Management Service. +It must be the fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. +Cannot be updated + EOT type = string - default = "composer-2.0.22-airflow-2.2.5" + default = null +} + +variable "maintenance_window" { + description = "The configuration settings for Cloud Composer maintenance windows" + type = object({ + start_time = string + end_time = string + recurrence = string + }) + default = null +} + +variable "recovery_config" { + description = "The configuration settings for recovery" + type = object({ + enabled = optional(bool) + snapshot_location = optional(string) + snapshot_creation_schedule = optional(string) + time_zone = optional(string) + }) + default = null +} + +variable "scheduler" { + description = "Configuration for resources used by Airflow schedulers" + type = object({ + cpu = optional(number) + memory_gb = optional(number) + storage_gb = optional(number) + count = optional(number) + }) + default = null +} + +variable "web_server" { + description = "Configuration for resources used by Airflow web server" + type = object({ + cpu = optional(number) + memory_gb = optional(number) + storage_gb = optional(number) + }) + default = null +} + +variable "worker" { + description = "value" + type = object({ + cpu = optional(number) + memory_gb = optional(number) + storage_gb = optional(number) + min_count = optional(number) + max_count = optional(number) + }) + default = null } variable "environment_size" { - description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database." + description = <<-EOT + The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. + Possible values are ENVIRONMENT_SIZE_SMALL, ENVIRONMENT_SIZE_MEDIUM, and ENVIRONMENT_SIZE_LARGE + EOT + type = string + default = null +} + +variable "resilience_mode" { + description = <<-EOT + The resilience mode states whether high resilience is enabled for the environment or not. + Value for resilience mode is HIGH_RESILIENCE. + If unspecified, defaults to standard resilience + EOT type = string - default = "ENVIRONMENT_SIZE_SMALL" + default = null } -variable "enable_private_endpoint" { - description = "If true, access to the public endpoint of the GKE cluster is denied." - type = bool - default = true +variable "master_authorized_networks_config" { + description = <<-EOT + Configuration options for the master authorized networks feature. + Enabled master authorized networks will disallow all external traffic to access Kubernetes master through HTTPS except traffic from the given CIDR blocks, Google Compute Engine Public IPs and Google Prod IPs. + EOT + type = object({ + enabled = bool + cidr_blocks = list(object({ + display_name = optional(string) + cidr_block = string + })) + }) + default = null } variable "labels" { diff --git a/versions.tf b/versions.tf index d272246..05cef8b 100644 --- a/versions.tf +++ b/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "4.24.0" + version = ">= 4.69.1" } } }