PLAT-2846: Obtain kubeconfig via gcloud (#77)

Use the gcloud CLI to create the kubeconfig.
Login if necessary

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
       CLOUDSDK_CORE_PROJECT: domino-eng-platform-dev
       CLOUDSDK_COMPUTE_ZONE: us-west1-a
       GOOGLE_APPLICATION_CREDENTIALS: /root/.config/gcloud/legacy_credentials/terraform-gke-test@domino-eng-platform-dev.iam.gserviceaccount.com/adc.json
-      TERRAFORM_VERSION: 1.0.5
+      TERRAFORM_VERSION: 1.0.6
 
     steps:
       - checkout
@@ -19,18 +19,13 @@ jobs:
             wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip
             unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin
 
-      - run:
-          name: Activate GCP Service Account
-          command: echo ${CLOUDSDK_SERVICE_KEY} | gcloud auth activate-service-account --key-file=-
-
       - run:
           name: Configure test cluster env
           command: |
             echo 'export TF_VAR_description="CircleCI Build for ${CIRCLE_PR_REPONAME}: ${CIRCLE_BUILD_URL}"' >> $BASH_ENV
             echo 'export TF_VAR_filestore_disabled="true"' >> $BASH_ENV
-            echo 'export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcp-${CIRCLE_BUILD_NUM}.json"' >> $BASH_ENV
+            echo 'export GOOGLE_CREDENTIALS="$CLOUDSDK_SERVICE_KEY"' >> $BASH_ENV
             echo 'export WORKSPACE=gcp-gke-circleci-${CIRCLE_BUILD_NUM}' >> $BASH_ENV
-            echo ${CLOUDSDK_SERVICE_KEY} > /tmp/gcp-${CIRCLE_BUILD_NUM}.json
 
       - run:
           name: Terraform fmt

gcr.tf

@@ -1,7 +1,23 @@
+provider "google-beta" {
+  project = var.project
+  region  = local.region
+}
+
 resource "google_artifact_registry_repository" "domino" {
   provider = google-beta
 
   location      = local.region
   repository_id = "${var.cluster_name}-domino"
   format        = "DOCKER"
 }
+
+resource "google_artifact_registry_repository_iam_member" "gcr" {
+  provider = google-beta
+
+  repository = google_artifact_registry_repository.domino.name
+  location   = google_artifact_registry_repository.domino.location
+
+  role   = "roles/artifactregistry.writer"
+  member = "serviceAccount:${google_service_account.accounts["gcr"].email}"
+}
+

main.tf

@@ -6,8 +6,6 @@ locals {
   region = length(split("-", var.location)) == 2 ? var.location : substr(var.location, 0, length(var.location) - 2)
   zone   = length(split("-", var.location)) == 3 ? var.location : format("%s-a", var.location)
 
-  authorized_networks = var.master_authorized_networks_config
-
   node_pools = {
     for node_pool, attrs in var.node_pools :
     node_pool => merge(attrs, lookup(var.node_pool_overrides, node_pool, {}))
@@ -20,11 +18,6 @@ provider "google" {
   region  = local.region
 }
 
-provider "google-beta" {
-  project = var.project
-  region  = local.region
-}
-
 data "google_project" "domino" {
   project_id = var.project
 }
@@ -135,25 +128,6 @@ resource "google_container_cluster" "domino_cluster" {
 
   enable_tpu = false
 
-  master_auth {
-    client_certificate_config {
-      issue_client_certificate = true
-    }
-  }
-
-  # This resource's provider has issues with reconciling the remote/local state
-  # of the `issue_client_certificate` field because we use channels to
-  # implicitly set a cluster version.
-  #
-  # We're going to ignore all changes to the `master_auth` block since we set
-  # these values statically. Hopefully, this issue will be resolved in a future
-  # version of the provider. See the following issue for more context.
-  #
-  # https://github.com/terraform-providers/terraform-provider-google/issues/3369#issuecomment-487226330
-  lifecycle {
-    ignore_changes = [master_auth]
-  }
-
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
   }
@@ -168,7 +142,7 @@ resource "google_container_cluster" "domino_cluster" {
 
   master_authorized_networks_config {
     dynamic "cidr_blocks" {
-      for_each = local.authorized_networks
+      for_each = var.master_authorized_networks_config
       content {
         cidr_block   = cidr_blocks.value.cidr_block
         display_name = cidr_blocks.value.display_name
@@ -199,6 +173,18 @@ resource "google_container_cluster" "domino_cluster" {
   pod_security_policy_config {
     enabled = var.enable_pod_security_policy
   }
+
+  provisioner "local-exec" {
+    environment = {
+      KUBECONFIG = var.kubeconfig_output_path
+    }
+    command = <<-EOF
+      if ! gcloud auth print-identity-token 2>/dev/null; then
+        printf "%s" "$GOOGLE_CREDENTIALS" | gcloud auth activate-service-account --project="${var.project}" --key-file=-
+      fi
+      gcloud container clusters get-credentials ${var.cluster_name} --zone ${local.zone}
+    EOF
+  }
 }
 
 resource "google_kms_key_ring" "key_ring" {

service-accounts.tf

@@ -35,16 +35,6 @@ resource "google_service_account_iam_binding" "platform_gcs" {
   ]
 }
 
-resource "google_artifact_registry_repository_iam_member" "gcr" {
-  provider = google-beta
-
-  repository = google_artifact_registry_repository.domino.name
-  location   = google_artifact_registry_repository.domino.location
-
-  role   = "roles/artifactregistry.writer"
-  member = "serviceAccount:${google_service_account.accounts["gcr"].email}"
-}
-
 resource "google_service_account_iam_binding" "gcr" {
   service_account_id = google_service_account.accounts["gcr"].name
   role               = "roles/iam.workloadIdentityUser"

tests/test.tf

@@ -8,12 +8,6 @@ terraform {
     google-beta = {
       version = ">=3.68"
     }
-    kubernetes = {
-      version = "~> 2.4"
-    }
-    local = {
-      version = ">=2.1"
-    }
     random = {
       version = ">=3.1"
     }
@@ -41,9 +35,10 @@ variable "filestore_disabled" {
 module "gke" {
   source = "./.."
 
-  cluster_name       = terraform.workspace
-  project            = "domino-eng-platform-dev"
-  description        = var.description
-  filestore_disabled = var.filestore_disabled
-  namespaces         = { platform = "domino-platform", compute = "domino-compute" }
+  cluster_name           = terraform.workspace
+  project                = "domino-eng-platform-dev"
+  description            = var.description
+  filestore_disabled     = var.filestore_disabled
+  namespaces             = { platform = "domino-platform", compute = "domino-compute" }
+  kubeconfig_output_path = "${path.cwd}/kubeconfig"
 }

variables.tf

@@ -11,8 +11,7 @@ variable "cluster_name" {
 
 variable "kubeconfig_output_path" {
   type        = string
-  default     = ""
-  description = "Specify where the cluster kubeconfig file should be generated. Defaults to current working directory."
+  description = "Specify where the cluster kubeconfig file should be generated."
 }
 
 variable "allowed_ssh_ranges" {