diff --git a/build/cloudbuild-cd.yaml b/build/cloudbuild-cd.yaml index 96dea667..04f6afcb 100644 --- a/build/cloudbuild-cd.yaml +++ b/build/cloudbuild-cd.yaml @@ -26,6 +26,8 @@ artifacts: location: gs://$_CACHE_BUCKET_NAME/artifacts/$BRANCH_NAME paths: - '/workspace/svcs-endpoints-filtered.json' + - '/workspace/zap_report_${_TARGET_ID}_${_RELEASE_ID}.html' +logsBucket: gs://$_CACHE_BUCKET_NAME/build_logs steps: ############################### Post-Deploy Checks ########################### @@ -39,7 +41,16 @@ steps: - -c - | gcloud config set project $_CLUSTER_PROJECT - gcloud container clusters get-credentials $_CLUSTER_NAME --region=$_DEFAULT_REGION + echo "Target Type ${_TARGET_TYPE}" + case ${_TARGET_TYPE} in + "anthos_cluster") + gcloud container fleet memberships get-credentials $_ANTHOS_MEMBERSHIP + ;; + *) + gcloud container clusters get-credentials $_CLUSTER_NAME --region=$_DEFAULT_REGION + ;; + esac + kubectl get svc -ojson > /workspace/svcs.json ### Below only grabs external IP'd svcs @@ -49,6 +60,9 @@ steps: ENDPOINTS=( $$(jq -r '.[].endpoint' /workspace/svcs-endpoints-filtered.json)) echo $$SVC_NAMES > /workspace/svc_names_env.txt echo $$ENDPOINTS > /workspace/endpoints_env.txt + volumes: + - name: 'zapvolume' + path: '/zap/wrk' # ZAProxy Scan - name: 'gcr.io/cloud-builders/docker' @@ -61,15 +75,14 @@ steps: ENDPOINTS=( $$(cat /workspace/endpoints_env.txt)) INDEX=0 - for SVC in "$${SVC_NAMES[@]}"; do - echo "$$SVC" - ENDPOINT="$${ENDPOINTS[$$INDEX]}" - echo "Checking $$ENDPOINT" - - docker run owasp/zap2docker-stable zap-cli quick-scan --self-contained --start-options '-config api.disablekey=true' http://$$ENDPOINT - - INDEX=$${INDEX}+1 - done + ENDPOINT="$${ENDPOINTS[$$INDEX]}" + echo "Checking $$ENDPOINT" + + docker container run --user root -v zapvolume:/zap/wrk/:rw -t owasp/zap2docker-stable zap-baseline.py -t http://$$ENDPOINT -r zap_report_${_TARGET_ID}_${_RELEASE_ID}.html -z '-config api.disablekey=true' -I + cp /zap/wrk/zap_report_${_TARGET_ID}_${_RELEASE_ID}.html /workspace/zap_report_${_TARGET_ID}_${_RELEASE_ID}.html + volumes: + - name: 'zapvolume' + path: '/zap/wrk' waitFor: - 'get-svc-endpoints' diff --git a/build/cloudbuild-ci.yaml b/build/cloudbuild-ci.yaml index 23aa4789..6486a167 100644 --- a/build/cloudbuild-ci.yaml +++ b/build/cloudbuild-ci.yaml @@ -12,19 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Pipeline Setup -# Maven cache to reduce calls to Maven Central -# test (in parallel) -# Secrets Scanner: secrets scanner -# License checking: ??? -# Build -# Docker build -# After Build -# Artifact Structure Test: Container structure test -# Image Vulnerabilities: CVE scanner -# Security Testing Gate -# Attestation - timeout: "3600s" # 1 hour tags: - "secure-cicd-ci" @@ -41,12 +28,9 @@ artifacts: paths: - 'build-artifacts.json' - 'build-artifacts-notag.json' +logsBucket: gs://$_CACHE_BUCKET_NAME/build_logs steps: -############################### Securing Source Code ########################### - -# Secrets Scanner (TODO: Switch to Talisman) - ############################### Build Containers ########################### # Create build-installation-image @@ -57,7 +41,7 @@ steps: - '-xe' - -c - | - ./mvnw validate + ./mvnw clean install skaffold config set --global local-cluster false skaffold build --default-repo=${_GAR_REPO_URI} --tag=$SHORT_SHA --cache-file='/.skaffold/cache' --file-output=/artifacts/build-artifacts.json diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index f287b562..f7c4f18e 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -13,6 +13,9 @@ # limitations under the License. steps: +- id: swap-module-refs + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && module-swapper'] - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' id: 'lint' args: ['/usr/local/bin/test_lint.sh'] diff --git a/examples/app_cicd/cloud-build-builder/Dockerfile b/examples/app_cicd/cloud-build-builder/Dockerfile index 88cf6b55..5cbccde6 100644 --- a/examples/app_cicd/cloud-build-builder/Dockerfile +++ b/examples/app_cicd/cloud-build-builder/Dockerfile @@ -24,17 +24,16 @@ WORKDIR /go/src/app RUN make out/signer RUN cp out/signer /signer - -FROM alpine:3.13 +FROM alpine:3.17 ### 1. Get Java via the package manager RUN apk update \ && apk upgrade \ -&& apk add --no-cache bash curl jq openjdk11-jre git openssh \ +&& apk add --no-cache bash curl jq openjdk17-jre git openssh \ && apk add --no-cache --virtual=build-dependencies unzip -#### Set JAVA_HOME -ENV JAVA_HOME="/usr/lib/jvm/java-11-openjdk" +### Set JAVA_HOME +ENV JAVA_HOME="/usr/lib/jvm/java-17-openjdk" ### 2. Get Python, PIP RUN apk add --no-cache python3 \ @@ -46,7 +45,7 @@ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi rm -r /root/.cache #### 3. Install gcloud -ENV CLOUD_SDK_VERSION="391.0.0" +ENV CLOUD_SDK_VERSION="423.0.0" ENV CLOUDSDK_INSTALL_DIR /usr/local/gcloud/ RUN wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ && tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ diff --git a/examples/app_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml b/examples/app_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml index 565e2eb7..6f4fa3d3 100644 --- a/examples/app_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml +++ b/examples/app_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml @@ -14,7 +14,7 @@ options: # Use higher CPU machines so the caching and build steps are faster. - machineType: 'N1_HIGHCPU_32' + machineType: 'E2_HIGHCPU_32' steps: - name: 'gcr.io/cloud-builders/docker' args: [ 'build', '-t', '$_DEFAULT_REGION-docker.pkg.dev/$PROJECT_ID/$_GAR_REPOSITORY/skaffold-builder', '.' ] diff --git a/examples/app_cicd/main.tf b/examples/app_cicd/main.tf index d4701b9c..66e702ca 100644 --- a/examples/app_cicd/main.tf +++ b/examples/app_cicd/main.tf @@ -39,12 +39,13 @@ module "cd_pipeline" { project_id = var.project_id primary_location = var.primary_location - gar_repo_name = module.ci_pipeline.app_artifact_repo - cloudbuild_cd_repo = "cloudbuild-cd-config" - deploy_branch_clusters = var.deploy_branch_clusters - app_deploy_trigger_yaml = "cloudbuild-cd.yaml" - cache_bucket_name = module.ci_pipeline.cache_bucket_name - clouddeploy_pipeline_name = local.clouddeploy_pipeline_name + gar_repo_name = module.ci_pipeline.app_artifact_repo + cloudbuild_cd_repo = "cloudbuild-cd-config" + deploy_branch_clusters = var.deploy_branch_clusters + app_deploy_trigger_yaml = "cloudbuild-cd.yaml" + cache_bucket_name = module.ci_pipeline.cache_bucket_name + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name + cloudbuild_service_account = module.ci_pipeline.build_sa_email depends_on = [ module.ci_pipeline ] diff --git a/examples/private_cluster_cicd/cloud-build-builder/Dockerfile b/examples/private_cluster_cicd/cloud-build-builder/Dockerfile index 88cf6b55..5cbccde6 100644 --- a/examples/private_cluster_cicd/cloud-build-builder/Dockerfile +++ b/examples/private_cluster_cicd/cloud-build-builder/Dockerfile @@ -24,17 +24,16 @@ WORKDIR /go/src/app RUN make out/signer RUN cp out/signer /signer - -FROM alpine:3.13 +FROM alpine:3.17 ### 1. Get Java via the package manager RUN apk update \ && apk upgrade \ -&& apk add --no-cache bash curl jq openjdk11-jre git openssh \ +&& apk add --no-cache bash curl jq openjdk17-jre git openssh \ && apk add --no-cache --virtual=build-dependencies unzip -#### Set JAVA_HOME -ENV JAVA_HOME="/usr/lib/jvm/java-11-openjdk" +### Set JAVA_HOME +ENV JAVA_HOME="/usr/lib/jvm/java-17-openjdk" ### 2. Get Python, PIP RUN apk add --no-cache python3 \ @@ -46,7 +45,7 @@ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi rm -r /root/.cache #### 3. Install gcloud -ENV CLOUD_SDK_VERSION="391.0.0" +ENV CLOUD_SDK_VERSION="423.0.0" ENV CLOUDSDK_INSTALL_DIR /usr/local/gcloud/ RUN wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ && tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ diff --git a/examples/private_cluster_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml b/examples/private_cluster_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml index 565e2eb7..6f4fa3d3 100644 --- a/examples/private_cluster_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml +++ b/examples/private_cluster_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml @@ -14,7 +14,7 @@ options: # Use higher CPU machines so the caching and build steps are faster. - machineType: 'N1_HIGHCPU_32' + machineType: 'E2_HIGHCPU_32' steps: - name: 'gcr.io/cloud-builders/docker' args: [ 'build', '-t', '$_DEFAULT_REGION-docker.pkg.dev/$PROJECT_ID/$_GAR_REPOSITORY/skaffold-builder', '.' ] diff --git a/examples/private_cluster_cicd/main.tf b/examples/private_cluster_cicd/main.tf index 189845e4..4ed6541f 100644 --- a/examples/private_cluster_cicd/main.tf +++ b/examples/private_cluster_cicd/main.tf @@ -42,13 +42,14 @@ module "cd_pipeline" { project_id = var.project_id primary_location = "us-central1" - gar_repo_name = module.ci_pipeline.app_artifact_repo - cloudbuild_cd_repo = "cloudbuild-cd-config-pc" - deploy_branch_clusters = var.deploy_branch_clusters - app_deploy_trigger_yaml = "cloudbuild-cd.yaml" - cache_bucket_name = module.ci_pipeline.cache_bucket_name - cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id - clouddeploy_pipeline_name = local.clouddeploy_pipeline_name + gar_repo_name = module.ci_pipeline.app_artifact_repo + cloudbuild_cd_repo = "cloudbuild-cd-config-pc" + deploy_branch_clusters = var.deploy_branch_clusters + app_deploy_trigger_yaml = "cloudbuild-cd.yaml" + cache_bucket_name = module.ci_pipeline.cache_bucket_name + cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name + cloudbuild_service_account = module.ci_pipeline.build_sa_email depends_on = [ module.ci_pipeline ] diff --git a/examples/standalone_single_project/README.md b/examples/standalone_single_project/README.md index e345023a..76c368a4 100644 --- a/examples/standalone_single_project/README.md +++ b/examples/standalone_single_project/README.md @@ -22,6 +22,8 @@ This example also creates GKE clusters and accompanying VPC networks for multipl | Name | Description | |------|-------------| | app\_source\_repo | URL of the created CSR app soure repo | +| cloudbuild\_cd\_repo\_name | URL of the created CSR app soure repo | | console\_walkthrough\_link | URL to open the in-console walkthrough. | +| gar\_repo | Artifact Registry repo | diff --git a/examples/standalone_single_project/cicd.tf b/examples/standalone_single_project/cicd.tf index c441b5ae..80f00662 100644 --- a/examples/standalone_single_project/cicd.tf +++ b/examples/standalone_single_project/cicd.tf @@ -18,27 +18,33 @@ locals { deploy_branch_clusters = { "01-${var.env1_name}" = { cluster = module.gke_cluster[var.env1_name].name, + anthos_membership = module.fleet_membership[var.env1_name].cluster_membership_id + target_type = "anthos_cluster" network = module.vpc.network_name - project_id = var.project_id, - location = var.region, + project_id = var.project_id + location = var.region required_attestations = [module.ci_pipeline.binauth_attestor_ids["build"]] env_attestation = module.ci_pipeline.binauth_attestor_ids["security"] next_env = "02-qa" }, "02-${var.env2_name}" = { cluster = module.gke_cluster[var.env2_name].name, + anthos_membership = module.fleet_membership[var.env2_name].cluster_membership_id + target_type = "anthos_cluster" network = module.vpc.network_name - project_id = var.project_id, - location = var.region, + project_id = var.project_id + location = var.region required_attestations = [module.ci_pipeline.binauth_attestor_ids["security"], module.ci_pipeline.binauth_attestor_ids["build"]] env_attestation = module.ci_pipeline.binauth_attestor_ids["quality"] next_env = "03-prod" }, "03-${var.env3_name}" = { cluster = module.gke_cluster[var.env3_name].name, + anthos_membership = module.fleet_membership[var.env3_name].cluster_membership_id + target_type = "anthos_cluster" network = module.vpc.network_name - project_id = var.project_id, - location = var.region, + project_id = var.project_id + location = var.region required_attestations = [module.ci_pipeline.binauth_attestor_ids["quality"], module.ci_pipeline.binauth_attestor_ids["security"], module.ci_pipeline.binauth_attestor_ids["build"]] env_attestation = "" next_env = "" @@ -77,14 +83,15 @@ module "cd_pipeline" { project_id = var.project_id primary_location = var.region - gar_repo_name = module.ci_pipeline.app_artifact_repo - cloudbuild_cd_repo = "${var.app_name}-cloudbuild-cd-config" - deploy_branch_clusters = local.deploy_branch_clusters - app_deploy_trigger_yaml = "cloudbuild-cd.yaml" - cache_bucket_name = module.ci_pipeline.cache_bucket_name - cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id - clouddeploy_pipeline_name = local.clouddeploy_pipeline_name - labels = var.labels + gar_repo_name = module.ci_pipeline.app_artifact_repo + cloudbuild_cd_repo = "${var.app_name}-cloudbuild-cd-config" + deploy_branch_clusters = local.deploy_branch_clusters + app_deploy_trigger_yaml = "cloudbuild-cd.yaml" + cache_bucket_name = module.ci_pipeline.cache_bucket_name + cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name + cloudbuild_service_account = module.ci_pipeline.build_sa_email + labels = var.labels depends_on = [ module.ci_pipeline ] diff --git a/examples/standalone_single_project/gke.tf b/examples/standalone_single_project/gke.tf index aee9ae8a..596f5525 100644 --- a/examples/standalone_single_project/gke.tf +++ b/examples/standalone_single_project/gke.tf @@ -95,3 +95,14 @@ module "gke_cluster" { module.vpc ] } + +module "fleet_membership" { + for_each = toset(local.envs) + source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-membership" + version = "~> 25.0.0" + + membership_name = "${module.gke_cluster[each.value].name}-membership" + project_id = var.project_id + location = var.region + cluster_name = module.gke_cluster[each.value].name +} diff --git a/examples/standalone_single_project/network.tf b/examples/standalone_single_project/network.tf index a8281a11..033a2130 100644 --- a/examples/standalone_single_project/network.tf +++ b/examples/standalone_single_project/network.tf @@ -60,30 +60,3 @@ resource "google_compute_network_peering_routes_config" "gke_peering_routes_conf import_custom_routes = true export_custom_routes = true } - -# Cloud Build Workerpool <-> GKE HA VPNs -module "gke_cloudbuild_vpn" { - source = "GoogleCloudPlatform/secure-cicd/google//modules/workerpool-gke-ha-vpn" - version = "~> 0.3" - - project_id = var.project_id - location = var.region - - gke_project = var.project_id - gke_network = module.vpc.network_name - gke_location = var.region - gke_control_plane_cidrs = { - (module.gke_cluster[var.env1_name].master_ipv4_cidr_block) = "GKE ${var.env1_name} control plane" - (module.gke_cluster[var.env2_name].master_ipv4_cidr_block) = "GKE ${var.env2_name} control plane", - (module.gke_cluster[var.env3_name].master_ipv4_cidr_block) = "GKE ${var.env3_name} control plane", - } - - workerpool_network = module.cloudbuild_private_pool.workerpool_network - workerpool_range = module.cloudbuild_private_pool.workerpool_range - gateway_1_asn = 65001 - gateway_2_asn = 65002 - bgp_range_1 = "169.254.1.0/30" - bgp_range_2 = "169.254.2.0/30" - - labels = var.labels -} diff --git a/examples/standalone_single_project/outputs.tf b/examples/standalone_single_project/outputs.tf index cd6bff37..c5288325 100644 --- a/examples/standalone_single_project/outputs.tf +++ b/examples/standalone_single_project/outputs.tf @@ -19,6 +19,16 @@ output "app_source_repo" { value = module.ci_pipeline.source_repo_urls["${var.app_name}-source"] } +output "cloudbuild_cd_repo_name" { + description = "URL of the created CSR app soure repo" + value = "${var.app_name}-cloudbuild-cd-config" +} + +output "gar_repo" { + description = "Artifact Registry repo" + value = module.ci_pipeline.app_artifact_repo +} + output "console_walkthrough_link" { description = "URL to open the in-console walkthrough." value = "https://shell.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fterraform-google-secure-cicd.git&cloudshell_git_branch=main&cloudshell_tutorial=examples%2Fstandalone_single_project%2Fwalkthrough.md" diff --git a/examples/standalone_single_project/walkthrough.md b/examples/standalone_single_project/walkthrough.md index 5e7994c6..df9c2a55 100644 --- a/examples/standalone_single_project/walkthrough.md +++ b/examples/standalone_single_project/walkthrough.md @@ -25,7 +25,7 @@ Estimated time to complete: To get started, click **Start**. ## Verify environment variables -Set the following environment variables based on your existing resources. +Set the following environment variables based on your existing resources. Replace the `< >` placeholder with the values you entered when deploying the solution. ``` export PROJECT_ID= @@ -33,7 +33,7 @@ export REGION= export APP_NAME= ``` -Run the following commands to set additional variables for the tutorial. +Run the following commands to set additional variables for the tutorial. If you set the above values correctly, you can run this entire block without modifying it. ```bash export GAR_REPOSITORY=$PROJECT_ID-$APP_NAME-image-repo export CLOUDBUILD_CD_REPO=$APP_NAME-cloudbuild-cd-config @@ -84,6 +84,7 @@ You will see the logs for the build process in the Cloud Shell Terminal. Once th ``` 1. Commit changes: ```bash + git add . git commit -m "initial commit" git push -u origin main ``` @@ -102,11 +103,11 @@ Click **Next**. cd bank-of-anthos git checkout -b main ``` -1. Copy the Cloud Build configuration to the app-source repo +1. Copy the Cloud Build configuration to the Bank of Anthos demo application folder ```bash cp ~/cloudshell_open/terraform-google-secure-cicd/build/cloudbuild-ci.yaml ~/bank-of-anthos/ ``` -1. Copy `policies` folder to app-source repo +1. Copy `policies` folder to the Bank of Anthos folder ```bash cp -R ~/cloudshell_open/terraform-google-secure-cicd/examples/app_cicd/policies ~/bank-of-anthos/policies ``` diff --git a/modules/secure-cd/README.md b/modules/secure-cd/README.md index e416003e..5465f1fa 100644 --- a/modules/secure-cd/README.md +++ b/modules/secure-cd/README.md @@ -62,8 +62,9 @@ The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build config | cache\_bucket\_name | cloud build artifact bucket name | `string` | n/a | yes | | cloudbuild\_cd\_repo | Name of repo that stores the Cloud Build CD phase configs - for post-deployment checks | `string` | n/a | yes | | cloudbuild\_private\_pool | Cloud Build private pool self-link | `string` | `""` | no | +| cloudbuild\_service\_account | Cloud Build SA email address | `string` | n/a | yes | | clouddeploy\_pipeline\_name | Cloud Deploy pipeline name | `string` | n/a | yes | -| deploy\_branch\_clusters | mapping of branch names to cluster deployments | 
map(object({
    cluster               = string
    project_id            = string
    location              = string
    required_attestations = list(string)
    env_attestation       = string
    next_env              = string
  }))
| `{}` | no | +| deploy\_branch\_clusters | mapping of branch names to cluster deployments. target\_type can be one of `gke`, `anthos_cluster`, or `run`. See [clouddeploy\_target Terraform docs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/clouddeploy_target) for more details | 
map(object({
    cluster               = optional(string)
    anthos_membership     = optional(string)
    project_id            = string
    location              = string
    required_attestations = list(string)
    env_attestation       = string
    next_env              = string
    target_type           = optional(string, "gke")
  }))
| `{}` | no | | gar\_repo\_name | Docker artifact registry repo to store app build images | `string` | n/a | yes | | labels | A set of key/value label pairs to assign to the resources deployed by this blueprint. | `map(string)` | `{}` | no | | primary\_location | Region used for key-ring | `string` | n/a | yes | diff --git a/modules/secure-cd/build.tf b/modules/secure-cd/build.tf index ec38d0a5..d0bc80f0 100644 --- a/modules/secure-cd/build.tf +++ b/modules/secure-cd/build.tf @@ -29,8 +29,12 @@ resource "google_cloudbuild_trigger" "deploy_trigger" { if config.next_env != "" } - project = var.project_id - name = "deploy-trigger-${each.value.cluster}" + project = var.project_id + location = var.primary_location + name = each.value.target_type == "gke" ? "deploy-trigger-${each.value.cluster}" : each.value.target_type == "anthos_cluster" ? "deploy-trigger-${each.value.anthos_membership}" : "deploy-trigger-${each.key}" + filename = "cloudbuild-cd.yaml" + + service_account = "projects/${var.project_id}/serviceAccounts/${var.cloudbuild_service_account}" pubsub_config { topic = google_pubsub_topic.clouddeploy_topic.id @@ -42,13 +46,13 @@ resource "google_cloudbuild_trigger" "deploy_trigger" { repo_type = "CLOUD_SOURCE_REPOSITORIES" } - filename = "cloudbuild-cd.yaml" - substitutions = merge( { _GAR_REPOSITORY = var.gar_repo_name _DEFAULT_REGION = each.value.location _CLUSTER_NAME = each.value.cluster + _ANTHOS_MEMBERSHIP = each.value.anthos_membership + _TARGET_TYPE = each.value.target_type _CLUSTER_PROJECT = each.value.project_id _CLOUDBUILD_FILENAME = var.app_deploy_trigger_yaml _CACHE_BUCKET_NAME = var.cache_bucket_name diff --git a/modules/secure-cd/iam.tf b/modules/secure-cd/iam.tf index e4e3708a..70900e90 100644 --- a/modules/secure-cd/iam.tf +++ b/modules/secure-cd/iam.tf @@ -29,10 +29,6 @@ locals { ] } -data "google_project" "app_cicd_project" { - project_id = var.project_id -} - # Cloud Deploy Execution Service Account # https://cloud.google.com/deploy/docs/cloud-deploy-service-account#execution_service_account resource "google_service_account" "clouddeploy_execution_sa" { @@ -67,7 +63,7 @@ resource "google_project_iam_member" "clouddeploy_service_agent_role" { resource "google_service_account_iam_member" "cloudbuild_clouddeploy_impersonation" { service_account_id = google_service_account.clouddeploy_execution_sa.name role = "roles/iam.serviceAccountUser" - member = "serviceAccount:${data.google_project.app_cicd_project.number}@cloudbuild.gserviceaccount.com" + member = "serviceAccount:${var.cloudbuild_service_account}" } # IAM membership for Cloud Deploy Execution SA deploy to GKE @@ -79,11 +75,41 @@ resource "google_project_iam_member" "clouddeploy_gke_dev" { } # IAM membership for Cloud Build SA to deploy to GKE -resource "google_project_iam_member" "gke_dev" { +resource "google_project_iam_member" "cloudbuild_gke_dev" { for_each = var.deploy_branch_clusters project = each.value.project_id role = "roles/container.developer" - member = "serviceAccount:${data.google_project.app_cicd_project.number}@cloudbuild.gserviceaccount.com" + member = "serviceAccount:${var.cloudbuild_service_account}" +} + +# IAM grants for deploying to GKE via Connect Gateway +# https://cloud.google.com/anthos/multicluster-management/gateway/setup#grant_roles_for_access_through_kubectl +# Cloud Deploy Execution SA deploy to cluster +resource "google_project_iam_member" "clouddeploy_gkehub_viewer" { + for_each = var.deploy_branch_clusters + project = each.value.project_id + role = "roles/gkehub.viewer" + member = "serviceAccount:${google_service_account.clouddeploy_execution_sa.email}" +} +resource "google_project_iam_member" "clouddeploy_gkehub_gatewayadmin" { + for_each = var.deploy_branch_clusters + project = each.value.project_id + role = "roles/gkehub.gatewayAdmin" + member = "serviceAccount:${google_service_account.clouddeploy_execution_sa.email}" +} + +# Cloud Build SA to deploy to cluster +resource "google_project_iam_member" "cloudbuild_gkehub_viewer" { + for_each = var.deploy_branch_clusters + project = each.value.project_id + role = "roles/gkehub.viewer" + member = "serviceAccount:${var.cloudbuild_service_account}" +} +resource "google_project_iam_member" "cloudbuild_gkehub_gatewayadmin" { + for_each = var.deploy_branch_clusters + project = each.value.project_id + role = "roles/gkehub.gatewayAdmin" + member = "serviceAccount:${var.cloudbuild_service_account}" } # IAM membership for Binary Authorization service agents in GKE projects on attestors diff --git a/modules/secure-cd/main.tf b/modules/secure-cd/main.tf index a7b1e910..f59b240c 100644 --- a/modules/secure-cd/main.tf +++ b/modules/secure-cd/main.tf @@ -30,15 +30,33 @@ locals { } resource "google_clouddeploy_target" "deploy_target" { + provider = google-beta for_each = var.deploy_branch_clusters - name = "${each.value.cluster}-target" + name = each.value.target_type == "anthos_cluster" ? "${each.value.anthos_membership}-target" : each.value.target_type == "gke" ? "${each.value.cluster}-target" : "${each.key}-target" description = "Target for ${each.key} environment" location = each.value.location project = var.project_id - gke { - cluster = "projects/${each.value.project_id}/locations/${each.value.location}/clusters/${each.value.cluster}" + dynamic "gke" { + for_each = lower(each.value.target_type) == "gke" ? [1] : [] + content { + cluster = "projects/${each.value.project_id}/locations/${each.value.location}/clusters/${each.value.cluster}" + } + } + + dynamic "anthos_cluster" { + for_each = lower(each.value.target_type) == "anthos_cluster" ? [1] : [] + content { + membership = "projects/${each.value.project_id}/locations/global/memberships/${each.value.anthos_membership}" + } + } + + dynamic "run" { + for_each = lower(each.value.target_type) == "run" ? [1] : [] + content { + location = "projects/${each.value.project_id}/locations/${each.value.location}" + } } execution_configs { diff --git a/modules/secure-cd/variables.tf b/modules/secure-cd/variables.tf index 0e5cbde6..48f17ec7 100644 --- a/modules/secure-cd/variables.tf +++ b/modules/secure-cd/variables.tf @@ -41,14 +41,16 @@ variable "app_deploy_trigger_yaml" { variable "deploy_branch_clusters" { type = map(object({ - cluster = string + cluster = optional(string) + anthos_membership = optional(string) project_id = string location = string required_attestations = list(string) env_attestation = string next_env = string + target_type = optional(string, "gke") })) - description = "mapping of branch names to cluster deployments" + description = "mapping of branch names to cluster deployments. target_type can be one of `gke`, `anthos_cluster`, or `run`. See [clouddeploy_target Terraform docs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/clouddeploy_target) for more details" default = {} } @@ -79,3 +81,8 @@ variable "labels" { type = map(string) default = {} } + +variable "cloudbuild_service_account" { + description = "Cloud Build SA email address" + type = string +} diff --git a/modules/secure-cd/versions.tf b/modules/secure-cd/versions.tf index 04f73900..7460b342 100644 --- a/modules/secure-cd/versions.tf +++ b/modules/secure-cd/versions.tf @@ -15,15 +15,15 @@ */ terraform { - required_version = ">= 0.13.0" + required_version = ">= 1.3" required_providers { google = { source = "hashicorp/google" - version = ">= 4.22, < 5.0" + version = ">= 4.44, < 5.0" } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.22, < 5.0" + version = ">= 4.44, < 5.0" } } diff --git a/modules/secure-ci/README.md b/modules/secure-ci/README.md index f4a030d1..425d80a2 100644 --- a/modules/secure-ci/README.md +++ b/modules/secure-ci/README.md @@ -62,6 +62,7 @@ The template [`cloudbuild-ci.yaml`](../../build/cloudbuild-ci.yaml) build config | binauth\_attestor\_ids | IDs of Attestors | | binauth\_attestor\_names | Names of Attestors | | binauth\_attestor\_project\_id | Project ID where attestors get created | +| build\_sa\_email | Cloud Build Service Account email address | | build\_trigger\_name | The name of the cloud build trigger for the app source repo. | | cache\_bucket\_name | The name of the storage bucket for cloud build. | | source\_repo\_names | Name of the created CSR repos | diff --git a/modules/secure-ci/main.tf b/modules/secure-ci/main.tf index ead300fa..ca5afe82 100644 --- a/modules/secure-ci/main.tf +++ b/modules/secure-ci/main.tf @@ -51,8 +51,9 @@ resource "google_storage_bucket_iam_member" "cloudbuild_artifacts_iam" { } resource "google_cloudbuild_trigger" "app_build_trigger" { - project = var.project_id - name = "${var.app_source_repo}-trigger" + project = var.project_id + name = "${var.app_source_repo}-trigger" + location = var.primary_location trigger_template { branch_name = var.trigger_branch_name repo_name = var.app_source_repo diff --git a/modules/secure-ci/outputs.tf b/modules/secure-ci/outputs.tf index 64be4f46..039bc861 100644 --- a/modules/secure-ci/outputs.tf +++ b/modules/secure-ci/outputs.tf @@ -53,3 +53,8 @@ output "source_repo_urls" { description = "URLS of the created CSR repos" value = { for repo in google_sourcerepo_repository.repos : repo.name => repo.url } } + +output "build_sa_email" { + description = "Cloud Build Service Account email address" + value = google_service_account.build_sa.email +} diff --git a/test/integration/app_cicd/app_cicd_test.go b/test/integration/app_cicd/app_cicd_test.go index 72e641ae..2be736ec 100644 --- a/test/integration/app_cicd/app_cicd_test.go +++ b/test/integration/app_cicd/app_cicd_test.go @@ -55,7 +55,7 @@ func TestAppCICDExample(t *testing.T) { /////// SECURE-CI /////// // Cloud Build Trigger - App Source - gcbCI := gcloud.Run(t, fmt.Sprintf("beta builds triggers describe %s --project %s", sourceTriggerName, projectID)) + gcbCI := gcloud.Run(t, fmt.Sprintf("builds triggers describe %s --project %s --region %s", sourceTriggerName, projectID, primaryLocation)) assert.Equal(sourceTriggerName, gcbCI.Get("name").String(), "Cloud Build Trigger name is valid") assert.Contains(gcbCI.Get("substitutions._DEFAULT_REGION").String(), primaryLocation, "Default Region trigger substitution is valid") @@ -91,7 +91,7 @@ func TestAppCICDExample(t *testing.T) { cdTriggers := [2]string{"deploy-trigger-dev-cluster", "deploy-trigger-qa-cluster"} for _, cdTrigger := range cdTriggers { - gcbCD := gcloud.Run(t, fmt.Sprintf("beta builds triggers describe %s --project %s", cdTrigger, projectID)) + gcbCD := gcloud.Run(t, fmt.Sprintf("builds triggers describe %s --project %s --region %s", cdTrigger, projectID, primaryLocation)) assert.Contains(gcbCD.Get("name").String(), cdTrigger, "Trigger name is valid") assert.Contains(gcbCD.Get("pubsubConfig.topic").String(), "clouddeploy-operations", "pubsub topic is valid") assert.Contains(gcbCD.Get("substitutions._CLUSTER_PROJECT").String(), "secure-cicd-gke-", "_CLUSTER_PROJECT trigger substitution is valid") diff --git a/test/integration/go.mod b/test/integration/go.mod index ef7d0c59..9a0e0519 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -48,6 +48,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.2-0.20210217184823-a52172cd2f64 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/otiai10/copy v1.9.0 github.com/pmezard/go-difflib v1.0.0 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/test/integration/go.sum b/test/integration/go.sum index d91fe737..eb7be257 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -1092,6 +1092,7 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/oracle/oci-go-sdk v7.1.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888= +github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= diff --git a/test/integration/standalone_single_project/standalone_single_project_test.go b/test/integration/standalone_single_project/standalone_single_project_test.go index fa6232a7..82e8efb8 100644 --- a/test/integration/standalone_single_project/standalone_single_project_test.go +++ b/test/integration/standalone_single_project/standalone_single_project_test.go @@ -18,30 +18,128 @@ package standalone_single_project import ( + "fmt" "testing" + "time" // import the blueprints test framework modules for testing and assertions + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/git" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" "github.com/stretchr/testify/assert" + + cp "github.com/otiai10/copy" ) // name the function as Test* func TestStandaloneSingleProjectExample(t *testing.T) { - // initialize Terraform test from the Blueprints test framework setupOutput := tft.NewTFBlueprintTest(t) projectID := setupOutput.GetTFSetupStringOutput("project_id_standalone") // wire setup output project_id_standalone to example var.project_id - standaloneSingleProjT := tft.NewTFBlueprintTest(t, tft.WithVars(map[string]interface{}{"project_id":projectID})) + standaloneSingleProjT := tft.NewTFBlueprintTest(t, tft.WithVars(map[string]interface{}{"project_id": projectID})) // define and write a custom verifier for this test case call the default verify for confirming no additional changes standaloneSingleProjT.DefineVerify(func(assert *assert.Assertions) { // perform default verification ensuring Terraform reports no additional changes on an applied blueprint standaloneSingleProjT.DefaultVerify(assert) + garRepo := standaloneSingleProjT.GetStringOutput("gar_repo") + appRepo := fmt.Sprintf("https://source.developers.google.com/p/%s/r/my-app-source", projectID) + cdRepo := standaloneSingleProjT.GetStringOutput("cloudbuild_cd_repo_name") + region := "us-central1" + pipelineName := "my-app-pipeline" + prodTarget := "my-app-cluster-prod-membership-target" + + // Create Builder image + gcloud.RunCmd(t, fmt.Sprintf("builds submit ../../../examples/private_cluster_cicd/cloud-build-builder --project %s --region %s --config=../../../examples/private_cluster_cicd/cloud-build-builder/cloudbuild-skaffold-build-image.yaml --substitutions=_DEFAULT_REGION=%s,_GAR_REPOSITORY=%s", projectID, region, region, garRepo)) + + // Configure Cloud Deploy post-deployment scans + tmpDirCD := t.TempDir() + gitCD := git.NewCmdConfig(t, git.WithDir(tmpDirCD)) + gitCDRun := func(args ...string) { + _, err := gitCD.RunCmdE(args...) + if err != nil { + t.Fatal(err) + } + } + + gcloud.Runf(t, "source repos clone %s %s --project %s", cdRepo, tmpDirCD, projectID) + gitCDRun("config", "user.email", "secure-cicd-robot@example.com") + gitCDRun("config", "user.name", "Secure CICD Robot") + gitCDRun("config", "--global", "init.defaultBranch", "main") + gitCDRun("checkout", "-b", "main") + err := cp.Copy("../../../build/cloudbuild-cd.yaml", fmt.Sprintf("%s/cloudbuild-cd.yaml", tmpDirCD)) + fmt.Println(err) + gitCDRun("add", ".") + gitCD.CommitWithMsg("initial commit", []string{"--allow-empty"}) + gitCDRun("push", "-u", "origin", "main", "-f") + + // Push demo app source code, CI config, and policies + tmpDirApp := t.TempDir() + gitApp := git.NewCmdConfig(t, git.WithDir(tmpDirApp)) + gitAppRun := func(args ...string) { + _, err := gitApp.RunCmdE(args...) + if err != nil { + t.Fatal(err) + } + } + + gitAppRun("clone", "--branch", "v0.5.11", "https://github.com/GoogleCloudPlatform/bank-of-anthos.git", tmpDirApp) + gitAppRun("config", "user.email", "secure-cicd-robot@example.com") + gitAppRun("config", "user.name", "Secure CICD Robot") + gitAppRun("config", "--global", "credential.https://source.developers.google.com.helper", "gcloud.sh") + gitAppRun("config", "--global", "init.defaultBranch", "main") + gitAppRun("checkout", "-b", "main") + err2 := cp.Copy("../../../build/cloudbuild-ci.yaml", fmt.Sprintf("%s/cloudbuild-ci.yaml", tmpDirApp)) + fmt.Println(err2) + err3 := cp.Copy("../../../examples/private_cluster_cicd/policies", fmt.Sprintf("%s/policies", tmpDirApp)) + fmt.Println(err3) + gitAppRun("remote", "add", "google", appRepo) + gitAppRun("add", ".") + gitApp.CommitWithMsg("initial commit", []string{"--allow-empty"}) + gitAppRun("push", "--all", "google", "-f") + + lastCommit := gitApp.GetLatestCommit() + // filter builds triggered based on pushed commit sha + buildListCmd := fmt.Sprintf("builds list --region=%s --filter substitutions.COMMIT_SHA='%s' --project %s", region, lastCommit, projectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil + } + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + return false, nil + } + return true, nil + } + } + utils.Poll(t, pollCloudBuild(buildListCmd), 25, 30*time.Second) + releaseName := fmt.Sprintf("release-%s", lastCommit[0:7]) + fmt.Println(releaseName) + rolloutListCmd := fmt.Sprintf("deploy rollouts list --project=%s --delivery-pipeline=%s --region=%s --release=%s --filter targetId=%s", projectID, pipelineName, region, releaseName, prodTarget) + // Poll CD rollouts until prod rollout is successful + pollCloudDeploy := func(cmd string) func() (bool, error) { + return func() (bool, error) { + rollouts := gcloud.Runf(t, cmd).Array() + if len(rollouts) < 1 { + return true, nil + } + latestRolloutState := rollouts[0].Get("state").String() + if latestRolloutState == "SUCCEEDED" { + return false, nil + } + return true, nil + } + } + utils.Poll(t, pollCloudDeploy(rolloutListCmd), 30, 60*time.Second) }) // call the test function to execute the integration test standaloneSingleProjT.Test() diff --git a/test/setup/iam.tf b/test/setup/iam.tf index d19e3569..a8beda20 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -25,6 +25,7 @@ locals { "roles/cloudkms.publicKeyViewer", "roles/containeranalysis.notes.editor", "roles/compute.networkAdmin", + "roles/gkehub.editor", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", "roles/pubsub.editor", diff --git a/test/setup/main.tf b/test/setup/main.tf index 6c81b38a..7abd9f6c 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -281,7 +281,9 @@ module "project_standalone" { "cloudtrace.googleapis.com", "monitoring.googleapis.com", "logging.googleapis.com", - "compute.googleapis.com" + "compute.googleapis.com", + "gkehub.googleapis.com", + "connectgateway.googleapis.com", ] activate_api_identities = [ {