-
Notifications
You must be signed in to change notification settings - Fork 0
/
cloudbuild-provision-cluster.yaml
185 lines (167 loc) · 7.36 KB
/
cloudbuild-provision-cluster.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## The steps in this Cloud Build script set up the cluster
## to be used by the project. This script is separate
## from the regular infrastructure setup because it is intended
## to be a one-time setup, whereas the steps in cloudbuild.yaml
## are repeatable.
steps:
## CLUSTER CREATE ##
# Create a new cluster with the appropriate configuration for this application
- name: 'gcr.io/cloud-builders/gcloud'
id: Create Cluster
waitFor: ['-']
args: [
'container', 'clusters', 'create', '${_CLUSTER_NAME}',
'--addons', 'HorizontalPodAutoscaling,HttpLoadBalancing,CloudRun',
'--machine-type', 'e2-standard-4',
'--num-nodes', '3',
'--cluster-version', '${_CLUSTER_GKE_VERSION}',
'--enable-stackdriver-kubernetes',
'--workload-pool', '${PROJECT_ID}.svc.id.goog',
'--enable-ip-alias',
'--zone', '${_CLUSTER_LOCATION}'
]
## SERVICE ACCOUNT SETUP
# Create a service account for Config Connecter (if one does not already exist)
- name: 'gcr.io/cloud-builders/gcloud'
id: Create Config Connector service account
waitFor: ['-']
entrypoint: /bin/bash
args:
- '-c'
- |
gcloud iam service-accounts describe cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com || gcloud iam service-accounts create cnrm-system
# Set up IAM role bindings for Config Connector
# See: https://cloud.google.com/iam/docs/understanding-roles#primitive_role_definitions
- name: 'gcr.io/cloud-builders/gcloud'
id: Grant Owner permissions
waitFor: ['Create Config Connector service account']
args: [
'projects','add-iam-policy-binding','${PROJECT_ID}',
'--member=serviceAccount:cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com',
'--role=roles/owner'
]
# Set up the Workload Identity binding for Config Connector on the cluster
# See: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity
- name: 'gcr.io/cloud-builders/gcloud'
id: Create IAM policy binding
waitFor: ['Create Config Connector service account', 'Create Cluster']
args: [
'iam','service-accounts','add-iam-policy-binding','cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com',
'--member=serviceAccount:${PROJECT_ID}.svc.id.goog[cnrm-system/cnrm-controller-manager]',
'--role=roles/iam.workloadIdentityUser'
]
## CONFIG CONNECTOR INSTALL
# See: https://cloud.google.com/config-connector/docs/how-to/install-upgrade-uninstall#installing_kcc
# Download the latest release of Config Connector
- name: 'gcr.io/cloud-builders/gsutil'
id: Download Config Connector
waitFor: ['-']
args: ['cp', 'gs://cnrm/latest/release-bundle.tar.gz', 'release-bundle.tar.gz']
# Extract the release artifacts
- name: 'alpine'
id: Extract Config Connector
waitFor: ['Download Config Connector']
entrypoint: /bin/tar
args: ['zxvf','release-bundle.tar.gz']
# Substitute project ID into the YAML definitions
- name: 'alpine'
id: Process Manifest
waitFor: ['Extract Config Connector']
entrypoint: /bin/ash
args:
- '-c'
- |
sed -i 's/${PROJECT_ID?}/${PROJECT_ID}/' install-bundle-workload-identity/0-cnrm-system.yaml
# Install Config Connector on the cluster
- name: 'gcr.io/cloud-builders/kubectl'
id: Install Config Connector
waitFor: ['Create Cluster','Process Manifest']
args: ['apply', '-f', 'install-bundle-workload-identity/']
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_CLUSTER_LOCATION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}'
# Verify Config Connector Installation
- name: 'gcr.io/cloud-builders/kubectl'
id: Wait Config Connector
waitFor: ['Install Config Connector']
args: ['wait', '-n', 'cnrm-system', '--for=condition=Ready', 'pod', '--all', '--timeout=5m']
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_CLUSTER_LOCATION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}'
# Create the web application namespace
- name: 'gcr.io/cloud-builders/kubectl'
id: Create Namespace
waitFor: ['Create Cluster']
args: ['create', 'namespace', '${_NAMESPACE}']
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_CLUSTER_LOCATION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}'
# Mark the namespace as managed by Config Connector
- name: 'gcr.io/cloud-builders/kubectl'
id: Annotate Namespace
waitFor: ['Create Namespace']
args: ['annotate', 'namespace', '${_NAMESPACE}', 'cnrm.cloud.google.com/project-id=${PROJECT_ID}']
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_CLUSTER_LOCATION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}'
# Install Istio 1.5 on the cluster so we can use Istio Authorization Policies.
# TODO(#97): Remove this step once CRfA comes bundled with Istio 1.5+
- name: 'gcr.io/cloud-builders/kubectl'
id: Install Istio 1.5
waitFor: ['Create Cluster','Process Manifest']
entrypoint: /bin/bash
args:
- '-c'
- |
kubectl apply -f https://raw.githubusercontent.com/knative/serving/release-0.15/third_party/istio-1.5.4/istio-crds.yaml
kubectl apply -f https://raw.githubusercontent.com/knative/serving/release-0.15/third_party/istio-1.5.4/istio-ci-mesh.yaml
kubectl wait -n istio-system pod --all --for=condition=Ready --timeout=5m
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_CLUSTER_LOCATION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}'
# Add a Standalone NEG to the cluster ingress
# See: https://cloud.google.com/kubernetes-engine/docs/how-to/standalone-neg
- name: 'gcr.io/cloud-builders/kubectl'
id: Create NEG
waitFor: ['Create Cluster', 'Install Istio 1.5']
args: ['annotate', '--overwrite', '--namespace=${_ISTIO_INGRESS_NAMESPACE}', 'service', '${_ISTIO_INGRESS_SERVICE}', 'cloud.google.com/neg={"exposed_ports": {"80":{}}}']
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_CLUSTER_LOCATION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}'
# Restrict cluster ingress firewall rules to just GCLB's published ranges:
# https://cloud.google.com/load-balancing/docs/https#source_ip_addresses
- name: 'gcr.io/cloud-builders/gcloud'
id: Restrict Cluster Ingress
entrypoint: /bin/bash
args:
- '-c'
- |
# Wait for the firewall rule associated with the current cluster creation to be created
# so that it's restricted along with any other cluster ingress firewalls that might apply.
CLUSTER_CREATE_TIME=$(gcloud container clusters describe ${_CLUSTER_NAME} --zone=${_CLUSTER_LOCATION} --format="value(createTime)")
while [[ -z $(gcloud compute firewall-rules list \
--filter="name:k8s-fw AND targetTags.list()~^gke-${_CLUSTER_NAME}-[0-9a-z]*-node$ AND creationTimestamp>=$${CLUSTER_CREATE_TIME}" \
--format="value(name)") ]]; \
do \
echo "unable to find cluster ingress firewall. Sleeping 10s..."; \
sleep 10s; \
done; \
for firewall in $(gcloud compute firewall-rules list \
--filter="name:k8s-fw AND targetTags.list()~^gke-${_CLUSTER_NAME}-[0-9a-z]*-node$" \
--format="value(name)");
do \
gcloud compute firewall-rules update "$firewall" --source-ranges=35.191.0.0/16,130.211.0.0/22; \
done;