Use vSphere projects from Boskos

Signed-off-by: Stefan Büringer buringerst@vmware.com
kubernetes-sigs · Jun 7, 2024 · 48bac89 · 48bac89
1 parent 8b7b506
commit 48bac89
Showing 1 changed file with 89 additions and 15 deletions.
diff --git a/images/capi/scripts/ci-ova.sh b/images/capi/scripts/ci-ova.sh
@@ -24,10 +24,22 @@ cd "${CAPI_ROOT}" || exit 1
 export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
 TARGETS=("ubuntu-2004" "ubuntu-2204" "photon-3" "photon-4" "photon-5" "rockylinux-8" "flatcar")
 
+export BOSKOS_RESOURCE_OWNER=image-builder
+if [[ "${JOB_NAME}" != "" ]]; then
+  export BOSKOS_RESOURCE_OWNER="${JOB_NAME}/${BUILD_ID}"
+fi
+export BOSKOS_RESOURCE_TYPE=vsphere-project-image-builder
+
 on_exit() {
   #Cleanup VMs
   cleanup_build_vm
 
+  # Stop boskos heartbeat
+  [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+
+  # If Boskos is being used then release the vsphere project.
+  [ -z "${BOSKOS_HOST:-}" ] || docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --vsphere-username="${VSPHERE_USERNAME}" --vsphere-password="${VSPHERE_PASSWORD}" --vsphere-server="${VSPHERE_SERVER}" --vsphere-tls-thumbprint="${VSPHERE_TLS_THUMBPRINT}" --vsphere-folder="${BOSKOS_RESOURCE_FOLDER}" --vsphere-resource-pool="${BOSKOS_RESOURCE_POOL}"
+
   # kill the VPN
   docker kill vpn
 }
@@ -42,24 +54,94 @@ cleanup_build_vm() {
   for target in ${TARGETS[@]};
   do
     # Adding || true to both commands so it does not exit after not being able to cleanup one target.
-    govc vm.power -off -force -wait /${GOVC_DATACENTER}/vm/${FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
-    govc object.destroy /${GOVC_DATACENTER}/vm/${FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
+    govc vm.power -off -force -wait /${GOVC_DATACENTER}/vm/${VSPHERE_FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
+    govc object.destroy /${GOVC_DATACENTER}/vm/${VSPHERE_FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
   done
 
 }
 
 trap on_exit EXIT
 
+# For Boskos
+export VSPHERE_SERVER="${GOVC_URL:-}"
+export VSPHERE_USERNAME="${GOVC_USERNAME:-}"
+export VSPHERE_PASSWORD="${GOVC_PASSWORD:-}"
+
 export PATH=${PWD}/.local/bin:$PATH
 export PATH=${PYTHON_BIN_DIR:-"/root/.local/bin"}:$PATH
 export GC_KIND="false"
 export TIMESTAMP="$(date -u '+%Y%m%dT%H%M%S')"
 export GOVC_DATACENTER="SDDC-Datacenter"
+export GOVC_CLUSTER="Cluster-1"
 export GOVC_INSECURE=true
-export FOLDER="Workloads/image-builder"
+
+# FIXME(sbueringer) before merge this should be moved to the ProwJob config in test-infra (so we can move jobs incrementally to the new environment / boskos)
+export BOSKOS_HOST=http://192.168.6.138:32222
+
+# Run the vpn client in container
+docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
+  -w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
+  gcr.io/k8s-staging-capi-vsphere/extra/openvpn:latest
+
+# Tail the vpn logs
+docker logs vpn
+
+# Wait until the VPN connection is active.
+function wait_for_vpn_up() {
+  local n=0
+  until [ $n -ge 30 ]; do
+    curl "https://${VSPHERE_SERVER}" --connect-timeout 2 -k -v && RET=$? || RET=$?
+    if [[ "$RET" -eq 0 ]]; then
+      break
+    fi
+    n=$((n + 1))
+    sleep 1
+  done
+  return "$RET"
+}
+wait_for_vpn_up
+
+# If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
+if [ -n "${BOSKOS_HOST:-}" ]; then
+  # Check out the account from Boskos and store the produced environment
+  # variables in a temporary file.
+  account_env_var_file="$(mktemp)"
+  docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest acquire --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-type="${BOSKOS_RESOURCE_TYPE}" 1>"${account_env_var_file}"
+  checkout_account_status="${?}"
+
+  # If the checkout process was a success then load the account's
+  # environment variables into this process.
+  # shellcheck disable=SC1090
+  [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+  export BOSKOS_RESOURCE_NAME=${BOSKOS_RESOURCE_NAME}
+  # Drop absolute prefix because packer needs the relative path.
+  export VSPHERE_FOLDER="$(echo "${BOSKOS_RESOURCE_FOLDER}" | sed "s@/${GOVC_DATACENTER}/vm/@@")"
+  export VSPHERE_RESOURCE_POOL="$(echo "${BOSKOS_RESOURCE_POOL}" | sed "s@/${GOVC_DATACENTER}/host/${GOVC_CLUSTER}/@@")"
+
+  # Always remove the account environment variable file. It contains
+  # sensitive information.
+  rm -f "${account_env_var_file}"
+
+  if [ ! "${checkout_account_status}" = "0" ]; then
+    echo "error getting vsphere project from Boskos" 1>&2
+    exit "${checkout_account_status}"
+  fi
+
+  # Run the heartbeat to tell boskos periodically that we are still
+  # using the checked out account.
+  docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest heartbeat --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"${ARTIFACTS}/boskos-heartbeat.log" 2>&1 &
+  HEART_BEAT_PID=$!
+else
+  echo "error getting vsphere project from Boskos, BOSKOS_HOST not set" 1>&2
+  exit 1
+fi
 
 echo "Running build with timestamp ${TIMESTAMP}"
 
+echo "Using user: ${GOVC_USERNAME}"
+echo "Using relative folder: ${VSPHERE_FOLDER}"
+echo "Using relative resource pool: ${VSPHERE_RESOURCE_POOL}"
+
 cat << EOF > packer/ova/vsphere.json
 {
     "vcenter_server":"${GOVC_URL}",
@@ -68,10 +150,10 @@ cat << EOF > packer/ova/vsphere.json
     "password":"${GOVC_PASSWORD}",
     "datastore":"WorkloadDatastore",
     "datacenter":"${GOVC_DATACENTER}",
-    "resource_pool": "Compute-ResourcePool/image-builder",
-    "cluster": "Cluster-1",
-    "network": "sddc-cgw-network-8",
-    "folder": "${FOLDER}"
+    "resource_pool": "${VSPHERE_RESOURCE_POOL}",
+    "cluster": "${GOVC_CLUSTER}",
+    "network": "sddc-cgw-network-10",
+    "folder": "${VSPHERE_FOLDER}"
 }
 EOF
 
@@ -81,14 +163,6 @@ cat packer/ova/packer-node.json | jq  'del(.builders[] | select( .name == "vsphe
 cat packer/ova/packer-node.json | jq  'del(.builders[] | select( .name == "vsphere-clone" ).export)' > packer/ova/packer-node.json.tmp && mv packer/ova/packer-node.json.tmp packer/ova/packer-node.json
 cat packer/ova/packer-node.json | jq  'del(."post-processors"[])' > packer/ova/packer-node.json.tmp && mv packer/ova/packer-node.json.tmp packer/ova/packer-node.json
 
-# Run the vpn client in container
-docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
-  -w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
-  gcr.io/k8s-staging-capi-vsphere/extra/openvpn:latest
-
-# Tail the vpn logs
-docker logs vpn
-
 # install deps and build all images
 make deps-ova