Conformance External Workloads (ci-external-workloads)

Conformance External Workloads (ci-external-workloads) #582

Workflow file for this run

.github/workflows/conformance-externalworkloads.yaml at 736337c

	name: Conformance External Workloads (ci-external-workloads)

	# Any change in triggers needs to be reflected in the concurrency group.
	on:
	workflow_dispatch:
	inputs:
	PR-number:
	description: "Pull request number."
	required: true
	context-ref:
	description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
	required: true
	SHA:
	description: "SHA under test (head of the PR branch)."
	required: true
	extra-args:
	description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
	required: false
	default: '{}'
	push:
	branches:
	- 'renovate/main-**'
	# Run every 8 hours
	schedule:
	- cron: '0 4/8 * * *'

	# By specifying the access of one of the scopes, all of those that are not
	# specified are set to 'none'.
	permissions:
	# To read actions state with catchpoint/workflow-telemetry-action
	actions: read
	# To be able to access the repository with actions/checkout
	contents: read
	# To allow retrieving information from the PR API
	pull-requests: read
	# To be able to set commit status
	statuses: write
	# To be able to request the JWT from GitHub's OIDC provider
	id-token: write

	concurrency:
	# Structure:
	# - Workflow name
	# - Event type
	# - A unique identifier depending on event type:
	# - schedule: SHA
	# - workflow_dispatch: PR number
	#
	# This structure ensures a unique concurrency group name is generated for each
	# type of testing, such that re-runs will cancel the previous run.
	group: \|
	${{ github.workflow }}
	${{ github.event_name }}
	${{
	(github.event_name == 'push' && github.sha) \|\|
	(github.event_name == 'schedule' && github.sha) \|\|
	(github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
	}}
	cancel-in-progress: true

	env:
	clusterName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
	vmName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
	vmStartupScript: .github/gcp-vm-startup.sh
	cilium_cli_ci_version:
	USE_GKE_GCLOUD_AUTH_PLUGIN: True
	# renovate: datasource=docker depName=google/cloud-sdk
	gcloud_version: 492.0.0

	jobs:
	echo-inputs:
	if: ${{ github.event_name == 'workflow_dispatch' }}
	name: Echo Workflow Dispatch Inputs
	runs-on: ubuntu-22.04
	steps:
	- name: Echo Workflow Dispatch Inputs
	run: \|
	echo '${{ tojson(inputs) }}'

	commit-status-start:
	name: Commit Status Start
	runs-on: ubuntu-latest
	steps:
	- name: Set initial commit status
	uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
	with:
	sha: ${{ inputs.SHA \|\| github.sha }}

	generate-matrix:
	name: Generate Matrix
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.set-matrix.outputs.matrix }}
	empty: ${{ steps.set-matrix.outputs.empty }}
	steps:
	- name: Checkout context ref (trusted)
	uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	ref: ${{ inputs.context-ref \|\| github.sha }}
	persist-credentials: false

	- name: Convert YAML to JSON
	run: \|
	work_dir=".github/actions/gke"
	destination_directory="/tmp/generated/gke"
	mkdir -p "${destination_directory}"

	yq -o=json ${work_dir}/k8s-versions.yaml \| jq . > "${destination_directory}/gke.json"

	- name: Generate Matrix
	run: \|
	cd /tmp/generated/gke

	# Use complete matrix in case of scheduled run
	# main -> event_name = schedule
	# other stable branches -> PR-number starting with v (e.g. v1.14)
	if [[ "${{ github.event_name }}" == "schedule" \|\| "${{ inputs.PR-number }}" == v* ]];then
	jq '{ "include": [ .k8s[] ] }' gke.json > /tmp/matrix.json
	else
	jq '{ "include": [ .k8s[] \| select(.default) ] }' gke.json > /tmp/matrix.json
	fi

	echo "Generated matrix:"
	cat /tmp/matrix.json

	- name: Set up gcloud credentials
	id: 'auth'
	uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
	with:
	workload_identity_provider: ${{ secrets.GCP_PR_WORKLOAD_IDENTITY_PROVIDER }}
	service_account: ${{ secrets.GCP_PR_SA }}
	create_credentials_file: true
	export_environment_variables: true

	- name: Set up gcloud CLI
	uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
	with:
	project_id: ${{ secrets.GCP_PROJECT_ID }}
	version: ${{ env.gcloud_version }}

	- name: Filter Matrix
	id: set-matrix
	run: \|
	cp /tmp/matrix.json /tmp/result.json
	jq -c '.include[]' /tmp/matrix.json \| while read i; do
	VERSION=$(echo $i \| jq -r '.version')
	ZONE=$(echo $i \| jq -r '.zone')
	gcloud --quiet container get-server-config \
	--flatten="channels" \
	--format="yaml(channels.validVersions)" --zone $ZONE > /tmp/output
	if grep -q -F $VERSION /tmp/output; then
	echo "Version $VERSION is valid for zone $ZONE"
	else
	echo "::notice::Removing version $VERSION as it's not valid for zone $ZONE"
	jq 'del(.include[] \| select(.version == "'$VERSION'"))' /tmp/result.json > /tmp/result.json.tmp
	mv /tmp/result.json.tmp /tmp/result.json
	fi
	done
	echo "Filtered matrix:"
	cat /tmp/result.json

	echo "matrix=$(jq -c . < /tmp/result.json)" >> $GITHUB_OUTPUT
	echo "empty=$(jq '(.include \| length) == 0' /tmp/result.json)" >> $GITHUB_OUTPUT

	installation-and-connectivity:
	name: Installation and Connectivity Test
	needs: generate-matrix
	if: ${{ needs.generate-matrix.outputs.empty == 'false' }}
	runs-on: ubuntu-latest
	timeout-minutes: 45
	env:
	job_name: "Installation and Connectivity Test"
	strategy:
	fail-fast: false
	matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}

	steps:
	- name: Collect Workflow Telemetry
	uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
	with:
	comment_on_pr: false

	- name: Checkout context ref (trusted)
	uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	ref: ${{ inputs.context-ref \|\| github.sha }}
	persist-credentials: false

	- name: Cleanup Disk space in runner
	uses: ./.github/actions/disk-cleanup

	- name: Set Environment Variables
	uses: ./.github/actions/set-env-variables

	- name: Get Cilium's default values
	id: default_vars
	uses: ./.github/actions/helm-default
	with:
	image-tag: ${{ inputs.SHA }}
	chart-dir: ./untrusted/install/kubernetes/cilium

	- name: Set up job variables
	id: vars
	run: \|
	if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
	OWNER="${{ inputs.PR-number }}"
	else
	OWNER="${{ github.ref_name }}"
	OWNER="${OWNER//[.\/]/-}"
	fi

	CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
	--helm-set cluster.name=${{ env.clusterName }} \
	--helm-set cluster.id=1 \
	--datapath-mode=tunnel \
	--helm-set kubeProxyReplacement=true"
	CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled --hubble=false --collect-sysdump-on-failure \
	--external-target google.com. --external-cidr 8.0.0.0/8 --external-ip 8.8.4.4 --external-other-ip 8.8.8.8"
	# Explicitly specify LoadBalancer service type since the default type is NodePort in Helm mode.
	# Ref: https://github.com/cilium/cilium-cli/pull/1527#discussion_r1177244379
	#
	# In Helm mode, externalWorkloads.enabled is set to false by default. You need to pass
	# --enable-external-workloads flag to enable it.
	# Ref: https://github.com/cilium/cilium/pull/25259
	CLUSTERMESH_ENABLE_DEFAULTS="--service-type LoadBalancer --enable-external-workloads"
	echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
	echo connectivity_test_defaults=${CONNECTIVITY_TEST_DEFAULTS} >> $GITHUB_OUTPUT
	echo clustermesh_enable_defaults=${CLUSTERMESH_ENABLE_DEFAULTS} >> $GITHUB_OUTPUT
	echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT
	echo owner=${OWNER} >> $GITHUB_OUTPUT

	- name: Set up gcloud credentials
	id: 'auth'
	uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
	with:
	workload_identity_provider: ${{ secrets.GCP_PR_WORKLOAD_IDENTITY_PROVIDER }}
	service_account: ${{ secrets.GCP_PR_SA }}
	create_credentials_file: true
	export_environment_variables: true

	- name: Set up gcloud CLI
	uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
	with:
	project_id: ${{ secrets.GCP_PROJECT_ID }}
	version: ${{ env.gcloud_version }}

	- name: Install gke-gcloud-auth-plugin
	run: \|
	gcloud components install gke-gcloud-auth-plugin

	- name: Display gcloud CLI info
	run: \|
	gcloud info

	- name: Create GCP VM
	uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
	with:
	retry_on: error
	timeout_minutes: 1
	max_attempts: 10
	command: \|
	gcloud compute instances create ${{ env.vmName }}-${{ matrix.vmIndex }} \
	--labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
	--zone ${{ matrix.zone }} \
	--machine-type e2-custom-2-4096 \
	--boot-disk-type pd-standard \
	--boot-disk-size 10GB \
	--image-project ubuntu-os-cloud \
	--image-family ubuntu-2004-lts \
	--metadata hostname=${{ env.vmName }}-${{ matrix.vmIndex }} \
	--metadata-from-file startup-script=${{ env.vmStartupScript}}

	- name: Create GKE cluster
	run: \|
	gcloud container clusters create ${{ env.clusterName }} \
	--labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
	--zone ${{ matrix.zone }} \
	--cluster-version ${{ matrix.version }} \
	--enable-ip-alias \
	--node-taints node.cilium.io/agent-not-ready=true:NoExecute \
	--cluster-ipv4-cidr="/21" \
	--services-ipv4-cidr="/24" \
	--image-type COS_CONTAINERD \
	--num-nodes 2 \
	--machine-type e2-custom-2-4096 \
	--disk-type pd-standard \
	--disk-size 20GB

	- name: Get cluster credentials
	run: \|
	gcloud container clusters get-credentials ${{ env.clusterName }} --zone ${{ matrix.zone }}

	- name: Install Cilium CLI
	uses: cilium/cilium-cli@fff38e882846c03f1720dad476e459323275ab9c # v0.16.17
	with:
	skip-build: ${{ env.CILIUM_CLI_SKIP_BUILD }}
	image-repo: ${{ env.CILIUM_CLI_IMAGE_REPO }}
	image-tag: ${{ steps.vars.outputs.sha }}

	# Warning: since this is a privileged workflow, subsequent workflow job
	# steps must take care not to execute untrusted code.
	- name: Checkout pull request branch (NOT TRUSTED)
	uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	ref: ${{ steps.vars.outputs.sha }}
	persist-credentials: false
	path: untrusted
	sparse-checkout: \|
	install/kubernetes/cilium

	- name: Wait for images to be available
	timeout-minutes: 30
	shell: bash
	run: \|
	for image in cilium-ci operator-generic-ci hubble-relay-ci clustermesh-apiserver-ci ; do
	until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
	done

	- name: Install Cilium in cluster
	id: install-cilium
	run: \|
	cilium install ${{ steps.vars.outputs.cilium_install_defaults }}

	- name: Enable cluster mesh
	run: \|
	cilium clustermesh enable ${{ steps.vars.outputs.clustermesh_enable_defaults }}

	- name: Wait for cluster mesh status to be ready
	run: \|
	cilium clustermesh status --wait

	- name: Add VM to cluster mesh
	run: \|
	cilium clustermesh vm create ${{ env.vmName }}-${{ matrix.vmIndex }} -n default --ipv4-alloc-cidr 10.192.1.0/30
	cilium clustermesh vm status

	- name: Install Cilium on VM
	run: \|
	extra_args=${{ env.CILIUM_RUNTIME_EXTRA_ARGS }}
	cilium clustermesh vm install install-external-workload.sh --config debug$([ ! -z $extra_args ] && echo ",${extra_args}")
	gcloud compute scp install-external-workload.sh ${{ env.vmName }}-${{ matrix.vmIndex }}:~/ --zone ${{ matrix.zone }}
	gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
	--command "~/install-external-workload.sh"
	sleep 5s
	gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
	--command "sudo cilium-dbg status"

	- name: Verify cluster DNS on VM
	# Limit nslookup to the first (global) DNS server setting
	run: \|
	gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
	--command "nslookup -d2 -retry=10 -timeout=5 -norecurse clustermesh-apiserver.kube-system.svc.cluster.local \$(systemd-resolve --status \| grep -m 1 \"Current DNS Server:\" \| cut -d':' -f2)"

	- name: Ping clustermesh-apiserver from VM
	run: \|
	gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
	--command "ping -c 3 \$(sudo cilium-dbg service list get -o jsonpath='{[?(@.spec.flags.name==\"clustermesh-apiserver\")].spec.backend-addresses[0].ip}')"

	- name: Make JUnit report directory
	run: \|
	mkdir -p cilium-junits

	- name: Run connectivity test (${{ join(matrix.*, ', ') }})
	run: \|
	cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
	--junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
	--junit-property github_job_step="Run connectivity test (${{ join(matrix.*, ', ') }})"

	- name: Post-test information gathering
	if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
	run: \|
	kubectl get pods --all-namespaces -o wide
	kubectl get cew --all-namespaces -o wide
	kubectl get cep --all-namespaces -o wide
	cilium status
	cilium clustermesh status
	cilium clustermesh vm status
	gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} --command "sudo cilium status"
	gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} --command "sudo docker logs cilium --timestamps"
	cilium sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
	shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently

	- name: Clean up GKE cluster and VM
	if: ${{ always() }}
	run: \|
	while [ "$(gcloud container operations list --zone ${{ matrix.zone }} --filter="status=RUNNING AND targetLink~${{ env.clusterName }}" --format="value(name)")" ];do
	echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
	done
	gcloud container clusters delete ${{ env.clusterName }} --zone ${{ matrix.zone }} --quiet --async
	gcloud compute instances delete ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} --quiet
	shell: bash {0} # Disable default fail-fast behavior so that all commands run independently

	- name: Upload artifacts
	if: ${{ !success() }}
	uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: cilium-sysdumps-${{ matrix.vmIndex }}
	path: cilium-sysdump-*.zip

	- name: Upload JUnits [junit]
	if: ${{ always() }}
	uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: cilium-junits-${{ matrix.vmIndex }}
	path: cilium-junits/*.xml

	- name: Publish Test Results As GitHub Summary
	if: ${{ always() }}
	uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
	with:
	junit-directory: "cilium-junits"

	merge-upload:
	if: ${{ always() && needs.installation-and-connectivity.result != 'skipped' }}
	name: Merge and Upload Artifacts
	runs-on: ubuntu-latest
	needs: installation-and-connectivity
	steps:
	- name: Merge Sysdumps
	if: ${{ needs.installation-and-connectivity.result == 'failure' }}
	uses: actions/upload-artifact/merge@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: cilium-sysdumps
	pattern: cilium-sysdumps-*
	retention-days: 5
	delete-merged: true
	continue-on-error: true
	- name: Merge JUnits
	uses: actions/upload-artifact/merge@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: cilium-junits
	pattern: cilium-junits-*
	retention-days: 5
	delete-merged: true

	commit-status-final:
	if: ${{ always() }}
	name: Commit Status Final
	needs: installation-and-connectivity
	runs-on: ubuntu-latest
	steps:
	- name: Set final commit status
	if: ${{ needs.installation-and-connectivity.result != 'skipped' }}
	uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
	with:
	sha: ${{ inputs.SHA \|\| github.sha }}
	status: ${{ needs.installation-and-connectivity.result }}
	- name: Set final commit status
	if: ${{ needs.installation-and-connectivity.result == 'skipped' }}
	uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
	with:
	sha: ${{ inputs.SHA \|\| github.sha }}
	status: ${{ github.event_name != 'schedule' && 'success' \|\| 'failure' }}
	description: 'Skipped'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Conformance External Workloads (ci-external-workloads) #582

Workflow file

Conformance External Workloads (ci-external-workloads) #582

Jobs

Run details

Workflow file for this run