Integrate kind for local testing #1106
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 Google LLC | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # https://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License | |
| name: Build Tests | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: # By default this runs for types assigned, opened and synchronize. | |
| env: | |
| # Names must be unique in parallel running tests. | |
| TPU_CLUSTER_NAME: build-xpk-2-v4-8-nodepools | |
| WORKLOAD_NAME: xpktest-build-${{ github.run_attempt }} | |
| PATHWAYS_WORKLOAD_NAME: xpkpw-build-${{ github.run_attempt }} | |
| CLUSTER_ARGUMENTS: "--network=${{secrets.NETWORK_NAME}}" | |
| RUN_ID: "pr-${{ github.event.number }}" | |
| jobs: | |
| cluster-create-and-delete: | |
| runs-on: [ubuntu-22.04] | |
| concurrency: # We support one nightly test and one build test for each branch to run at a time currently. | |
| group: build-test-cluster-group-${{ github.ref }} | |
| cancel-in-progress: false | |
| steps: | |
| - name: Change RUN_ID env var if merge to main | |
| run: echo "RUN_ID=main" >> $GITHUB_ENV | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Update cluster name with RUN_ID | |
| run: echo "TPU_CLUSTER_NAME=$TPU_CLUSTER_NAME-$RUN_ID" >> $GITHUB_ENV | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| - uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: '>= 363.0.0' | |
| install_components: 'beta,gke-gcloud-auth-plugin' | |
| - name: Verify gcp setup | |
| run: gcloud info | |
| - name: Set Google Cloud CLI properties to a unused zone to verify --zone arg is passed properly in commands. | |
| run: | | |
| gcloud config set compute/zone us-east4-a | |
| gcloud config get compute/zone | |
| - name: Install xpk dependencies | |
| run: | | |
| make install | |
| echo $PWD/bin >> "$GITHUB_PATH" | |
| - name: Check xpk installation | |
| run: xpk --help | |
| - name: Create a private Pathways-enabled XPK Cluster with 2x v4-8 nodepools. Larger num-nodes to avoid master resizing. | |
| run: python xpk.py cluster create-pathways --cluster $TPU_CLUSTER_NAME --private --tpu-type=v4-8 --num-slices=2 --zone=us-central2-b --default-pool-cpu-machine-type=n1-standard-16 --default-pool-cpu-num-nodes=16 --reservation='${{ secrets.GCP_TPU_V4_RESERVATION }}' --custom-cluster-arguments="${CLUSTER_ARGUMENTS}" | |
| - name: Verify the created cluster is private | |
| run: gcloud container clusters describe $TPU_CLUSTER_NAME --region=us-central2 --format="value(privateClusterConfig.enablePrivateNodes)" | grep 'True' || (echo 'The created cluster is not private.' && exit 1) | |
| - name: Authenticate Docker | |
| run: gcloud auth configure-docker --quiet | |
| - name: Create test script to execute in workloads | |
| run: echo -e '#!/bin/bash \n echo "Hello world from a test script!"' > test.sh | |
| - name: Run a base-docker-image workload | |
| run: python xpk.py workload create --cluster $TPU_CLUSTER_NAME --workload $WORKLOAD_NAME --command "bash test.sh" --tpu-type=v4-8 --num-slices=2 --zone=us-central2-b | |
| - name: Run xpk inspector with the workload created above | |
| run: python3 xpk.py inspector --cluster $TPU_CLUSTER_NAME --zone=us-central2-b --workload $WORKLOAD_NAME | |
| - name: Wait for workload completion and confirm it succeeded | |
| run: python3 xpk.py workload list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $WORKLOAD_NAME --timeout 300 | |
| - name: Run a Pathways workload on Ubuntu base image | |
| run: python xpk.py workload create-pathways --cluster $TPU_CLUSTER_NAME --workload $PATHWAYS_WORKLOAD_NAME --docker-image='marketplace.gcr.io/google/ubuntu2004' --tpu-type=v4-8 --num-slices=2 --zone=us-central2-b --command "echo \"Hello world from a test script! \"" | |
| - name: Wait for Pathways workload completion and confirm it succeeded | |
| run: python3 xpk.py workload list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $PATHWAYS_WORKLOAD_NAME --timeout 300 | |
| - name: List out the workloads on the cluster | |
| run: python3 xpk.py workload list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | |
| - name: Run xpk info | |
| run: python3 xpk.py info --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | tee output.txt | grep -P '^(?=.*QUEUE)(?=.*PENDING_WORKLOADS)(?=.*ADMITTED_WORKLOADS)(?=.*2xv4-8:google.com/tpu)(?=.*cpu-rm:cpu)(?=.*cpu-rm:memory)(?=.*cpu-proxy:cpu)(?=.*cpu-proxy:memory)(?=.*cpu-user:cpu)(?=.*cpu-user:memory)' || (echo 'Invalid command output' && cat output.txt && exit 1) | |
| - name: Delete the workload on the cluster | |
| run: python3 xpk.py workload delete --workload $WORKLOAD_NAME --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | |
| - name: Delete the Pathways workload on the cluster | |
| run: python3 xpk.py workload delete --workload $PATHWAYS_WORKLOAD_NAME --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | |
| - name: Run a batch job on the cluster | |
| run: python3 xpk.py batch --cluster $TPU_CLUSTER_NAME --zone=us-central2-b test.sh | |
| - name: List out the jobs on the cluster | |
| run: python3 xpk.py job ls | grep 'xpk-def-app-profile-slurm-' | |
| - name: Cancel the batch job on the cluster | |
| run: | | |
| job_name=$(python3 xpk.py job ls | grep 'xpk-def-app-profile-slurm-' | head -1 | awk '{print $1}') | |
| python3 xpk.py job cancel ${job_name} | grep "job.batch/${job_name} deleted" | |
| - name: Delete the cluster created | |
| if: always() | |
| run: python xpk.py cluster delete --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | |