Build VM Image workflow_run_id:0 download_ya_archive:false remove_old_images:false update_image_id:true #13
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build VM Image | |
| run-name: Build VM Image ${{ github.event.workflow_run.event }} workflow_run_id:${{ inputs.workflow_run_id }} download_ya_archive:${{ inputs.download_ya_archive }} remove_old_images:${{ inputs.remove_old_images }} update_image_id:${{ inputs.update_image_id }} | |
| on: | |
| # schedule: | |
| # - cron: "0 5 * * *" | |
| workflow_call: | |
| inputs: | |
| build: | |
| description: "Build VM image" | |
| required: false | |
| default: true | |
| type: boolean | |
| download_ya_archive: | |
| description: "Download .ya archive to resulting image" | |
| required: false | |
| default: true | |
| type: boolean | |
| workflow_run_id: | |
| description: "Workflow run id (optional)" | |
| required: false | |
| default: 0 | |
| type: number | |
| file: | |
| description: "File to download" | |
| required: false | |
| default: "ya_web_url_file" | |
| type: string | |
| images_to_keep: | |
| description: "Number of images to keep" | |
| required: false | |
| default: 3 | |
| type: number | |
| remove_old_images: | |
| description: "Delete old images" | |
| required: false | |
| default: false | |
| type: boolean | |
| update_image_id: | |
| description: "Update image id" | |
| required: false | |
| default: false | |
| type: boolean | |
| image_prefix: | |
| description: "Image prefix" | |
| required: false | |
| default: "gh-2204-auto" | |
| type: string | |
| check_vm_creation: | |
| description: "Check VM creation" | |
| required: false | |
| default: false | |
| type: boolean | |
| nebius: | |
| description: "Run on new nebius runners" | |
| required: false | |
| default: "no" | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| build: | |
| description: "Build VM image" | |
| required: false | |
| default: true | |
| type: boolean | |
| download_ya_archive: | |
| description: "Download .ya archive to resulting image" | |
| required: false | |
| default: true | |
| type: boolean | |
| workflow_run_id: | |
| description: "Workflow run id (optional)" | |
| required: false | |
| default: 0 | |
| type: number | |
| file: | |
| description: "File to download" | |
| required: false | |
| default: "ya_web_url_file" | |
| type: string | |
| images_to_keep: | |
| description: "Number of images to keep" | |
| required: false | |
| default: 3 | |
| type: number | |
| remove_old_images: | |
| description: "Delete old images" | |
| required: false | |
| default: false | |
| type: boolean | |
| update_image_id: | |
| description: "Update image id" | |
| required: false | |
| default: false | |
| type: boolean | |
| image_prefix: | |
| description: "Image prefix" | |
| required: false | |
| default: "gh-2204-auto" | |
| type: string | |
| check_vm_creation: | |
| description: "Check VM creation" | |
| required: false | |
| default: false | |
| type: boolean | |
| nebius: | |
| description: "Run on new nebius runners" | |
| required: false | |
| default: "no" | |
| type: choice | |
| options: | |
| - "yes" | |
| - "no" | |
| env: | |
| build: ${{ github.event_name == 'schedule' && 'true' || inputs.build }} | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -eo pipefail -x {0} | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| IMAGE_ID_2204: ${{ steps.build.outputs.IMAGE_ID_2204 }} | |
| IMAGE_ALREADY_EXISTS: ${{ steps.check-image.outputs.image_already_exists }} | |
| BUILD: ${{ env.build }} | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| - name: Set up Packer | |
| if: inputs.nebius == 'no' | |
| uses: hashicorp/setup-packer@v3 | |
| with: | |
| version: 1.10.2 | |
| - name: install dependencies | |
| run: | | |
| pip install https://github.com/librarian/python-sdk/releases/download/v0.1.1/nebiusai-0.1.1-py3-none-any.whl | |
| pip install PyGithub==2.2.0 grpcio grpcio-tools yandexcloud==0.258.0 | |
| echo "PATH=$PATH:$HOME/nebius-cloud/bin:$HOME/.nebius/bin" >> $GITHUB_ENV | |
| - name: Set up NCP | |
| if: inputs.nebius == 'no' | |
| run: | | |
| curl -sSL https://storage.ai.nebius.cloud/ncp/install.sh | bash | |
| cat <<EOF > sa.json | |
| ${sa_json} | |
| EOF | |
| ncp config profile create nbs-github-user-sa | |
| ncp config set service-account-key sa.json | |
| ncp config set endpoint api.ai.nebius.cloud:443 | |
| env: | |
| sa_json: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }} | |
| - name: Set up Nebius CLI | |
| if: inputs.nebius == 'yes' | |
| run: | | |
| curl -sSL https://storage.ai.nebius.cloud/nebius/install.sh | bash | |
| cat <<EOF > sa.json | |
| ${sa_json} | |
| EOF | |
| cat sa.json | jq -r '."subject-credentials"."private-key"' > private.pem | |
| echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."kid"')" | |
| public_key_id=$(cat sa.json | jq -r '."subject-credentials"."kid"') | |
| echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."iss"')" | |
| service_account_id=$(cat sa.json | jq -r '."subject-credentials"."iss"') | |
| echo "::add-mask::tenant-e00en3r863f7me6wtd" | |
| nebius profile create --endpoint api.eu-north1.nebius.cloud \ | |
| --profile nbs-github-user-sa \ | |
| --parent-id tenant-e00en3r863f7me6wtd \ | |
| --public-key-id "${public_key_id}" \ | |
| --private-key-file private.pem \ | |
| --service-account-id "${service_account_id}" | |
| env: | |
| sa_json: ${{ secrets.NEW_NEBIUS_SA_JSON_CREDENTIALS }} | |
| - name: Get IAM token | |
| if: inputs.nebius == 'no' | |
| id: get-iam-token | |
| run: | | |
| YC_TOKEN=$(ncp --profile=nbs-github-user-sa iam create-token --format=json | jq -r .iam_token) | |
| echo "::add-mask::$YC_TOKEN" | |
| echo "YC_TOKEN=$YC_TOKEN" >> $GITHUB_ENV | |
| - name: Get .ya archive url | |
| if: inputs.download_ya_archive == true && inputs.nebius == 'no' | |
| id: get-ya-archive-url | |
| run: | | |
| python .github/scripts/workflow-artifact-download.py \ | |
| --artifact "${ARTIFACT_NAME}" \ | |
| --workflow-run-id "${WORKFLOW_RUN_ID}" | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| ARTIFACT_NAME: ${{ inputs.file }} | |
| WORKFLOW_RUN_ID: ${{ inputs.workflow_run_id }} | |
| - name: Check whether we have image with this workflow_run_id | |
| if: inputs.download_ya_archive == true && inputs.nebius == 'no' | |
| id: check-image | |
| run: | | |
| ncp --profile=nbs-github-user-sa compute images list --folder-id "${FOLDER_ID}" --format=json | tee -a images.txt | |
| exit_code=0 | |
| jq -r 'first( | |
| .[].name | | |
| if .== "${{ inputs.image_prefix }}-${{ steps.get-ya-archive-url.outputs.run_id }}" | |
| then | |
| halt_error(1) | |
| else | |
| empty | |
| end | |
| )' < images.txt || { | |
| exit_code=$? | |
| } | |
| if [ "$exit_code" -eq 1 ]; then | |
| echo "Image with this run_id already exists, if you want to overwrite it, rename old one" | |
| echo "image_already_exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Image with this run_id doesn't exist" | |
| echo "image_already_exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| env: | |
| FOLDER_ID: bjeuq5o166dq4ukv3eec | |
| - name: Run `packer init` and create variables file | |
| if: steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| id: init | |
| run: | | |
| VARIABLES_FILE=$(mktemp XXXXXXX.hcl) | |
| IMAGE_FAMILY_NAME=${GITHUB_REPOSITORY//\//-} | |
| echo "VARIABLES_FILE=$VARIABLES_FILE" >> $GITHUB_ENV | |
| packer init .github/packer/config.pkr.hcl | |
| cat << EOF > $VARIABLES_FILE | |
| RUNNER_VERSION="${RUNNER_VERSION}" | |
| USER_TO_CREATE="${USER_NAME}" | |
| PASSWORD_HASH="${PASSWORD_HASH}" | |
| ${YA_ARCHIVE_PARAMETER} | |
| FOLDER_ID="${FOLDER_ID}" | |
| ZONE="${ZONE}" | |
| SUBNET_ID="${SUBNET_ID}" | |
| ORG="${ORG}" | |
| TEAM="${TEAM}" | |
| GITHUB_TOKEN="${GITHUB_TOKEN}" | |
| AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" | |
| AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" | |
| REPOSITORY="${GITHUB_REPOSITORY}" | |
| IMAGE_FAMILY_NAME="${IMAGE_FAMILY_NAME}" | |
| IMAGE_PREFIX="${IMAGE_PREFIX}" | |
| BUILD_RUN_ID="${BUILD_RUN_ID}" | |
| EOF | |
| cat $VARIABLES_FILE | |
| env: | |
| PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUNNER_VERSION: ${{ vars.RUNNER_VERSION || '2.308.0' }} | |
| FOLDER_ID: bjeuq5o166dq4ukv3eec | |
| ZONE: eu-north1-c | |
| SUBNET_ID: f8uh0ml4rhb45nde9p75 | |
| USER_NAME: github | |
| GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| PASSWORD_HASH: ${{ secrets.VM_USER_PASSWD }} | |
| ORG: "ydb-platform" | |
| TEAM: "nbs" | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| IMAGE_PREFIX: ${{ inputs.image_prefix }} | |
| BUILD_RUN_ID: ${{ github.run_id }} | |
| YA_ARCHIVE_PARAMETER: | | |
| ${{ (inputs.download_ya_archive && steps.get-ya-archive-url.outputs.latest_url != '') | |
| && format( | |
| 'YA_ARCHIVE_URL="{0}" | |
| NIGHTLY_RUN_ID="{1}"', | |
| steps.get-ya-archive-url.outputs.latest_url, steps.get-ya-archive-url.outputs.run_id | |
| ) || '' | |
| }} | |
| - name: Run `packer validate` | |
| if: steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| id: validate | |
| run: | | |
| packer validate -var-file=$VARIABLES_FILE .github/packer/github-runner.pkr.hcl | |
| env: | |
| PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run `packer build` | |
| id: build | |
| if: env.build == 'true' && steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| timeout-minutes: 120 | |
| run: | | |
| packer build -timestamp-ui -var-file=$VARIABLES_FILE .github/packer/github-runner.pkr.hcl | |
| echo "IMAGE_ID_2204=$(jq -r '.builds[0].artifact_id' manifest.json)" >> $GITHUB_OUTPUT | |
| env: | |
| PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Print IMAGE_ID_2204 | |
| if: env.build == 'true' && steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| run: echo ${{ steps.build.outputs.IMAGE_ID_2204 }} | |
| - name: Start runner | |
| id: start-runner | |
| if: env.build == 'true' && steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| uses: ./.github/actions/runner_create | |
| timeout-minutes: 60 | |
| with: | |
| repo_owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| service_account_key: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }} | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| vm_folder: bjeuq5o166dq4ukv3eec | |
| vm_name: ${{ format('packer-run-{0}-{1}', github.run_id, github.run_attempt) }} | |
| vm_zone: eu-north1-c | |
| vm_cpu: 60 | |
| vm_memory: 420 | |
| vm_disk_type: 'network-ssd-nonreplicated' | |
| vm_disk_size: 1023 | |
| vm_subnet: f8uh0ml4rhb45nde9p75 | |
| vm_image: ${{ steps.build.outputs.IMAGE_ID_2204 }} | |
| vm_labels: ${{ format('run={0}-{1},repo={2},owner={3}', github.run_id, github.run_attempt, github.event.repository.name, github.repository_owner) }} | |
| vm_user_passwd: ${{ secrets.VM_USER_PASSWD }} | |
| - name: Stop runner | |
| uses: ./.github/actions/runner_remove | |
| if: always() && env.build == 'true' && steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| timeout-minutes: 60 | |
| with: | |
| service_account_key: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }} | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| repo_owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| vm_id: ${{ steps.start-runner.outputs.instance-id }} | |
| - name: Set new image id | |
| if: env.build == 'true' && steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'no' | |
| run: | | |
| python ./.github/scripts/manage-images.py \ | |
| --service-account-key sa.json \ | |
| --new-image-id ${NEW_IMAGE_ID} \ | |
| --image-variable-name ${IMAGE_VARIABLE_NAME} \ | |
| --images-to-keep ${IMAGES_TO_KEEP} ${UPDATE_IMAGE_ID} ${REMOVE_OLD_IMAGES} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| IMAGE_VARIABLE_NAME: IMAGE_ID_2204 | |
| NEW_IMAGE_ID: ${{ steps.build.outputs.IMAGE_ID_2204 }} | |
| IMAGES_TO_KEEP: ${{ inputs.images_to_keep }} | |
| UPDATE_IMAGE_ID: ${{ inputs.update_image_id && '--update-image-id' || '' }} | |
| REMOVE_OLD_IMAGES: ${{ inputs.remove_old_images && '--remove-old-images' || '' }} | |
| - name: Get latest image ID from Nebius | |
| if: inputs.nebius == 'yes' && env.build == 'true' | |
| id: get-image-id-nebius | |
| run: | | |
| image_id=$(nebius compute image get-latest-by-family --image-family ubuntu22.04-nbs-github-ci --parent-id ${{ vars.NEBIUS_PARENT_ID }} --format json | jq -r .metadata.id) | |
| echo "IMAGE_ID_2204=$image_id" >> $GITHUB_OUTPUT | |
| - name: Start runner (nebius) | |
| id: start-runner-nebius | |
| if: inputs.nebius == 'yes' && env.build == 'true' && steps.get-image-id-nebius.outputs.IMAGE_ID_2204 != vars.NEBIUS_IMAGE_ID_2204 | |
| uses: ./.github/actions/nebius_runner_create | |
| timeout-minutes: 60 | |
| with: | |
| org: ydb-platform | |
| team: nbs | |
| repo_owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| service_account_key: ${{ secrets.NEW_NEBIUS_SA_JSON_CREDENTIALS }} | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| vm_parent_id: ${{ vars.NEBIUS_PARENT_ID }} | |
| vm_name: ${{ format('packer-run-{0}-{1}', github.run_id, github.run_attempt) }} | |
| vm_preset: "2vcpu-8gb" | |
| vm_disk_type: 'network-ssd-nonreplicated' | |
| vm_disk_size: 93 | |
| vm_subnet: ${{ vars.NEBIUS_SUBNET_ID }} | |
| vm_image: ${{ steps.get-image-id-nebius.outputs.IMAGE_ID_2204 }} | |
| vm_labels: ${{ format('run={0}-{1},repo={2},owner={3}', github.run_id, github.run_attempt, github.event.repository.name, github.repository_owner) }} | |
| vm_user_passwd: ${{ secrets.VM_USER_PASSWD }} | |
| - name: Stop runner (nebius) | |
| uses: ./.github/actions/nebius_runner_remove | |
| if: always() && inputs.nebius == 'yes' && env.build == 'true' && steps.get-image-id-nebius.outputs.IMAGE_ID_2204 != vars.NEBIUS_IMAGE_ID_2204 | |
| timeout-minutes: 60 | |
| with: | |
| service_account_key: ${{ secrets.NEW_NEBIUS_SA_JSON_CREDENTIALS }} | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| repo_owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| vm_parent_id: ${{ vars.NEBIUS_PARENT_ID }} | |
| vm_id: ${{ steps.start-runner-nebius.outputs.instance-id }} | |
| - name: Set new image id (nebius) | |
| if: env.build == 'true' && steps.check-image.outputs.image_already_exists != 'true' && inputs.nebius == 'yes' | |
| run: | | |
| python ./.github/scripts/nebius-manage-images.py \ | |
| --service-account-key sa.json \ | |
| --new-image-id ${NEW_IMAGE_ID} \ | |
| --image-variable-name ${IMAGE_VARIABLE_NAME} ${UPDATE_IMAGE_ID} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| IMAGE_VARIABLE_NAME: NEBIUS_IMAGE_ID_2204 | |
| NEW_IMAGE_ID: ${{ steps.get-image-id-nebius.outputs.IMAGE_ID_2204 }} | |
| UPDATE_IMAGE_ID: ${{ inputs.update_image_id && '--update-image-id' || '' }} |