From 6e8c2cf123be259598d3183effc491eeda2c8946 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 24 Sep 2024 17:45:23 -0400 Subject: [PATCH] Make CI smarter about when to rebuild the base images --- .github/actions/base_images/action.yml | 100 ++++++++++++++++------ .github/actions/build_image/action.yml | 42 ++------- .github/actions/publish_images/action.yml | 5 +- .github/actions/test_image/action.yml | 23 +++-- .github/workflows/ci.yml | 6 +- .github/workflows/release.yml | 8 +- 6 files changed, 101 insertions(+), 83 deletions(-) diff --git a/.github/actions/base_images/action.yml b/.github/actions/base_images/action.yml index 0a90d2cd..96b65ce5 100644 --- a/.github/actions/base_images/action.yml +++ b/.github/actions/base_images/action.yml @@ -1,15 +1,15 @@ --- name: Build Base Images -description: Build the base images (pulp/base & pulp/pulp-ci-centos9) +description: Build the base images (pulp/base & pulp/pulp-ci-centos9) if needed # Both ARM64 & x86-64 versions of each are built -# Save the images to a tar and upload to a cache using output 'pulp_ci_centos_id' as the key +# Use hashFiles(base_image_files, pulp-ci_image_files) + python_version as the key to the cache inputs: python_version: required: true description: "Python Version to use to build, e.g '3.9'" outputs: - pulp_ci_centos_id: - value: ${{ steps.pulp_ci_centos_id.outputs.pulp_ci_centos_id }} + base_cache_key: + value: ${{ steps.hash_key.outputs.base_cache_key }} description: "The cache key the built images were uploaded to." runs: @@ -17,24 +17,76 @@ runs: steps: - uses: actions/checkout@v4 - - name: Set the temporary image tag + - name: Calculate base images hash + id: hash_key run: | - temp_base_tag="${GITHUB_REF_NAME%/*}" - python_version="${{ inputs.python_version }}" - echo "Building $temp_base_tag with python $python_version" - echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV - echo "PYTHON_VERSION=${python_version}" >> $GITHUB_ENV + hash=${{ hashFiles('images/Containerfile.core.base', 'images/pulp_ci_centos/Containerfile', 'images/assets/**', 'images/s6_assets/**') }} + echo "base image hash is ${hash}, python version is ${{ inputs.python_version }}" + echo "base_cache_key=${hash}-${{ inputs.python_version }}" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore previously cached images + id: cache + uses: actions/cache/restore@v3 + with: + key: base-images=${{ steps.hash_key.outputs.base_cache_key }} + path: base-images.tar.gz + + - name: Extract images if cached + if: steps.cache.outputs.cache-hit == 'true' + run: | + echo "Base Images were in cache" + podman load -i base-images.tar.gz + shell: bash + + - name: Check for updates on cached images + if: steps.cache.outputs.cache-hit == 'true' + run: | + # Enable running/building ARM64 images: https://github.com/multiarch/qemu-user-static + sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes + IMAGES=() + for IMAGE in base pulp-ci-centos9; do + for ARCH in arm64 amd64; do + echo "Checking if rebuild needed for ${IMAGE}:${ARCH}" + podman run --pull=never pulp/${IMAGE}:ci-${ARCH} bash -c "dnf check-upgrade" + if [ $? -gt 0 ]; then + echo "Rebuild needed for ${IMAGE}:${ARCH}" + IMAGES+=('${IMAGE}:${ARCH}') + fi + done + done + if [ ${#IMAGES[@]} -eq 0 ]; then + echo "No rebuilds needed :)" + else + echo "BUILD_IMAGES=[$(echo ${IMAGES[@]} | sed 's/ /, /g')]" >> $GITHUB_ENV + fi + shell: bash + + - name: Set images to build on cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + echo "BUILD_IMAGES=['base:arm64', 'base:amd64', 'pulp-ci-centos9:arm64', 'pulp-ci-centos9:amd64']" >> $GITHUB_ENV + sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes shell: bash - name: Build images + if: env.BUILD_IMAGES run: | + IMAGES=(${{ join(fromJSON(env.BUILD_IMAGES), ' ') }}) + echo "Going to build images: ${IMAGES[@]}" podman version buildah version - sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes - for ARCH in arm64 amd64 - do - podman build --platform linux/$ARCH --format docker --file images/Containerfile.core.base --tag pulp/base:${TEMP_BASE_TAG}-${ARCH} --build-arg PYTHON_VERSION=${PYTHON_VERSION} . - podman build --platform linux/$ARCH --format docker --file images/pulp_ci_centos/Containerfile --tag pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} . + for IMAGE in "${IMAGES[@]}"; do + echo "Building image ${IMAGE}" + ARCH=${IMAGE##*:} + case $IMAGE in + base:*) + podman build --platform "linux/${ARCH}" --format docker --file images/Containerfile.core.base --tag "pulp/base:ci-${ARCH}" --build-arg PYTHON_VERSION=${{ inputs.python_version }} . + ;; + pulp-ci-centos9:*) + podman build --platform "linux/${ARCH}" --format docker --file images/pulp_ci_centos/Containerfile --tag "pulp/pulp-ci-centos9:ci-${ARCH}" --build-arg FROM_TAG="ci-${ARCH}" . + ;; + esac done shell: bash # we use the docker format (default), even though it may not be the fastest, @@ -43,20 +95,16 @@ runs: # We should look into whether its possible to export just pulp-ci-centos, # and tag the base image manually. - name: Save podman images to tarball - id: pulp_ci_centos_id + if: env.BUILD_IMAGES run: | - podman save -m -o base-images.tar pulp/base:${TEMP_BASE_TAG}-arm64 pulp/base:${TEMP_BASE_TAG}-amd64 pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-arm64 pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-amd64 - # The id is unique to the image build (not the Containerfile) and will be used in the cache key - # If a workflow completes successfully, every workflow will generate a new cache. - # And if we re-run the entire workflow ("Re-run all jobs"), it will generate a new cache too. - # If we re-run a failed app-images job, it will use the existing cache from base-images - id=$(podman image inspect --format '{{ .Id }}' pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-amd64) - echo "pulp_ci_centos_id=${id}" >> "$GITHUB_OUTPUT" - echo "pulp_ci_centos_id=${id}" >> "$GITHUB_ENV" + rm -f base-images.tar.gz + podman save -m -o base-images.tar pulp/base:ci-arm64 pulp/base:ci-amd64 pulp/pulp-ci-centos9:ci-arm64 pulp/pulp-ci-centos9:ci-amd64 + gzip base-images.tar shell: bash - name: Cache podman images + if: env.BUILD_IMAGES uses: actions/cache/save@v3 with: - key: base-images=${{ env.pulp_ci_centos_id }} - path: base-images.tar + key: base-images=${{ steps.hash_key.outputs.base_cache_key }} + path: base-images.tar.gz diff --git a/.github/actions/build_image/action.yml b/.github/actions/build_image/action.yml index efb3d4c5..cfbda5b5 100644 --- a/.github/actions/build_image/action.yml +++ b/.github/actions/build_image/action.yml @@ -19,30 +19,12 @@ outputs: app_branch: value: ${{ steps.image_version_branch.outputs.app_branch }} description: 'The pulpcore version branch that the built image matches' - app_arch_tag: - value: ${{ steps.image_tags.outputs.app_arch_tag }} - description: 'The temporary amd64 arch tag of the app image' runs: using: "composite" steps: - uses: actions/checkout@v4 - - name: Set the temporary image tags - id: image_tags - run: | - if [ "${{ inputs.image_variant }}" == "nightly" ]; then - temp_app_tag="nightly" - else - temp_app_tag="${GITHUB_REF_NAME%/*}" - fi - temp_base_tag="${GITHUB_REF_NAME%/*}" - echo "Building $temp_app_tag from base $temp_base_tag" - echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV - echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV - echo "app_arch_tag=${temp_app_tag}-amd64" >> $GITHUB_OUTPUT - shell: bash - - name: Set up Python uses: actions/setup-python@v4 @@ -50,23 +32,11 @@ runs: uses: actions/cache/restore@v3 with: key: base-images=${{ inputs.image_cache_key }} - path: base-images.tar + path: base-images.tar.gz - name: Load podman images from tarball run: | - podman load -i base-images.tar - shell: bash - - - name: Install httpie and podman-compose - run: | - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - echo "Working around https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394" - curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/golang-github-containernetworking-plugins/containernetworking-plugins_1.1.1+ds1-3_amd64.deb - sudo dpkg -i containernetworking-plugins_1.1.1+ds1-3_amd64.deb - # Ubuntu 22.04 has old podman 3.4.4, we need podman-compose==1.0.3 to avoid an - # error with dependency contianers not being detected as running. - # "error generating dependency graph for container" - pip install httpie podman-compose==1.0.3 + podman load -i base-images.tar.gz shell: bash - name: Build images @@ -78,10 +48,10 @@ runs: do if [[ "${{ inputs.image_name }}" == "pulp-minimal" || "${{ inputs.image_name }}" == "galaxy-minimal" ]]; then base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1) - podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.core --tag pulp/${{ inputs.image_name }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} . - podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.webserver --tag pulp/${base_image}-web:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_APP_TAG}-${ARCH} . + podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.core --tag pulp/${{ inputs.image_name }}:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} . + podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.webserver --tag pulp/${base_image}-web:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} . else - podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile --tag pulp/${{ inputs.image_name }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} . + podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile --tag pulp/${{ inputs.image_name }}:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} . fi done podman images -a @@ -96,7 +66,7 @@ runs: else pip_name="galaxy-ng" fi - app_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:${TEMP_APP_TAG}-amd64 bash -c "pip3 show ${pip_name} | sed -n -e 's/Version: //p'") + app_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci-amd64 bash -c "pip3 show ${pip_name} | sed -n -e 's/Version: //p'") app_branch=$(echo ${app_version} | grep -oP '\d+\.\d+') echo "APP_VERSION: ${app_version}" diff --git a/.github/actions/publish_images/action.yml b/.github/actions/publish_images/action.yml index 9cd9fd00..c09ab46c 100644 --- a/.github/actions/publish_images/action.yml +++ b/.github/actions/publish_images/action.yml @@ -5,9 +5,6 @@ inputs: image_names: description: 'Names of the images to be published' required: true - build_tag: - description: 'Current tag of the local built container' - required: true tags: description: 'List of tags the images are to be published under' required: true @@ -56,7 +53,7 @@ runs: for registry in ghcr.io docker.io quay.io; do for image in ${{ inputs.image_names }}; do for tag in ${{ inputs.tags }}; do - podman manifest create ${registry}/pulp/${image}:${tag} containers-storage:localhost/pulp/${image}:${{ inputs.build_tag }}-amd64 containers-storage:localhost/pulp/${image}:${{ inputs.build_tag }}-arm64 + podman manifest create ${registry}/pulp/${image}:${tag} containers-storage:localhost/pulp/${image}:ci-amd64 containers-storage:localhost/pulp/${image}:ci-arm64 podman manifest push --all ${registry}/pulp/${image}:${tag} ${registry}/pulp/${image}:${tag} done done diff --git a/.github/actions/test_image/action.yml b/.github/actions/test_image/action.yml index 2e24f264..365dce96 100644 --- a/.github/actions/test_image/action.yml +++ b/.github/actions/test_image/action.yml @@ -12,18 +12,27 @@ inputs: app_branch: description: 'The branch the app was built on' required: true - app_arch_tag: - description: 'The temporary amd64 arch tag of the built app image' - required: true runs: using: "composite" steps: + - name: Install httpie and podman-compose + run: | + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "Working around https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394" + curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/golang-github-containernetworking-plugins/containernetworking-plugins_1.1.1+ds1-3_amd64.deb + sudo dpkg -i containernetworking-plugins_1.1.1+ds1-3_amd64.deb + # Ubuntu 22.04 has old podman 3.4.4, we need podman-compose==1.0.3 to avoid an + # error with dependency contianers not being detected as running. + # "error generating dependency graph for container" + pip install httpie podman-compose==1.0.3 + shell: bash + - name: Test image with upgrade in s6 mode (pulp) if: inputs.image_name == 'pulp' run: | # 3.20 has postgres 12 rather than 13 - images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:${{ inputs.app_arch_tag }}" http "quay.io/pulp/all-in-one-pulp:3.20" + images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:ci-amd64" http "quay.io/pulp/all-in-one-pulp:3.20" podman stop pulp podman rm pulp shell: bash @@ -31,7 +40,7 @@ runs: - name: Test the image in s6 mode (galaxy) if: inputs.image_name == 'galaxy' run: | - images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:${{ inputs.app_arch_tag }}" https + images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:ci-amd64" https podman stop pulp podman rm pulp shell: bash @@ -48,8 +57,8 @@ runs: fi else FILE="compose.yml" - WEB_TAG="${{ inputs.app_arch_tag }}" + WEB_TAG="ci-amd64" fi base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1) - images/compose/test.sh "${{ inputs.image_name }}:${{ inputs.app_arch_tag }}" "${base_image}-web:${WEB_TAG}" $FILE + images/compose/test.sh "${{ inputs.image_name }}:ci-amd64" "${base_image}-web:${WEB_TAG}" $FILE shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45aeae7b..7daefa98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: runs-on: ubuntu-latest outputs: image_variants: "${{ steps.image_variants.outputs.image_variants }}" - pulp_ci_centos_id: "${{ steps.build_base_images.outputs.pulp_ci_centos_id }}" + base_cache_key: "${{ steps.build_base_images.outputs.base_cache_key }}" steps: # We do not want to build nightly images unless it's a PR to the latest branch, # or a branch/dispatch build on the latest branch. @@ -76,7 +76,6 @@ jobs: outputs: app_version: ${{ steps.build_image.outputs.app_version }} app_branch: ${{ steps.build_image.outputs.app_branch }} - app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }} strategy: fail-fast: false matrix: @@ -95,7 +94,7 @@ jobs: with: image_name: ${{ matrix.image_name }} image_variant: ${{ matrix.image_variant }} - image_cache_key: ${{ needs.base-images.outputs.pulp_ci_centos_id }} + image_cache_key: ${{ needs.base-images.outputs.base_cache_key }} - name: Test App Image uses: "./.github/actions/test_image" @@ -103,7 +102,6 @@ jobs: image_name: ${{ matrix.image_name }} image_variant: ${{ matrix.image_variant }} app_branch: ${{ steps.build_image.outputs.app_branch }} - app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }} - name: Logs if: always() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e6f715e..3751d51b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: base-images: runs-on: ubuntu-latest outputs: - pulp_ci_centos_id: ${{ steps.build_base_images.outputs.pulp_ci_centos_id }} + base_cache_key: ${{ steps.build_base_images.outputs.base_cache_key }} image_variants: ${{ steps.image_variants.outputs.image_variants }} steps: @@ -138,7 +138,6 @@ jobs: with: image_names: "base pulp-ci-centos9" tags: ${{ env.TAGS }} - build_tag: ${{ github.ref_name }} github_token: ${{ secrets.GITHUB_TOKEN }} docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }} docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }} @@ -151,7 +150,6 @@ jobs: outputs: app_version: ${{ steps.build_image.outputs.app_version }} app_branch: ${{ steps.build_image.outputs.app_branch }} - app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }} strategy: fail-fast: false matrix: @@ -170,7 +168,7 @@ jobs: with: image_name: ${{ matrix.image_name }} image_variant: ${{ matrix.image_variant }} - image_cache_key: ${{ needs.base-images.outputs.pulp_ci_centos_id }} + image_cache_key: ${{ needs.base-images.outputs.base_cache_key }} - name: Test App Image uses: "./.github/actions/test_image" @@ -178,7 +176,6 @@ jobs: image_name: ${{ matrix.image_name }} image_variant: ${{ matrix.image_variant }} app_branch: ${{ steps.build_image.outputs.app_branch }} - app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }} - name: Set tags run: | @@ -211,7 +208,6 @@ jobs: with: image_names: ${{ env.IMAGES }} tags: ${{ env.TAGS }} - build_tag: ${{ matrix.image_variant == 'nightly' && 'nightly' || github.ref_name }} github_token: ${{ secrets.GITHUB_TOKEN }} docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }} docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }}