pulp_images.yml

name: pulp-oci-images CI

on:
  push:
    branches:
      - latest
  schedule:
    - cron: '0 1 * * *'
  workflow_dispatch:
  pull_request:
env:
  COLORTERM: 'yes'
  TERM: 'xterm-256color'
  PYTEST_ADDOPTS: '--color=yes'

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          # by default, it uses a depth of 1
          # this fetches all history so that we can read each commit
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v4

      - name: Check commit message
        if: github.event_name == 'pull_request'
        env:
          GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
        run: |
          echo ::group::REQUESTS
          pip install pygithub
          echo ::endgroup::
          for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')
          do
            python .ci/scripts/validate_commit_message.py $sha
            VALUE=$?
            if [ "$VALUE" -gt 0 ]; then
              exit $VALUE
            fi
          done
        shell: bash

      - name: Update to the latest pip
        run: python -m pip install --upgrade pip

  base-images:
    needs: lint
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    outputs:
      image_variants: "${{ steps.image_variants.outputs.image_variants }}"
      pulp_ci_centos_id: "${{ steps.pulp_ci_centos_id.outputs.pulp_ci_centos_id }}"
    steps:
      # We do not want to build nightly images unless it's a PR to the latest branch,
      # or a branch/dispatch build on the latest branch.
      - name: Set the list of image_variants for later jobs
        id: image_variants
        run: |
          if [ "${{ github.base_ref }}" == "latest" ] || [ "${{ github.ref_name }}" == "latest" ]; then
            echo "image_variants=[\"nightly\",\"stable\"]" >> "$GITHUB_OUTPUT"
          else
            echo "image_variants=[\"stable\"]" >> "$GITHUB_OUTPUT"
          fi

      - name: Set the temporary image tag
        run: |
          temp_base_tag="${GITHUB_REF_NAME%/*}"
          echo "Building $temp_base_tag"
          echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV

      - uses: actions/checkout@v4
        with:
          # by default, it uses a depth of 1
          # this fetches all history so that we can read each commit
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v4

      - name: Update to the latest pip
        run: python -m pip install --upgrade pip

      - name: Dispatch workflows on stable branches
        if: github.event_name == 'schedule'
        run: |
          echo ::group::PYDEPS
          pip install gitpython requests packaging jinja2 pyyaml
          echo ::endgroup::
          python .ci/scripts/update_ci_branches.py
        env:
          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}

      - name: Build images
        run: |
          podman version
          buildah version
          sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes
          for ARCH in arm64 amd64
          do
            podman build --platform linux/$ARCH --format docker --file images/Containerfile.core.base --tag pulp/base:${TEMP_BASE_TAG}-${ARCH} .
            podman build --platform linux/$ARCH --format docker --file images/pulp_ci_centos/Containerfile --tag pulp/pulp-ci-centos:${TEMP_BASE_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} .
          done
      # we use the docker format (default), even though it may not be the fastest,
      # because it supports saving both images at once.
      # However, it seems to export the common layers twice.
      # We should look into whether its possible to export just pulp-ci-centos,
      # and tag the base image manually.
      - name: Save podman images to tarball
        id: pulp_ci_centos_id
        run: |
           podman save -m -o base-images.tar pulp/base:${TEMP_BASE_TAG}-arm64 pulp/base:${TEMP_BASE_TAG}-amd64 pulp/pulp-ci-centos:${TEMP_BASE_TAG}-arm64 pulp/pulp-ci-centos:${TEMP_BASE_TAG}-amd64
           # The id is unique to the image build (not the Containerfile) and will be used in the cache key
           # If a workflow completes successfully, every workflow will generate a new cache.
           # And if we re-run the entire workflow ("Re-run all jobs"), it will generate a new cache too.
           # If we re-run a failed app-images job, it will use the existing cache from base-images
           id=$(podman image inspect --format '{{ .Id }}' pulp/pulp-ci-centos:${TEMP_BASE_TAG}-amd64)
           echo "pulp_ci_centos_id=${id}" >> "$GITHUB_OUTPUT"
           echo "pulp_ci_centos_id=${id}" >> "$GITHUB_ENV"

      - name: Cache podman images
        uses: actions/cache/save@v3
        with:
          key: base-images=${{ env.pulp_ci_centos_id }}
          path: base-images.tar
          fail-on-cache-miss: true

  app-images:
    needs: base-images
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        image_variant: ${{ fromJSON(needs.base-images.outputs.image_variants) }}
        app:
          - image_name: pulp-minimal
            web_image: pulp-web
            pip_name: pulpcore
          - image_name: pulp
            web_image: pulp-web
            pip_name: pulpcore
          - image_name: galaxy-minimal
            web_image: galaxy-web
            pip_name: galaxy-ng
          - image_name: galaxy
            web_image: galaxy-web
            pip_name: galaxy-ng
    steps:
      - name: Set the temporary image tags
        run: |
          if [ "${{ matrix.image_variant }}" == "nightly" ]; then
            temp_app_tag="nightly"
          else
            temp_app_tag="${GITHUB_REF_NAME%/*}"
          fi
          temp_base_tag="${GITHUB_REF_NAME%/*}"
          echo "Building $temp_app_tag from base $temp_base_tag"
          echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV
          echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV

      - uses: actions/checkout@v4
        with:
          # by default, it uses a depth of 1
          # this fetches all history so that we can read each commit
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v4

      - name: Install python dependencies
        if: github.event_name == 'schedule'
        run: |
          echo ::group::PYDEPS
          pip install gitpython requests packaging jinja2 pyyaml
          echo ::endgroup::

      - name: Verify needs.base-images.outputs.pulp_ci_centos_id is not blank
        run: |
          if [ -z "${{ needs.base-images.outputs.pulp_ci_centos_id }}" ]; then
            exit 1
          fi

      - name: Restore podman images from cache
        uses: actions/cache/restore@v3
        with:
          key: base-images=${{ needs.base-images.outputs.pulp_ci_centos_id }}
          path: base-images.tar
          fail-on-cache-miss: true

      - name: Load podman images from tarball
        run: |
          podman load -i base-images.tar

      - name: Install httpie and podman-compose
        run: |
          echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
          echo "Working around https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394"
          curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/golang-github-containernetworking-plugins/containernetworking-plugins_1.1.1+ds1-3_amd64.deb
          sudo dpkg -i containernetworking-plugins_1.1.1+ds1-3_amd64.deb
          # Ubuntu 22.04 has old podman 3.4.4, we need podman-compose==1.0.3 to avoid an
          # error with dependency contianers not being detected as running.
          # "error generating dependency graph for container"
          pip install httpie podman-compose==1.0.3
        shell: bash

      - name: Build images
        run: |
          podman version
          buildah version
          sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes
          for ARCH in arm64 amd64
          do
            if [[ "${{ matrix.app.image_name }}" == "pulp-minimal" || "${{ matrix.app.image_name }}" == "galaxy-minimal" ]]; then
              podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ matrix.app.image_name }}/${{ matrix.image_variant }}/Containerfile.core --tag pulp/${{ matrix.app.image_name }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} .
              podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ matrix.app.image_name }}/${{ matrix.image_variant }}/Containerfile.webserver --tag pulp/${{ matrix.app.web_image }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_APP_TAG}-${ARCH} .
            else
              podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ matrix.app.image_name }}/${{ matrix.image_variant }}/Containerfile --tag pulp/${{ matrix.app.image_name }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} .
            fi
          done
          podman images -a

      - name: Set version and branch image tags
        run: |
          app_version=$(podman run --pull=never pulp/${{ matrix.app.image_name }}:${TEMP_APP_TAG}-amd64 bash -c "pip3 show ${{ matrix.app.pip_name }} | sed -n -e 's/Version: //p'")
          app_branch=$(echo ${app_version} | grep -oP '\d+\.\d+')

          echo "APP_VERSION: ${app_version}"
          echo "APP_BRANCH: ${app_branch}"
          echo "APP_VERSION=${app_version}" >> $GITHUB_ENV
          echo "APP_BRANCH=${app_branch}" >> $GITHUB_ENV

          base_version=$(podman run --pull=never pulp/${{ matrix.app.image_name }}:${TEMP_APP_TAG}-amd64 bash -c "pip3 show pulpcore | sed -n -e 's/Version: //p'")
          base_branch=$(echo ${base_version} | grep -oP '\d+\.\d+')

          echo "BASE_VERSION: ${base_version}"
          echo "BASE_BRANCH: ${base_branch}"
          echo "BASE_VERSION=${base_version}" >> $GITHUB_ENV
          echo "BASE_BRANCH=${base_branch}" >> $GITHUB_ENV

      - name: Test image with upgrade in s6 mode (pulp)
        if: matrix.app.image_name == 'pulp'
        run: |
          # 3.20 has postgres 12 rather than 13
          images/s6_assets/test.sh "pulp/${{  matrix.app.image_name }}:${TEMP_APP_TAG}-amd64" http "quay.io/pulp/all-in-one-pulp:3.20"
          podman stop pulp
          podman rm pulp

      - name: Test the image in s6 mode (galaxy)
        if: matrix.app.image_name == 'galaxy'
        run: |
          images/s6_assets/test.sh "pulp/${{  matrix.app.image_name }}:${TEMP_APP_TAG}-amd64" https
          podman stop pulp
          podman rm pulp

      - name: Compose up
        run: |
          if [[ "${{ matrix.app.image_name }}" == "pulp" || "${{ matrix.app.image_name }}" == "galaxy" ]]; then

            # Reuse the folders from the s6 mode tests
            FILE="compose.folders.yml"
            echo "host all all 10.0.0.0/8 trust" | sudo tee -a pgsql/data/pg_hba.conf > /dev/null
            echo "listen_addresses = '*'" | sudo tee -a pgsql/data/postgresql.conf > /dev/null

            # We'll pull the web image from a registry since we didn't build it.
            if [ "${{ matrix.image_variant }}" == "nightly" ]; then
              WEB_TAG="nightly"
            else
              WEB_TAG="${APP_BRANCH}"
            fi
          else
            FILE="compose.yml"
            WEB_TAG="${TEMP_APP_TAG}-amd64"
          fi
          cd images/compose
          sed -i "s/pulp-minimal:latest/${{ matrix.app.image_name }}:${TEMP_APP_TAG}-amd64/g" $FILE
          sed -i "s/pulp-web:latest/${{ matrix.app.web_image }}:${WEB_TAG}/g" $FILE
          id | grep "(root)" || sudo usermod -G root $(whoami)
          podman-compose -f $FILE up -d
          podman exec compose_pulp_api_1 /usr/bin/wait_on_database_migrations.sh
          for _ in $(seq 20)
          do
            if curl --fail http://localhost:8080/pulp/api/v3/status/ > /dev/null 2>&1
            then
              break
            fi
            sleep 3
          done
          curl --fail http://localhost:8080/pulp/api/v3/status/ | jq
        shell: bash

      - name: Github login
        if: github.event_name != 'pull_request'
        env:
          PULP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PULP_GITHUB_USERNAME: ${{ github.actor }}
        run: echo "$PULP_GITHUB_TOKEN" | podman login -u "$PULP_GITHUB_USERNAME" --password-stdin ghcr.io

      - name: Docker login
        if: github.event_name != 'pull_request'
        env:
          DOCKER_BOT_PASSWORD: ${{ secrets.DOCKER_BOT_PASSWORD }}
          DOCKER_BOT_USERNAME: ${{ secrets.DOCKER_BOT_USERNAME }}
        run: echo "$DOCKER_BOT_PASSWORD" | podman login -u "$DOCKER_BOT_USERNAME" --password-stdin docker.io

      - name: Quay login
        if: github.event_name != 'pull_request'
        env:
          QUAY_BOT_PASSWORD: ${{ secrets.QUAY_BOT_PASSWORD }}
          QUAY_BOT_USERNAME: ${{ secrets.QUAY_BOT_USERNAME }}
        run: echo "$QUAY_BOT_PASSWORD" | podman login -u "$QUAY_BOT_USERNAME" --password-stdin quay.io

      - name: Push base images to registries
        if: github.event_name != 'pull_request'
        run: |
          for registry in ghcr.io docker.io quay.io; do
            # Technically multiple jobs will build and push these 2 images
            # but it is created once in the base-images job, and will be identical
            for image_name_looped in base pulp-ci-centos; do
              if [ "${TEMP_BASE_TAG}" == "latest" ]; then
                tags="${BASE_BRANCH} ${BASE_VERSION} latest"
              else
                tags="${BASE_BRANCH} ${BASE_VERSION}"
              fi
              # Workaround the fact that the latest branch no longer pushes pulp-ci-centos (CentOS 8)
              # so let's use this 3.39 branch for it.
              # But let's not set the latest tag for the base image, that is EL9.
              if [ "$image_name_looped" == "pulp-ci-centos" ]; then
                if [ "${TEMP_BASE_TAG}" == "3.39" ]; then
                  tags="${tags} latest"
                fi
              fi
              for tag in $tags; do
                podman manifest create ${registry}/pulp/${image_name_looped}:${tag} containers-storage:localhost/pulp/${image_name_looped}:${TEMP_BASE_TAG}-amd64 containers-storage:localhost/pulp/${image_name_looped}:${TEMP_BASE_TAG}-arm64
                podman manifest push --all ${registry}/pulp/${image_name_looped}:${tag} ${registry}/pulp/${image_name_looped}:${tag}
              done
            done
          done
          unset tag
        shell: bash

      - name: Push app images to registries
        if: github.event_name != 'pull_request'
        run: |
          if [[ "${{ matrix.app.image_name }}" == "pulp" || "${{ matrix.app.image_name }}" == "galaxy" ]]; then
            images="${{ matrix.app.image_name }}"
          else
            images="${{ matrix.app.image_name }} ${{ matrix.app.web_image }}"
          fi
          for registry in ghcr.io docker.io quay.io; do

            for image_name_looped in $images; do
              if [ "${{ matrix.image_variant }}" == "stable" ]; then
                # latest branch stable variant gets tagged as both "latest" and "stable"
                if [ "${TEMP_APP_TAG}" == "latest" ]; then
                  tags="${APP_BRANCH} ${APP_VERSION} ${{ matrix.image_variant }} latest"
                else
                  tags="${APP_BRANCH} ${APP_VERSION}"
                fi
              else
                # The matrix should ensure that the nightly variant only ever pertains to the "latest" branch, but let's be extra safe
                if [ "${GITHUB_REF_NAME%/*}" == "latest" ]; then
                  # "nightly" is a special case, no version / branch tags
                  tags="nightly"
                fi
              fi
              for tag in $tags; do
                podman manifest create ${registry}/pulp/${image_name_looped}:${tag} containers-storage:localhost/pulp/${image_name_looped}:${TEMP_APP_TAG}-amd64 containers-storage:localhost/pulp/${image_name_looped}:${TEMP_APP_TAG}-arm64
                podman manifest push --all ${registry}/pulp/${image_name_looped}:${tag} ${registry}/pulp/${image_name_looped}:${tag}
              done
            done
          done
        shell: bash

      - name: Logs
        if: always()
        run: |
          set +e
          podman ps -a
          podman images -a
          podman logs pulp
          cd images/compose
          podman-compose logs
          podman logs --tail=10000 compose_pulp_api_1
          podman logs --tail=10000 compose_pulp_content_1
          podman logs --tail=10000 compose_pulp_worker_1
          podman logs --tail=10000 compose_pulp_worker_2
          podman logs --tail=10000 compose_pulp_web_1
          VOLUME_PATH=$(podman volume inspect pulpdev | jq -r .[].Mountpoint)
          sudo ls -al $VOLUME_PATH
          sudo tree $VOLUME_PATH
          http --follow --timeout 30 --check-status --pretty format --print hb http://localhost:8080/pulp/api/v3/status/ || true