Updated CI with the nested container fix

Related: pulp/pulp-oci-images#540 [noissue]
pulp · Nov 1, 2023 · 9ead8f7 · 9ead8f7
1 parent eab1797
commit 9ead8f7
Show file tree

Hide file tree

Showing 12 changed files with 84 additions and 52 deletions.
diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2
@@ -8,28 +8,39 @@ ADD ./{{ item.name }} ./{{ item.name }}
 {% endfor %}
 
 # Install python packages
-# Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
 RUN pip3 install
-{%- if s3_test | default(false) -%}
-{{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
-{%- endif -%}
-{%- if azure_test | default(false) -%}
-{{ " " }}django-storages[azure]>=1.12.2
-{%- endif -%}
-{%- if gcp_test | default(false) -%}
-{{ " " }}django-storages[google]>=1.13.2
-{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm
 {%- endif -%}
 {{ " " }}{{ item.source }}
+{%- if item.name == "pulpcore" -%}
+{%- if s3_test | default(false) -%}
+[s3]
+{%- elif azure_test | default(false) -%}
+[azure]
+{%- elif gcp_test | default(false) -%}
+[google]
+{%- endif -%}
+{%- endif -%}
 {%- if item.ci_requirements | default(false) -%}
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
 
+{% if pulp_env is defined and pulp_env %}
+{% for key, value in pulp_env.items() %}
+ENV {{ key | upper }}={{ value }}
+{% endfor %}
+{% endif %}
+
+{% if pulp_scenario_env is defined and pulp_scenario_env %}
+{% for key, value in pulp_scenario_env.items() %}
+ENV {{ key | upper }}={{ value }}
+{% endfor %}
+{% endif %}
+
 USER pulp:pulp
 RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
        /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link

diff --git a/.github/template_gitref b/.github/template_gitref
@@ -1 +1 @@
-2021.08.26-241-g9cfc63e
+2021.08.26-249-gca8f021
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -24,11 +24,11 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,10 +18,10 @@ jobs:
   ready-to-ship:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       - name: Install requirements
@@ -45,7 +45,7 @@ jobs:
     name: Assert single commit
     if: github.base_ref == 'main'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Checkout main
@@ -59,10 +59,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       # lint_requirements contains tools needed for flake8, etc.
@@ -112,11 +112,11 @@ jobs:
       deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 

diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml
@@ -9,10 +9,6 @@
 name: Create New Release Branch
 on:
   workflow_dispatch:
-    inputs:
-      name:
-        description: "Branch name (e.g. 3.14)"
-        required: true
 
 env:
   RELEASE_WORKFLOW: true
@@ -25,11 +21,12 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
-          fetch-depth: 1
+          fetch-depth: 0
+          path: pulp_container
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -40,36 +37,43 @@ jobs:
           echo ::endgroup::
 
       - name: Setting secrets
+        working-directory: pulp_container
         run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
         env:
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
 
-      - name: Verify that branch name matches current version string on main branch
+      - name: Determine new branch name
+        working-directory: pulp_container
         run: |
-          X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0((a[0-9]+)|(b[0-9]+))?\.dev",/\1/p' | awk '{$1=$1};1')
-          if [[ "$X_Y_VERSION" != "${{ inputs.name }}" ]]
+          # Just to be sure...
+          git checkout main
+          NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+          if [ -z "$NEW_BRANCH" ]
           then
-            echo "Branch name doesn't match the current version string $X_Y_VERSION."
+            echo Could not determine the new branch name.
             exit 1
           fi
+          echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV"
 
-      - name: Create ${{ inputs.name }} release branch
+      - name: Create release branch
+        working-directory: pulp_container
         run: |
-          git checkout -b ${{ inputs.name }}
-          git push origin ${{ inputs.name }}
+          git branch "${NEW_BRANCH}"
 
       - name: Bump version on main branch
+        working-directory: pulp_container
         run: |
-          git checkout main
           bump2version --no-commit minor
 
       - name: Remove entries from CHANGES directory
+        working-directory: pulp_container
         run: |
           find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} +
 
       - name: Make a PR with version bump and without CHANGES/*
         uses: peter-evans/create-pull-request@v4
         with:
+          path: pulp_container
           token: ${{ secrets.RELEASE_TOKEN }}
           committer: pulpbot <pulp-infra@redhat.com>
           author: pulpbot <pulp-infra@redhat.com>
@@ -81,3 +85,8 @@ jobs:
             Bump minor version
             [noissue]
           delete-branch: true
+
+      - name: Push release branch
+        working-directory: pulp_container
+        run: |
+          git push origin "${NEW_BRANCH}"
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -35,11 +35,11 @@ jobs:
           - TEST: lowerbounds
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -168,11 +168,11 @@ jobs:
   changelog:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
 
@@ -207,11 +207,11 @@ jobs:
       TEST: publish
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -30,11 +30,11 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -79,7 +79,7 @@ jobs:
         with:
           name: pulp_container.tar
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       - uses: ruby/setup-ruby@v1
@@ -201,7 +201,7 @@ jobs:
         with:
           name: pulp_container.tar
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 

diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh
@@ -21,8 +21,6 @@ PIP_REQUIREMENTS=("pulp-cli")
 if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]
 then
   PIP_REQUIREMENTS+=("-r" "doc_requirements.txt")
-  git clone https://github.com/pulp/pulpcore.git ../pulpcore
-  PIP_REQUIREMENTS+=("psycopg2-binary" "-r" "../pulpcore/doc_requirements.txt")
 fi
 
 pip install ${PIP_REQUIREMENTS[*]}
@@ -65,13 +63,15 @@ services:
       - ../../../pulp-openapi-generator:/root/pulp-openapi-generator
     env:
       PULP_WORKERS: "4"
+      PULP_HTTPS: "true"
 VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
+pulp_env: {}
 pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "flatpak_index": true}
 pulp_scheme: https
 
-pulp_container_tag: https-pulp_container-ci
+pulp_container_tag: "latest"
 
 VARSYAML
 
@@ -89,6 +89,7 @@ if [ "$TEST" = "s3" ]; then
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
 pulp_scenario_settings: {"flatpak_index": false}\
+pulp_scenario_env: {}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
 fi
@@ -108,6 +109,7 @@ if [ "$TEST" = "azure" ]; then
     command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml
   sed -i -e '$a azure_test: true\
 pulp_scenario_settings: {"flatpak_index": true}\
+pulp_scenario_env: {}\
 ' vars/main.yaml
 fi
 

diff --git a/.github/workflows/scripts/pre_before_install.sh b/.github/workflows/scripts/pre_before_install.sh
@@ -25,3 +25,10 @@ then
   " | sudo tee -a /etc/containers/registries.conf
 fi
 
+# Configure the GHA host for buildah/skopeo running within the pulp container.
+# Default UID & GID range is 165536-231071, which is 64K long.
+# But nested buildah/skopeo always needs more than needs 64K.
+# The Pulp image is configured for 64K + 10000 .
+sudo sed -i "s\runner:165536:65536\runner:165536:75536\g" /etc/subuid /etc/subgid
+podman system migrate
+
diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml
@@ -19,7 +19,7 @@ jobs:
   update_backport_labels:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       - name: Configure Git with pulpbot name and email

diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml
@@ -30,7 +30,7 @@ jobs:
           path: plugin_template
           fetch-depth: 0
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 

diff --git a/template_config.yml b/template_config.yml
@@ -1,7 +1,7 @@
 # This config represents the latest values used when running the plugin-template. Any settings that
 # were not present before running plugin-template have been added with their default values.
 
-# generated with plugin_template@2021.08.26-241-g9cfc63e
+# generated with plugin_template@2021.08.26-249-gca8f021
 
 additional_repos: []
 api_root: /pulp/
@@ -51,6 +51,10 @@ plugin_snake: pulp_container
 post_job_template: null
 pre_job_template: null
 publish_docs_to_pulpprojectdotorg: true
+pulp_env: {}
+pulp_env_azure: {}
+pulp_env_gcp: {}
+pulp_env_s3: {}
 pulp_scheme: https
 pulp_settings:
   allowed_content_checksums:
@@ -69,7 +73,6 @@ pulp_settings_azure:
 pulp_settings_gcp: null
 pulp_settings_s3:
   flatpak_index: false
-pulp_settings_stream: null
 pulpprojectdotorg_key_id: aa499d7938ed
 pydocstyle: true
 pypi_username: pulp