From 61d1fd3b9e84c1f908a617ec54d3d05cc31fb5c0 Mon Sep 17 00:00:00 2001 From: David Luong Date: Thu, 1 Aug 2024 14:58:23 -0700 Subject: [PATCH] [noissue] Change apiVersion to avoid failure while deploying hub [noissue] --- .ci/scripts/backup_and_restore.sh | 4 +- .ci/scripts/pulp_ansible/distribution_repo.sh | 3 ++ .../pulp_ansible/distribution_repo_version.sh | 3 ++ .ci/scripts/pulp_ansible/quickstart.sh | 17 +++++++++ .../pulp_ansible/remote-collection-token.sh | 8 ++++ .ci/scripts/pulp_ansible/remote-collection.sh | 7 ++++ .ci/scripts/pulp_ansible/remote.sh | 2 + .ci/scripts/pulp_ansible/repo.sh | 2 + .ci/scripts/pulp_ansible/setup.sh | 23 +++++++++++ .ci/scripts/pulp_ansible/sync-collection.sh | 9 +++++ .ci/scripts/pulp_ansible/sync.sh | 9 +++++ .ci/scripts/pulp_container/base.sh | 32 ++++++++++++++++ .../pulp_container/build_containerfile.sh | 16 ++++++++ .ci/scripts/pulp_container/cleanup_export.sh | 5 +++ .../pulp_container/create_containerfile.sh | 11 ++++++ .ci/scripts/pulp_container/create_example.sh | 12 ++++++ .ci/scripts/pulp_container/distribution.sh | 22 +++++++++++ .ci/scripts/pulp_container/docs_check.sh | 38 +++++++++++++++++++ .../pulp_container/download_after_sync.sh | 11 ++++++ .../pulp_container/download_after_tagging.sh | 22 +++++++++++ .../pulp_container/export_repository.sh | 20 ++++++++++ .ci/scripts/pulp_container/image_tagging.sh | 18 +++++++++ .ci/scripts/pulp_container/image_untagging.sh | 18 +++++++++ .../pulp_container/import_repository.sh | 14 +++++++ .ci/scripts/pulp_container/manifest_copy.sh | 15 ++++++++ .../pulp_container/recursive_add_tag.sh | 19 ++++++++++ .../pulp_container/recursive_remove_tag.sh | 15 ++++++++ .ci/scripts/pulp_container/remote.sh | 11 ++++++ .ci/scripts/pulp_container/repo.sh | 9 +++++ .ci/scripts/pulp_container/second_repo.sh | 9 +++++ .ci/scripts/pulp_container/sync.sh | 15 ++++++++ .ci/scripts/pulp_container/tag_copy.sh | 16 ++++++++ .ci/scripts/pulp_tests.sh | 19 +++------- .github/workflows/ci.yml | 2 - Dockerfile | 3 ++ roles/postgres/tasks/main.yml | 2 +- roles/postgres/tasks/upgrade_postgres.yml | 2 +- roles/postgres/templates/postgres.yaml.j2 | 2 +- .../templates/pulp-api.deployment.yaml.j2 | 19 +++++++++- .../templates/pulp-content.deployment.yaml.j2 | 2 +- roles/pulp-status/tasks/main.yml | 2 +- .../templates/pulp-web.deployment.yaml.j2 | 2 +- .../templates/pulp-worker.deployment.yaml.j2 | 2 +- .../redis/templates/redis.deployment.yaml.j2 | 2 +- 44 files changed, 468 insertions(+), 26 deletions(-) create mode 100755 .ci/scripts/pulp_ansible/distribution_repo.sh create mode 100755 .ci/scripts/pulp_ansible/distribution_repo_version.sh create mode 100755 .ci/scripts/pulp_ansible/quickstart.sh create mode 100755 .ci/scripts/pulp_ansible/remote-collection-token.sh create mode 100755 .ci/scripts/pulp_ansible/remote-collection.sh create mode 100755 .ci/scripts/pulp_ansible/remote.sh create mode 100755 .ci/scripts/pulp_ansible/repo.sh create mode 100755 .ci/scripts/pulp_ansible/setup.sh create mode 100644 .ci/scripts/pulp_ansible/sync-collection.sh create mode 100755 .ci/scripts/pulp_ansible/sync.sh create mode 100755 .ci/scripts/pulp_container/base.sh create mode 100644 .ci/scripts/pulp_container/build_containerfile.sh create mode 100644 .ci/scripts/pulp_container/cleanup_export.sh create mode 100644 .ci/scripts/pulp_container/create_containerfile.sh create mode 100644 .ci/scripts/pulp_container/create_example.sh create mode 100755 .ci/scripts/pulp_container/distribution.sh create mode 100755 .ci/scripts/pulp_container/docs_check.sh create mode 100755 .ci/scripts/pulp_container/download_after_sync.sh create mode 100644 .ci/scripts/pulp_container/download_after_tagging.sh create mode 100644 .ci/scripts/pulp_container/export_repository.sh create mode 100644 .ci/scripts/pulp_container/image_tagging.sh create mode 100644 .ci/scripts/pulp_container/image_untagging.sh create mode 100644 .ci/scripts/pulp_container/import_repository.sh create mode 100755 .ci/scripts/pulp_container/manifest_copy.sh create mode 100755 .ci/scripts/pulp_container/recursive_add_tag.sh create mode 100755 .ci/scripts/pulp_container/recursive_remove_tag.sh create mode 100755 .ci/scripts/pulp_container/remote.sh create mode 100755 .ci/scripts/pulp_container/repo.sh create mode 100755 .ci/scripts/pulp_container/second_repo.sh create mode 100755 .ci/scripts/pulp_container/sync.sh create mode 100755 .ci/scripts/pulp_container/tag_copy.sh diff --git a/.ci/scripts/backup_and_restore.sh b/.ci/scripts/backup_and_restore.sh index 27d544897..ab2067358 100755 --- a/.ci/scripts/backup_and_restore.sh +++ b/.ci/scripts/backup_and_restore.sh @@ -93,10 +93,12 @@ base_url = "$BASE_ADDR" api_root = "$API_ROOT" verify_ssl = false format = "json" +username = "admin" +password = "password" EOF fi -cat ~/.config/pulp/cli.toml | tee ~/.config/pulp/settings.toml +cat ~/.config/pulp/cli.toml pulp content list CONTENT_LENGTH=$(pulp content list | jq length) diff --git a/.ci/scripts/pulp_ansible/distribution_repo.sh b/.ci/scripts/pulp_ansible/distribution_repo.sh new file mode 100755 index 000000000..0636ccb50 --- /dev/null +++ b/.ci/scripts/pulp_ansible/distribution_repo.sh @@ -0,0 +1,3 @@ +# Distributions are created asynchronously. Create one, and specify the repository that will +# be served at the base path specified. +pulp ansible distribution create --name "baz" --base-path "my_content" --repository "foo" diff --git a/.ci/scripts/pulp_ansible/distribution_repo_version.sh b/.ci/scripts/pulp_ansible/distribution_repo_version.sh new file mode 100755 index 000000000..ba18ac9db --- /dev/null +++ b/.ci/scripts/pulp_ansible/distribution_repo_version.sh @@ -0,0 +1,3 @@ +# Distributions are created asynchronously. Create one, and specify the repository version that will +# be served at the base path specified. +pulp ansible distribution create --name "bar" --base-path "some_content" --repository "foo" --version 0 diff --git a/.ci/scripts/pulp_ansible/quickstart.sh b/.ci/scripts/pulp_ansible/quickstart.sh new file mode 100755 index 000000000..32f41d97e --- /dev/null +++ b/.ci/scripts/pulp_ansible/quickstart.sh @@ -0,0 +1,17 @@ +# This script will execute the component scripts and ensure that the documented examples +# work as expected. + +# From the _scripts directory, run with `source quickstart.sh` (source to preserve the environment +# variables) +source setup.sh + +echo "Role - Workflows" +source repo.sh +source distribution_repo.sh +source distribution_repo_version.sh +source remote.sh +source sync.sh + +echo "Collection - Workflows" +source remote-collection.sh +source sync-collection.sh diff --git a/.ci/scripts/pulp_ansible/remote-collection-token.sh b/.ci/scripts/pulp_ansible/remote-collection-token.sh new file mode 100755 index 000000000..3dc89b0cc --- /dev/null +++ b/.ci/scripts/pulp_ansible/remote-collection-token.sh @@ -0,0 +1,8 @@ +# Create a remote that syncs some versions of django into your repository. +pulp ansible remote -t "collection" create \ + --name "abar" \ + --auth-url "https://sso.qa.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token" \ + --token "$ANSIBLE_TOKEN_AUTH" \ + --tls-validation false \ + --url "https://cloud.redhat.com/api/automation-hub/" \ + --requirements $'collections:\n - testing.ansible_testing_content' diff --git a/.ci/scripts/pulp_ansible/remote-collection.sh b/.ci/scripts/pulp_ansible/remote-collection.sh new file mode 100755 index 000000000..0248540ee --- /dev/null +++ b/.ci/scripts/pulp_ansible/remote-collection.sh @@ -0,0 +1,7 @@ +# Create a remote that syncs some versions of django into your repository. +pulp ansible remote -t "collection" create \ + --name "cbar" \ + --url "https://galaxy-dev.ansible.com/" \ + --requirements $'collections:\n - testing.ansible_testing_content' +# If requirements are in a file +# you can use the option '--requirements @' instead. diff --git a/.ci/scripts/pulp_ansible/remote.sh b/.ci/scripts/pulp_ansible/remote.sh new file mode 100755 index 000000000..0e874ee49 --- /dev/null +++ b/.ci/scripts/pulp_ansible/remote.sh @@ -0,0 +1,2 @@ +# Create a remote that syncs some versions of django into your repository. +pulp ansible remote -t "role" create --name "bar" --url "https://galaxy.ansible.com/api/v1/roles/?owner__username=elastic" diff --git a/.ci/scripts/pulp_ansible/repo.sh b/.ci/scripts/pulp_ansible/repo.sh new file mode 100755 index 000000000..621e9a24b --- /dev/null +++ b/.ci/scripts/pulp_ansible/repo.sh @@ -0,0 +1,2 @@ +# Start by creating a new repository named foo +pulp ansible repository create --name "foo" diff --git a/.ci/scripts/pulp_ansible/setup.sh b/.ci/scripts/pulp_ansible/setup.sh new file mode 100755 index 000000000..25ca119b9 --- /dev/null +++ b/.ci/scripts/pulp_ansible/setup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -e + +export BASE_ADDR=${BASE_ADDR:-http://pulp:80} + +if [ -z "$(pip freeze | grep pulp-cli)" ]; then + echo "Installing pulp-cli" + pip install pulp-cli[pygments] +fi + +# Set up CLI config file +if [ ! -f ~/.config/pulp/settings.toml ]; then + echo "Configuring pulp-cli" + mkdir -p ~/.config/pulp + cat > ~/.config/pulp/cli.toml << EOF +[cli] +base_url = "$BASE_ADDR" +verify_ssl = false +format = "json" +username = "admin" +password = "password" +EOF +fi diff --git a/.ci/scripts/pulp_ansible/sync-collection.sh b/.ci/scripts/pulp_ansible/sync-collection.sh new file mode 100644 index 000000000..c083662e1 --- /dev/null +++ b/.ci/scripts/pulp_ansible/sync-collection.sh @@ -0,0 +1,9 @@ +# Sync repository foo using remote cbar +pulp ansible repository sync --name "foo" --remote "cbar" + +# Use the -b option to have the sync task complete in the background +# e.g. pulp -b ansible repository sync --name "foo" --remote "cbar" + +# After the task is complete, it gives us a new repository version +# Inspecting new repository version +pulp ansible repository version show --repository "foo" diff --git a/.ci/scripts/pulp_ansible/sync.sh b/.ci/scripts/pulp_ansible/sync.sh new file mode 100755 index 000000000..f81e0c65d --- /dev/null +++ b/.ci/scripts/pulp_ansible/sync.sh @@ -0,0 +1,9 @@ +# Sync repository foo using remote bar +pulp ansible repository sync --name "foo" --remote "role:bar" + +# Use the -b option to have the sync task complete in the background +# e.g. pulp -b ansible repository sync --name "foo" --remote "bar" + +# After the task is complete, it gives us a new repository version +# Inspecting new repository version +pulp ansible repository version show --repository "foo" --version 1 diff --git a/.ci/scripts/pulp_container/base.sh b/.ci/scripts/pulp_container/base.sh new file mode 100755 index 000000000..744768a79 --- /dev/null +++ b/.ci/scripts/pulp_container/base.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +echo "Setting environment variables for default hostname/port for the API and the Content app" +BASE_ADDR=${BASE_ADDR:-http://localhost:24817} +REGISTRY_ADDR=${BASE_ADDR#*//} + +# Poll a Pulp task until it is finished. +wait_until_task_finished() { + echo "Polling the task until it has reached a final state." + local task_url=$1 + local timeout=10 + while [ "$timeout" -gt 0 ] + do + local response=$(http $task_url) + local state=$(jq -r .state <<< ${response}) + jq . <<< "${response}" || exit 1 + case ${state} in + failed|canceled) + echo "Task in final state: ${state}" + exit 1 + ;; + completed) + echo "$task_url complete." + break + ;; + *) + echo "Still waiting..." + sleep 2 + timeout=$(($timeout-1)) + ;; + esac + done +} diff --git a/.ci/scripts/pulp_container/build_containerfile.sh b/.ci/scripts/pulp_container/build_containerfile.sh new file mode 100644 index 000000000..0176541b6 --- /dev/null +++ b/.ci/scripts/pulp_container/build_containerfile.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +echo "Create a task that will build a container image from a Containerfile." + +TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' containerfile@./Containerfile \ +artifacts="{\"$ARTIFACT_HREF\": \"foo/bar/example.txt\"}" | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +echo "Set REPOVERSION_HREF from finished task." +REPOVERSION_HREF=$(http $BASE_ADDR$TASK_HREF| jq -r '.created_resources | first') + +echo "Inspecting RepositoryVersion." +http $BASE_ADDR$REPOVERSION_HREF diff --git a/.ci/scripts/pulp_container/cleanup_export.sh b/.ci/scripts/pulp_container/cleanup_export.sh new file mode 100644 index 000000000..e0579089a --- /dev/null +++ b/.ci/scripts/pulp_container/cleanup_export.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# remove the repository and its content from the current filesystem +pulp container distribution destroy --name "test/fixture" +pulp orphan cleanup --protection-time 0 diff --git a/.ci/scripts/pulp_container/create_containerfile.sh b/.ci/scripts/pulp_container/create_containerfile.sh new file mode 100644 index 000000000..ccd24b79c --- /dev/null +++ b/.ci/scripts/pulp_container/create_containerfile.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +echo "Create a Containerfile that expects foo/bar/example.txt inside /pulp_working_directory." + +echo 'FROM centos:7 + +# Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter. +COPY foo/bar/example.txt /inside-image.txt + +# Print the content of the file when the container starts +CMD ["cat", "/inside-image.txt"]' >> Containerfile \ No newline at end of file diff --git a/.ci/scripts/pulp_container/create_example.sh b/.ci/scripts/pulp_container/create_example.sh new file mode 100644 index 000000000..a1df9c915 --- /dev/null +++ b/.ci/scripts/pulp_container/create_example.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +echo "Create a text file and upload it to Pulp" + +echo 'Hello world!' > example.txt + +ARTIFACT_HREF=$(http --form POST http://localhost/pulp/api/v3/artifacts/ \ + file@./example.txt \ + | jq -r '.pulp_href') + +echo "Inspecting new artifact." +http $BASE_ADDR$ARTIFACT_HREF diff --git a/.ci/scripts/pulp_container/distribution.sh b/.ci/scripts/pulp_container/distribution.sh new file mode 100755 index 000000000..ae09e17ed --- /dev/null +++ b/.ci/scripts/pulp_container/distribution.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DIST_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) +DIST_BASE_PATH='test' + +# Distributions are created asynchronously. +echo "Creating distribution \ + (name=$DIST_NAME, base_path=$DIST_BASE_PATH repository=$REPO_HREF)." +TASK_HREF=$(http POST $BASE_ADDR/pulp/api/v3/distributions/container/container/ \ + name=$DIST_NAME \ + base_path=$DIST_BASE_PATH \ + repository=$REPO_HREF | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +echo "Setting DISTRIBUTION_HREF from the completed task." +# DISTRIBUTION_HREF is the pulp-api HREF, not the content app href +DISTRIBUTION_HREF=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspecting Distribution." +http $BASE_ADDR$DISTRIBUTION_HREF diff --git a/.ci/scripts/pulp_container/docs_check.sh b/.ci/scripts/pulp_container/docs_check.sh new file mode 100755 index 000000000..097aa9a17 --- /dev/null +++ b/.ci/scripts/pulp_container/docs_check.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euv + +# This script will execute the component scripts and ensure that the documented examples +# work as expected. + +# From the _scripts directory, run with `source docs_check_sync_publish.sh` (source to preserve the +# environment variables) +source base.sh + +# Check Sync +source repo.sh +source remote.sh +source sync.sh +source distribution.sh +source download_after_sync.sh + +# Check add/remove +source second_repo.sh +source recursive_add_tag.sh +source recursive_remove_tag.sh + +# Check Copy +source second_repo.sh +source manifest_copy.sh + +source second_repo.sh +source tag_copy.sh + +# Check tag/untag +source image_tagging.sh +source download_after_tagging.sh +source image_untagging.sh + +# Check import/export +source export_repository.sh +source cleanup_export.sh +source import_repository.sh diff --git a/.ci/scripts/pulp_container/download_after_sync.sh b/.ci/scripts/pulp_container/download_after_sync.sh new file mode 100755 index 000000000..0b3c4e727 --- /dev/null +++ b/.ci/scripts/pulp_container/download_after_sync.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CONTAINER_TAG='manifest_a' + +echo "Setting REGISTRY_PATH, which can be used directly with the Docker Client." +REGISTRY_PATH=$(http $BASE_ADDR$DISTRIBUTION_HREF | jq -r '.registry_path') + +echo "Next we pull and run the image from pulp" +echo "$REGISTRY_PATH:$CONTAINER_TAG" +sudo docker login -u admin -p password $REGISTRY_PATH +sudo docker run $REGISTRY_PATH:$CONTAINER_TAG diff --git a/.ci/scripts/pulp_container/download_after_tagging.sh b/.ci/scripts/pulp_container/download_after_tagging.sh new file mode 100644 index 000000000..22310747e --- /dev/null +++ b/.ci/scripts/pulp_container/download_after_tagging.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +TAG_NAME='custom_tag' + +DIST_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) +DIST_BASE_PATH='tag' + +echo "Publishing the latest repository." +TASK_URL=$(http POST $BASE_ADDR/pulp/api/v3/distributions/container/container/ \ + name=$DIST_NAME base_path=$DIST_BASE_PATH repository=$REPO_HREF \ + | jq -r '.task') + +wait_until_task_finished $BASE_ADDR$TASK_URL + +DISTRIBUTION_HREF=$(http $BASE_ADDR$TASK_URL \ + | jq -r '.created_resources | first') +REGISTRY_PATH=$(http $BASE_ADDR$DISTRIBUTION_HREF \ + | jq -r '.registry_path') + +echo "Running ${REGISTRY_PATH}:${TAG_NAME}." +sudo docker login -u admin -p password $REGISTRY_PATH +sudo docker run $REGISTRY_PATH:$TAG_NAME diff --git a/.ci/scripts/pulp_container/export_repository.sh b/.ci/scripts/pulp_container/export_repository.sh new file mode 100644 index 000000000..21703b81c --- /dev/null +++ b/.ci/scripts/pulp_container/export_repository.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +podman pull ghcr.io/pulp/test-fixture-1:manifest_a + +# push a tagged image to the registry +podman login ${REGISTRY_ADDR} -u admin -p password --tls-verify=false +podman tag ghcr.io/pulp/test-fixture-1:manifest_a \ + ${REGISTRY_ADDR}/test/fixture:manifest_a +podman push ${REGISTRY_ADDR}/test/fixture:manifest_a --tls-verify=false + +# a repository of the push type is automatically created +REPOSITORY_HREF=$(pulp container repository -t push show \ + --name "test/fixture" | jq -r ".pulp_href") + +# export the repository to the directory '/tmp/exports/test-fixture' +EXPORTER_HREF=$(http ${BASE_ADDR}/pulp/api/v3/exporters/core/pulp/ \ + name=both repositories:="[\"${REPOSITORY_HREF}\"]" \ + path=/tmp/exports/test-fixture | jq -r ".pulp_href") +TASK_HREF=$(http POST ${BASE_ADDR}${EXPORTER_HREF}exports/ | jq -r ".task") +wait_until_task_finished ${BASE_ADDR}${TASK_HREF} diff --git a/.ci/scripts/pulp_container/image_tagging.sh b/.ci/scripts/pulp_container/image_tagging.sh new file mode 100644 index 000000000..40f38a767 --- /dev/null +++ b/.ci/scripts/pulp_container/image_tagging.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TAG_NAME='custom_tag' +MANIFEST_DIGEST=$(http $BASE_ADDR'/pulp/api/v3/content/container/manifests/?repository_version='$REPOVERSION_HREF \ + | jq -r '.results | first | .digest') + +echo "Tagging the manifest." +TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'tag/' tag=$TAG_NAME digest=$MANIFEST_DIGEST \ + | jq -r '.task') + +wait_until_task_finished $BASE_ADDR$TASK_URL + +echo "Getting a reference to a newly created tag." +CREATED_TAG=$(http $BASE_ADDR$TASK_URL \ + | jq -r '.created_resources | .[] | select(test("content"))') + +echo "Display properties of the created tag." +http $BASE_ADDR$CREATED_TAG diff --git a/.ci/scripts/pulp_container/image_untagging.sh b/.ci/scripts/pulp_container/image_untagging.sh new file mode 100644 index 000000000..e5f537d31 --- /dev/null +++ b/.ci/scripts/pulp_container/image_untagging.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TAG_NAME='custom_tag' + +echo "Untagging a manifest which is labeled with ${TAG_NAME}" +TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'untag/' tag=$TAG_NAME \ + | jq -r '.task') + +wait_until_task_finished $BASE_ADDR$TASK_URL + +echo "Getting a reference to all removed tags." +REPO_VERSION=$(http $BASE_ADDR$TASK_URL \ + | jq -r '.created_resources | first') +REMOVED_TAGS=$(http $BASE_ADDR$REPO_VERSION \ + | jq -r '.content_summary | .removed | ."container.tag" | .href') + +echo "List removed tags from the latest repository version." +http $BASE_ADDR$REMOVED_TAGS diff --git a/.ci/scripts/pulp_container/import_repository.sh b/.ci/scripts/pulp_container/import_repository.sh new file mode 100644 index 000000000..306293e0e --- /dev/null +++ b/.ci/scripts/pulp_container/import_repository.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# create a repository with the same name as the exported one +http ${BASE_ADDR}/pulp/api/v3/repositories/container/container/ \ + name="test/fixture" | jq -r ".pulp_href" + +# import the exported repository stored in '/tmp/exports/test-fixture' +IMPORTER_HREF=$(http ${BASE_ADDR}/pulp/api/v3/importers/core/pulp/ \ + name="test/fixture" | jq -r ".pulp_href") +EXPORTED_REPO_PATH=$(find "/tmp/exports/test-fixture" -type f -name \ + "*.tar.gz" | head -n 1) +GROUP_HREF=$(http ${BASE_ADDR}${IMPORTER_HREF}imports/ \ + path=${EXPORTED_REPO_PATH} | jq -r ".task_group") +echo ${BASE_ADDR}${GROUP_HREF} diff --git a/.ci/scripts/pulp_container/manifest_copy.sh b/.ci/scripts/pulp_container/manifest_copy.sh new file mode 100755 index 000000000..92ff2bda9 --- /dev/null +++ b/.ci/scripts/pulp_container/manifest_copy.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Create a task to copy all manifests from source to destination repo." +TASK_HREF=$(http POST $BASE_ADDR$SECOND_REPO_HREF'copy_manifests/' \ + source_repository=$REPO_HREF \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +MANIFEST_COPY_VERSION=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$MANIFEST_COPY_VERSION diff --git a/.ci/scripts/pulp_container/recursive_add_tag.sh b/.ci/scripts/pulp_container/recursive_add_tag.sh new file mode 100755 index 000000000..2794a2f4d --- /dev/null +++ b/.ci/scripts/pulp_container/recursive_add_tag.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +echo "Retrieve the href of Tag manifest_a in the synced repository." +TAG_HREF=$(http $BASE_ADDR'/pulp/api/v3/content/container/tags/?repository_version='$REPOVERSION_HREF'&name=manifest_a' \ + | jq -r '.results | first | .pulp_href') + +echo "Create a task to recursively add a tag to the repo." +TASK_HREF=$(http POST $BASE_ADDR$SECOND_REPO_HREF'add/' \ + content_units:="[\"$TAG_HREF\"]" \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +ADDED_VERSION=$(http $BASE_ADDR$TASK_HREF| jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$ADDED_VERSION diff --git a/.ci/scripts/pulp_container/recursive_remove_tag.sh b/.ci/scripts/pulp_container/recursive_remove_tag.sh new file mode 100755 index 000000000..3d39f6e48 --- /dev/null +++ b/.ci/scripts/pulp_container/recursive_remove_tag.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Create a task to recursively remove the same tag to the repo." +TASK_HREF=$(http POST $BASE_ADDR$SECOND_REPO_HREF'remove/' \ + content_units:="[\"$TAG_HREF\"]" \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +REMOVED_VERSION=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$REMOVED_VERSION diff --git a/.ci/scripts/pulp_container/remote.sh b/.ci/scripts/pulp_container/remote.sh new file mode 100755 index 000000000..37eb3b37f --- /dev/null +++ b/.ci/scripts/pulp_container/remote.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +REMOTE_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) + +echo "Creating $REMOTE_NAME remote that points to an external source of container images." +REMOTE_HREF=$(http POST $BASE_ADDR/pulp/api/v3/remotes/container/container/ \ + name=$REMOTE_NAME \ + url='https://registry-1.docker.io' \ + upstream_name='pulp/test-fixture-1' | jq -r '.pulp_href') + +echo "Inspecting new Remote." +http $BASE_ADDR$REMOTE_HREF diff --git a/.ci/scripts/pulp_container/repo.sh b/.ci/scripts/pulp_container/repo.sh new file mode 100755 index 000000000..19140cb31 --- /dev/null +++ b/.ci/scripts/pulp_container/repo.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +REPO_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) + +echo "Creating a new repository named $REPO_NAME." +REPO_HREF=$(http POST $BASE_ADDR/pulp/api/v3/repositories/container/container/ name=$REPO_NAME \ + | jq -r '.pulp_href') + +echo "Inspecting repository." +http $BASE_ADDR$REPO_HREF diff --git a/.ci/scripts/pulp_container/second_repo.sh b/.ci/scripts/pulp_container/second_repo.sh new file mode 100755 index 000000000..a34c44a49 --- /dev/null +++ b/.ci/scripts/pulp_container/second_repo.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +DEST_REPO_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) + +echo "Create a second repository so we can add content to it." +SECOND_REPO_HREF=$(http POST $BASE_ADDR/pulp/api/v3/repositories/container/container/ name=$DEST_REPO_NAME \ + | jq -r '.pulp_href') + +echo "Inspect repository." +http $BASE_ADDR$SECOND_REPO_HREF diff --git a/.ci/scripts/pulp_container/sync.sh b/.ci/scripts/pulp_container/sync.sh new file mode 100755 index 000000000..9ca91fc7f --- /dev/null +++ b/.ci/scripts/pulp_container/sync.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Create a task to sync the repository using the remote." +TASK_HREF=$(http POST $BASE_ADDR$REPO_HREF'sync/' remote=$REMOTE_HREF mirror=False \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +echo "Set REPOVERSION_HREF from finished task." +REPOVERSION_HREF=$(http $BASE_ADDR$TASK_HREF| jq -r '.created_resources | first') + +echo "Inspecting RepositoryVersion." +http $BASE_ADDR$REPOVERSION_HREF diff --git a/.ci/scripts/pulp_container/tag_copy.sh b/.ci/scripts/pulp_container/tag_copy.sh new file mode 100755 index 000000000..fdf094eb2 --- /dev/null +++ b/.ci/scripts/pulp_container/tag_copy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +echo "Create a task to copy a tag to the repo." +TASK_HREF=$(http POST $BASE_ADDR$SECOND_REPO_HREF'copy_tags/' \ + source_repository=$REPO_HREF \ + names:="[\"manifest_a\"]" \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +TAG_COPY_VERSION=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$TAG_COPY_VERSION diff --git a/.ci/scripts/pulp_tests.sh b/.ci/scripts/pulp_tests.sh index 0c42c9cef..d986d84b4 100755 --- a/.ci/scripts/pulp_tests.sh +++ b/.ci/scripts/pulp_tests.sh @@ -11,6 +11,7 @@ if [[ "${1-}" == "--minikube" ]] || [[ "${1-}" == "-m" ]]; then if [[ "$CI_TEST" == "true" ]]; then SVC_NAME="example-pulp-web-svc" WEB_PORT="24880" + sudo pkill -f "port-forward" || true kubectl port-forward service/$SVC_NAME $WEB_PORT:$WEB_PORT & fi fi @@ -34,27 +35,19 @@ if [ ! -f ~/.config/pulp/settings.toml ]; then base_url = "$BASE_ADDR" verify_ssl = false format = "json" +username = "admin" +password = "password" EOF fi -cat ~/.config/pulp/cli.toml | tee ~/.config/pulp/settings.toml +cat ~/.config/pulp/cli.toml pulp status | jq -pushd pulp_ansible/docs/_scripts -timeout 5m bash -x quickstart.sh || { - YLATEST=$(git ls-remote --heads https://github.com/pulp/pulp_ansible.git | grep -o "[[:digit:]]\.[[:digit:]]*" | sort -V | tail -1) - git fetch --depth=1 origin heads/$YLATEST:$YLATEST - git checkout $YLATEST +pushd $(dirname $0)/pulp_ansible/ timeout 5m bash -x quickstart.sh -} popd -pushd pulp_container/docs/_scripts -timeout 5m bash -x docs_check.sh || { - YLATEST=$(git ls-remote --heads https://github.com/pulp/pulp_container.git | grep -o "[[:digit:]]\.[[:digit:]]*" | sort -V | tail -1) - git fetch --depth=1 origin heads/$YLATEST:$YLATEST - git checkout $YLATEST +pushd $(dirname $0)/pulp_container/ timeout 5m bash -x docs_check.sh -} popd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae493d263..f52bf2a32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,8 +139,6 @@ jobs: sudo popeye --kubeconfig ~/.kube/config || true - name: Test all components run: | - git clone --depth=1 https://github.com/pulp/pulp_ansible.git - git clone --depth=1 https://github.com/pulp/pulp_container.git .ci/scripts/retry.sh 3 ".ci/scripts/pulp_tests.sh -m" shell: bash env: diff --git a/Dockerfile b/Dockerfile index 630c9ca8b..16a88418d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,9 @@ COPY requirements.yml ${HOME}/requirements.yml RUN ansible-galaxy collection install --force -r ${HOME}/requirements.yml \ && chmod -R ug+rwx ${HOME}/.ansible +RUN pip install --upgrade kubernetes && \ + rm -rf /root/.cache/pip + COPY watches.yaml ${HOME}/watches.yaml COPY roles/ ${HOME}/roles/ COPY playbooks/ ${HOME}/playbooks/ diff --git a/roles/postgres/tasks/main.yml b/roles/postgres/tasks/main.yml index ad78e96a8..7164a3d68 100644 --- a/roles/postgres/tasks/main.yml +++ b/roles/postgres/tasks/main.yml @@ -218,7 +218,7 @@ - name: Check PostgreSQL status k8s_info: - api_version: v1 + api_version: apps/v1 kind: StatefulSet namespace: '{{ ansible_operator_meta.namespace }}' name: '{{ ansible_operator_meta.name }}-postgres-{{ postgres_version }}' diff --git a/roles/postgres/tasks/upgrade_postgres.yml b/roles/postgres/tasks/upgrade_postgres.yml index 2578cc905..c20892f2d 100644 --- a/roles/postgres/tasks/upgrade_postgres.yml +++ b/roles/postgres/tasks/upgrade_postgres.yml @@ -218,7 +218,7 @@ - name: Remove old Postgres StatefulSet k8s: kind: StatefulSet - api_version: v1 + api_version: apps/v1 namespace: "{{ ansible_operator_meta.namespace }}" name: "{{ ansible_operator_meta.name }}-postgres" state: absent diff --git a/roles/postgres/templates/postgres.yaml.j2 b/roles/postgres/templates/postgres.yaml.j2 index a8eafa446..e65264197 100644 --- a/roles/postgres/templates/postgres.yaml.j2 +++ b/roles/postgres/templates/postgres.yaml.j2 @@ -1,6 +1,6 @@ # Postgres StatefulSet. --- -apiVersion: v1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: '{{ ansible_operator_meta.name }}-postgres-{{ postgres_version }}' diff --git a/roles/pulp-api/templates/pulp-api.deployment.yaml.j2 b/roles/pulp-api/templates/pulp-api.deployment.yaml.j2 index a19dd912a..66d428a21 100644 --- a/roles/pulp-api/templates/pulp-api.deployment.yaml.j2 +++ b/roles/pulp-api/templates/pulp-api.deployment.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: v1 +apiVersion: apps/v1 kind: Deployment metadata: name: "{{ ansible_operator_meta.name }}-api" @@ -131,7 +131,22 @@ spec: image: "{{ _image }}" imagePullPolicy: "{{ image_pull_policy }}" # We set args, not command, so as to not override the entrypoint script - args: ["pulp-api"] + command: + - /bin/sh + - -c + - | + mkdir -p /var/lib/pulp/media /var/lib/pulp/assets /var/lib/pulp/tmp + pulpcore-manager migrate + ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password + if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then + echo "pulp admin can be initialized." + PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE) + fi + + if [ -n "${PULP_ADMIN_PASSWORD}" ]; then + /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}" + fi + pulp-api env: - name: POSTGRES_SERVICE_HOST value: "{{ postgres_host }}" diff --git a/roles/pulp-content/templates/pulp-content.deployment.yaml.j2 b/roles/pulp-content/templates/pulp-content.deployment.yaml.j2 index 418d1cb7f..a9c7c5c50 100644 --- a/roles/pulp-content/templates/pulp-content.deployment.yaml.j2 +++ b/roles/pulp-content/templates/pulp-content.deployment.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: v1 +apiVersion: apps/v1 kind: Deployment metadata: name: "{{ ansible_operator_meta.name }}-content" diff --git a/roles/pulp-status/tasks/main.yml b/roles/pulp-status/tasks/main.yml index 331b1dcfa..b6192ff5f 100644 --- a/roles/pulp-status/tasks/main.yml +++ b/roles/pulp-status/tasks/main.yml @@ -103,7 +103,7 @@ - name: Remove resource manager deployment kubernetes.core.k8s: - api_version: v1 + api_version: apps/v1 kind: Deployment name: "{{ ansible_operator_meta.name }}-resource-manager" namespace: "{{ ansible_operator_meta.namespace }}" diff --git a/roles/pulp-web/templates/pulp-web.deployment.yaml.j2 b/roles/pulp-web/templates/pulp-web.deployment.yaml.j2 index 4b7093865..9f35d79b8 100644 --- a/roles/pulp-web/templates/pulp-web.deployment.yaml.j2 +++ b/roles/pulp-web/templates/pulp-web.deployment.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: v1 +apiVersion: apps/v1 kind: Deployment metadata: name: "{{ ansible_operator_meta.name }}-web" diff --git a/roles/pulp-worker/templates/pulp-worker.deployment.yaml.j2 b/roles/pulp-worker/templates/pulp-worker.deployment.yaml.j2 index 5c2c13a0a..b389cf897 100644 --- a/roles/pulp-worker/templates/pulp-worker.deployment.yaml.j2 +++ b/roles/pulp-worker/templates/pulp-worker.deployment.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: v1 +apiVersion: apps/v1 kind: Deployment metadata: name: "{{ ansible_operator_meta.name }}-worker" diff --git a/roles/redis/templates/redis.deployment.yaml.j2 b/roles/redis/templates/redis.deployment.yaml.j2 index 6ac36ba1e..ef2aa22c4 100644 --- a/roles/redis/templates/redis.deployment.yaml.j2 +++ b/roles/redis/templates/redis.deployment.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: v1 +apiVersion: apps/v1 kind: Deployment metadata: name: "{{ ansible_operator_meta.name }}-redis"