From ee0d8be4303080234b124c86e2e370ada443ffbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20=22decko=22=20de=20Brito?= Date: Tue, 19 Sep 2023 16:24:58 -0300 Subject: [PATCH] Split up some responsabilities from the entrypoint to a service container. Closes #544 --- .github/workflows/pulp_images.yml | 10 ++-- CHANGES/544.misc | 1 + images/Containerfile.core.base | 2 + images/assets/add_signing_service.sh | 9 ++++ images/assets/pulp-api | 41 +--------------- images/assets/pulp-content | 2 +- images/assets/pulp-worker | 2 +- images/assets/set_init_password.sh | 25 ++++++++++ images/assets/wait_on_database_migrations.sh | 2 +- images/compose/compose.folders.yml | 49 ++++++++++++++++++- images/compose/compose.yml | 51 +++++++++++++++++++- 11 files changed, 143 insertions(+), 51 deletions(-) create mode 100644 CHANGES/544.misc create mode 100755 images/assets/add_signing_service.sh create mode 100755 images/assets/set_init_password.sh diff --git a/.github/workflows/pulp_images.yml b/.github/workflows/pulp_images.yml index 7f2206dd..eed5a55e 100644 --- a/.github/workflows/pulp_images.yml +++ b/.github/workflows/pulp_images.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # by default, it uses a depth of 1 # this fetches all history so that we can read each commit @@ -75,7 +75,7 @@ jobs: echo "Building $temp_base_tag" echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # by default, it uses a depth of 1 # this fetches all history so that we can read each commit @@ -162,7 +162,7 @@ jobs: echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # by default, it uses a depth of 1 # this fetches all history so that we can read each commit @@ -271,14 +271,14 @@ jobs: sed -i "s/pulp-web:latest/${{ matrix.app.web_image }}:${WEB_TAG}/g" $FILE id | grep "(root)" || sudo usermod -G root $(whoami) podman-compose -f $FILE up -d - sleep 30 + podman exec compose_pulp_api_1 /usr/bin/wait_on_database_migrations.sh for _ in $(seq 20) do - sleep 3 if curl --fail http://localhost:8080/pulp/api/v3/status/ > /dev/null 2>&1 then break fi + sleep 3 done curl --fail http://localhost:8080/pulp/api/v3/status/ | jq shell: bash diff --git a/CHANGES/544.misc b/CHANGES/544.misc new file mode 100644 index 00000000..7584c2c0 --- /dev/null +++ b/CHANGES/544.misc @@ -0,0 +1 @@ +Split some responsabilities into different service containers. diff --git a/images/Containerfile.core.base b/images/Containerfile.core.base index 6957fb46..14967f3e 100644 --- a/images/Containerfile.core.base +++ b/images/Containerfile.core.base @@ -105,6 +105,8 @@ COPY images/assets/readyz.py /usr/bin/readyz.py COPY images/assets/route_paths.py /usr/bin/route_paths.py COPY images/assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py COPY images/assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh +COPY images/assets/set_init_password.sh /usr/bin/set_init_password.sh +COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh COPY images/assets/pulp-api /usr/bin/pulp-api COPY images/assets/pulp-content /usr/bin/pulp-content COPY images/assets/pulp-worker /usr/bin/pulp-worker diff --git a/images/assets/add_signing_service.sh b/images/assets/add_signing_service.sh new file mode 100755 index 00000000..c40b07b1 --- /dev/null +++ b/images/assets/add_signing_service.sh @@ -0,0 +1,9 @@ +#!/bin/bash -eu + +/usr/bin/wait_on_postgres.py +/usr/bin/wait_on_database_migrations.sh + +if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then + /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" + /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService +fi diff --git a/images/assets/pulp-api b/images/assets/pulp-api index 655d4ea5..f01feb4a 100755 --- a/images/assets/pulp-api +++ b/images/assets/pulp-api @@ -1,44 +1,7 @@ -#!/bin/bash -x - -mkdir -p /var/lib/pulp/media \ - /var/lib/pulp/assets \ - /var/lib/pulp/tmp +#!/bin/bash -eu /usr/bin/wait_on_postgres.py - -# Get list of installed plugins via pip -# Assumes they are all named like "pulp-file" -> "file", with no 2nd dash. -# (Was previously needed when we ran `pulpcore-manager makemigrations`) -# PLUGINS=$(pip list | awk -F '[[:space:]]+|[-]' '/pulp-/{printf $2 " " }') - -/usr/local/bin/pulpcore-manager migrate --noinput - -set +x - -if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then - /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" - /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService -fi - -if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]] -then - PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())") - if [ "$PASSWORD_SET" = "False" ] - then - /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}" - fi -else - ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password - if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then - echo "pulp admin can be initialized." - PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE) - fi - - if [ -n "${PULP_ADMIN_PASSWORD}" ]; then - /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}" - fi -fi -set -x +/usr/bin/wait_on_database_migrations.sh if which pulpcore-api then diff --git a/images/assets/pulp-content b/images/assets/pulp-content index 12550287..863151f5 100755 --- a/images/assets/pulp-content +++ b/images/assets/pulp-content @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash -eu /usr/bin/wait_on_postgres.py /usr/bin/wait_on_database_migrations.sh diff --git a/images/assets/pulp-worker b/images/assets/pulp-worker index f63e33b2..8bd46918 100755 --- a/images/assets/pulp-worker +++ b/images/assets/pulp-worker @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash -eu /usr/bin/wait_on_postgres.py /usr/bin/wait_on_database_migrations.sh diff --git a/images/assets/set_init_password.sh b/images/assets/set_init_password.sh new file mode 100755 index 00000000..262f232e --- /dev/null +++ b/images/assets/set_init_password.sh @@ -0,0 +1,25 @@ +#!/bin/bash -eu + +/usr/bin/wait_on_postgres.py +/usr/bin/wait_on_database_migrations.sh + +if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]] +then + PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())") + if [ "$PASSWORD_SET" = "False" ] + then + /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}" + fi +else + ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password + if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then + echo "pulp admin can be initialized." + PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE) + fi + + if [ -n "${PULP_ADMIN_PASSWORD}" ]; then + /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}" + fi +fi +set -x + diff --git a/images/assets/wait_on_database_migrations.sh b/images/assets/wait_on_database_migrations.sh index fa653386..25b7cdff 100755 --- a/images/assets/wait_on_database_migrations.sh +++ b/images/assets/wait_on_database_migrations.sh @@ -2,7 +2,7 @@ echo "Checking for database migrations" while true; do - /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]' + /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]' &> /dev/null exit_code=$? if [ $exit_code -eq 1 ]; then # grep returning 1 means that the searched-for string was not found. diff --git a/images/compose/compose.folders.yml b/images/compose/compose.folders.yml index ca07d7d9..37cb2b50 100644 --- a/images/compose/compose.folders.yml +++ b/images/compose/compose.folders.yml @@ -31,6 +31,43 @@ services: timeout: 5s retries: 5 + migration_service: + image: "pulp/pulp-minimal:latest" + depends_on: + postgres: + condition: service_healthy + command: pulpcore-manager migrate --noinput + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "../../pulp_storage:/var/lib/pulp:z" + + signing_key_service: + image: "pulp/pulp-minimal:latest" + command: sh -c "add_signing_service.sh" + depends_on: + postgres: + condition: service_healthy + migration_service: + condition: service_completed_successfully + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "../../pulp_storage:/var/lib/pulp:z" + + set_init_password_service: + image: "pulp/pulp-minimal:latest" + command: set_init_password.sh + depends_on: + postgres: + condition: service_healthy + environment: + PULP_DEFAULT_ADMIN_PASSWORD: password + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "../../pulp_storage:/var/lib/pulp:z" + pulp_web: image: "pulp/pulp-web:latest" command: ['/usr/bin/nginx.sh'] @@ -56,14 +93,18 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully + set_init_password_service: + condition: service_completed_successfully + signing_key_service: + condition: service_completed_successfully hostname: pulp-api user: pulp volumes: - "./assets/settings.py:/etc/pulp/settings.py:z" - "./assets/certs:/etc/pulp/certs:z" - "../../pulp_storage:/var/lib/pulp:z" - environment: - PULP_DEFAULT_ADMIN_PASSWORD: password restart: always healthcheck: test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ] @@ -81,6 +122,8 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully hostname: pulp-content user: pulp volumes: @@ -104,6 +147,8 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully user: pulp volumes: - "./assets/settings.py:/etc/pulp/settings.py:z" diff --git a/images/compose/compose.yml b/images/compose/compose.yml index 6eace6d6..187c5f7e 100644 --- a/images/compose/compose.yml +++ b/images/compose/compose.yml @@ -31,12 +31,51 @@ services: timeout: 5s retries: 5 + migration_service: + image: "pulp/pulp-minimal:latest" + depends_on: + postgres: + condition: service_healthy + command: pulpcore-manager migrate --noinput + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + + signing_key_service: + image: "pulp/pulp-minimal:latest" + command: sh -c "add_signing_service.sh" + depends_on: + postgres: + condition: service_healthy + migration_service: + condition: service_completed_successfully + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + + set_init_password_service: + image: "pulp/pulp-minimal:latest" + command: set_init_password.sh + depends_on: + postgres: + condition: service_healthy + environment: + PULP_DEFAULT_ADMIN_PASSWORD: password + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + pulp_web: image: "pulp/pulp-web:latest" command: ['/usr/bin/nginx.sh'] depends_on: pulp_api: + condition: service_healthy pulp_content: + condition: service_healthy ports: - "8080:8080" hostname: pulp @@ -56,14 +95,18 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully + set_init_password_service: + condition: service_completed_successfully + signing_key_service: + condition: service_completed_successfully hostname: pulp-api user: pulp volumes: - "./assets/settings.py:/etc/pulp/settings.py:z" - "./assets/certs:/etc/pulp/certs:z" - "pulp:/var/lib/pulp" - environment: - PULP_DEFAULT_ADMIN_PASSWORD: password restart: always healthcheck: test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ] @@ -81,6 +124,8 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully hostname: pulp-content user: pulp volumes: @@ -104,6 +149,8 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully user: pulp volumes: - "./assets/settings.py:/etc/pulp/settings.py:z"