From 718ceb2c3b089cc98a63d9f64167a3cd252fb668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20=22decko=22=20de=20Brito?= Date: Tue, 19 Sep 2023 16:24:58 -0300 Subject: [PATCH] Split up some responsabilities from the entrypoint to a service container. Closes #544 --- .github/workflows/pulp_images.yml | 6 +-- CHANGES/544.misc | 1 + images/assets/add_signing_service.sh | 9 ++++ images/assets/pulp-api | 41 +---------------- images/assets/pulp-content | 2 +- images/assets/pulp-worker | 2 +- images/assets/set_init_password.sh | 21 +++++++++ images/assets/wait_on_database_migrations.sh | 6 +-- images/compose/compose.yml | 46 ++++++++++++++++++- .../pulp-minimal/nightly/Containerfile.core | 10 ++++ images/pulp-minimal/stable/Containerfile.core | 12 ++++- 11 files changed, 106 insertions(+), 50 deletions(-) create mode 100644 CHANGES/544.misc create mode 100755 images/assets/add_signing_service.sh create mode 100755 images/assets/set_init_password.sh diff --git a/.github/workflows/pulp_images.yml b/.github/workflows/pulp_images.yml index 7f2206dd..669467c3 100644 --- a/.github/workflows/pulp_images.yml +++ b/.github/workflows/pulp_images.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # by default, it uses a depth of 1 # this fetches all history so that we can read each commit @@ -75,7 +75,7 @@ jobs: echo "Building $temp_base_tag" echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # by default, it uses a depth of 1 # this fetches all history so that we can read each commit @@ -162,7 +162,7 @@ jobs: echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # by default, it uses a depth of 1 # this fetches all history so that we can read each commit diff --git a/CHANGES/544.misc b/CHANGES/544.misc new file mode 100644 index 00000000..7584c2c0 --- /dev/null +++ b/CHANGES/544.misc @@ -0,0 +1 @@ +Split some responsabilities into different service containers. diff --git a/images/assets/add_signing_service.sh b/images/assets/add_signing_service.sh new file mode 100755 index 00000000..a0023f56 --- /dev/null +++ b/images/assets/add_signing_service.sh @@ -0,0 +1,9 @@ +#!/bin/bash -xeu + +/usr/bin/wait_on_postgres.py +/usr/bin/wait_on_database_migrations.sh + +if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then + /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" + /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService +fi diff --git a/images/assets/pulp-api b/images/assets/pulp-api index 655d4ea5..f01feb4a 100755 --- a/images/assets/pulp-api +++ b/images/assets/pulp-api @@ -1,44 +1,7 @@ -#!/bin/bash -x - -mkdir -p /var/lib/pulp/media \ - /var/lib/pulp/assets \ - /var/lib/pulp/tmp +#!/bin/bash -eu /usr/bin/wait_on_postgres.py - -# Get list of installed plugins via pip -# Assumes they are all named like "pulp-file" -> "file", with no 2nd dash. -# (Was previously needed when we ran `pulpcore-manager makemigrations`) -# PLUGINS=$(pip list | awk -F '[[:space:]]+|[-]' '/pulp-/{printf $2 " " }') - -/usr/local/bin/pulpcore-manager migrate --noinput - -set +x - -if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then - /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" - /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService -fi - -if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]] -then - PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())") - if [ "$PASSWORD_SET" = "False" ] - then - /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}" - fi -else - ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password - if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then - echo "pulp admin can be initialized." - PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE) - fi - - if [ -n "${PULP_ADMIN_PASSWORD}" ]; then - /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}" - fi -fi -set -x +/usr/bin/wait_on_database_migrations.sh if which pulpcore-api then diff --git a/images/assets/pulp-content b/images/assets/pulp-content index 12550287..863151f5 100755 --- a/images/assets/pulp-content +++ b/images/assets/pulp-content @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash -eu /usr/bin/wait_on_postgres.py /usr/bin/wait_on_database_migrations.sh diff --git a/images/assets/pulp-worker b/images/assets/pulp-worker index f63e33b2..8bd46918 100755 --- a/images/assets/pulp-worker +++ b/images/assets/pulp-worker @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash -eu /usr/bin/wait_on_postgres.py /usr/bin/wait_on_database_migrations.sh diff --git a/images/assets/set_init_password.sh b/images/assets/set_init_password.sh new file mode 100755 index 00000000..f0b4faa2 --- /dev/null +++ b/images/assets/set_init_password.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eu +if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]] +then + PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())") + if [ "$PASSWORD_SET" = "False" ] + then + /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}" + fi +else + ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password + if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then + echo "pulp admin can be initialized." + PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE) + fi + + if [ -n "${PULP_ADMIN_PASSWORD}" ]; then + /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}" + fi +fi +set -x + diff --git a/images/assets/wait_on_database_migrations.sh b/images/assets/wait_on_database_migrations.sh index fa653386..054a991b 100755 --- a/images/assets/wait_on_database_migrations.sh +++ b/images/assets/wait_on_database_migrations.sh @@ -1,8 +1,8 @@ -#!/bin/bash +#!/bin/bash -eu echo "Checking for database migrations" while true; do - /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]' + /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]' &> /dev/null exit_code=$? if [ $exit_code -eq 1 ]; then # grep returning 1 means that the searched-for string was not found. @@ -16,5 +16,5 @@ while true; do # which is probably because the database is not "up enough" to continue yet. echo "Waiting for migration, last exit code $exit_code" fi - sleep 5 + sleep 1 done diff --git a/images/compose/compose.yml b/images/compose/compose.yml index 6eace6d6..8bd4854e 100644 --- a/images/compose/compose.yml +++ b/images/compose/compose.yml @@ -20,6 +20,40 @@ services: timeout: 5s retries: 5 + migration_service: + image: "pulp/pulp-minimal:latest" + restart: never + depends_on: + postgres: + condition: service_healthy + command: pulpcore-manager migrate --noinput + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + + signing_key_service: + image: "pulp/pulp-minimal:latest" + restart: never + command: sh -c "add_signing_service.sh" + depends_on: + migration_service: + condition: service_completed_successfully + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + + admin_password_service: + image: "pulp/pulp-minimal:latest" + command: set_init_password.sh + environment: + PULP_DEFAULT_ADMIN_PASSWORD: password + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + redis: image: "docker.io/library/redis:latest" volumes: @@ -36,7 +70,9 @@ services: command: ['/usr/bin/nginx.sh'] depends_on: pulp_api: + condition: service_healthy pulp_content: + condition: service_healthy ports: - "8080:8080" hostname: pulp @@ -56,14 +92,16 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully + # signing_key_service: + # condition: service_completed_successfully hostname: pulp-api user: pulp volumes: - "./assets/settings.py:/etc/pulp/settings.py:z" - "./assets/certs:/etc/pulp/certs:z" - "pulp:/var/lib/pulp" - environment: - PULP_DEFAULT_ADMIN_PASSWORD: password restart: always healthcheck: test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ] @@ -81,6 +119,8 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully hostname: pulp-content user: pulp volumes: @@ -104,6 +144,8 @@ services: condition: service_healthy postgres: condition: service_healthy + migration_service: + condition: service_completed_successfully user: pulp volumes: - "./assets/settings.py:/etc/pulp/settings.py:z" diff --git a/images/pulp-minimal/nightly/Containerfile.core b/images/pulp-minimal/nightly/Containerfile.core index 69cb582b..bf685f4e 100644 --- a/images/pulp-minimal/nightly/Containerfile.core +++ b/images/pulp-minimal/nightly/Containerfile.core @@ -14,6 +14,16 @@ RUN pip3 install git+https://github.com/pulp/pulpcore.git#egg=pulpcore \ git+https://github.com/pulp/pulp_python.git \ git+https://github.com/pulp/pulp_rpm.git +COPY images/assets/readyz.py /usr/bin/readyz.py +COPY images/assets/route_paths.py /usr/bin/route_paths.py +COPY images/assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py +COPY images/assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh +COPY images/assets/set_init_password.sh /usr/bin/set_init_password.sh +COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh +COPY images/assets/pulp-api /usr/bin/pulp-api +COPY images/assets/pulp-content /usr/bin/pulp-content +COPY images/assets/pulp-worker /usr/bin/pulp-worker + USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link diff --git a/images/pulp-minimal/stable/Containerfile.core b/images/pulp-minimal/stable/Containerfile.core index 8adf72d7..8ec1abc0 100644 --- a/images/pulp-minimal/stable/Containerfile.core +++ b/images/pulp-minimal/stable/Containerfile.core @@ -27,7 +27,17 @@ RUN pip3 install --upgrade \ # Prevent pip-installed /usr/local/bin/pulp-content from getting run instead of # our /usr/bin/pulp-content script. -RUN rm -f /usr/local/bin/pulp-content +# RUN rm -r /usr/local/bin/pulp-content + +COPY images/assets/readyz.py /usr/bin/readyz.py +COPY images/assets/route_paths.py /usr/bin/route_paths.py +COPY images/assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py +COPY images/assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh +COPY images/assets/set_init_password.sh /usr/bin/set_init_password.sh +COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh +COPY images/assets/pulp-api /usr/bin/pulp-api +COPY images/assets/pulp-content /usr/bin/pulp-content +COPY images/assets/pulp-worker /usr/bin/pulp-worker USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \