Merge pull request #551 from decko/new_services_container

Split up some responsabilities from the entrypoint to a service container.

.github/workflows/pulp_images.yml

@@ -17,7 +17,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # by default, it uses a depth of 1
           # this fetches all history so that we can read each commit
@@ -75,7 +75,7 @@ jobs:
           echo "Building $temp_base_tag"
           echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # by default, it uses a depth of 1
           # this fetches all history so that we can read each commit
@@ -162,7 +162,7 @@ jobs:
           echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV
           echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # by default, it uses a depth of 1
           # this fetches all history so that we can read each commit
@@ -271,14 +271,14 @@ jobs:
           sed -i "s/pulp-web:latest/${{ matrix.app.web_image }}:${WEB_TAG}/g" $FILE
           id | grep "(root)" || sudo usermod -G root $(whoami)
           podman-compose -f $FILE up -d
-          sleep 30
+          podman exec compose_pulp_api_1 /usr/bin/wait_on_database_migrations.sh
           for _ in $(seq 20)
           do
-            sleep 3
             if curl --fail http://localhost:8080/pulp/api/v3/status/ > /dev/null 2>&1
             then
               break
             fi
+            sleep 3
           done
           curl --fail http://localhost:8080/pulp/api/v3/status/ | jq
         shell: bash

CHANGES/544.misc

@@ -0,0 +1 @@
+Split some responsabilities into different service containers.

images/Containerfile.core.base

@@ -105,6 +105,8 @@ COPY images/assets/readyz.py /usr/bin/readyz.py
 COPY images/assets/route_paths.py /usr/bin/route_paths.py
 COPY images/assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py
 COPY images/assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh
+COPY images/assets/set_init_password.sh /usr/bin/set_init_password.sh
+COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh
 COPY images/assets/pulp-api /usr/bin/pulp-api
 COPY images/assets/pulp-content /usr/bin/pulp-content
 COPY images/assets/pulp-worker /usr/bin/pulp-worker

images/assets/add_signing_service.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -eu
+
+/usr/bin/wait_on_postgres.py
+/usr/bin/wait_on_database_migrations.sh
+
+if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then
+  /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}"
+  /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService
+fi

images/assets/pulp-api

@@ -1,44 +1,7 @@
-#!/bin/bash -x
-
-mkdir -p /var/lib/pulp/media \
-         /var/lib/pulp/assets \
-         /var/lib/pulp/tmp
+#!/bin/bash -eu
 
 /usr/bin/wait_on_postgres.py
-
-# Get list of installed plugins via pip
-# Assumes they are all named like "pulp-file" -> "file", with no 2nd dash.
-# (Was previously needed when we ran `pulpcore-manager makemigrations`)
-# PLUGINS=$(pip list | awk -F '[[:space:]]+|[-]' '/pulp-/{printf $2 " " }')
-
-/usr/local/bin/pulpcore-manager migrate --noinput
-
-set +x
-
-if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then
-    /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}"
-    /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService
-fi
-
-if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]]
-then
-  PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())")
-  if [ "$PASSWORD_SET" = "False" ]
-  then
-    /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}"
-  fi
-else
-  ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password
-  if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then
-     echo "pulp admin can be initialized."
-     PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE)
-  fi
-
-  if [ -n "${PULP_ADMIN_PASSWORD}" ]; then
-      /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}"
-  fi
-fi
-set -x
+/usr/bin/wait_on_database_migrations.sh
 
 if which pulpcore-api
 then

images/assets/pulp-content

@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -eu
 
 /usr/bin/wait_on_postgres.py
 /usr/bin/wait_on_database_migrations.sh

images/assets/pulp-worker

@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -eu
 
 /usr/bin/wait_on_postgres.py
 /usr/bin/wait_on_database_migrations.sh

images/assets/set_init_password.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -eu
+
+/usr/bin/wait_on_postgres.py
+/usr/bin/wait_on_database_migrations.sh
+
+if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]]
+then
+  PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())")
+  if [ "$PASSWORD_SET" = "False" ]
+  then
+    /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}"
+  fi
+else
+  ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password
+  if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then
+     echo "pulp admin can be initialized."
+     PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE)
+  fi
+
+  if [ -n "${PULP_ADMIN_PASSWORD}" ]; then
+      /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}"
+  fi
+fi
+set -x
+

images/assets/wait_on_database_migrations.sh

@@ -2,7 +2,7 @@
 
 echo "Checking for database migrations"
 while true; do
-  /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]'
+  /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]' &> /dev/null
   exit_code=$?
   if [ $exit_code -eq 1 ]; then
     # grep returning 1 means that the searched-for string was not found.

images/compose/compose.folders.yml

@@ -31,6 +31,43 @@ services:
       timeout: 5s
       retries: 5
 
+  migration_service:
+    image: "pulp/pulp-minimal:latest"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    command: pulpcore-manager migrate --noinput
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "../../pulp_storage:/var/lib/pulp:z"
+
+  signing_key_service:
+    image: "pulp/pulp-minimal:latest"
+    command: sh -c "add_signing_service.sh"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      migration_service: 
+        condition: service_completed_successfully
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "../../pulp_storage:/var/lib/pulp:z"
+
+  set_init_password_service:
+    image: "pulp/pulp-minimal:latest"
+    command: set_init_password.sh
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PULP_DEFAULT_ADMIN_PASSWORD: password
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "../../pulp_storage:/var/lib/pulp:z"
+
   pulp_web:
     image: "pulp/pulp-web:latest"
     command: ['/usr/bin/nginx.sh']
@@ -56,14 +93,18 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
+      set_init_password_service:
+        condition: service_completed_successfully
+      signing_key_service:
+        condition: service_completed_successfully
     hostname: pulp-api
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"
       - "./assets/certs:/etc/pulp/certs:z"
       - "../../pulp_storage:/var/lib/pulp:z"
-    environment:
-      PULP_DEFAULT_ADMIN_PASSWORD: password
     restart: always
     healthcheck:
       test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ]
@@ -81,6 +122,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     hostname: pulp-content
     user: pulp
     volumes:
@@ -104,6 +147,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"

images/compose/compose.yml

@@ -31,12 +31,51 @@ services:
       timeout: 5s
       retries: 5
 
+  migration_service:
+    image: "pulp/pulp-minimal:latest"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    command: pulpcore-manager migrate --noinput
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "pulp:/var/lib/pulp"
+
+  signing_key_service:
+    image: "pulp/pulp-minimal:latest"
+    command: sh -c "add_signing_service.sh"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      migration_service: 
+        condition: service_completed_successfully
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "pulp:/var/lib/pulp"
+
+  set_init_password_service:
+    image: "pulp/pulp-minimal:latest"
+    command: set_init_password.sh
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PULP_DEFAULT_ADMIN_PASSWORD: password
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "pulp:/var/lib/pulp"
+
   pulp_web:
     image: "pulp/pulp-web:latest"
     command: ['/usr/bin/nginx.sh']
     depends_on:
       pulp_api:
+        condition: service_healthy
       pulp_content:
+        condition: service_healthy
     ports:
       - "8080:8080"
     hostname: pulp
@@ -56,14 +95,18 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
+      set_init_password_service:
+        condition: service_completed_successfully
+      signing_key_service:
+        condition: service_completed_successfully
     hostname: pulp-api
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"
       - "./assets/certs:/etc/pulp/certs:z"
       - "pulp:/var/lib/pulp"
-    environment:
-      PULP_DEFAULT_ADMIN_PASSWORD: password
     restart: always
     healthcheck:
       test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ]
@@ -81,6 +124,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     hostname: pulp-content
     user: pulp
     volumes:
@@ -104,6 +149,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"