Split up some responsabilities from the entrypoint to a service conta…

…iner. Closes #544
pulp · Oct 4, 2023 · 194dac4 · 194dac4
1 parent 4d07c02
commit 194dac4
Show file tree

Hide file tree

Showing 11 changed files with 172 additions and 50 deletions.
diff --git a/.github/workflows/pulp_images.yml b/.github/workflows/pulp_images.yml
@@ -17,7 +17,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # by default, it uses a depth of 1
           # this fetches all history so that we can read each commit
@@ -75,7 +75,7 @@ jobs:
           echo "Building $temp_base_tag"
           echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # by default, it uses a depth of 1
           # this fetches all history so that we can read each commit
@@ -162,7 +162,7 @@ jobs:
           echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV
           echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # by default, it uses a depth of 1
           # this fetches all history so that we can read each commit
@@ -251,6 +251,30 @@ jobs:
           podman stop pulp
           podman rm pulp
 
+      # - name: Apply Django Migrations
+      #   run: |
+      #     if [[ "${{ matrix.app.image_name }}" == "pulp" || "${{ matrix.app.image_name }}" == "galaxy" ]]; then
+      #       # Reuse the folders from the s6 mode tests
+      #       FILE="compose.folders.yml"
+      #       # We'll pull the web image from a registry since we didn't build it.
+      #       if [ "${{ matrix.image_variant }}" == "nightly" ]; then
+      #         WEB_TAG="nightly"
+      #       else
+      #         WEB_TAG="${APP_BRANCH}"
+      #       fi
+      #     else
+      #       FILE="compose.yml"
+      #       WEB_TAG="${TEMP_APP_TAG}"
+      #     fi
+      #     cd images/compose
+      #     sed -i "s/pulp-minimal:latest/${{ matrix.app.image_name }}:${TEMP_APP_TAG}/g" $FILE
+      #     sed -i "s/pulp-web:latest/${{ matrix.app.web_image }}:${WEB_TAG}/g" $FILE
+      #     id | grep "(root)" || sudo usermod -G root $(whoami)
+      #     set -x
+      #     podman-compose -f $FILE up migration_service
+      #   shell: bash
+
+
       - name: Compose up
         run: |
           if [[ "${{ matrix.app.image_name }}" == "pulp" || "${{ matrix.app.image_name }}" == "galaxy" ]]; then
@@ -270,8 +294,10 @@ jobs:
           sed -i "s/pulp-minimal:latest/${{ matrix.app.image_name }}:${TEMP_APP_TAG}/g" $FILE
           sed -i "s/pulp-web:latest/${{ matrix.app.web_image }}:${WEB_TAG}/g" $FILE
           id | grep "(root)" || sudo usermod -G root $(whoami)
+          set -x
           podman-compose -f $FILE up -d
-          sleep 30
+          # sleep 400
+          podman exec compose_pulp_api_1 /usr/bin/wait_on_database_migrations.sh
           for _ in $(seq 20)
           do
             sleep 3
@@ -283,6 +309,10 @@ jobs:
           curl --fail http://localhost:8080/pulp/api/v3/status/ | jq
         shell: bash
 
+      - name: tmate
+        if: failure()
+        uses: mxschmitt/action-tmate@v3
+
       - name: Github login
         if: github.event_name != 'pull_request'
         env:

diff --git a/CHANGES/544.misc b/CHANGES/544.misc
@@ -0,0 +1 @@
+Split some responsabilities into different service containers.
diff --git a/images/Containerfile.core.base b/images/Containerfile.core.base
@@ -105,6 +105,8 @@ COPY images/assets/readyz.py /usr/bin/readyz.py
 COPY images/assets/route_paths.py /usr/bin/route_paths.py
 COPY images/assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py
 COPY images/assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh
+COPY images/assets/set_init_password.sh /usr/bin/set_init_password.sh
+COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh
 COPY images/assets/pulp-api /usr/bin/pulp-api
 COPY images/assets/pulp-content /usr/bin/pulp-content
 COPY images/assets/pulp-worker /usr/bin/pulp-worker

diff --git a/images/assets/add_signing_service.sh b/images/assets/add_signing_service.sh
@@ -0,0 +1,9 @@
+#!/bin/bash -eu
+
+/usr/bin/wait_on_postgres.py
+/usr/bin/wait_on_database_migrations.sh
+
+if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then
+  /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}"
+  /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService
+fi
diff --git a/images/assets/pulp-api b/images/assets/pulp-api
@@ -1,44 +1,7 @@
-#!/bin/bash -x
-
-mkdir -p /var/lib/pulp/media \
-         /var/lib/pulp/assets \
-         /var/lib/pulp/tmp
+#!/bin/bash -eu
 
 /usr/bin/wait_on_postgres.py
-
-# Get list of installed plugins via pip
-# Assumes they are all named like "pulp-file" -> "file", with no 2nd dash.
-# (Was previously needed when we ran `pulpcore-manager makemigrations`)
-# PLUGINS=$(pip list | awk -F '[[:space:]]+|[-]' '/pulp-/{printf $2 " " }')
-
-/usr/local/bin/pulpcore-manager migrate --noinput
-
-set +x
-
-if [ -n "${PULP_SIGNING_KEY_FINGERPRINT}" ]; then
-    /usr/local/bin/pulpcore-manager add-signing-service "${COLLECTION_SIGNING_SERVICE}" /var/lib/pulp/scripts/collection_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}"
-    /usr/local/bin/pulpcore-manager add-signing-service "${CONTAINER_SIGNING_SERVICE}" /var/lib/pulp/scripts/container_sign.sh "${PULP_SIGNING_KEY_FINGERPRINT}" --class container:ManifestSigningService
-fi
-
-if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]]
-then
-  PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())")
-  if [ "$PASSWORD_SET" = "False" ]
-  then
-    /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}"
-  fi
-else
-  ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password
-  if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then
-     echo "pulp admin can be initialized."
-     PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE)
-  fi
-
-  if [ -n "${PULP_ADMIN_PASSWORD}" ]; then
-      /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}"
-  fi
-fi
-set -x
+/usr/bin/wait_on_database_migrations.sh
 
 if which pulpcore-api
 then

diff --git a/images/assets/pulp-content b/images/assets/pulp-content
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -eu
 
 /usr/bin/wait_on_postgres.py
 /usr/bin/wait_on_database_migrations.sh

diff --git a/images/assets/pulp-worker b/images/assets/pulp-worker
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -xeu
 
 /usr/bin/wait_on_postgres.py
 /usr/bin/wait_on_database_migrations.sh

diff --git a/images/assets/set_init_password.sh b/images/assets/set_init_password.sh
@@ -0,0 +1,25 @@
+#!/bin/bash -eu
+
+/usr/bin/wait_on_postgres.py
+/usr/bin/wait_on_database_migrations.sh
+
+if [[ -n "$PULP_DEFAULT_ADMIN_PASSWORD" ]]
+then
+  PASSWORD_SET=$(/usr/local/bin/pulpcore-manager shell -c "from django.contrib.auth import get_user_model; print(get_user_model().objects.filter(username=\"admin\").exists())")
+  if [ "$PASSWORD_SET" = "False" ]
+  then
+    /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_DEFAULT_ADMIN_PASSWORD}"
+  fi
+else
+  ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password
+  if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then
+     echo "pulp admin can be initialized."
+     PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE)
+  fi
+
+  if [ -n "${PULP_ADMIN_PASSWORD}" ]; then
+      /usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}"
+  fi
+fi
+set -x
+
diff --git a/images/assets/wait_on_database_migrations.sh b/images/assets/wait_on_database_migrations.sh
@@ -2,7 +2,7 @@
 
 echo "Checking for database migrations"
 while true; do
-  /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]'
+  /usr/local/bin/pulpcore-manager showmigrations | grep '\[ \]' &> /dev/null
   exit_code=$?
   if [ $exit_code -eq 1 ]; then
     # grep returning 1 means that the searched-for string was not found.

diff --git a/images/compose/compose.folders.yml b/images/compose/compose.folders.yml
@@ -31,6 +31,43 @@ services:
       timeout: 5s
       retries: 5
 
+  migration_service:
+    image: "pulp/pulp-minimal:latest"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    command: pulpcore-manager migrate --noinput
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "../../pulp_storage:/var/lib/pulp:z"
+
+  signing_key_service:
+    image: "pulp/pulp-minimal:latest"
+    command: sh -c "add_signing_service.sh"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      migration_service: 
+        condition: service_completed_successfully
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "../../pulp_storage:/var/lib/pulp:z"
+
+  set_init_password_service:
+    image: "pulp/pulp-minimal:latest"
+    command: set_init_password.sh
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PULP_DEFAULT_ADMIN_PASSWORD: password
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "../../pulp_storage:/var/lib/pulp:z"
+
   pulp_web:
     image: "pulp/pulp-web:latest"
     command: ['/usr/bin/nginx.sh']
@@ -56,14 +93,18 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
+      set_init_password_service:
+        condition: service_completed_successfully
+      signing_key_service:
+        condition: service_completed_successfully
     hostname: pulp-api
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"
       - "./assets/certs:/etc/pulp/certs:z"
       - "../../pulp_storage:/var/lib/pulp:z"
-    environment:
-      PULP_DEFAULT_ADMIN_PASSWORD: password
     restart: always
     healthcheck:
       test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ]
@@ -81,6 +122,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     hostname: pulp-content
     user: pulp
     volumes:
@@ -104,6 +147,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"

diff --git a/images/compose/compose.yml b/images/compose/compose.yml
@@ -31,12 +31,51 @@ services:
       timeout: 5s
       retries: 5
 
+  migration_service:
+    image: "pulp/pulp-minimal:latest"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    command: pulpcore-manager migrate --noinput
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "pulp:/var/lib/pulp"
+
+  # signing_key_service:
+  #   image: "pulp/pulp-minimal:latest"
+  #   command: sh -c "add_signing_service.sh"
+  #   depends_on:
+  #     postgres:
+  #       condition: service_healthy
+  #     migration_service: 
+  #       condition: service_completed_successfully
+  #   volumes:
+  #     - "./assets/settings.py:/etc/pulp/settings.py:z"
+  #     - "./assets/certs:/etc/pulp/certs:z"
+  #     - "pulp:/var/lib/pulp"
+
+  set_init_password_service:
+    image: "pulp/pulp-minimal:latest"
+    command: set_init_password.sh
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PULP_DEFAULT_ADMIN_PASSWORD: password
+    volumes:
+      - "./assets/settings.py:/etc/pulp/settings.py:z"
+      - "./assets/certs:/etc/pulp/certs:z"
+      - "pulp:/var/lib/pulp"
+
   pulp_web:
     image: "pulp/pulp-web:latest"
     command: ['/usr/bin/nginx.sh']
     depends_on:
       pulp_api:
+        condition: service_healthy
       pulp_content:
+        condition: service_healthy
     ports:
       - "8080:8080"
     hostname: pulp
@@ -56,14 +95,18 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
+      set_init_password_service:
+        condition: service_completed_successfully
+      # signing_key_service:
+      #   condition: service_completed_successfully
     hostname: pulp-api
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"
       - "./assets/certs:/etc/pulp/certs:z"
       - "pulp:/var/lib/pulp"
-    environment:
-      PULP_DEFAULT_ADMIN_PASSWORD: password
     restart: always
     healthcheck:
       test: [ "CMD-SHELL", "readyz.py /pulp/api/v3/status/" ]
@@ -81,6 +124,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     hostname: pulp-content
     user: pulp
     volumes:
@@ -104,6 +149,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      migration_service:
+        condition: service_completed_successfully
     user: pulp
     volumes:
       - "./assets/settings.py:/etc/pulp/settings.py:z"