From 186aca947736e97daf56f518cff950de0710a8a0 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 2 Oct 2024 18:54:25 +1300 Subject: [PATCH] ci: Use PostgreSQL 16 and fix database issues in local/sandboxes (#513) * ci: Copy the new databse dump * build: Use postgresql dump files * build: Use PostgreSQL 16 * ci: force remove image build container --------- Co-authored-by: Nicolas Giard --- .github/workflows/build.yml | 2 +- README.md | 2 +- dev/deploy-to-container/cli.js | 6 +++-- docker-compose.yml | 2 +- ..._convert_native_dump_to_sql_and_restore.sh | 23 ++++++------------- docker/db.Dockerfile | 6 ++--- 6 files changed, 17 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7bb5a18..5b84d27d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -168,7 +168,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.DO_SPACES_DEV_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACES_DEV_ACCESS_KEY }} run: | - aws s3 cp --quiet --endpoint-url=${{ secrets.DO_SPACES_DEV_ENDPOINT }} s3://wagtail/ietfwww.dump ietfwww.dump + aws s3 cp --quiet --endpoint-url=${{ secrets.DO_SPACES_DEV_ENDPOINT }} s3://wagtail/ietfwww.dump ./docker/database/ietfwww.dump aws s3 cp --quiet --endpoint-url=${{ secrets.DO_SPACES_DEV_ENDPOINT }} s3://wagtail/media.tgz media.tgz - name: Deploy to containers diff --git a/README.md b/README.md index 9d0ce360..c2e55539 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ So the only requirement to run it locally is a recent version of Docker with doc ##### How to run (with a database dump) -1. Obtain a recent database dump with name like `ietfa.*.gz` and place in `docker/database/` directory. Otherwise, it will start with a fresh database. +1. Obtain a recent database dump with name like `ietf*.dump` and place in `docker/database/` directory. Otherwise, it will start with a fresh database. 2. Obtain and unarchive media files into `media/` folder. 3. Run `docker compose up`. It will build and start the frontend builder (`yarn run start`) and the backend (`python manage.py runserver` analog), along with a Postgresql database. The first run will take a while because the database dump needs to be restored. 4. After the frontend compilation finishes, the website should become available at http://localhost:8001 diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 30288882..85f8b23d 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -47,11 +47,13 @@ async function main () { src: [ 'docker/db.Dockerfile', 'docker/scripts/db-import.sh', - 'ietfwww.dump' + 'docker/database/ietfwww.dump' ] }, { dockerfile: 'docker/db.Dockerfile', - t: 'ws-db:latest' + t: 'ws-db:latest', + rm: true, + forcerm: true }) await new Promise((resolve, reject) => { dock.modem.followProgress(dbImageBuildStream, (err, res) => err ? reject(err) : resolve(res)) diff --git a/docker-compose.yml b/docker-compose.yml index 1fc897d7..4f656584 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: database: restart: on-failure - image: postgres:14.6 + image: postgres:16 volumes: - "./docker/database:/docker-entrypoint-initdb.d/" environment: diff --git a/docker/database/02_convert_native_dump_to_sql_and_restore.sh b/docker/database/02_convert_native_dump_to_sql_and_restore.sh index 452bc5b5..105c5997 100755 --- a/docker/database/02_convert_native_dump_to_sql_and_restore.sh +++ b/docker/database/02_convert_native_dump_to_sql_and_restore.sh @@ -5,29 +5,20 @@ cd /docker-entrypoint-initdb.d set +e -FILE=$(ls -1 ietfa.*.gz | head) +FILE=$(ls -1 *.dump | head) set -e restore_dump() { - # use docker_process_sql function from docker-entrypoint.sh in the Postgres container - # pg_restore -xO means restore no owner, no permissions - pg_restore -xO -f - | sed -e '/CREATE SCHEMA public/d' | docker_process_sql + # use docker_process_sql function from docker-entrypoint.sh in the Postgres container + # pg_restore -xO means restore no owner, no permissions + cat "$1" | pg_restore -xO -f - | sed -e '/CREATE SCHEMA public/d' | docker_process_sql } if [ -s "${FILE}" ]; then - echo "Found ${FILE}, converting..." - case $FILE in - *.gz) - echo "Restoring the gzipped archive..." - gzip -d -c $FILE | restore_dump - ;; - *) - echo "Restoring the archive..." - cat $FILE | restore_dump - ;; - esac + echo "Restoring the archive..." + restore_dump "$FILE" else - echo "No dump, starting fresh." + echo "No dump, starting fresh." fi diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile index 95c05ec3..f910a1b7 100644 --- a/docker/db.Dockerfile +++ b/docker/db.Dockerfile @@ -1,7 +1,7 @@ # ===================== # --- Builder Stage --- # ===================== -FROM postgres:14.6 AS builder +FROM postgres:16 AS builder ENV POSTGRES_PASSWORD=password ENV POSTGRES_USER=postgres @@ -10,7 +10,7 @@ ENV POSTGRES_HOST_AUTH_METHOD=trust ENV PGDATA=/data COPY docker/scripts/db-import.sh /docker-entrypoint-initdb.d/ -COPY ietfa.torchbox.latest.gz / +COPY docker/database/ietfwww.dump / RUN ["sed", "-i", "s/exec \"$@\"/echo \"skipping...\"/", "/usr/local/bin/docker-entrypoint.sh"] RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"] @@ -18,7 +18,7 @@ RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"] # =================== # --- Final Image --- # =================== -FROM postgres:14.6 +FROM postgres:16 LABEL maintainer="IETF Tools Team " COPY --from=builder /data $PGDATA