Skip to content

Commit

Permalink
ci: merge main to release (#7449)
Browse files Browse the repository at this point in the history
ci: merge main to release
  • Loading branch information
rjsparks authored May 21, 2024
2 parents be763a6 + 6f8df32 commit 36446f3
Show file tree
Hide file tree
Showing 79 changed files with 3,274 additions and 1,562 deletions.
6 changes: 6 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,9 @@ indent_size = 2

[ietf/**.html]
insert_final_newline = false

# Settings for Kubernetes yaml
# ---------------------------------------------------------
# Use 2-space indents
[k8s/**.yaml]
indent_size = 2
101 changes: 39 additions & 62 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ on:
default: true
required: true
type: boolean
sandboxNoDbRefresh:
description: 'Sandbox Disable Daily DB Refresh'
deploy:
description: 'Deploy to Staging / Prod'
default: false
required: true
type: boolean
legacySandbox:
description: 'Deploy to Legacy Sandbox'
sandboxNoDbRefresh:
description: 'Sandbox Disable Daily DB Refresh'
default: false
required: false
required: true
type: boolean
skiptests:
description: 'Skip Tests'
Expand Down Expand Up @@ -161,7 +161,7 @@ jobs:

- name: Download a Coverage Results
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
uses: actions/download-artifact@v4.1.4
uses: actions/download-artifact@v4.1.7
with:
name: coverage

Expand Down Expand Up @@ -220,7 +220,7 @@ jobs:
.devcontainer
.github
.vscode
helm
k8s
playwright
svn-history
docker-compose.yml
Expand Down Expand Up @@ -323,7 +323,7 @@ jobs:
steps:
- name: Notify on Slack (Success)
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
uses: slackapi/slack-github-action@v1.25.0
uses: slackapi/slack-github-action@v1.26.0
with:
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
payload: |
Expand All @@ -346,7 +346,7 @@ jobs:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
- name: Notify on Slack (Failure)
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
uses: slackapi/slack-github-action@v1.25.0
uses: slackapi/slack-github-action@v1.26.0
with:
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
payload: |
Expand Down Expand Up @@ -385,7 +385,7 @@ jobs:
- uses: actions/checkout@v4

- name: Download a Release Artifact
uses: actions/download-artifact@v4.1.4
uses: actions/download-artifact@v4.1.7
with:
name: release-${{ env.PKG_VERSION }}

Expand All @@ -407,62 +407,39 @@ jobs:
DEBIAN_FRONTEND: noninteractive
run: |
docker image prune -a -f
legacySandbox:
name: Deploy to Legacy Sandbox
if: ${{ !failure() && !cancelled() && github.event.inputs.legacySandbox == 'true' }}
# -----------------------------------------------------------------
# STAGING
# -----------------------------------------------------------------
staging:
name: Deploy to Staging
if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }}
needs: [prepare, release]
runs-on: [self-hosted, legacy-sandbox-server]
runs-on: ubuntu-latest
environment:
name: legacy-sandbox
url: "https://sandbox.ietf.org"
name: staging
env:
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}

steps:
- name: Download a Release Artifact
uses: actions/download-artifact@v4.1.4
with:
name: release-${{ env.PKG_VERSION }}
path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
- name: Deploy to staging
run: |
curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}'
- name: Extract Release
env:
DEBIAN_FRONTEND: noninteractive
working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
run: |
echo "Extracting release tarball..."
tar xzf release.tar.gz
echo "Deleting release tarball..."
rm -rf release.tar.gz
- name: Setup Environment
env:
DEBIAN_FRONTEND: noninteractive
working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
run: |
echo "Copying settings from previous deploy..."
cp ../web/ietf/settings_local.py ietf/
rsync -a ../web/test/ test/
echo "Installing Python dependencies..."
python3.9 -mvenv env
source env/bin/activate
pip install -r requirements.txt
pip freeze > frozen-requirements.txt
echo "Collecting static..."
ietf/manage.py collectstatic
echo "Running checks..."
ietf/manage.py check
- name: Update Docker Containers
env:
DEBIAN_FRONTEND: noninteractive
working-directory: /a/docker/datatracker
run: |
echo "Pulling latest docker images..."
docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback
docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback
docker-compose pull
# echo "Shutting down containers..."
# docker-compose down -t 300
# -----------------------------------------------------------------
# PROD
# -----------------------------------------------------------------
prod:
name: Deploy to Production
if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }}
needs: [staging]
runs-on: ubuntu-latest
environment:
name: production
env:
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}

steps:
- name: Deploy to production
run: |
curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}'
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
path: geckodriver.log

- name: Upload Coverage Results to Codecov
uses: codecov/codecov-action@v4.2.0
uses: codecov/codecov-action@v4.3.1
with:
files: coverage.xml

Expand Down
10 changes: 0 additions & 10 deletions bin/daily
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,10 @@ cd $DTDIR/

logger -p user.info -t cron "Running $DTDIR/bin/daily"

# Run the hourly jobs first
$DTDIR/bin/hourly

# Set up the virtual environment
source $DTDIR/env/bin/activate


# Update our information about the current version of some commands we use
$DTDIR/ietf/manage.py update_external_command_info

# Get IANA-registered yang models
#YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
# Hardcode the rsync target to avoid any unwanted deletes:
Expand All @@ -43,9 +37,5 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0
# Re-run yang checks on active documents
$DTDIR/ietf/manage.py run_yang_model_checks -v0

# Expire last calls
# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
$DTDIR/ietf/bin/expire-last-calls

# Purge older PersonApiKeyEvents
$DTDIR/ietf/manage.py purge_old_personal_api_key_events 14
2 changes: 1 addition & 1 deletion bin/dump-to-names-json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

set -x
ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \
mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \
mailtrigger.MailTrigger mailtrigger.Recipient name \
group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \
| jq --sort-keys "sort_by(.model, .pk)" \
| jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json
40 changes: 0 additions & 40 deletions bin/hourly

This file was deleted.

56 changes: 39 additions & 17 deletions dev/build/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,17 +1,39 @@
FROM ghcr.io/ietf-tools/datatracker-app-base:latest
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"

ENV DEBIAN_FRONTEND=noninteractive

COPY . .
COPY ./dev/build/start.sh ./start.sh
RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt
RUN chmod +x start.sh && \
chmod +x docker/scripts/app-create-dirs.sh && \
sh ./docker/scripts/app-create-dirs.sh

VOLUME [ "/assets" ]

EXPOSE 8000

CMD ["./start.sh"]
FROM ghcr.io/ietf-tools/datatracker-app-base:latest
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"

ENV DEBIAN_FRONTEND=noninteractive

# uid 498 = wwwrun and gid 496 = www on ietfa
RUN groupadd -g 1000 datatracker && \
useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker

RUN apt-get purge -y imagemagick imagemagick-6-common

# Install libreoffice (needed via PPT2PDF_COMMAND)
RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \
apt-get update && \
apt-get -qyt bullseye-backports install libreoffice-nogui

COPY . .
COPY ./dev/build/start.sh ./start.sh
COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh
COPY ./dev/build/celery-start.sh ./celery-start.sh

RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \
echo '# empty' > ietf/settings_local.py && \
ietf/manage.py patch_libraries && \
rm -f ietf/settings_local.py

RUN chmod +x start.sh && \
chmod +x datatracker-start.sh && \
chmod +x celery-start.sh && \
chmod +x docker/scripts/app-create-dirs.sh && \
sh ./docker/scripts/app-create-dirs.sh

RUN mkdir -p /a

VOLUME [ "/a" ]

EXPOSE 8000

CMD ["./start.sh"]
22 changes: 22 additions & 0 deletions dev/build/celery-start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash
#
# Run a celery worker
#
echo "Running Datatracker checks..."
./ietf/manage.py check

cleanup () {
# Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
if [[ -n "${celery_pid}" ]]; then
echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..."
kill -TERM "${celery_pid}"
wait "${celery_pid}"
fi
}

trap 'trap "" TERM; cleanup' TERM

# start celery in the background so we can trap the TERM signal
celery "$@" &
celery_pid=$!
wait "${celery_pid}"
17 changes: 17 additions & 0 deletions dev/build/datatracker-start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

echo "Running Datatracker checks..."
./ietf/manage.py check

echo "Running Datatracker migrations..."
./ietf/manage.py migrate --settings=settings_local

echo "Starting Datatracker..."

gunicorn \
--workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \
--max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \
--timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \
--bind :8000 \
--log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \
ietf.wsgi:application
28 changes: 19 additions & 9 deletions dev/build/start.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
#!/bin/bash

echo "Running Datatracker checks..."
./ietf/manage.py check

echo "Running Datatracker migrations..."
./ietf/manage.py migrate --settings=settings_local

echo "Starting Datatracker..."
./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local
#
# Environment config:
#
# CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker)
#
case "${CONTAINER_ROLE:-datatracker}" in
datatracker)
exec ./datatracker-start.sh
;;
celery)
exec ./celery-start.sh --app=ietf worker
;;
beat)
exec ./celery-start.sh --app=ietf beat
;;
*)
echo "Unknown role '${CONTAINER_ROLE}'"
exit 255
esac
2 changes: 1 addition & 1 deletion dev/deploy-to-container/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import Docker from 'dockerode'
import path from 'path'
import fs from 'fs-extra'
import tar from 'tar'
import * as tar from 'tar'
import yargs from 'yargs/yargs'
import { hideBin } from 'yargs/helpers'
import slugify from 'slugify'
Expand Down
Loading

0 comments on commit 36446f3

Please sign in to comment.