Merge branch 'main' into dependabot/pip/elasticsearch-gte-7-and-lt-9

.github/workflows/ci.yml

@@ -17,6 +17,18 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v2
+      - name: Install platform dependencies
+        run: |
+          sudo apt -y update
+          sudo apt -y install --no-install-recommends \
+              texlive-latex-base \
+              texlive-latex-recommended \
+              texlive-plain-generic \
+              lmodern
+      - name: Install pandoc
+        run: |
+          wget https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-amd64.deb
+          sudo dpkg -i pandoc-2.17.1.1-1-amd64.deb
       - uses: actions/setup-python@v2
         with:
           python-version: 3.9.16

Dockerfile

@@ -1,9 +1,48 @@
-FROM python:3.9-bullseye
+FROM python:3.9-bookworm
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
+
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
+
+# Install System level build requirements, this is done before
+# everything else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        texlive-latex-base \
+        texlive-latex-recommended \
+        texlive-fonts-recommended \
+        texlive-plain-generic \
+        lmodern
+
+RUN case $(uname -m) in \
+         "x86_64")  ARCH=amd64 ;; \
+         "aarch64")  ARCH=arm64 ;; \
+    esac \
+    && wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \
+    && dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb
+
 RUN mkdir /code
 WORKDIR /code
+
 COPY dev-requirements.txt /code/
 COPY base-requirements.txt /code/
-RUN pip install -r dev-requirements.txt
+COPY prod-requirements.txt /code/
+COPY requirements.txt /code/
+
+RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    set -x \
+    && pip --disable-pip-version-check \
+        install \
+        -r dev-requirements.txt
+
 COPY . /code/

Dockerfile.cabotage

@@ -0,0 +1,49 @@
+FROM python:3.9-bullseye
+COPY --from=ewdurbin/nginx-static:1.25.x /usr/bin/nginx /usr/bin/nginx
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
+
+# Install System level build requirements, this is done before
+# everything else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        texlive-latex-base \
+        texlive-latex-recommended \
+        texlive-fonts-recommended \
+        texlive-plain-generic \
+        lmodern
+
+RUN case $(uname -m) in \
+         "x86_64")  ARCH=amd64 ;; \
+         "aarch64")  ARCH=arm64 ;; \
+    esac \
+    && wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \
+    && dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb
+
+RUN mkdir /code
+WORKDIR /code
+
+COPY dev-requirements.txt /code/
+COPY base-requirements.txt /code/
+COPY prod-requirements.txt /code/
+COPY requirements.txt /code/
+
+RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    set -x \
+    && pip --disable-pip-version-check \
+        install \
+        -r requirements.txt -r prod-requirements.txt
+COPY . /code/
+RUN DJANGO_SETTINGS_MODULE=pydotorg.settings.static python manage.py collectstatic --noinput

Procfile

@@ -1,2 +1,4 @@
 release: python manage.py migrate --noinput
 web: bin/start-nginx gunicorn -c gunicorn.conf pydotorg.wsgi
+worker: celery -A pydotorg worker -l INFO
+worker-beat: celery -A pydotorg beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler

base-requirements.txt

@@ -19,6 +19,8 @@ feedparser==6.0.8
 beautifulsoup4==4.11.2
 icalendar==4.0.7
 chardet==4.0.0
+celery[redis]==5.3.6
+django-celery-beat==2.5.0
 # TODO: We may drop 'django-imagekit' completely.
 django-imagekit==4.0.2
 django-haystack==3.2.1
@@ -44,12 +46,11 @@ django-filter==2.4.0
 django-ordered-model==3.4.3
 django-widget-tweaks==1.4.8
 django-countries==7.2.1
-xhtml2pdf==0.2.5
-django-easy-pdf3==0.1.2
 num2words==0.5.10
 django-polymorphic==3.0.0
 sorl-thumbnail==12.7.0
-docxtpl==0.12.0
-reportlab==3.6.6
 django-extensions==3.1.4
 django-import-export==2.7.1
+
+pypandoc==1.12
+panflute==2.3.0

bin/start-nginx

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+psmgr=/tmp/nginx-buildpack-wait
+rm -f $psmgr
+mkfifo $psmgr
+
+n=1
+while getopts :f option ${@:1:2}
+do
+  case "${option}"
+  in
+    f) FORCE=$OPTIND; n=$((n+1));;
+  esac
+done
+
+# Initialize log directory.
+mkdir -p /tmp/logs/nginx
+touch /tmp/logs/nginx/access.log /tmp/logs/nginx/error.log
+echo 'buildpack=nginx at=logs-initialized'
+
+# Start log redirection.
+(
+  # Redirect nginx logs to stdout.
+  tail -qF -n 0 /tmp/logs/nginx/*.log
+  echo 'logs' >$psmgr
+) &
+
+# Start App Server
+(
+  # Take the command passed to this bin and start it.
+  # E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb
+  COMMAND=${@:$n}
+  echo "buildpack=nginx at=start-app cmd=$COMMAND"
+  $COMMAND
+  echo 'app' >$psmgr
+) &
+
+if [[ -z "$FORCE" ]]
+then
+  FILE="/tmp/app-initialized"
+
+  # We block on app-initialized so that when nginx binds to $PORT
+  # are app is ready for traffic.
+  while [[ ! -f "$FILE" ]]
+  do
+    echo 'buildpack=nginx at=app-initialization'
+    sleep 1
+  done
+  echo 'buildpack=nginx at=app-initialized'
+fi
+
+# Start nginx
+(
+  # We expect nginx to run in foreground.
+  # We also expect a socket to be at /tmp/nginx.socket.
+  echo 'buildpack=nginx at=nginx-start'
+  cd /tmp
+  /usr/bin/nginx -p . -c /code/config/nginx.conf
+  echo 'nginx' >$psmgr
+) &
+
+# This read will block the process waiting on a msg to be put into the fifo.
+# If any of the processes defined above should exit,
+# a msg will be put into the fifo causing the read operation
+# to un-block. The process putting the msg into the fifo
+# will use it's process name as a msg so that we can print the offending
+# process to stdout.
+read exit_process <$psmgr
+echo "buildpack=nginx at=exit process=$exit_process"
+exit 1

config/mime.types

@@ -0,0 +1,98 @@
+types {
+    text/html                                        html htm shtml;
+    text/css                                         css;
+    text/xml                                         xml;
+    image/gif                                        gif;
+    image/jpeg                                       jpeg jpg;
+    application/javascript                           js;
+    application/atom+xml                             atom;
+    application/rss+xml                              rss;
+
+    text/mathml                                      mml;
+    text/plain                                       txt;
+    text/vnd.sun.j2me.app-descriptor                 jad;
+    text/vnd.wap.wml                                 wml;
+    text/x-component                                 htc;
+
+    image/avif                                       avif;
+    image/png                                        png;
+    image/svg+xml                                    svg svgz;
+    image/tiff                                       tif tiff;
+    image/vnd.wap.wbmp                               wbmp;
+    image/webp                                       webp;
+    image/x-icon                                     ico;
+    image/x-jng                                      jng;
+    image/x-ms-bmp                                   bmp;
+
+    font/woff                                        woff;
+    font/woff2                                       woff2;
+
+    application/java-archive                         jar war ear;
+    application/json                                 json;
+    application/mac-binhex40                         hqx;
+    application/msword                               doc;
+    application/pdf                                  pdf;
+    application/postscript                           ps eps ai;
+    application/rtf                                  rtf;
+    application/vnd.apple.mpegurl                    m3u8;
+    application/vnd.google-earth.kml+xml             kml;
+    application/vnd.google-earth.kmz                 kmz;
+    application/vnd.ms-excel                         xls;
+    application/vnd.ms-fontobject                    eot;
+    application/vnd.ms-powerpoint                    ppt;
+    application/vnd.oasis.opendocument.graphics      odg;
+    application/vnd.oasis.opendocument.presentation  odp;
+    application/vnd.oasis.opendocument.spreadsheet   ods;
+    application/vnd.oasis.opendocument.text          odt;
+    application/vnd.openxmlformats-officedocument.presentationml.presentation
+                                                     pptx;
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+                                                     xlsx;
+    application/vnd.openxmlformats-officedocument.wordprocessingml.document
+                                                     docx;
+    application/vnd.wap.wmlc                         wmlc;
+    application/wasm                                 wasm;
+    application/x-7z-compressed                      7z;
+    application/x-cocoa                              cco;
+    application/x-java-archive-diff                  jardiff;
+    application/x-java-jnlp-file                     jnlp;
+    application/x-makeself                           run;
+    application/x-perl                               pl pm;
+    application/x-pilot                              prc pdb;
+    application/x-rar-compressed                     rar;
+    application/x-redhat-package-manager             rpm;
+    application/x-sea                                sea;
+    application/x-shockwave-flash                    swf;
+    application/x-stuffit                            sit;
+    application/x-tcl                                tcl tk;
+    application/x-x509-ca-cert                       der pem crt;
+    application/x-xpinstall                          xpi;
+    application/xhtml+xml                            xhtml;
+    application/xspf+xml                             xspf;
+    application/zip                                  zip;
+
+    application/octet-stream                         bin exe dll;
+    application/octet-stream                         deb;
+    application/octet-stream                         dmg;
+    application/octet-stream                         iso img;
+    application/octet-stream                         msi msp msm;
+
+    audio/midi                                       mid midi kar;
+    audio/mpeg                                       mp3;
+    audio/ogg                                        ogg;
+    audio/x-m4a                                      m4a;
+    audio/x-realaudio                                ra;
+
+    video/3gpp                                       3gpp 3gp;
+    video/mp2t                                       ts;
+    video/mp4                                        mp4;
+    video/mpeg                                       mpeg mpg;
+    video/quicktime                                  mov;
+    video/webm                                       webm;
+    video/x-flv                                      flv;
+    video/x-m4v                                      m4v;
+    video/x-mng                                      mng;
+    video/x-ms-asf                                   asx asf;
+    video/x-ms-wmv                                   wmv;
+    video/x-msvideo                                  avi;
+}

config/nginx.conf.erb → config/nginx.conf

@@ -1,6 +1,5 @@
 daemon off;
-#Heroku dynos have at least 4 cores.
-worker_processes <%= ENV['NGINX_WORKERS'] || 4 %>;
+worker_processes 2;
 
 events {
 	use epoll;
@@ -15,9 +14,8 @@ http {
 
 	server_tokens off;
 
-	log_format l2met 'measure#nginx.service=$request_time request_id=$http_x_request_id';
-	access_log logs/nginx/access.log l2met;
-	error_log logs/nginx/error.log;
+	access_log /tmp/logs/nginx/access.log;
+	error_log /tmp/logs/nginx/error.log;
 
 	include mime.types;
 	default_type application/octet-stream;
@@ -29,11 +27,11 @@ http {
         client_max_body_size 32m;
 
 	upstream app_server {
-		server unix:/tmp/nginx.socket fail_timeout=0;
+		server unix:/var/run/cabotage/nginx.sock fail_timeout=0;
  	}
 
 	server {
-		listen <%= ENV["PORT"] %>;
+		listen unix:/var/run/cabotage/cabotage.sock;
 		server_name _;
 		keepalive_timeout 5;
 
@@ -208,19 +206,19 @@ http {
                   return 301 https://www.python.org/download/windows/;
                 }
 
-                location /download/ {
+                location ~ ^/download/$ {
                   return 301 https://www.python.org/downloads/;
                 }
 
-                location /download/source/ {
+                location ~ ^/download/source/$ {
                   return 301 https://www.python.org/downloads/source/;
                 }
 
-                location /download/mac/ {
+                location ~ ^/download/mac/$ {
                   return 301 https://www.python.org/downloads/macos/;
                 }
 
-                location /download/windows/ {
+                location ~ ^/download/windows/$ {
                   return 301 https://www.python.org/downloads/windows/;
                 }
 
@@ -317,17 +315,17 @@ http {
                 }
 
                 location /static/ {
-                    alias /app/static-root/;
+                    alias /code/static-root/;
                     add_header Cache-Control "max-age=604800, public";  # 604800 is 7 days
                 }
 
                 location /images/ {
-                    alias /app/static-root/images/;
+                    alias /code/static-root/images/;
                     add_header Cache-Control "max-age=604800, public";  # 604800 is 7 days
                 }
 
                 location /favicon.ico {
-                    alias /app/static-root/favicon.ico;
+                    alias /code/static-root/favicon.ico;
                     add_header Cache-Control "max-age=604800, public";  # 604800 is 7 days
                 }