Merge branch 'main' into release

Dockerfile

@@ -16,7 +16,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     set -x \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
-        pandoc \
         texlive-latex-base \
         texlive-latex-recommended \
         texlive-fonts-recommended \
@@ -35,6 +34,8 @@ WORKDIR /code
 
 COPY dev-requirements.txt /code/
 COPY base-requirements.txt /code/
+COPY prod-requirements.txt /code/
+COPY requirements.txt /code/
 
 RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
 

Dockerfile.cabotage

@@ -0,0 +1,49 @@
+FROM python:3.9-bullseye
+COPY --from=ewdurbin/nginx-static:1.25.x /usr/bin/nginx /usr/bin/nginx
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
+
+# Install System level build requirements, this is done before
+# everything else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        texlive-latex-base \
+        texlive-latex-recommended \
+        texlive-fonts-recommended \
+        texlive-plain-generic \
+        lmodern
+
+RUN case $(uname -m) in \
+         "x86_64")  ARCH=amd64 ;; \
+         "aarch64")  ARCH=arm64 ;; \
+    esac \
+    && wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \
+    && dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb
+
+RUN mkdir /code
+WORKDIR /code
+
+COPY dev-requirements.txt /code/
+COPY base-requirements.txt /code/
+COPY prod-requirements.txt /code/
+COPY requirements.txt /code/
+
+RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    set -x \
+    && pip --disable-pip-version-check \
+        install \
+        -r requirements.txt -r prod-requirements.txt
+COPY . /code/
+RUN DJANGO_SETTINGS_MODULE=pydotorg.settings.static python manage.py collectstatic --noinput

Procfile

@@ -1,2 +1,4 @@
 release: python manage.py migrate --noinput
 web: bin/start-nginx gunicorn -c gunicorn.conf pydotorg.wsgi
+worker: celery -A pydotorg worker -l INFO
+worker-beat: celery -A pydotorg beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler

bin/start-nginx

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+psmgr=/tmp/nginx-buildpack-wait
+rm -f $psmgr
+mkfifo $psmgr
+
+n=1
+while getopts :f option ${@:1:2}
+do
+  case "${option}"
+  in
+    f) FORCE=$OPTIND; n=$((n+1));;
+  esac
+done
+
+# Initialize log directory.
+mkdir -p /tmp/logs/nginx
+touch /tmp/logs/nginx/access.log /tmp/logs/nginx/error.log
+echo 'buildpack=nginx at=logs-initialized'
+
+# Start log redirection.
+(
+  # Redirect nginx logs to stdout.
+  tail -qF -n 0 /tmp/logs/nginx/*.log
+  echo 'logs' >$psmgr
+) &
+
+# Start App Server
+(
+  # Take the command passed to this bin and start it.
+  # E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb
+  COMMAND=${@:$n}
+  echo "buildpack=nginx at=start-app cmd=$COMMAND"
+  $COMMAND
+  echo 'app' >$psmgr
+) &
+
+if [[ -z "$FORCE" ]]
+then
+  FILE="/tmp/app-initialized"
+
+  # We block on app-initialized so that when nginx binds to $PORT
+  # are app is ready for traffic.
+  while [[ ! -f "$FILE" ]]
+  do
+    echo 'buildpack=nginx at=app-initialization'
+    sleep 1
+  done
+  echo 'buildpack=nginx at=app-initialized'
+fi
+
+# Start nginx
+(
+  # We expect nginx to run in foreground.
+  # We also expect a socket to be at /tmp/nginx.socket.
+  echo 'buildpack=nginx at=nginx-start'
+  cd /tmp
+  /usr/bin/nginx -p . -c /code/config/nginx.conf
+  echo 'nginx' >$psmgr
+) &
+
+# This read will block the process waiting on a msg to be put into the fifo.
+# If any of the processes defined above should exit,
+# a msg will be put into the fifo causing the read operation
+# to un-block. The process putting the msg into the fifo
+# will use it's process name as a msg so that we can print the offending
+# process to stdout.
+read exit_process <$psmgr
+echo "buildpack=nginx at=exit process=$exit_process"
+exit 1

config/mime.types

@@ -0,0 +1,98 @@
+types {
+    text/html                                        html htm shtml;
+    text/css                                         css;
+    text/xml                                         xml;
+    image/gif                                        gif;
+    image/jpeg                                       jpeg jpg;
+    application/javascript                           js;
+    application/atom+xml                             atom;
+    application/rss+xml                              rss;
+
+    text/mathml                                      mml;
+    text/plain                                       txt;
+    text/vnd.sun.j2me.app-descriptor                 jad;
+    text/vnd.wap.wml                                 wml;
+    text/x-component                                 htc;
+
+    image/avif                                       avif;
+    image/png                                        png;
+    image/svg+xml                                    svg svgz;
+    image/tiff                                       tif tiff;
+    image/vnd.wap.wbmp                               wbmp;
+    image/webp                                       webp;
+    image/x-icon                                     ico;
+    image/x-jng                                      jng;
+    image/x-ms-bmp                                   bmp;
+
+    font/woff                                        woff;
+    font/woff2                                       woff2;
+
+    application/java-archive                         jar war ear;
+    application/json                                 json;
+    application/mac-binhex40                         hqx;
+    application/msword                               doc;
+    application/pdf                                  pdf;
+    application/postscript                           ps eps ai;
+    application/rtf                                  rtf;
+    application/vnd.apple.mpegurl                    m3u8;
+    application/vnd.google-earth.kml+xml             kml;
+    application/vnd.google-earth.kmz                 kmz;
+    application/vnd.ms-excel                         xls;
+    application/vnd.ms-fontobject                    eot;
+    application/vnd.ms-powerpoint                    ppt;
+    application/vnd.oasis.opendocument.graphics      odg;
+    application/vnd.oasis.opendocument.presentation  odp;
+    application/vnd.oasis.opendocument.spreadsheet   ods;
+    application/vnd.oasis.opendocument.text          odt;
+    application/vnd.openxmlformats-officedocument.presentationml.presentation
+                                                     pptx;
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+                                                     xlsx;
+    application/vnd.openxmlformats-officedocument.wordprocessingml.document
+                                                     docx;
+    application/vnd.wap.wmlc                         wmlc;
+    application/wasm                                 wasm;
+    application/x-7z-compressed                      7z;
+    application/x-cocoa                              cco;
+    application/x-java-archive-diff                  jardiff;
+    application/x-java-jnlp-file                     jnlp;
+    application/x-makeself                           run;
+    application/x-perl                               pl pm;
+    application/x-pilot                              prc pdb;
+    application/x-rar-compressed                     rar;
+    application/x-redhat-package-manager             rpm;
+    application/x-sea                                sea;
+    application/x-shockwave-flash                    swf;
+    application/x-stuffit                            sit;
+    application/x-tcl                                tcl tk;
+    application/x-x509-ca-cert                       der pem crt;
+    application/x-xpinstall                          xpi;
+    application/xhtml+xml                            xhtml;
+    application/xspf+xml                             xspf;
+    application/zip                                  zip;
+
+    application/octet-stream                         bin exe dll;
+    application/octet-stream                         deb;
+    application/octet-stream                         dmg;
+    application/octet-stream                         iso img;
+    application/octet-stream                         msi msp msm;
+
+    audio/midi                                       mid midi kar;
+    audio/mpeg                                       mp3;
+    audio/ogg                                        ogg;
+    audio/x-m4a                                      m4a;
+    audio/x-realaudio                                ra;
+
+    video/3gpp                                       3gpp 3gp;
+    video/mp2t                                       ts;
+    video/mp4                                        mp4;
+    video/mpeg                                       mpeg mpg;
+    video/quicktime                                  mov;
+    video/webm                                       webm;
+    video/x-flv                                      flv;
+    video/x-m4v                                      m4v;
+    video/x-mng                                      mng;
+    video/x-ms-asf                                   asx asf;
+    video/x-ms-wmv                                   wmv;
+    video/x-msvideo                                  avi;
+}

config/nginx.conf.erb → config/nginx.conf

@@ -1,6 +1,5 @@
 daemon off;
-#Heroku dynos have at least 4 cores.
-worker_processes <%= ENV['NGINX_WORKERS'] || 4 %>;
+worker_processes 2;
 
 events {
 	use epoll;
@@ -15,9 +14,8 @@ http {
 
 	server_tokens off;
 
-	log_format l2met 'measure#nginx.service=$request_time request_id=$http_x_request_id';
-	access_log logs/nginx/access.log l2met;
-	error_log logs/nginx/error.log;
+	access_log /tmp/logs/nginx/access.log;
+	error_log /tmp/logs/nginx/error.log;
 
 	include mime.types;
 	default_type application/octet-stream;
@@ -29,11 +27,11 @@ http {
         client_max_body_size 32m;
 
 	upstream app_server {
-		server unix:/tmp/nginx.socket fail_timeout=0;
+		server unix:/var/run/cabotage/nginx.sock fail_timeout=0;
  	}
 
 	server {
-		listen <%= ENV["PORT"] %>;
+		listen unix:/var/run/cabotage/cabotage.sock;
 		server_name _;
 		keepalive_timeout 5;
 
@@ -317,17 +315,17 @@ http {
                 }
 
                 location /static/ {
-                    alias /app/static-root/;
+                    alias /code/static-root/;
                     add_header Cache-Control "max-age=604800, public";  # 604800 is 7 days
                 }
 
                 location /images/ {
-                    alias /app/static-root/images/;
+                    alias /code/static-root/images/;
                     add_header Cache-Control "max-age=604800, public";  # 604800 is 7 days
                 }
 
                 location /favicon.ico {
-                    alias /app/static-root/favicon.ico;
+                    alias /code/static-root/favicon.ico;
                     add_header Cache-Control "max-age=604800, public";  # 604800 is 7 days
                 }
 

gunicorn.conf

@@ -1,4 +1,4 @@
-bind = 'unix:/tmp/nginx.socket'
+bind = 'unix:/var/run/cabotage/nginx.sock'
 backlog = 1024
 preload_app = True
 max_requests = 2048

pydotorg/settings/heroku.py → pydotorg/settings/cabotage.py

@@ -29,6 +29,9 @@
         'ENGINE': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
         'URL': HAYSTACK_SEARCHBOX_SSL_URL,
         'INDEX_NAME': config('HAYSTACK_INDEX', default='haystack-prod'),
+        'KWARGS': {
+            'ca_certs': '/var/run/secrets/cabotage.io/ca.crt',
+        }
     },
 }
 
@@ -68,7 +71,7 @@
 
 RAVEN_CONFIG = {
     "dsn": config('SENTRY_DSN'),
-    "release": config('SOURCE_VERSION'),
+    "release": config('SOURCE_COMMIT'),
 }
 
 AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID')

pydotorg/settings/static.py

@@ -0,0 +1,25 @@
+import os
+
+import dj_database_url
+import raven
+from decouple import Csv
+
+from .base import *
+
+DEBUG = TEMPLATE_DEBUG = False
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine',
+        'URL': 'http://127.0.0.1:9200',
+        'INDEX_NAME': 'haystack-null',
+    },
+}
+
+MIDDLEWARE = [
+    'whitenoise.middleware.WhiteNoiseMiddleware',
+] + MIDDLEWARE
+
+MEDIAFILES_LOCATION = 'media'
+DEFAULT_FILE_STORAGE = 'custom_storages.storages.MediaStorage'
+STATICFILES_STORAGE = 'custom_storages.storages.PipelineManifestStorage'

pydotorg/urls.py

@@ -16,6 +16,7 @@
 urlpatterns = [
     # homepage
     path('', views.IndexView.as_view(), name='home'),
+    re_path(r'^_health/?', views.health, name='health'),
     path('authenticated', views.AuthenticatedView.as_view(), name='authenticated'),
     re_path(r'^humans.txt$', TemplateView.as_view(template_name='humans.txt', content_type='text/plain')),
     re_path(r'^robots.txt$', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),

pydotorg/views.py

@@ -1,10 +1,15 @@
 from django.conf import settings
+from django.http import HttpResponse
 from django.views.generic.base import RedirectView, TemplateView
 
 from codesamples.models import CodeSample
 from downloads.models import Release
 
 
+def health(request):
+    return HttpResponse('OK')
+
+
 class IndexView(TemplateView):
     template_name = "python/index.html"