diff --git a/Dockerfile b/Dockerfile index f8aca13a2..a3c351f5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ set -x \ && apt-get update \ && apt-get install --no-install-recommends -y \ - pandoc \ texlive-latex-base \ texlive-latex-recommended \ texlive-fonts-recommended \ @@ -35,6 +34,8 @@ WORKDIR /code COPY dev-requirements.txt /code/ COPY base-requirements.txt /code/ +COPY prod-requirements.txt /code/ +COPY requirements.txt /code/ RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel diff --git a/Dockerfile.cabotage b/Dockerfile.cabotage new file mode 100644 index 000000000..d96e002a7 --- /dev/null +++ b/Dockerfile.cabotage @@ -0,0 +1,49 @@ +FROM python:3.9-bullseye +COPY --from=ewdurbin/nginx-static:1.25.x /usr/bin/nginx /usr/bin/nginx +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +# By default, Docker has special steps to avoid keeping APT caches in the layers, which +# is good, but in our case, we're going to mount a special cache volume (kept between +# builds), so we WANT the cache to persist. +RUN set -eux; \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; + +# Install System level build requirements, this is done before +# everything else because these are rarely ever going to change. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + set -x \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + texlive-latex-base \ + texlive-latex-recommended \ + texlive-fonts-recommended \ + texlive-plain-generic \ + lmodern + +RUN case $(uname -m) in \ + "x86_64") ARCH=amd64 ;; \ + "aarch64") ARCH=arm64 ;; \ + esac \ + && wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \ + && dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb + +RUN mkdir /code +WORKDIR /code + +COPY dev-requirements.txt /code/ +COPY base-requirements.txt /code/ +COPY prod-requirements.txt /code/ +COPY requirements.txt /code/ + +RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel + +RUN --mount=type=cache,target=/root/.cache/pip \ + set -x \ + && pip --disable-pip-version-check \ + install \ + -r requirements.txt -r prod-requirements.txt +COPY . /code/ +RUN DJANGO_SETTINGS_MODULE=pydotorg.settings.static python manage.py collectstatic --noinput diff --git a/Procfile b/Procfile index 651bc19b8..16deb5f5b 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,4 @@ release: python manage.py migrate --noinput web: bin/start-nginx gunicorn -c gunicorn.conf pydotorg.wsgi +worker: celery -A pydotorg worker -l INFO +worker-beat: celery -A pydotorg beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/base-requirements.txt b/base-requirements.txt index 779389fa9..bee3181b7 100644 --- a/base-requirements.txt +++ b/base-requirements.txt @@ -13,12 +13,14 @@ psycopg2-binary==2.8.6 python3-openid==3.2.0 python-decouple==3.4 # lxml used by BeautifulSoup. -lxml==4.6.3 +lxml==4.9.2 cssselect==1.1.0 feedparser==6.0.8 beautifulsoup4==4.11.2 icalendar==4.0.7 chardet==4.0.0 +celery[redis]==5.3.6 +django-celery-beat==2.5.0 # TODO: We may drop 'django-imagekit' completely. django-imagekit==4.0.2 django-haystack==3.2.1 diff --git a/bin/start-nginx b/bin/start-nginx new file mode 100755 index 000000000..6ffacb572 --- /dev/null +++ b/bin/start-nginx @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +psmgr=/tmp/nginx-buildpack-wait +rm -f $psmgr +mkfifo $psmgr + +n=1 +while getopts :f option ${@:1:2} +do + case "${option}" + in + f) FORCE=$OPTIND; n=$((n+1));; + esac +done + +# Initialize log directory. +mkdir -p /tmp/logs/nginx +touch /tmp/logs/nginx/access.log /tmp/logs/nginx/error.log +echo 'buildpack=nginx at=logs-initialized' + +# Start log redirection. +( + # Redirect nginx logs to stdout. + tail -qF -n 0 /tmp/logs/nginx/*.log + echo 'logs' >$psmgr +) & + +# Start App Server +( + # Take the command passed to this bin and start it. + # E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb + COMMAND=${@:$n} + echo "buildpack=nginx at=start-app cmd=$COMMAND" + $COMMAND + echo 'app' >$psmgr +) & + +if [[ -z "$FORCE" ]] +then + FILE="/tmp/app-initialized" + + # We block on app-initialized so that when nginx binds to $PORT + # are app is ready for traffic. + while [[ ! -f "$FILE" ]] + do + echo 'buildpack=nginx at=app-initialization' + sleep 1 + done + echo 'buildpack=nginx at=app-initialized' +fi + +# Start nginx +( + # We expect nginx to run in foreground. + # We also expect a socket to be at /tmp/nginx.socket. + echo 'buildpack=nginx at=nginx-start' + cd /tmp + /usr/bin/nginx -p . -c /code/config/nginx.conf + echo 'nginx' >$psmgr +) & + +# This read will block the process waiting on a msg to be put into the fifo. +# If any of the processes defined above should exit, +# a msg will be put into the fifo causing the read operation +# to un-block. The process putting the msg into the fifo +# will use it's process name as a msg so that we can print the offending +# process to stdout. +read exit_process <$psmgr +echo "buildpack=nginx at=exit process=$exit_process" +exit 1 diff --git a/blogs/tests/test_views.py b/blogs/tests/test_views.py index ee7df723b..5c6c5053f 100644 --- a/blogs/tests/test_views.py +++ b/blogs/tests/test_views.py @@ -27,12 +27,3 @@ def test_blog_home(self): latest = BlogEntry.objects.latest() self.assertEqual(resp.context['latest_entry'], latest) - - def test_blog_redirects(self): - """ - Test that when '/blog/' is hit, it redirects '/blogs/' - """ - response = self.client.get('/blog/') - self.assertRedirects(response, - '/blogs/', - status_code=301) diff --git a/config/mime.types b/config/mime.types new file mode 100644 index 000000000..8d37c8636 --- /dev/null +++ b/config/mime.types @@ -0,0 +1,98 @@ +types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/avif avif; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + + font/woff woff; + font/woff2 woff2; + + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.oasis.opendocument.graphics odg; + application/vnd.oasis.opendocument.presentation odp; + application/vnd.oasis.opendocument.spreadsheet ods; + application/vnd.oasis.opendocument.text odt; + application/vnd.openxmlformats-officedocument.presentationml.presentation + pptx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + xlsx; + application/vnd.openxmlformats-officedocument.wordprocessingml.document + docx; + application/vnd.wap.wmlc wmlc; + application/wasm wasm; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} diff --git a/config/nginx.conf.erb b/config/nginx.conf similarity index 89% rename from config/nginx.conf.erb rename to config/nginx.conf index 527fdc0df..5ee6aaf6b 100644 --- a/config/nginx.conf.erb +++ b/config/nginx.conf @@ -1,6 +1,5 @@ daemon off; -#Heroku dynos have at least 4 cores. -worker_processes <%= ENV['NGINX_WORKERS'] || 4 %>; +worker_processes 2; events { use epoll; @@ -15,9 +14,8 @@ http { server_tokens off; - log_format l2met 'measure#nginx.service=$request_time request_id=$http_x_request_id'; - access_log logs/nginx/access.log l2met; - error_log logs/nginx/error.log; + access_log /tmp/logs/nginx/access.log; + error_log /tmp/logs/nginx/error.log; include mime.types; default_type application/octet-stream; @@ -29,11 +27,11 @@ http { client_max_body_size 32m; upstream app_server { - server unix:/tmp/nginx.socket fail_timeout=0; + server unix:/var/run/cabotage/nginx.sock fail_timeout=0; } server { - listen <%= ENV["PORT"] %>; + listen unix:/var/run/cabotage/cabotage.sock; server_name _; keepalive_timeout 5; @@ -84,6 +82,10 @@ http { return 301 https://www.python.org/psf; } + location ~ ^/community-landing/?(.*)$ { + return 301 https://www.python.org/community/; + } + location /doc/Summary { return 301 http://legacy.python.org/doc/intros/summary; } @@ -204,6 +206,22 @@ http { return 301 https://www.python.org/download/windows/; } + location ~ ^/download/$ { + return 301 https://www.python.org/downloads/; + } + + location ~ ^/download/source/$ { + return 301 https://www.python.org/downloads/source/; + } + + location ~ ^/download/mac/$ { + return 301 https://www.python.org/downloads/macos/; + } + + location ~ ^/download/windows/$ { + return 301 https://www.python.org/downloads/windows/; + } + location /Mirrors.html { return 301 https://www.python.org/mirrors/; } @@ -292,18 +310,22 @@ http { return 302 /blogs/; } + location /blog/ { + return 301 https://python.org/blogs/; + } + location /static/ { - alias /app/static-root/; + alias /code/static-root/; add_header Cache-Control "max-age=604800, public"; # 604800 is 7 days } location /images/ { - alias /app/static-root/images/; + alias /code/static-root/images/; add_header Cache-Control "max-age=604800, public"; # 604800 is 7 days } location /favicon.ico { - alias /app/static-root/favicon.ico; + alias /code/static-root/favicon.ico; add_header Cache-Control "max-age=604800, public"; # 604800 is 7 days } diff --git a/docker-compose.yml b/docker-compose.yml index 22221629c..2e5f8bf16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,8 +14,17 @@ services: test: ["CMD", "pg_isready", "-U", "pythondotorg", "-d", "pythondotorg"] interval: 1s + redis: + image: redis:7-bullseye + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli","ping"] + interval: 1s + web: build: . + image: pythondotorg:docker-compose command: python manage.py runserver 0.0.0.0:8000 volumes: - .:/code @@ -27,3 +36,19 @@ services: depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy + + worker: + image: pythondotorg:docker-compose + command: celery -A pydotorg worker -B -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler + volumes: + - .:/code + environment: + DATABASE_URL: postgresql://pythondotorg:pythondotorg@postgres:5432/pythondotorg + DJANGO_SETTINGS_MODULE: pydotorg.settings.local + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy diff --git a/gunicorn.conf b/gunicorn.conf index a68960607..74207d515 100644 --- a/gunicorn.conf +++ b/gunicorn.conf @@ -1,4 +1,4 @@ -bind = 'unix:/tmp/nginx.socket' +bind = 'unix:/var/run/cabotage/nginx.sock' backlog = 1024 preload_app = True max_requests = 2048 diff --git a/pydotorg/__init__.py b/pydotorg/__init__.py index e69de29bb..3307b5134 100644 --- a/pydotorg/__init__.py +++ b/pydotorg/__init__.py @@ -0,0 +1,3 @@ +from pydotorg.celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/pydotorg/celery.py b/pydotorg/celery.py new file mode 100644 index 000000000..51062cf9b --- /dev/null +++ b/pydotorg/celery.py @@ -0,0 +1,15 @@ +import os + +from celery import Celery +from django.core import management + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pydotorg.settings.local") + +app = Celery("pydotorg") +app.config_from_object("django.conf:settings", namespace="CELERY") + +@app.task(bind=True) +def run_management_command(self, command_name, args, kwargs): + management.call_command(command_name, *args, **kwargs) + +app.autodiscover_tasks() diff --git a/pydotorg/settings/base.py b/pydotorg/settings/base.py index ccbf3acab..a602dff7e 100644 --- a/pydotorg/settings/base.py +++ b/pydotorg/settings/base.py @@ -31,6 +31,23 @@ ) } +# celery settings +_REDIS_URL = config("REDIS_URL", default="redis://redis:6379/0") + +CELERY_BROKER_URL = _REDIS_URL +CELERY_RESULT_BACKEND = _REDIS_URL + +CELERY_BEAT_SCHEDULE = { + # "example-management-command": { + # "task": "pydotorg.celery.run_management_command", + # "schedule": crontab(hour=12, minute=0), + # "args": ("daily_volunteer_reminder", [], {}), + # }, + # 'example-task': { + # 'task': 'users.tasks.example_task', + # }, +} + ### Locale settings TIME_ZONE = 'UTC' @@ -163,6 +180,7 @@ 'django.contrib.admin', 'django.contrib.admindocs', + 'django_celery_beat', 'django_translation_aliases', 'pipeline', 'sitetree', diff --git a/pydotorg/settings/heroku.py b/pydotorg/settings/cabotage.py similarity index 94% rename from pydotorg/settings/heroku.py rename to pydotorg/settings/cabotage.py index 83e975cb1..d73beb83c 100644 --- a/pydotorg/settings/heroku.py +++ b/pydotorg/settings/cabotage.py @@ -29,6 +29,9 @@ 'ENGINE': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine', 'URL': HAYSTACK_SEARCHBOX_SSL_URL, 'INDEX_NAME': config('HAYSTACK_INDEX', default='haystack-prod'), + 'KWARGS': { + 'ca_certs': '/var/run/secrets/cabotage.io/ca.crt', + } }, } @@ -68,7 +71,7 @@ RAVEN_CONFIG = { "dsn": config('SENTRY_DSN'), - "release": config('SOURCE_VERSION'), + "release": config('SOURCE_COMMIT'), } AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID') diff --git a/pydotorg/settings/static.py b/pydotorg/settings/static.py new file mode 100644 index 000000000..3d93e113e --- /dev/null +++ b/pydotorg/settings/static.py @@ -0,0 +1,25 @@ +import os + +import dj_database_url +import raven +from decouple import Csv + +from .base import * + +DEBUG = TEMPLATE_DEBUG = False + +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine', + 'URL': 'http://127.0.0.1:9200', + 'INDEX_NAME': 'haystack-null', + }, +} + +MIDDLEWARE = [ + 'whitenoise.middleware.WhiteNoiseMiddleware', +] + MIDDLEWARE + +MEDIAFILES_LOCATION = 'media' +DEFAULT_FILE_STORAGE = 'custom_storages.storages.MediaStorage' +STATICFILES_STORAGE = 'custom_storages.storages.PipelineManifestStorage' diff --git a/pydotorg/urls.py b/pydotorg/urls.py index d1496dc03..f6ee8001d 100644 --- a/pydotorg/urls.py +++ b/pydotorg/urls.py @@ -3,7 +3,7 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.conf.urls.static import static from django.urls import path, re_path -from django.views.generic.base import TemplateView, RedirectView +from django.views.generic.base import TemplateView from django.conf import settings from cms.views import custom_404 @@ -16,6 +16,7 @@ urlpatterns = [ # homepage path('', views.IndexView.as_view(), name='home'), + re_path(r'^_health/?', views.health, name='health'), path('authenticated', views.AuthenticatedView.as_view(), name='authenticated'), re_path(r'^humans.txt$', TemplateView.as_view(template_name='humans.txt', content_type='text/plain')), re_path(r'^robots.txt$', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')), @@ -24,18 +25,11 @@ # python section landing pages path('about/', TemplateView.as_view(template_name="python/about.html"), name='about'), - # Redirect old download links to new downloads pages - path('download/', RedirectView.as_view(url='https://www.python.org/downloads/', permanent=True)), - path('download/source/', RedirectView.as_view(url='https://www.python.org/downloads/source/', permanent=True)), - path('download/mac/', RedirectView.as_view(url='https://www.python.org/downloads/macos/', permanent=True)), - path('download/windows/', RedirectView.as_view(url='https://www.python.org/downloads/windows/', permanent=True)), - # duplicated downloads to getit to bypass China's firewall. See # https://github.com/python/pythondotorg/issues/427 for more info. path('getit/', include('downloads.urls', namespace='getit')), path('downloads/', include('downloads.urls', namespace='download')), path('doc/', views.DocumentationIndexView.as_view(), name='documentation'), - path('blog/', RedirectView.as_view(url='/blogs/', permanent=True)), path('blogs/', include('blogs.urls')), path('inner/', TemplateView.as_view(template_name="python/inner.html"), name='inner'), @@ -54,7 +48,6 @@ name='account_change_password'), path('accounts/', include('allauth.urls')), path('box/', include('boxes.urls')), - re_path(r'^community-landing(/.*)?$', RedirectView.as_view(url='/community/', permanent=True)), path('community/', include('community.urls', namespace='community')), path('community/microbit/', TemplateView.as_view(template_name="community/microbit.html"), name='microbit'), path('events/', include('events.urls', namespace='events')), diff --git a/pydotorg/views.py b/pydotorg/views.py index 476e62fd9..9777cf1aa 100644 --- a/pydotorg/views.py +++ b/pydotorg/views.py @@ -1,10 +1,15 @@ from django.conf import settings +from django.http import HttpResponse from django.views.generic.base import RedirectView, TemplateView from codesamples.models import CodeSample from downloads.models import Release +def health(request): + return HttpResponse('OK') + + class IndexView(TemplateView): template_name = "python/index.html" diff --git a/sponsors/contracts.py b/sponsors/contracts.py index 7e72cde39..2720ec1c8 100644 --- a/sponsors/contracts.py +++ b/sponsors/contracts.py @@ -31,6 +31,7 @@ def _contract_context(contract, **context): "sponsorship": contract.sponsorship, "benefits": _clean_split(contract.benefits_list.raw), "legal_clauses": _clean_split(contract.legal_clauses.raw), + "renewal": True if contract.sponsorship.renewal else False, } ) previous_effective = contract.sponsorship.previous_effective_date @@ -41,8 +42,6 @@ def _contract_context(contract, **context): def render_markdown_from_template(contract, **context): template = "sponsors/admin/contracts/sponsorship-agreement.md" - if contract.sponsorship.renewal: - template = "sponsors/admin/contracts/renewal-agreement.md" context = _contract_context(contract, **context) return render_to_string(template, context) diff --git a/sponsors/pandoc_filters/pagebreak.py b/sponsors/pandoc_filters/pagebreak.py index 525b89c57..22a786a2b 100644 --- a/sponsors/pandoc_filters/pagebreak.py +++ b/sponsors/pandoc_filters/pagebreak.py @@ -62,7 +62,6 @@ def action(self, elem, doc): if isinstance(elem, pf.RawBlock): if elem.text == r"\newpage": if (doc.format == "docx"): - pf.debug("Page Break") elem = self.pagebreak # elif elem.text == r"\newsection": # if (doc.format == "docx"): diff --git a/sponsors/reference.docx b/sponsors/reference.docx index bf094ca4f..9e2df1a41 100644 Binary files a/sponsors/reference.docx and b/sponsors/reference.docx differ diff --git a/templates/base.html b/templates/base.html index cfa0f34ce..424df06f6 100644 --- a/templates/base.html +++ b/templates/base.html @@ -236,7 +236,7 @@

  • Socialize