Merge pull request #313 from zms-publishing/docker-deployment

Update Docker development and prepare to use these docker containers in our production deployments.

.devcontainer/devcontainer.json

@@ -0,0 +1,78 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
+{
+  "name": "ZMS Development Environment",
+  // Update the 'dockerComposeFile' list if you have more compose files or use different names.
+  // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+  "dockerComposeFile": ["../docker-compose.yml", "docker-compose.yml"],
+  // The 'service' property is the name of the service for the container that VS Code should
+  // use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+  "service": "zope",
+  // The optional 'workspaceFolder' property is the path VS Code should open by default when
+  // connected. This is typically a file mount in .devcontainer/docker-compose.yml
+  "workspaceFolder": "/home/zope/",
+  // Features to add to the dev container. More info: https://containers.dev/features.
+  // "features": {},
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+  // Uncomment the next line if you want start specific services in your Docker Compose config.
+  // "runServices": [],
+  // Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+  "shutdownAction": "stopCompose",
+  // Uncomment the next line to run commands after the container is created.
+  // "postCreateCommand": "cat /etc/os-release",
+  // Configure tool-specific properties.
+  // "customizations": {},
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+        // "ms-python.autopep8"
+      ],
+      "settings": {
+        "terminal.integrated.defaultProfile.linux": "bash",
+        "python.defaultInterpreterPath": "/home/zope/venv/bin/python",
+        "window.zoomLevel": 0,
+        "git.ignoreMissingGitWarning": true,
+        "editor.minimap.enabled": false,
+        "editor.renderWhitespace": "all",
+        "editor.renderControlCharacters": false,
+        "workbench.iconTheme": "vs-minimal",
+        "files.associations": {
+          "*.zpt": "html",
+          "*.zcml": "xml"
+        },
+        "scm.alwaysShowActions": true,
+        "files.exclude": {
+          "*.pyc": true,
+          "*-all.min.*": true,
+          "**/cache/**": true,
+          "**/Data.*": true
+        },
+        "search.exclude": {
+          "**/apidocs/**": true
+        },
+        "files.eol": "\n",
+        "files.autoSave": "afterDelay",
+        "workbench.colorTheme": "Visual Studio Light",
+        "python.linting.enabled": false,
+        "python.formatting.provider": "none"
+        // "python.testing.pytestEnabled": false,
+        // "python.testing.unittestEnabled": true,
+        // "python.testing.unittestArgs": [
+        //   "-v",
+        //   "-s",
+        //   "./tests",
+        //   "-p",
+        //   "test*.py"
+        // ],
+        //   "[python]": {
+        //     "editor.defaultFormatter": "ms-python.autopep8"
+        //   }
+      }
+    }
+  }
+  // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "devcontainer"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,6 @@
+services:
+  zope:
+    volumes:
+      - .:/home/zope/workspace/:cached
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity

.vscode/Docker.code-workspace

@@ -0,0 +1,40 @@
+{
+	"folders": [
+		{
+			"name": "ZMS-Docker",
+			"path": "../.."
+		},
+	],
+
+	"launch": {
+		"version": "0.2.0",
+		"configurations": [
+			{
+				"name": "ZMS-Docker",
+				"type": "debugpy",
+				"request": "launch",
+				"justMyCode": false,
+				"console": "integratedTerminal",
+				"program": "/home/zope/venv/bin/runwsgi",
+				"args": [
+					"--debug",
+					"--verbose",
+					"/home/zope/etc/zope.ini",
+				],
+				"env": {
+					"PYTHONUNBUFFERED":"1",
+					"CONFIG_FILE": "/home/zope/etc/zope.ini",
+					"INSTANCE_HOME": "/home/zope/",
+					"CLIENT_HOME": "/home/zope/",
+					"PYTHON": "/home/zope/venv/bin/python",
+					"SOFTWARE_HOME": "/home/zope/venv/bin"
+				},
+				"serverReadyAction":{
+					"pattern":"Serving on http://0.0.0.0:80",
+					"uriFormat": "http://admin:admin@127.0.0.1:80/manage_main",
+					"action": "openExternally",
+				},
+			},
+		]
+	}
+}

docker-compose.yml

@@ -0,0 +1,25 @@
+services:
+  zope:
+    build: ./docker/base
+    image: zope:latest
+    depends_on:
+      - zeo
+    stop_grace_period: 1s # SIGKILL after 1s, as zope is always taking the full 10 seconds
+    ports:
+      - 80:80
+    volumes:
+      - .:/home/zope/venv/src/zms/
+      - ./docker/zope/etc/:/home/zope/etc/
+      - ./docker/zope/var/:/home/zope/var/
+      # TODO we may want to map these in from outside to ease debugging
+      # - ./docker/zope/Extensions/:/home/zope/Extensions/
+    # allow attaching to the container to debug with `breakpoint()`
+    stdin_open: true
+    tty: true
+
+  zeo:
+    image: zope:latest
+    command: runzeo --configure etc/zeo.conf
+    volumes:
+      - ./docker/zeo/etc/:/home/zope/etc/
+      - ./docker/zeo/var/:/home/zope/var/

docker/Readme.md

@@ -0,0 +1,34 @@
+# Running ZMS in a Docker container
+
+Important: *The presented Docker environment is not yet recommended for production, just for testing and exploration.* We do plan to evolve these to be production ready, but we are not there yet.
+
+The ZMS source folder `./docker` contains two minimalistic Docker files:
+
+1. the [Dockerfile](zms-base/Dockerfile) for creating a Docker *image* and
+2. the [docker-compose.yml](../docker-compose.yml) file for building the Docker *containers*.
+
+The image utilizes a Linux with a fresh Python3 and some additional software packages (like mariadb and openldap). The ZMS installation happens with pip in a virtual python environment (`/home/zope/venv`) and provides the ZMS code in the pip-"editable" mode. Both the ZMS source code (`/home/zope/venv/src/ZMS/.git`) and the Zope instance are placed in the virtual python environment folder (`/home/zope/`)
+
+To make Zope running there are some crucial config files needed which usually (created by `mkwsgiinstance`) are set on default values. In a Docker environment these defaults must be modified; moreover the setup contains a ZEO-server for running multiple Zope processes in parallel (e.g. for debugging). That is why a small set of config files is provided as presets via  the the source-folders
+1. ./docker/{zope,zeo}/etc
+1. ./docker/{zope,zeo}/var
+1. ./docker/zope/Extensions
+
+These sources are mapped into the respective *containers*
+
+## Overview of Docker- and all Zope config-files
+
+*Hint: to ease the file access from the container the config files are not restricted:* `chmod -R 777 ./docker/`
+
+## Running the ZMS Container with VSCode
+
+The VSCode Docker Extension [ms-azuretools.vscode-docker](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker) is a perfect tool for handling containers. A right mouse click on the file ´docker-compose.yaml´ starts composing the container. Initially ZEO will be started and Zope will run on http://localhost/, the management interface on http://admin:admin@localhost/manage_main.
+
+![Running the ZMS Container with VSCode](../docs/images/admin_docker_run.gif)
+
+## Attach VSCode to the ZMS Container
+
+Another right click on the running container-ID allows to intrude the container with VSCode and launch a new Zope instance in debugging mode.
+Hint: For this purpose the  docker-container folder `/home/zope/venv/src/zms/docker/.vscode/` contains a prepared VSCode-workspace file and a launch file for starting Zope in debug-mode within the container  [launch.json](https://github.com/zms-publishing/ZMS/blob/main/docker/.vscode/launch.json). The thus launched Zope instance will run port 8087.
+
+![Attach VSCode to the ZMS Container](../docs/images/admin_docker_debug_zeo.gif)

docker/TODO.md

@@ -0,0 +1,21 @@
+# Goals
+
+- [x] one process per container
+- [X] Starting the bare docker file will give you a basic zope / zms
+- [x] everything as similar to our server deployment as possible to allow easy migration
+- [x] modern os and python
+- [ ] simple to use and develop in vscode -> .devcontainer!
+- [x] all mutable data in mounted volumes
+- [ ] example systemd files to run everything
+  - [ ] this should show how automated container updates are done!
+- [ ] example nginx config so you get the same experience as on the server
+- [x] Allow working on zms inside the container
+- [ ] Full development experience with all dependennt services locally (mariadb, memcached, …)
+
+# TODOs
+
+- [x] Create basic Dockerfile for the project
+- [x] specialize them for zeoserver and zope
+- [x] create docker-compose file that runs each server separately
+- [ ] add devcontainer.json to develop and run everything from vscode
+- [ ] mount the zms source live into the container so working within it becomes possible

docker/base/Dockerfile

@@ -0,0 +1,72 @@
+FROM python:3.12
+
+EXPOSE 80
+
+ARG CI_COMMIT_SHA=development
+ENV CI_COMMIT_SHA=$CI_COMMIT_SHA
+
+# Ensure all system packages are up to date
+ENV DEBIAN_FRONTEND=noninteractive
+RUN \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    <<EOR
+    apt update
+    apt --yes upgrade
+EOR
+
+# Install Zope/ZEO Dependencies
+RUN \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    <<EOR
+    apt update
+    apt --yes install libldap2-dev libsasl2-dev
+EOR
+
+# Drop root privileges
+ENV UID=1000
+RUN useradd --system --create-home --uid $UID zope
+USER zope:zope
+WORKDIR /home/zope/
+
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+ENV UV_LINK_MODE=copy
+
+# Create venv and permanently enable it
+# ARG CREATE_VENV_COMMAND="uv venv"
+ARG CREATE_VENV_COMMAND="python3 -m venv"
+ENV VIRTUAL_ENV=/home/zope/venv
+RUN $CREATE_VENV_COMMAND $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Enable caching of pip packages to speed up rebuild times
+# cannot use uv as long as so many editable packages need to be installed
+# ARG PIP_INSTALL="uv pip install"
+ARG PIP_INSTALL="pip install --config-settings editable_mode=compat"
+ENV PIP_CACHE_DIR=/home/zope/venv/cache
+RUN --mount=type=cache,uid=$UID,target=$PIP_CACHE_DIR,sharing=locked <<EOR
+    set -ex
+
+    if echo ${PIP_INSTALL} | grep -qv "\buv\b"; then
+        $PIP_INSTALL --cache-dir $PIP_CACHE_DIR --upgrade pip wheel setuptools
+    fi
+    $PIP_INSTALL --cache-dir $PIP_CACHE_DIR --upgrade --editable git+https://github.com/zms-publishing/ZMS.git#egg=ZMS
+    $PIP_INSTALL --cache-dir $PIP_CACHE_DIR --requirement https://raw.githubusercontent.com/zms-publishing/ZMS5/master/requirements-full.txt
+    $PIP_INSTALL --cache-dir $PIP_CACHE_DIR --upgrade --editable git+https://github.com/sntl-projects/Products.zmsPluggableAuthService.git#egg=Products.zmsPluggableAuthService[nginx-sso]
+    $PIP_INSTALL --cache-dir $PIP_CACHE_DIR ZEO
+    # Allow remote debugging from vscode
+    $PIP_INSTALL --cache-dir $PIP_CACHE_DIR debugpy
+EOR
+
+# Create Zope Instance
+RUN mkwsgiinstance --dir . --user admin:admin
+
+# Configure Zope Instance
+COPY --chown=zope:zope --chmod=500 start.sh bin/start.sh
+COPY --chown=zope:zope zope.ini etc/zope.ini
+COPY --chown=zope:zope zope.conf etc/zope.conf
+RUN mkdir cache Extensions
+
+CMD ["/home/zope/bin/start.sh"]

docker/base/start.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+instance_dir="/home/zope/"
+venv_bin_dir="/home/zope/venv/bin"
+
+# Regarding debug-mode=on
+# @see https://zope.readthedocs.io/en/latest/zopebook/InstallingZope.html#the-debug-mode-directive
+# recommend to disable debug-mode in production as it auto reloads templates on change
+# @see https://zope.readthedocs.io/en/latest/zdgbook/TestingAndDebugging.html#debug-mode
+# mentions that this also shows backtraces to users
+# @see https://zope.readthedocs.io/en/latest/news.html#wsgi-as-the-new-default-server-type
+# mentions that this disables a httpexceptions midleware and shows backtraces on the console>
+# TODO talk to FH if this makes sense. Might require a slightly different workflow from them.
+# Currently always enabled on our servers.
+
+# Regarding --debug
+# @see https://zope.readthedocs.io/en/latest/operation.html#running-zope-in-the-foreground
+# seems to sugges that this also enables the debug-mode directive
+# TODO talk to FH to allow me to check that
+
+exec $venv_bin_dir/runwsgi --debug --verbose $instance_dir/etc/zope.ini debug-mode=on http_port=80