build: docker images for latest release and main branch (#2116)

This PR adds docker image building to the github actions workflow.

Two images are built:
- unstable: This is built from the current branch
- latest: This is built from the version of rdflib in
  `docker/latest/requirements.txt`, but as this will be updated by
  dependabot it will essentially always be the latest version.

The resulting image references are:

- `ghcr.io/rdflib/rdflib:unstable`
- `ghcr.io/rdflib/rdflib:latest` with an alias
  `ghcr.io/rdflib/rdflib:${rdflib_version}`, e.g.
  `ghcr.io/rdflib/rdflib:6.2.0`.

As dependabot is conifgured for both pyton and docker dependencies, any
change/update to dependencies will result in a new image being built.

To avoid spurious changes in image digests the existing image is used as
the cache source, so if the requirements.txt and Dockerfile for
`ghcr.io/rdflib/rdflib:latest` has not changed then a new image should
not be published for it even if the build runs.

I have tested this in my forked repo, but there may be some kinks to
still work out.

.dockerignore

@@ -3,3 +3,5 @@
 .mypy_cache
 .pytest_cache
 .git
+docs/
+test/

.github/dependabot.yml

@@ -1,16 +1,32 @@
 version: 2
 updates:
-- package-ecosystem: pip
-  directory: "/"
-  schedule:
-    interval: weekly
-  open-pull-requests-limit: 10
-  ignore:
-  - dependency-name: sphinx
-    versions:
-    - 3.4.3
-    - 3.5.2
-- package-ecosystem: github-actions
-  directory: "/"
-  schedule:
-    interval: weekly
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: sphinx
+        versions:
+          - 3.4.3
+          - 3.5.2
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: docker/latest/
+    schedule:
+      interval: weekly
+  - package-ecosystem: pip
+    directory: docker/latest/
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: docker/unstable/
+    schedule:
+      interval: weekly
+  - package-ecosystem: pip
+    directory: docker/unstable/
+    schedule:
+      interval: weekly

.github/workflows/docker-images.yaml

@@ -0,0 +1,81 @@
+name: Docker Images
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    outputs:
+      docker-push: ${{ steps.vars.outputs.docker-push }}
+      oci-reference: ${{ steps.vars.outputs.oci-reference }}
+    steps:
+      - id: vars
+        run: |
+          echo ::set-output name=github-ref::${{ github.ref }}
+          echo ::set-output name=docker-push::${{ github.ref == 'refs/heads/main' }}
+          echo ::set-output name=oci-reference::ghcr.io/$( echo "${{ github.repository}}" | tr '[:upper:]' '[:lower:]' )
+      - id: vars-dump
+        run: |
+          echo "steps.vars.github-ref = ${{ steps.vars.outputs.github-ref }}"
+          echo "steps.vars.docker-push = ${{ steps.vars.outputs.docker-push }}"
+          echo "steps.vars.oci-reference = ${{ steps.vars.outputs.oci-reference }}"
+
+  build:
+    needs: setup
+    permissions:
+      contents: read
+      packages: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build images
+        shell: bash
+        run: |
+          task \
+            OCI_REFERENCE="${{ needs.setup.outputs.oci-reference }}" \
+            docker:latest docker:unstable
+  push:
+    if: "${{ needs.setup.outputs.docker-push == 'true' }}"
+    needs: [ setup, build ]
+    permissions:
+      id-token: write
+      packages: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Push images
+        shell: bash
+        run: |
+          task \
+            DOCKER_PUSH=true \
+            OCI_REFERENCE="${{ needs.setup.outputs.oci-reference }}" \
+            docker:latest docker:unstable

.github/workflows/validate.yaml

@@ -11,6 +11,10 @@ env:
   XDG_CACHE_HOME: ${{ github.workspace }}/cache
   PIP_CACHE_DIR: ${{ github.workspace }}/pip-cache
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   validate:
     permissions:

Taskfile.yml

@@ -16,7 +16,7 @@ vars:
   VENV_PYTHON: '{{if (eq OS "windows")}}{{.VENV_BINPREFIX}}python.exe{{else}}{{.VENV_BINPREFIX}}python{{end}}'
   # The python interpreter to use.
   PYTHON: python
-  _PYTHON: '{{if (mustFromJson .WITH_VENV)}}{{.VENV_PYTHON}}{{else}}{{.PYTHON}}{{end}}'
+  _PYTHON: "{{if (mustFromJson .WITH_VENV)}}{{.VENV_PYTHON}}{{else}}{{.PYTHON}}{{end}}"
   # Truthish values ("true", "1", etc.) results in java being installed as a
   # system dependency.
   INSTALL_SYSTEM_DEPS_JAVA: "false"
@@ -36,7 +36,9 @@ vars:
   # Truthish values ("true", "1", etc.) results in coverage being generated for
   # relevant commands.
   WITH_COVERAGE: "false"
-
+  PIP_COMPILE: pip-compile
+  DOCKER: docker
+  OCI_REFERENCE: ghcr.io/rdflib/rdflib
 tasks:
   install:system-deps:
     desc: Install system dependencies
@@ -59,12 +61,12 @@ tasks:
   install:tox:
     desc: Install tox
     cmds:
-      - '{{._PYTHON | shellQuote}} -m pip install tox {{if (mustFromJson .WITH_GITHUB_ACTIONS)}}tox-gh-actions{{end}}'
+      - "{{._PYTHON | shellQuote}} -m pip install tox {{if (mustFromJson .WITH_GITHUB_ACTIONS)}}tox-gh-actions{{end}}"
 
   install:pip-deps:
     desc: Install pip dependencies
     cmds:
-      - '{{._PYTHON | shellQuote}} -m pip install --upgrade -r requirements.txt -r requirements.dev.txt -r docs/sphinx-requirements.txt {{if (mustFromJson .INSTALL_PIP_EXTRAS)}}-r requirements.dev-extra.txt{{end}}'
+      - "{{._PYTHON | shellQuote}} -m pip install --upgrade -r requirements.txt -r requirements.dev.txt -r docs/sphinx-requirements.txt {{if (mustFromJson .INSTALL_PIP_EXTRAS)}}-r requirements.dev-extra.txt{{end}}"
 
   install:flake8:
     desc: Install flake8 dependencies
@@ -78,7 +80,6 @@ tasks:
       - task: install:tox
       - task: install:pip-deps
 
-
   venv:create:
     desc: Create a venv to VENV_PATH(={{.VENV_PATH}})
     cmds:
@@ -275,7 +276,6 @@ tasks:
           COVERAGE_FILE: ".coverage"
       - cmd: coveralls
 
-
   gha:flake8:
     desc: GitHub Actions flake8 workflow
     cmds:
@@ -287,6 +287,67 @@ tasks:
     cmds:
       - cmd: "{{._PYTHON | shellQuote}} -m rdflib.tools.rdfpipe {{.CLI_ARGS}}"
 
+  pip-compile:
+    cmds:
+      - cmd: "{{.PIP_COMPILE}} --quiet --annotate --emit-options {{.CLI_ARGS}}"
+  docker:prepare:
+    cmds:
+      - task: pip-compile
+        vars:
+          { CLI_ARGS: "-o docker/unstable/requirements.txt requirements.txt" }
+      - task: pip-compile
+        vars: { CLI_ARGS: "docker/latest/requirements.in" }
+  docker:unstable:
+    desc: ...
+    cmds:
+      - cmd: |
+          # fetch for caching ...
+          {{.DOCKER}} image pull {{.OCI_REFERENCE}}:unstable || :
+
+          set -eo pipefail
+
+          pipx run --spec=build build --wheel
+          {{.DOCKER}} buildx build \
+            --cache-to=type=inline \
+            --cache-from=type=registry,ref={{.OCI_REFERENCE}}:unstable \
+            --tag {{.OCI_REFERENCE}}:unstable \
+            --progress plain \
+            -f docker/unstable/Dockerfile \
+            {{.DOCKER_BUILD_ARGS}} \
+            .
+
+          if {{.DOCKER_PUSH | default "false"}}
+          then
+            {{.DOCKER}} image push {{.OCI_REFERENCE}}:unstable
+          fi
+  docker:latest:
+    desc: ...
+    cmds:
+      - cmd: |
+          # fetch for caching ...
+          {{.DOCKER}} image pull {{.OCI_REFERENCE}}:latest || :
+
+          set -eo pipefail
+
+          {{.DOCKER}} buildx build \
+            --cache-to=type=inline \
+            --cache-from=type=registry,ref={{.OCI_REFERENCE}}:latest \
+            --tag {{.OCI_REFERENCE}}:latest \
+            --progress plain \
+            -f docker/latest/Dockerfile \
+            {{.DOCKER_BUILD_ARGS}} \
+            .
+
+          _latest_rdflib_version=$({{.DOCKER}} run --entrypoint= {{.OCI_REFERENCE}}:latest bash -c 'pip show rdflib | sed -n "s/^Version: //gp"')
+          echo "_latest_rdflib_version=${_latest_rdflib_version}"
+
+          {{.DOCKER}} image tag {{.OCI_REFERENCE}}:latest {{.OCI_REFERENCE}}:${_latest_rdflib_version}
+
+          if {{.DOCKER_PUSH | default "false"}}
+          then
+            {{.DOCKER}} image push {{.OCI_REFERENCE}}:latest
+            {{.DOCKER}} image push {{.OCI_REFERENCE}}:${_latest_rdflib_version}
+          fi
   _rimraf:
     # This task is a utility task for recursively removing directories, it is
     # similar to rm -rf but not identical and it should work wherever there is

docker/latest/Dockerfile

@@ -0,0 +1,9 @@
+FROM docker.io/library/python:3.10-slim@sha256:c212a18a48efd4de1d6455f197538563d9cc312775761f14c50866c9a7eabd0e
+
+COPY docker/latest/requirements.txt /var/tmp/build/
+
+RUN \
+    pip install --no-deps -r /var/tmp/build/requirements.txt && \
+    rdfpipe --version
+
+ENTRYPOINT [ ]

docker/latest/requirements.in

@@ -0,0 +1,2 @@
+rdflib==6.2.0
+html5lib

docker/latest/requirements.txt

@@ -0,0 +1,22 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile docker/latest/requirements.in
+#
+html5lib==1.1
+    # via -r docker/latest/requirements.in
+isodate==0.6.1
+    # via rdflib
+pyparsing==3.0.9
+    # via rdflib
+rdflib==6.2.0
+    # via -r docker/latest/requirements.in
+six==1.16.0
+    # via html5lib
+webencodings==0.5.1
+    # via html5lib
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+

docker/unstable/Dockerfile

@@ -0,0 +1,14 @@
+FROM docker.io/library/python:3.10-slim@sha256:c212a18a48efd4de1d6455f197538563d9cc312775761f14c50866c9a7eabd0e
+
+COPY docker/unstable/requirements.txt /var/tmp/build/
+
+RUN \
+    pip install -r /var/tmp/build/requirements.txt
+
+COPY dist/*.whl /var/tmp/build/
+
+RUN \
+    pip install --no-deps /var/tmp/build/*.whl && \
+    rdfpipe --version
+
+ENTRYPOINT [ ]

docker/unstable/requirements.txt

@@ -0,0 +1,19 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile --output-file=docker/unstable/requirements.txt requirements.txt
+#
+html5lib==1.1
+    # via -r requirements.txt
+isodate==0.6.1
+    # via -r requirements.txt
+pyparsing==3.0.9
+    # via -r requirements.txt
+six==1.16.0
+    # via
+    #   html5lib
+    #   isodate
+webencodings==0.5.1
+    # via html5lib
+

requirements.txt

@@ -1,3 +1,4 @@
 html5lib
 isodate
 pyparsing
+importlib-metadata; python_version < '3.8.0'