Merge pull request #6 from mjanez/main

Improve Dockerfiles

.github/drafts/docker.yaml

@@ -0,0 +1,51 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - main
+      - latest
+    tags: 
+      - 'v*.*.*'
+  pull_request:
+    branches:
+      - main
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/docker-manual.yaml

@@ -0,0 +1,73 @@
+name: Build and release image manually
+
+on: workflow_dispatch
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  CONTEXT: .
+  BRANCH: ${{ github.ref_name }}
+  DOCKERFILE_PATH: /ckan-mqa
+  DOCKERFILE: Dockerfile
+
+jobs:
+  docker:
+    name: manual/build-docker-push:${{ github.ref_name }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: |
+            org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md
+            org.opencontainers.image.version=${{ env.BRANCH }}
+
+      - name: Replace slashes in BRANCH to avoid errors
+        run: echo "BRANCH=${BRANCH////_}" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }}
+          labels: ${{ steps.meta.outputs.labels }}
+          context: ${{ env.CONTEXT }}
+          file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
+
+      - name: Linting Dockerfile with hadolint in GH Actions
+        uses: hadolint/hadolint-action@v3.1.0
+        with:
+          dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
+          no-fail: true
+
+      - name: Run Trivy container image vulnerability scanner
+        uses: aquasecurity/trivy-action@0.17.0
+        with:
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }}
+          format: sarif
+          output: trivy-results.sarif
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: trivy-results.sarif

.github/workflows/docker-pr.yaml

@@ -0,0 +1,49 @@
+name: Test ckan-mqa images (PR)
+
+on:
+  pull_request:
+    branches:
+        - master
+        - 'ckan-mqa-*.*.*'
+        - '!dev/ckan-mqa-*.*.*'
+        - '!feature/*'
+        - '!fix/*'
+
+env:
+    REGISTRY: ghcr.io
+    IMAGE_NAME: ckan-mqa
+    CONTEXT: .
+    BRANCH: ${{ github.head_ref }}
+    DOCKERFILE_PATH: /ckan-mqa
+    DOCKERFILE: Dockerfile
+    HADOLINT_VERSION: 2.12.0
+
+jobs:
+  docker:
+    name: runner/test-ckan-mqa-pr:${{ github.head_ref }}
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Linting Dockerfiles and annotate code inline in the github PR viewer
+        id: hadolint
+        uses: jbergstroem/hadolint-gh-action@v1.12.2
+        with:
+          dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
+          version: ${{ env.HADOLINT_VERSION }}
+          annotate: true
+          error_level: -1

.github/workflows/docker.yaml

@@ -1,51 +1,83 @@
-name: Docker
-
-on:
-  push:
-    branches:
-      - main
-      - latest
-    tags: 
-      - 'v*.*.*'
-  pull_request:
-    branches:
-      - main
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Login to registry
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Extract Docker metadata
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      -
-        name: Build and push
-        uses: docker/build-push-action@v3
-        with:
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+name: Build and push ckan-mqa image from PR Merge
+
+on:
+  pull_request:
+    types:
+        - closed
+    branches:
+        - master
+        - 'ckan-mqa-*.*.*'
+        - '!dev/ckan-mqa-*.*.*'
+        - '!feature/*'
+        - '!fix/*'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  CONTEXT: .
+  BRANCH: ${{ github.head_ref }}
+  DOCKERFILE_PATH: /ckan-mqa
+  DOCKERFILE: Dockerfile
+
+jobs:
+  docker:
+    name: runner/build-docker-push:${{ github.head_ref }}
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: |
+            org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md
+            org.opencontainers.image.version=${{ env.BRANCH }}
+
+      - name: Replace slashes in BRANCH to avoid errors
+        run: echo "BRANCH=${BRANCH////_}" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          context: ${{ env.CONTEXT }}
+          file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
+
+      - name: Linting Dockerfile with hadolint in GH Actions
+        uses: hadolint/hadolint-action@v3.1.0
+        with:
+          dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
+          no-fail: true
+
+      - name: Run Trivy container image vulnerability scanner
+        uses: aquasecurity/trivy-action@0.12.0
+        with:
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }}
+          format: sarif
+          output: trivy-results.sarif
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: trivy-results.sarif

Dockerfile → ckan-mqa/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.13.0a3-slim
 LABEL maintainer="mnl.janez@gmail.com"
 
 ENV APP_DIR=/app
@@ -17,11 +17,11 @@ RUN apt-get -q -y update && \
     python3 -m pip install pdm
 
 WORKDIR ${APP_DIR}
-COPY pyproject.toml pdm.lock .
+COPY pyproject.toml pdm.lock ./
 
 RUN pdm install --no-self --group prod
 
-COPY ckan-mqa/entrypoint.sh entrypoint.sh
-COPY ckan2mqa ckan2mqa
+COPY ckan-mqa/docker-entrypoint.d/entrypoint.sh entrypoint.sh
+COPY ./ckan2mqa /ckan2mqa
 
 ENTRYPOINT ["/bin/bash", "./entrypoint.sh"]

Dockerfile.dev → ckan-mqa/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.13.0a3-slim
 LABEL maintainer="mnl.janez@gmail.com"
 
 ENV APP_DIR=/app
@@ -17,15 +17,15 @@ RUN apt-get -q -y update && \
     python3 -m pip install pdm debugpy
 
 WORKDIR ${APP_DIR}
-COPY pyproject.toml pdm.lock .
+COPY pyproject.toml pdm.lock ./
 
 RUN pdm install --no-self --group prod
 
-COPY ckan-mqa/entrypoint_dev.sh entrypoint.sh
-COPY ckan2mqa ckan2mqa
+COPY ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh entrypoint.sh
+COPY ./ckan2mqa /ckan2mqa
 
 EXPOSE 5678/TCP
 
 # Set entrypoint with debugpy
-ENTRYPOINT ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client", "./entrypoint_dev.sh"]
+ENTRYPOINT ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client", "./ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh"]
 CMD ["tail", "-f", "/dev/null"]

docker-compose.dev.yml

@@ -4,7 +4,7 @@ services:
     container_name: ckan-mqa-dev
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: ckan-mqa/Dockerfile.dev
     env_file:
       - .env
     logging:

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     container_name: ckan-mqa
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: ckan-mqa/Dockerfile
     image: ghcr.io/mjanez/ckan-mqa:latest
     env_file:
       - .env

log/mqa/README.md

@@ -1,2 +1,2 @@
 # Log folder
-This folder is used to store logs generated by `ckan2pycsw` script and `catalog.rdf` file downloaded from URL. Logs are essential for debugging and tracing the execution of the code. By keeping logs, you can track the behavior of your code over time, identify errors and performance issues, and improve the overall quality of your code.
+This folder is used to store logs generated by `ckan2pycsw` script and `catalog.rdf` file downloaded from `CKAN_CATALOG_URL`. Logs are essential for debugging and tracing the execution of the code. By keeping logs, you can track the behavior of your code over time, identify errors and performance issues, and improve the overall quality of your code.