docker.yml

name: Docker

on:
  push:
    branches: [master, abdo-dev, kaspar-dev, review]
  # using "pull_request_target" instead of "pull_request" runs this in the
  # "master" branch context and makes secrets work with external pull requests
  # (incl. dependabot). we check for author_association below so first-time
  # contributors can't run this workflow.
  pull_request_target:
    branches: [master]
jobs:
  get_version:
    if: ${{ github.event_name == 'push' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'CONTRIBUTOR' }}
    name: Get git version
    runs-on: ubuntu-20.04
    outputs:
      frontend_sha: ${{ steps.get_sha.outputs.frontend_sha }}
      processor_sha: ${{ steps.get_sha.outputs.processor_sha }}
      nginx_sha: ${{ steps.get_sha.outputs.nginx_sha }}
      matrix: ${{ steps.set_matrix.outputs.matrix }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4
        with:
          # check out the pull-request when triggered by "pull_request_target",
          # when triggered on "push" this is empty and checks out the branch
          # pushed to
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: 'recursive'
          # 0 means fetch all commits, which we need for getting the right git versions of each folder
          fetch-depth: 0

      - name: Get commit SHA
        id: get_sha
        shell: bash
        run: |
          echo "frontend_sha=$(git log --format=%H -n1 frontend/)" >> $GITHUB_OUTPUT
          echo "processor_sha=$(git log --format=%H -n1 processor/)" >> $GITHUB_OUTPUT
          echo "nginx_sha=$(git log --format=%H -n1 nginx/)" >> $GITHUB_OUTPUT

      - name: Create e2e matrix
        # For master run e2e across all browsers otherwise, run it only in chrome
        id: set_matrix
        # Run e2e across all tests on `push` event to the `master` branch (merging another branch into master triggers push too).
        # Run e2e in chrome only otherwise.
        run: |
          if [[ "${{ github.event_name }}" == "push" ]] && [[ "${{ github.ref }}" == "refs/heads/master" ]];
          then
            matrix='{"browser":["chrome", "edge", "electron", "firefox"],"containers":[1,2,3]}'
          else
            matrix='{"browser":["chrome"],"containers":[1,2,3]}'
          fi
          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT";

  build:
    if: ${{ github.event_name == 'push' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'CONTRIBUTOR' }}
    name: Build image
    runs-on: ubuntu-20.04
    needs:
      - get_version
    strategy:
      fail-fast: true
      matrix:
        service: [frontend, processor, nginx]

    steps:
      - name: Set SHA output
        id: set_sha
        shell: bash
        run: |
          case '${{ matrix.service }}' in
            frontend)
              sha="${{ needs.get_version.outputs.frontend_sha }}";;
            processor)
              sha="${{ needs.get_version.outputs.processor_sha }}";;
            nginx)
              sha="${{ needs.get_version.outputs.nginx_sha }}";;
          esac
          echo "sha=${sha}" >> $GITHUB_OUTPUT

      - name: Check for ${{ matrix.service }} image
        id: check_for_image
        shell: bash
        run: |
          if [ "$(curl \
           https://ghcr.io/v2/kitspace/${{ matrix.service }}/manifests/${{ steps.set_sha.outputs.sha }} \
           -H "Authorization: Bearer $(echo '${{ secrets.GITHUB_TOKEN }}' | base64)"  \
           -H 'Accept: application/vnd.oci.image.index.v1+json' \
            | jq .errors)" == 'null' ];
            then image_exists='true'
            else image_exists='false'
          fi
          echo "image_exists=${image_exists}" >> $GITHUB_OUTPUT

      - name: Check out the repo
        uses: actions/checkout@v4
        if: steps.check_for_image.outputs.image_exists == 'false'
        with:
          # check out the pull-request when triggered by "pull_request_target",
          # when triggered on "push" this is empty and checks out the branch
          # pushed to
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: 'recursive'

      - name: Set up Docker Buildx
        if: steps.check_for_image.outputs.image_exists == 'false'
        uses: docker/setup-buildx-action@v3

      - name: Login to ghcr.io
        if: steps.check_for_image.outputs.image_exists == 'false'
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GHCR_PAT }}

      # to avoid being rate-limited on pulls
      - name: Login to hub.docker.com
        if: steps.check_for_image.outputs.image_exists == 'false'
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_PAT }}
        continue-on-error: true

      - name: Build and push ${{ matrix.service }}
        if: steps.check_for_image.outputs.image_exists == 'false'
        uses: docker/build-push-action@v4
        with:
          context: ./${{ matrix.service }}
          file: ./${{ matrix.service }}/Dockerfile
          push: true
          tags: ghcr.io/kitspace/${{ matrix.service }}:${{ steps.set_sha.outputs.sha }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Report "review" deploy failure
        if: ${{ failure() && github.ref == 'refs/heads/review' }}
        run: scripts/report_auto_merge_deploy.py '${{ secrets.GITHUB_TOKEN }}' 'failure'

  deploy:
    if: ${{ github.event_name == 'push' }}
    name: Deploy to staging
    runs-on: ubuntu-22.04
    environment: staging
    needs:
      - get_version
      - build
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4
        with:
          submodules: 'recursive'
          # 0 means fetch all commits, which we need for "review" branch deploy
          # reporting
          fetch-depth: 0

      - name: Report "review" deploy pending
        if: ${{ github.ref == 'refs/heads/review' }}
        run: scripts/report_auto_merge_deploy.py '${{ secrets.GITHUB_TOKEN }}' 'pending'

      - name: Login to ghcr.io
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
        # we don't really need to log in to pull the images, just doing it in
        # case an image is accidentally set to private
        continue-on-error: true

      # to avoid being rate-limited on pulls
      - name: Login to hub.docker.com
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_PAT }}
        continue-on-error: true

      - name: 'Set S3 env variables depending on branch name'
        run: |
          case '${{ github.ref_name }}' in
            kaspar-dev)
              echo 'S3_ACCESS_KEY=${{ secrets.STAGING_S3_ACCESS_KEY_KASPAR_DEV }}' >> $GITHUB_ENV
              echo 'S3_SECRET_KEY=${{ secrets.STAGING_S3_SECRET_KEY_KASPAR_DEV }}' >> $GITHUB_ENV
              echo 'S3_PROCESSOR_BUCKET_NAME=kitspace-staging-kaspar-dev' >> $GITHUB_ENV;;
            abdo-dev)
              echo 'S3_ACCESS_KEY=${{ secrets.STAGING_S3_ACCESS_KEY_ABDO_DEV }}' >> $GITHUB_ENV
              echo 'S3_SECRET_KEY=${{ secrets.STAGING_S3_SECRET_KEY_ABDO_DEV }}' >> $GITHUB_ENV
              echo 'S3_PROCESSOR_BUCKET_NAME=kitspace-staging-abdo-dev' >> $GITHUB_ENV;;
            review)
              echo 'S3_ACCESS_KEY=${{ secrets.STAGING_S3_ACCESS_KEY_REVIEW }}' >> $GITHUB_ENV
              echo 'S3_SECRET_KEY=${{ secrets.STAGING_S3_SECRET_KEY_REVIEW }}' >> $GITHUB_ENV
              echo 'S3_PROCESSOR_BUCKET_NAME=kitspace-staging-review' >> $GITHUB_ENV;;
            master)
              echo 'S3_ACCESS_KEY=${{ secrets.STAGING_S3_ACCESS_KEY_MASTER }}' >> $GITHUB_ENV
              echo 'S3_SECRET_KEY=${{ secrets.STAGING_S3_SECRET_KEY_MASTER }}' >> $GITHUB_ENV
              echo 'S3_PROCESSOR_BUCKET_NAME=kitspace-staging-master' >> $GITHUB_ENV;;
          esac

      - name: Deploy to ${{ github.ref_name }}.staging.kitspace.dev
        id: deploy_to_staging
        env:
          POSTGRES_PASSWORD: '${{ secrets.STAGING_POSTGRES_ROOT_PASSWORD }}'
          POSTGRES_READONLY_USER_PASSWORD: '${{ secrets.STAGING_POSTGRES_READONLY_PASSWORD }}'
          GITEA_SECRET_KEY: '${{ secrets.STAGING_GITEA_SECRET }}'
          GITEA_INTERNAL_TOKEN: '${{ secrets.STAGING_GITEA_INTERNAL_TOKEN }}'
          MEILI_MASTER_KEY: '${{ secrets.STAGING_MEILI_MASTER_KEY }}'
          REDIS_PASSWORD: '${{ secrets.STAGING_REDIS_PASSWORD }}'
          KITSPACE_PROCESSOR_REMOTE_API_TOKENS: '${{ secrets.STAGING_KITSPACE_PROCESSOR_REMOTE_API_TOKEN }}'
          KITSPACE_PROCESSOR_ASSET_VERSION: v1
          KITSPACE_DOMAIN: '${{ github.ref_name }}.staging.kitspace.dev'
          KITSPACE_SCHEME: https
          KITSPACE_DEV_PORT: '80'
          KITSPACE_EXTERNAL_PORT: ''
          CERTBOT_ENABLED: 'true'
          CERTBOT_EMAIL: 'kaspar@kitspace.org'
          COMPOSE_PROJECT_NAME: 'kitspace-${{ github.ref_name }}'
          FRONTEND_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.frontend_sha }}'
          PROCESSOR_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.processor_sha }}'
          NGINX_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.nginx_sha }}'
          KITSPACE_PROCESSOR_LOG_LEVEL: debug
          # The maximum allowable file size that can be uploaded to gitea or the processor
          MAX_FILE_SIZE: '6M'
          MAXIMUM_REPO_MIGRATION_TIME: '300000'
          KITSPACE_ROBOTS_TXT: "User-agent: *\\\\nDisallow: \\/\\\\n"
          S3_ENDPOINT: https://s3.eu-west-2.amazonaws.com
          SENTRY_DSN: '${{ secrets.SENTRY_DSN }}'
          SENTRY_ENVIRONMENT: staging
          SENTRY_RELEASE: '${{ github.sha }}'
        run: |
          mkdir -p ~/.ssh
          echo -e '${{ secrets.STAGING_SSH_PRIVATE_KEY }}' > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          ssh-keyscan '${{ github.ref_name }}.staging.kitspace.dev' >> ~/.ssh/known_hosts
          docker context create staging --docker 'host=ssh://deploy@${{ github.ref_name }}.staging.kitspace.dev'
          docker context use staging
          docker --host='ssh://deploy@${{ github.ref_name }}.staging.kitspace.dev' compose --compatibility -f docker-compose.yml -f docker-compose.deploy.yml up -d

      - name: Report "review" deploy failure
        if: ${{ failure() && github.ref == 'refs/heads/review' }}
        run: scripts/report_auto_merge_deploy.py '${{ secrets.GITHUB_TOKEN }}' 'failure'

      - name: Report "review" deploy success
        if: ${{ success() && github.ref == 'refs/heads/review' }}
        run: scripts/report_auto_merge_deploy.py '${{ secrets.GITHUB_TOKEN }}' 'success'

  sentry_release:
    if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
    environment: staging
    name: Create Sentry release
    runs-on: ubuntu-20.04
    needs:
      - deploy
    steps:
      - name: Check out the repo
        # https://github.com/getsentry/action-release#troubleshooting
        uses: actions/checkout@v4
        with:
          submodules: 'recursive'
          fetch-depth: 0

      - name: Create Sentry release
        uses: getsentry/action-release@v1
        with:
          environment: staging
        env:
          SENTRY_AUTH_TOKEN: ${{ secrets.STAGING_SENTRY_AUTH_TOKEN }}
          SENTRY_ORG: kitspace
          SENTRY_PROJECT: nginx

  test:
    name: Test E2E
    runs-on: ubuntu-20.04
    needs:
      - get_version
      - build
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.get_version.outputs.matrix) }}
    steps:
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Check out the repo
        uses: actions/checkout@v4
        with:
          # check out the pull-request when triggered by "pull_request_target",
          # when triggered on "push" this is empty and checks out the branch
          # pushed to
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: 'recursive'

      - name: Set environment variables
        uses: c-py/action-dotenv-to-setenv@v2
        with:
          env-file: .env.example

      - name: Configure /etc/hosts
        shell: bash
        run: cat ./config/hosts | sudo tee -a /etc/hosts

      - name: Install Gitea
        shell: bash
        timeout-minutes: 5
        env:
          FRONTEND_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.frontend_sha }}'
          PROCESSOR_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.processor_sha }}'
          NGINX_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.nginx_sha }}'
        run: |
          scripts/install_gitea.sh

      - name: Setup Deno
        uses: denoland/setup-deno@v1
        with:
          deno-version: v1.x

      - name: Pre e2e
        env:
          FRONTEND_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.frontend_sha }}'
          PROCESSOR_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.processor_sha }}'
          NGINX_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.nginx_sha }}'
        shell: bash
        run: |
          scripts/generate_e2e_fixtures.sh

          # start gitea so we can generate the admin token required for e2e
          docker compose --compatibility -f docker-compose.yml -f docker-compose.deploy.yml up -d
          until docker logs kitspace_gitea_1 | grep -q 'ORM engine initialization successful' ; do sleep 3s; done
          token="$(deno run --allow-env --allow-net --allow-run ./scripts/importBoardsTxt.ts --tokenOnly)"
          docker compose stop

          echo "CYPRESS_GITEA_ADMIN_TOKEN=${token}" >> $GITHUB_ENV

      - name: e2e
        timeout-minutes: 40
        env:
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
          # pass GitHub token to allow accurately detecting a build vs a re-run build
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          FRONTEND_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.frontend_sha }}'
          PROCESSOR_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.processor_sha }}'
          NGINX_DEPLOY_IMAGE_TAG: ':${{ needs.get_version.outputs.nginx_sha }}'
          browser: ${{ matrix.browser }}
          group: ${{ matrix.browser }}
          BUILD_ID: '${{ github.run_id }}-${{ github.run_attempt }}'
          COMMIT_INFO_MESSAGE: '${{ github.event.head_commit.message }}'
          COMMIT_INFO_EMAIL: '${{ github.event.head_commit.author.email }}'
          COMMIT_INFO_AUTHOR: '${{ github.event.head_commit.author.name }}'
          COMMIT_INFO_SHA: '${{ github.event.head_commit.id }}'
          COMMIT_INFO_TIMESTAMP: '${{ github.event.head_commit.timestamp }}'
          COMMIT_INFO_BRANCH: '${{ github.ref_name || github.head_ref }}'
        run: |
          docker compose --compatibility -f docker-compose.yml -f docker-compose.deploy.yml -f docker-compose.e2e.yml up \
            --abort-on-container-exit --exit-code-from e2e

  tag_docker_images:
    if: ${{ github.event_name == 'push' }}
    name: Tag with branch name
    runs-on: ubuntu-22.04
    needs:
      - get_version
      - build
    strategy:
      fail-fast: false
      matrix:
        service: [frontend, processor, nginx]

    steps:
      - name: Tag '${{ matrix.service }}:${{ github.ref_name }}'
        run: |
          case '${{ matrix.service }}' in
            frontend)
              sha='${{ needs.get_version.outputs.frontend_sha }}';;
            processor)
              sha='${{ needs.get_version.outputs.processor_sha }}';;
            nginx)
              sha='${{ needs.get_version.outputs.nginx_sha }}';;
          esac
          curl \
            "https://ghcr.io/v2/kitspace/${{ matrix.service }}/manifests/${sha}" \
            -H "Authorization: Bearer $(echo '${{ secrets.GITHUB_TOKEN }}' | base64)"  \
            -H 'Accept: application/vnd.oci.image.index.v1+json' \
            > manifest.json
          cat manifest.json
          [ "$(cat manifest.json | jq .errors)" == 'null' ] && \
            curl -X PUT \
              https://ghcr.io/v2/kitspace/${{ matrix.service }}/manifests/${{ github.ref_name }} \
              -H "Authorization: Bearer $(echo '${{ secrets.GITHUB_TOKEN }}' | base64)"  \
              -H "Content-Type: application/vnd.oci.image.index.v1+json" \
              -d '@manifest.json'