From 2a7199cc9984fe99269f71138708369f5bf334d6 Mon Sep 17 00:00:00 2001 From: philmcmahon Date: Thu, 3 Oct 2024 10:00:22 +0100 Subject: [PATCH] Prepare for building multiple containers Co-authored by : Marjan Kalanaki --- ...ocker.yml => build-publish-containers.yml} | 55 ++++++++++++------- containers/media-download.Dockerfile | 17 ++++++ .../whisper.Dockerfile | 0 packages/backend-common/src/process.ts | 0 4 files changed, 51 insertions(+), 21 deletions(-) rename .github/workflows/{build-whisper-docker.yml => build-publish-containers.yml} (63%) create mode 100644 containers/media-download.Dockerfile rename whisper_container/Dockerfile => containers/whisper.Dockerfile (100%) create mode 100644 packages/backend-common/src/process.ts diff --git a/.github/workflows/build-whisper-docker.yml b/.github/workflows/build-publish-containers.yml similarity index 63% rename from .github/workflows/build-whisper-docker.yml rename to .github/workflows/build-publish-containers.yml index a0d10adc..323f826a 100644 --- a/.github/workflows/build-whisper-docker.yml +++ b/.github/workflows/build-publish-containers.yml @@ -6,19 +6,20 @@ on: workflow_dispatch: push: paths: - - 'whisper_container/Dockerfile' - - '.github/workflows/build-whisper-docker.yml' + - '../../containers/*' + - '.github/workflows/build-publish-containers.yml' # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: - IMAGE_NAME: ${{ github.repository }} + WHISPER_IMAGE_NAME: ${{ github.repository }} + MEDIA_DOWNLOAD_IMAGE_NAME: ${{ github.repository }}_media-download BUILD_NUMBER: ${{ github.run_number }} GITHUB_REGISTRY: ghcr.io # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: build-and-push-image: - runs-on: ubuntu-latest + runs-on: 2core-ubuntu-latest-arm # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read @@ -38,18 +39,6 @@ jobs: registry: ${{ env.GITHUB_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - # Note - this step is the thing where we are indicating which repositories we want to push the container to - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.GITHUB_REGISTRY }}/${{ env.IMAGE_NAME }} - ${{ secrets.TRANSCRIPTION_SERVICE_ECR_URI }} - tags: | - type=sha - type=raw,value=latest - uses: aws-actions/configure-aws-credentials@v4 name: Configure AWS credentials for pushing to ECR with: @@ -61,14 +50,38 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image + # - name: Build and push whisper Docker image + # uses: docker/build-push-action@v5 + # with: + # context: . + # file: containers/whisper.Dockerfile + # # to add x86: linux/amd64 + # platforms: linux/arm64 + # push: true + # tags: ${{ github.ref_name }},${{ env.GITHUB_REGISTRY }}/${{ env.WHISPER_IMAGE_NAME }} + # cache-from: type=gha + # cache-to: type=gha,mode=max + + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version-file: .nvmrc + cache: 'npm' + - name: Install dependencies + run: | + npm install + - name: Build media-download + run: | + npm run media-download::build + - name: Build and push media-download Docker image uses: docker/build-push-action@v5 with: - context: whisper_container/ - # to add x86: linux/amd64 + context: . + file: containers/media-download.Dockerfile platforms: linux/arm64 push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ env.GITHUB_REGISTRY }}/${{ env.MEDIA_DOWNLOAD_IMAGE_NAME }}:${{ github.ref_name }},${{ secrets.TRANSCRIPTION_SERVICE_ECR_URI }}:${{ github.ref_name }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + node_version=20.11.0 diff --git a/containers/media-download.Dockerfile b/containers/media-download.Dockerfile new file mode 100644 index 00000000..a3c1ea2b --- /dev/null +++ b/containers/media-download.Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-bookworm +WORKDIR /opt +LABEL com.theguardian.transcription-service.media-download-container="Media download container with yt-dlp, associated dependnencies and media download app" + +ARG node_version + +RUN pip install yt-dlp + +RUN apt-get update +RUN apt-get install -y ffmpeg git nodejs npm +RUN npm install -g n +RUN echo "node version: $node_version" +RUN n $node_version + +COPY ./packages/media-download/dist/index.js /opt/media-download.js + +CMD node /opt/media-download.js diff --git a/whisper_container/Dockerfile b/containers/whisper.Dockerfile similarity index 100% rename from whisper_container/Dockerfile rename to containers/whisper.Dockerfile diff --git a/packages/backend-common/src/process.ts b/packages/backend-common/src/process.ts new file mode 100644 index 00000000..e69de29b