ci: avoid installing the toolchain from scratch in each job

* Use GitHub Packages * Use container in a step, instead of a container job F4PGA_INSTALL_DIR changed to an absolute location due to actions/runner#1525 Signed-off-by: Unai Martinez-Corral <umartinezcorral@antmicro.com>
chipsalliance · Aug 3, 2022 · c233c83 · c233c83
1 parent 524dfb6
commit c233c83
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 55 deletions.
diff --git a/.github/scripts/generate_job_matrices.py → .github/scripts/generate_matrices.py b/.github/scripts/generate_job_matrices.py → .github/scripts/generate_matrices.py
@@ -19,6 +19,8 @@
 from os import environ
 from sys import argv as sys_argv
 
+registry = 'ghcr.io/chipsalliance/f4pga/dev/conda'
+
 isFork = len(sys_argv)>1 and sys_argv[1] != 'chipsalliance/f4pga-examples'
 
 runs_on = (
@@ -86,21 +88,19 @@ def get_jobs(
 
     for osver in osvers:
         jobs.extend([{
-            'name': "Surelog" if usesSurelog else "Default",
+            'name': ("Surelog" if usesSurelog else "Default") + f' | xc7 | {osver[0]}/{osver[1]} | {example}',
             'runs-on': runs_on,
             'fpga-fam': "xc7",
-            'os': osver[0],
-            'os-version': osver[1],
+            'image': f'{registry}/{osver[0]}/{osver[1]}/xc7',
             'example': example,
             'surelog': "-parse -DSYNTHESIS" if usesSurelog else ""
         } for example in examples])
 
     jobs.extend([{
-        'name': "Surelog" if usesSurelog else "Default",
+        'name': ("Surelog" if usesSurelog else "Default") + f' | eos-s3 | {osver[0]}/{osver[1]} | counter',
         'runs-on': runs_on,
         'fpga-fam': "eos-s3",
-        'os': osver[0],
-        'os-version': osver[1],
+        'image': f'{registry}/{osver[0]}/{osver[1]}/eos-s3',
         'example': "counter",
         'surelog': "-parse -DSYNTHESIS" if usesSurelog else ""
     } for osver in osvers])
@@ -111,3 +111,33 @@ def get_jobs(
     jobs = get_jobs(distribution, False) + get_jobs(distribution, True)
     print(f'::set-output name={distribution}::{jobs!s}')
     print(f"{distribution}: {jobs!s}")
+
+utils = {
+    'ubuntu': 'apt -qqy update && apt -qqy install wget locales patch && locale-gen $LANG',
+    'debian': 'apt -qqy update && apt -qqy install wget locales patch && locale-gen $LANG',
+    'centos': 'yum -y install wget patch',
+    'fedora': 'dnf install -y wget patch'
+}
+
+images = [
+    {
+        'registry': f'{registry}',
+        'image': f'{osver[0]}/{osver[1]}/{fam}',
+        'from': f'{osver[0]}:{osver[1]}',
+        'utils': utils[osver[0]],
+        'args': f'{fam} {osver[0]}'
+    } for osver in [
+        ("debian", "buster"),
+        ("debian", "bullseye"),
+        ("debian", "sid"),
+        ("fedora", "35"),
+        ("fedora", "36"),
+        ("ubuntu", "xenial"),
+        ("ubuntu", "bionic"),
+        ("ubuntu", "focal"),
+        ("centos", "7")
+    ] for fam in ['xc7', 'eos-s3']
+]
+
+print(f'::set-output name=images::{images!s}')
+print(f"images: {images!s}")
diff --git a/.github/workflows/Automerge.yml b/.github/workflows/Automerge.yml
@@ -22,32 +22,89 @@ jobs:
       ubuntu: ${{ steps.generate.outputs.ubuntu }}
       fedora: ${{ steps.generate.outputs.fedora }}
       centos: ${{ steps.generate.outputs.centos }}
+      images: ${{ steps.generate.outputs.images }}
     steps:
       - uses: actions/checkout@v3
       - id: generate
-        run: ./.github/scripts/generate_job_matrices.py '${{ github.repository }}'
+        run: ./.github/scripts/generate_matrices.py '${{ github.repository }}'
 
 
-  Debian:
+  Packages:
     needs: Matrices
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.Matrices.outputs.images) }}
+    name: 'Image: ${{ matrix.image }}'
+    env:
+      DOCKER_BUILDKIT: 1
+
+    steps:
+
+      - name: Setup repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Build container image
+        run: |
+          docker build -t ${{ matrix.registry }}/${{ matrix.image }} . -f- <<'EOF'
+          FROM ${{ matrix.from }}
+
+          ENV LANG en_US.UTF-8
+
+          # Install utils
+          RUN ${{ matrix.utils }}
+
+          # Install tuttest
+          RUN wget https://github.com/antmicro/tuttest/releases/download/v0.2-beta/tuttest -O /usr/bin/tuttest \
+           && chmod a+rx /usr/bin/tuttest
+
+          # Install F4PGA toolchain
+          RUN --mount=type=bind,target=/tmp/ctx cp -vr /tmp/ctx /tmp/repo \
+           && cd /tmp/repo && bash /tmp/repo/.github/scripts/install-toolchain.sh ${{ matrix.args }} \
+           && cd .. && rm -rf /tmp/repo
+          EOF
+
+      - name: Push container image
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: pyTooling/Actions/with-post-step@r0
+        with:
+          main: |
+            echo '${{ github.token }}' | docker login ghcr.io -u GitHub-Actions --password-stdin
+            docker push ${{ matrix.registry }}/${{ matrix.image }}
+          post: docker logout ghcr.io
+
+
+  Debian:
+    needs:
+      - Matrices
+      - Packages
     uses: ./.github/workflows/sphinx-tuttest.yml
     with:
       matrix: ${{ needs.Matrices.outputs.debian }}
 
   Ubuntu:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
     uses: ./.github/workflows/sphinx-tuttest.yml
     with:
       matrix: ${{ needs.Matrices.outputs.ubuntu }}
 
   Fedora:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
     uses: ./.github/workflows/sphinx-tuttest.yml
     with:
       matrix: ${{ needs.Matrices.outputs.fedora }}
 
   Centos:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
     uses: ./.github/workflows/sphinx-tuttest.yml
     with:
       matrix: ${{ needs.Matrices.outputs.centos }}

diff --git a/.github/workflows/sphinx-tuttest.yml b/.github/workflows/sphinx-tuttest.yml
@@ -22,65 +22,45 @@ jobs:
       matrix:
         include: ${{ fromJson(inputs.matrix) }}
     runs-on: ${{ matrix.runs-on }}
-    name: ${{ matrix.os-version }} | ${{ matrix.name }} | ${{ matrix.fpga-fam }} | ${{ matrix.example }}
+    name: ${{ matrix.name }}
 
     env:
-      LANG: "en_US.UTF-8"
-      DEBIAN_FRONTEND: "noninteractive"
       GHA_PREEMPTIBLE: "false"
       SURELOG_CMD: ${{ matrix.surelog }}
 
-    container: ${{matrix.os}}:${{matrix.os-version}}
-
     steps:
 
       - name: Setup repository
         uses: actions/checkout@v3
         with:
           submodules: recursive
 
-      - name: Install utils
-        run: |
-          case ${{ matrix.os }} in
-            debian|ubuntu) apt -qqy update && apt -qqy install git wget locales && locale-gen $LANG ;;
-            centos) yum -y install git wget ;;
-            fedora) dnf install -y git wget ;;
-          esac
-
-      - name: Install tuttest
-        run: |
-          wget https://github.com/antmicro/tuttest/releases/download/v0.2-beta/tuttest -O /usr/bin/tuttest
-          chmod a+rx /usr/bin/tuttest
+      - name: Pull container image
+        run: docker pull ${{matrix.image}}
 
       - name: Patch package URLs
         if: inputs.latest == true
         run: |
-          case ${{ matrix.os }} in
-            debian|ubuntu)
-              apt update -qq
-              apt install -y patch
-            ;;
-            centos)
-              yum -y install patch
-            ;;
-            fedora)
-              dnf install -y patch
-            ;;
-          esac
+          sudo apt update -qq
+          sudo apt install -y patch
           patch -p1 -i .github/latest.patch
 
-      - name: Install F4PGA toolchain
-        run: bash .github/scripts/install-toolchain.sh ${{matrix.fpga-fam}} ${{matrix.os}}
-
       - name: Build examples
-        run: bash .github/scripts/build-examples.sh ${{matrix.fpga-fam}} ${{matrix.example}}
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: f4pga-examples-bitstreams-${{ matrix.name }}
-          path: '**/*.bit'
+        # Workaround for 'uses: docker://${{matrix.image}}' not being supported.
+        # See https://github.com/github/feedback/discussions/9049
+        run: >-
+          docker run --rm
+          -v $(pwd):/wrk -w /wrk
+          -e DEBIAN_FRONTEND="noninteractive"
+          ${{matrix.image}}
+          bash .github/scripts/build-examples.sh ${{matrix.fpga-fam}} ${{matrix.example}}
 
-      - uses: actions/upload-artifact@v3
-        with:
-          name: f4pga-examples-plots-${{ matrix.name }}
-          path: '**/plot_*.svg'
+#      - uses: actions/upload-artifact@v3
+#        with:
+#          name: f4pga-examples-bitstreams-${{ matrix.name }}
+#          path: '**/*.bit'
+#
+#      - uses: actions/upload-artifact@v3
+#        with:
+#          name: f4pga-examples-plots-${{ matrix.name }}
+#          path: '**/plot_*.svg'
diff --git a/docs/building-examples.rst b/docs/building-examples.rst
@@ -9,7 +9,7 @@ set it to earlier, for example:
 .. code-block:: bash
    :name: export-install-dir
 
-   export F4PGA_INSTALL_DIR=~/opt/f4pga
+   export F4PGA_INSTALL_DIR=/opt/f4pga
 
 Select your FPGA family:
 

diff --git a/docs/getting.rst b/docs/getting.rst
@@ -84,7 +84,7 @@ and so you will need to add some ``sudo`` commands to the instructions below.
 .. code-block:: bash
    :name: conda-install-dir
 
-   export F4PGA_INSTALL_DIR=~/opt/f4pga
+   export F4PGA_INSTALL_DIR=/opt/f4pga
 
 Setup and download assets
 ~~~~~~~~~~~~~~~~~~~~~~~~~