From 55c21ddae5c5069d6f491db4537a48170ad81cea Mon Sep 17 00:00:00 2001 From: Hayden Roche Date: Wed, 27 Sep 2023 18:32:02 -0700 Subject: [PATCH] Make changes to CI so Docker image is reused. (#110) --- .devcontainer/devcontainer.json | 2 +- .github/actions/compile-examples/action.yml | 17 --- .github/actions/load-ci-image/action.yml | 17 +++ .../actions/run-tests-in-container/Dockerfile | 27 ---- .../actions/run-tests-in-container/action.yml | 6 - .github/workflows/note-arduino-ci.yml | 141 +++++++++++++++--- .../compile-examples/Dockerfile => Dockerfile | 18 +-- examples/build_example.sh | 19 +++ test/run_all_tests.sh | 11 +- 9 files changed, 174 insertions(+), 84 deletions(-) delete mode 100644 .github/actions/compile-examples/action.yml create mode 100644 .github/actions/load-ci-image/action.yml delete mode 100644 .github/actions/run-tests-in-container/Dockerfile delete mode 100644 .github/actions/run-tests-in-container/action.yml rename .github/actions/compile-examples/Dockerfile => Dockerfile (95%) create mode 100755 examples/build_example.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 00dc9c8..f6e37ac 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,7 +8,7 @@ // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. //"dockerFile": "../.github/actions/compile-examples/Dockerfile", - "dockerFile": "../.github/actions/run-tests-in-container/Dockerfile", + "dockerFile": "../Dockerfile", // Set *default* container specific settings.json values on container create. "settings": {}, diff --git a/.github/actions/compile-examples/action.yml b/.github/actions/compile-examples/action.yml deleted file mode 100644 index 7868c66..0000000 --- a/.github/actions/compile-examples/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: 'Compile Examples' -author: 'Zachary J. Fields' -description: 'Validate examples by compiling for supported platforms' -runs: - using: 'docker' - image: './Dockerfile' - entrypoint: 'bash' - args: - - -c - - "cp -r . /home/blues/Arduino/libraries/Blues_Wireless_Notecard \ - && HOME=/home/blues arduino-cli compile \ - --build-property compiler.cpp.extra_flags='-Wno-unused-parameter -Werror' \ - --fqbn ${{ inputs.fully-qualified-board-name }} \ - --log-level trace \ - --verbose \ - --warnings all \ - ${{ inputs.example-sketch }}" diff --git a/.github/actions/load-ci-image/action.yml b/.github/actions/load-ci-image/action.yml new file mode 100644 index 0000000..fb2c6b4 --- /dev/null +++ b/.github/actions/load-ci-image/action.yml @@ -0,0 +1,17 @@ +name: 'Load note-arduino CI Docker image' +runs: + using: 'composite' + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Download image artifact + uses: actions/download-artifact@v2 + with: + name: note_arduino_ci_image + path: /tmp + + - name: Load Docker image + shell: bash + run: | + docker load --input /tmp/note_arduino_ci_image.tar diff --git a/.github/actions/run-tests-in-container/Dockerfile b/.github/actions/run-tests-in-container/Dockerfile deleted file mode 100644 index 3f6be1e..0000000 --- a/.github/actions/run-tests-in-container/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Container with dependencies necessary to run unit-tests and code coverage - -# Build development environment -# $ docker build . --tag note-arduino-test - -# Launch development environment (mount source root as /note-arduino/) -# $ docker run --rm --volume $(pwd)/../../../:/note-arduino/ --workdir /note-arduino/ note-arduino-test - -# Base Image -FROM alpine:3.14 - - -# Testing Dependencies -RUN ["ash", "-c", "\ - apk add --no-cache \ - g++ \ - gdb \ - gzip \ - valgrind \ -"] - -RUN ["ash", "-c", "\ - apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing \ - lcov \ -"] - -ENTRYPOINT ["./test/run_all_tests.sh"] diff --git a/.github/actions/run-tests-in-container/action.yml b/.github/actions/run-tests-in-container/action.yml deleted file mode 100644 index ec6207b..0000000 --- a/.github/actions/run-tests-in-container/action.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: 'Containerized Test' -author: 'Zachary J. Fields' -description: 'Perform unit-test in containerized environment' -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/workflows/note-arduino-ci.yml b/.github/workflows/note-arduino-ci.yml index 228f614..78706bb 100644 --- a/.github/workflows/note-arduino-ci.yml +++ b/.github/workflows/note-arduino-ci.yml @@ -7,25 +7,92 @@ on: branches: [ master ] jobs: - validate_library: # job id + check_dockerfile_changed: runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: - - name: Checkout Code - id: checkout + - name: Checkout code uses: actions/checkout@v3 - - name: Run Tests In Container - id: containerized_tests - uses: ./.github/actions/run-tests-in-container - - name: Coveralls Action Bug Workaround - id: coveralls_bug_workaround - run: sudo sed -i 's/github\/workspace/home\/runner\/work\/note-arduino\/note-arduino/g' ./coverage/lcov.info - - name: Publish Test Coverage - id: publish_coverage + + # TODO: This is a 3rd party GitHub action from some dude. Ideally, we'd + # use something more "official". + - name: Check if Dockerfile changed + uses: dorny/paths-filter@v2 + id: filter + with: + base: 'master' + filters: | + changed: + - 'Dockerfile' + build_ci_docker_image: + runs-on: ubuntu-latest + needs: [check_dockerfile_changed] + if: ${{ needs.check_dockerfile_changed.outputs.changed == 'true' }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Rebuild image + uses: docker/build-push-action@v4 + with: + context: . + load: true + tags: ghcr.io/blues/note_arduino_ci:latest + outputs: type=docker,dest=/tmp/note_arduino_ci_image.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v3 + with: + name: note_arduino_ci_image + path: /tmp/note_arduino_ci_image.tar + + run_tests: + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [build_ci_docker_image] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Load CI Docker image + if: ${{ needs.build_ci_docker_image.result == 'success' }} + uses: ./.github/actions/load-ci-image + + - name: Run tests + run: | + docker run --rm --volume $(pwd):/note-arduino/ \ + --workdir /note-arduino/ \ + --entrypoint ./test/run_all_tests.sh \ + --user root \ + ghcr.io/blues/note_arduino_ci:latest + + - name: Adjust lcov source file paths for Coveralls + run: sudo sed -i 's/\/note-arduino\///g' ./coverage/lcov.info + + - name: Publish test coverage uses: coverallsapp/github-action@master with: github-token: ${{ secrets.GITHUB_TOKEN }} - validate_examples: # job id + path-to-lcov: ./coverage/lcov.info + + build_examples: runs-on: ubuntu-latest + if: ${{ always() }} + needs: [build_ci_docker_image] continue-on-error: true strategy: matrix: @@ -38,26 +105,60 @@ jobs: - ./examples/Example5_UsingTemplates/Example5_UsingTemplates.ino - ./examples/Example6_SensorTutorial/Example6_SensorTutorial.ino - ./examples/Example7_PowerControl/Example7_PowerControl.ino - - ./examples/Example8_BinarySendReceive/Example8_BinarySendReceive.ino - - ./examples/Example9_BinarySendReceiveChunked/Example9_BinarySendReceiveChunked.ino + # TODO: Uncomment these once note-c is updated with the necessary NoteBinary* functions. + # - ./examples/Example8_BinarySendReceive/Example8_BinarySendReceive.ino + # - ./examples/Example9_BinarySendReceiveChunked/Example9_BinarySendReceiveChunked.ino fully-qualified-board-name: - STMicroelectronics:stm32:BluesW:pnum=SWAN_R5 - esp32:esp32:featheresp32 - adafruit:samd:adafruit_feather_m4 - STMicroelectronics:stm32:GenF4:pnum=FEATHER_F405 - - arduino:mbed_nano:nano33ble + # TODO: Fix this. May require upstream fix. + # - arduino:mbed_nano:nano33ble - SparkFun:apollo3:sfe_artemis_thing_plus - STMicroelectronics:stm32:Nucleo_32:pnum=NUCLEO_L432KC - adafruit:nrf52:feather52840:softdevice=s140v6 - rp2040:rp2040:rpipico - arduino:avr:uno + steps: - - name: Checkout Code + - name: Checkout code id: checkout uses: actions/checkout@v3 + + - name: Load CI docker image + if: ${{ needs.build_ci_docker_image.result == 'success' }} + uses: ./.github/actions/load-ci-image + - name: Compile Examples - id: compile_examples - uses: ./.github/actions/compile-examples + run: | + docker run --rm --volume $(pwd):/note-arduino/ \ + --workdir /note-arduino/ \ + --entrypoint ./examples/build_example.sh \ + ghcr.io/blues/note_arduino_ci:latest \ + ${{ matrix.fully-qualified-board-name }} \ + ${{ matrix.example-sketch }} + + publish_ci_image: + runs-on: ubuntu-latest + # Make sure tests passed and examples built successfully before publishing. + needs: [build_ci_docker_image, run_tests, build_examples] + # Only publish the image if this is a push event and the Docker image was rebuilt. + if: ${{ github.event_name == 'push' && needs.build_ci_docker_image.result == 'success' }} + + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Push image to registry + uses: docker/build-push-action@v4 with: - example-sketch: ${{ matrix.example-sketch }} - fully-qualified-board-name: ${{ matrix.fully-qualified-board-name }} + push: true + tags: ghcr.io/blues/note_arduino_ci:latest diff --git a/.github/actions/compile-examples/Dockerfile b/Dockerfile similarity index 95% rename from .github/actions/compile-examples/Dockerfile rename to Dockerfile index 48a809c..78b762b 100644 --- a/.github/actions/compile-examples/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2022 Blues Inc. All rights reserved. +# Copyright 2023 Blues Inc. All rights reserved. # Use of this source code is governed by licenses granted by the # copyright holder including that found in the LICENSE file. @@ -15,7 +15,7 @@ ARG UID=1000 ARG USER="blues" # POSIX compatible (Linux/Unix) base image -FROM debian:bullseye-slim +FROM debian:bookworm-slim # Import global arguments ARG ARDUINO_CLI_VERSION @@ -55,14 +55,20 @@ RUN ["dash", "-c", "\ bash-completion \ ca-certificates \ curl \ + g++ \ + gdb \ + gzip \ + lcov \ python-is-python3 \ python3 \ python3-pip \ ssh \ tree \ + valgrind \ && pip install \ adafruit-nrfutil \ pyserial \ + --break-system-packages \ && apt-get clean \ && apt-get purge \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ @@ -104,11 +110,3 @@ RUN ["dash", "-c", "\ && arduino-cli core install STMicroelectronics:stm32 \ && arduino-cli lib install \"Blues Wireless Notecard Pseudo Sensor\" \ "] - -# Set Execution Environment -USER root -WORKDIR /host-volume - -ENTRYPOINT ["arduino-cli"] - -CMD ["help"] diff --git a/examples/build_example.sh b/examples/build_example.sh new file mode 100755 index 0000000..e391302 --- /dev/null +++ b/examples/build_example.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +NOTE_ARDUINO_DIR="$SCRIPT_DIR/.." + +# If this is being run inside a Docker container (i.e. for GitHub actions CI), +# copy the latest note-arduino code into the appropriate place so that it can +# be consumed when building the example. +if grep -sq 'docker\|lxc' /proc/1/cgroup; then + cp -r $NOTE_ARDUINO_DIR $HOME/Arduino/libraries/Blues_Wireless_Notecard +fi + +arduino-cli compile \ + --build-property compiler.cpp.extra_flags='-Wno-unused-parameter -Werror' \ + --fqbn $1 \ + --log-level trace \ + --verbose \ + --warnings all \ + $2 diff --git a/test/run_all_tests.sh b/test/run_all_tests.sh index 88fe4a4..f4f5189 100755 --- a/test/run_all_tests.sh +++ b/test/run_all_tests.sh @@ -7,6 +7,10 @@ DEFAULT='\x1B[0;0m' all_tests_result=0 +# This fixes a problem when running valgrind in a Docker container when the +# host machine is running Fedora. See https://stackoverflow.com/a/75293014. +ulimit -n 1024 + if [ 0 -eq $all_tests_result ]; then echo && echo -e "${YELLOW}Compiling and running Notecard Test Suite...${DEFAULT}" g++ -fprofile-arcs -ftest-coverage -Wall -Wextra -Werror -Wpedantic -std=c++11 -O0 -g \ @@ -164,12 +168,13 @@ if [ 0 -eq ${all_tests_result} ]; then # Run coverage if available if [ $(which lcov) ]; then - rm mock-*.gc?? *_Mock.gc?? *test.gc?? \ - && gcov --version \ + rm -f mock-*.gc?? *_Mock.gc?? *test.gc?? + gcov --version \ && lcov --version \ && mkdir -p ./coverage \ && lcov --capture \ --directory . \ + --no-external \ --exclude '/note-arduino/test/*' \ --output-file ./coverage/lcov.info \ --rc lcov_branch_coverage=1 @@ -177,7 +182,7 @@ if [ 0 -eq ${all_tests_result} ]; then echo -e "${YELLOW}COVERAGE REPORT NOT PRODUCED!!!${DEFAULT}"; all_tests_result=998 else - lcov --summary ./coverage/lcov.info + lcov --summary --rc lcov_branch_coverage=1 ./coverage/lcov.info fi fi rm -f failed_test_run