From 133b07e4747fd319945e19bf9f4a33fff2260430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 21:23:37 +0200 Subject: [PATCH 1/8] ci: github: save and restore ccache database --- .github/workflows/build_and_test.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4dba9f592c..9016a315de 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -10,6 +10,7 @@ on: env: BUILD_DIR: Dist GCOVR_FLAGS: --gcov-ignore-parse-errors --exclude-throw-branches --filter Common --filter Pcap --filter Packet --xml + CCACHE_DIR: ${{ github.workspace }}/.ccache permissions: contents: read @@ -90,6 +91,13 @@ jobs: . /opt/intel/oneapi/setvars.sh printenv >> $GITHUB_ENV + - name: Restore Ccache + id: ccache-restore + uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ matrix.image }}-ccache + - name: Configure PcapPlusPlus run: cmake -DLIGHT_PCAPNG_ZSTD=${{ matrix.config-zstd }} -DPCAPPP_BUILD_COVERAGE=ON ${{ matrix.additional-flags }} -S . -B "$BUILD_DIR" @@ -138,6 +146,12 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - name: Save Ccache + uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + dpdk: runs-on: ubuntu-latest container: seladb/${{ matrix.image }} @@ -154,6 +168,13 @@ jobs: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Restore Ccache + id: ccache-restore + uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ matrix.image }}-ccache + - name: Configure PcapPlusPlus run: cmake -DPCAPPP_USE_DPDK=ON ${{ matrix.additional-flags }} -S . -B "$BUILD_DIR" @@ -187,6 +208,12 @@ jobs: - name: Test Tutorials run: cd build_examples/tutorials_bin && ./Tutorial-HelloWorld + - name: Save Ccache + uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + pfring: runs-on: ubuntu-latest container: seladb/ubuntu2004-pfring @@ -199,6 +226,13 @@ jobs: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Restore Ccache + id: ccache-restore + uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ matrix.image }}-ccache + - name: Configure PcapPlusPlus run: ${{ matrix.configure }} @@ -229,6 +263,12 @@ jobs: - name: Test Tutorials run: cd build_examples/tutorials_bin && ./Tutorial-HelloWorld + - name: Save Ccache + uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + macos-x86: runs-on: ${{ matrix.os-version }} strategy: From dd6f3f662ab77a87dd8721584fa14b55c04ee374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 21:39:52 +0200 Subject: [PATCH 2/8] workflows: ensure BUILD_DIR is only one param --- .github/workflows/build_and_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9016a315de..95120d87fe 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -121,7 +121,7 @@ jobs: run: | cmake -DPCAPPP_BUILD_COVERAGE=OFF -S . -B "$BUILD_DIR" cmake --build "$BUILD_DIR" -j - cmake --install $BUILD_DIR + cmake --install "$BUILD_DIR" - name: Build Tutorials run: | @@ -198,7 +198,7 @@ jobs: run: | cmake -DPCAPPP_BUILD_COVERAGE=OFF -S . -B "$BUILD_DIR" cmake --build "$BUILD_DIR" -j - cmake --install $BUILD_DIR + cmake --install "$BUILD_DIR" - name: Build Tutorials run: | @@ -253,7 +253,7 @@ jobs: run: | cmake -DPCAPPP_BUILD_COVERAGE=OFF -S . -B "$BUILD_DIR" cmake --build "$BUILD_DIR" -j - cmake --install $BUILD_DIR + cmake --install "$BUILD_DIR" - name: Build Tutorials run: | @@ -334,7 +334,7 @@ jobs: run: | cmake -DPCAPPP_BUILD_COVERAGE=OFF -S . -B "$BUILD_DIR" cmake --build "$BUILD_DIR" -j - cmake --install $BUILD_DIR + cmake --install "$BUILD_DIR" - name: Build Tutorials run: | @@ -700,7 +700,7 @@ jobs: PCAPPLUSPLUS_INCLUDE_PATH="$TOYVPN_PCAPPLUSPLUS/include" cmake -DPCAPPP_BUILD_COVERAGE=OFF -S . -B "$BUILD_DIR" cmake --build "$BUILD_DIR" -j - cmake --install $BUILD_DIR --prefix ${TOYVPN_PCAPPLUSPLUS} + cmake --install "$BUILD_DIR" --prefix ${TOYVPN_PCAPPLUSPLUS} mkdir -p ${PCAPPLUSPLUS_LIBS_PATH} ${PCAPPLUSPLUS_INCLUDE_PATH} mv ${TOYVPN_PCAPPLUSPLUS}/lib/*.a ${PCAPPLUSPLUS_LIBS_PATH}/ mv ${TOYVPN_PCAPPLUSPLUS}/include/pcapplusplus/*.h ${PCAPPLUSPLUS_INCLUDE_PATH}/ From b32fb9a063eb98971373532b7c3566dd876dbe87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 21:54:24 +0200 Subject: [PATCH 3/8] workflow: merge MacOS jobs and use Ccache --- .github/workflows/build_and_test.yml | 94 ++++++---------------------- 1 file changed, 18 insertions(+), 76 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 95120d87fe..a9041f8a19 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -289,6 +289,9 @@ jobs: - os-version: macos-13 arch: arm64 config-zstd: OFF + - os-version: macos-14 + arch: arm64 + config-zstd: OFF steps: - name: Checkout code @@ -299,10 +302,21 @@ jobs: # support version: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json python-version: "3.12" + - name: Install Requirements Ccache & Tcpreplay + run: brew install ccache tcpreplay + + - name: Restore Ccache + id: ccache-restore + uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ matrix.os-version }}-ccache + - name: Configure PcapPlusPlus # Ensure user have access to network devices instead of giving super-user right run: | sudo chmod a+rw /dev/bpf* + sudo chmod a+rw /usr/local cmake -DLIGHT_PCAPNG_ZSTD=${{ matrix.config-zstd }} -DCMAKE_OSX_ARCHITECTURES=${{ matrix.arch }} -DPCAPPP_BUILD_COVERAGE=ON -S . -B "$BUILD_DIR" - name: Build PcapPlusPlus @@ -311,9 +325,6 @@ jobs: - name: Check architecture run: lipo $BUILD_DIR/Pcap++/libPcap++.a -verify_arch ${{ matrix.arch }} - - name: Install Tcpreplay - run: brew install tcpreplay - - name: Test PcapPlusPlus # We can't run cross compiled binaries if: ${{ matrix.arch == 'x86_64' }} @@ -361,80 +372,11 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - macos-m1: - runs-on: ${{ matrix.os-version }} - strategy: - matrix: - os-version: [macos-14] - - steps: - - name: Checkout code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - # support version: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json - python-version: "3.12" - - - name: Configure PcapPlusPlus - # Ensure user have access to network devices instead of giving super-user right - # Seems macos-14 has more strict file permission - run: | - sudo chmod a+rw /dev/bpf* - sudo chmod a+rw /usr/local - cmake -DPCAPPP_BUILD_COVERAGE=ON -S . -B "$BUILD_DIR" - - - name: Build PcapPlusPlus - run: cmake --build "$BUILD_DIR" -j - - - name: Check architecture - run: lipo $BUILD_DIR/Pcap++/libPcap++.a -verify_arch arm64 - - - name: Install Tcpreplay - run: brew install tcpreplay - - - name: Test PcapPlusPlus - run: | - python -m pip install -U pip - python -m pip install -r ci/run_tests/requirements.txt - python ci/run_tests/run_tests.py --interface en0 - - - name: Test Examples - run: | - cd Tests/ExamplesTest - python -m pip install -U pip - python -m pip install -r requirements.txt - python -m pytest --interface en0 --use-sudo --root-path=../../Dist/examples_bin - - - name: Check installation - run: | - cmake -DPCAPPP_BUILD_COVERAGE=OFF -S . -B "$BUILD_DIR" - cmake --build "$BUILD_DIR" -j - cmake --install "$BUILD_DIR" - - - name: Build Tutorials - run: | - mkdir -p build_examples - cmake -DPCAPPP_BUILD_TUTORIALS=ON -S Examples -B build_examples - cmake --build build_examples -j - - - name: Test Tutorials - run: cd build_examples/tutorials_bin && ./Tutorial-HelloWorld - - - name: Create Cobertura Report - run: | - python3 -m pip install gcovr - gcovr -v -r . $GCOVR_FLAGS -o coverage.xml - - - name: Upload Coverage Results - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0 + - name: Save Ccache + uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: - files: ./coverage.xml - flags: ${{ matrix.os-version }},unittest - fail_ci_if_error: false - verbose: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} mingw-w64: runs-on: windows-latest From 6d2dfad6c384dfcd1379d4b139dcecd0878875e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 21:56:09 +0200 Subject: [PATCH 4/8] workflows: uniformize python versions --- .github/workflows/build_and_test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a9041f8a19..2d71eca4c1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -297,7 +297,8 @@ jobs: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + - name: Setup Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: # support version: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json python-version: "3.12" @@ -405,7 +406,7 @@ jobs: - name: Setup Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: - python-version: "3.8.x" + python-version: "3.12" - name: Install NPcap env: @@ -486,9 +487,10 @@ jobs: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + - name: Setup Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: - python-version: "3.8.x" + python-version: "3.12" - name: Add MSBuild to PATH uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # v2.0.0 From 5c2c1d40c6cae73639e21c36199f7bfca3a69b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 22:17:12 +0200 Subject: [PATCH 5/8] workflows: add ccache for XDP --- .github/workflows/build_and_test.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 2d71eca4c1..becfce2830 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -665,7 +665,14 @@ jobs: - name: Install dependencies run: | - sudo apt update && sudo apt -y install libpcap-dev libbpf-dev tcpreplay + sudo apt update && sudo apt -y install ccache libpcap-dev libbpf-dev tcpreplay + + - name: Restore Ccache + id: ccache-restore + uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ matrix.image }}-ccache - name: Configure PcapPlusPlus run: cmake -DPCAPPP_USE_XDP=ON -DPCAPPP_BUILD_COVERAGE=ON -S . -B $BUILD_DIR @@ -694,6 +701,12 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - name: Save Ccache + uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + test_regressions: name: Run ${{ matrix.engine }}-${{ matrix.sanitizer }} fuzzer for regressions runs-on: ubuntu-latest From 04a0edc031cc3ece6dcd5157cccfe570740745ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 22:18:00 +0200 Subject: [PATCH 6/8] workflows: fix double space --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index becfce2830..3b77c897f3 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -683,8 +683,8 @@ jobs: - name: Test PcapPlusPlus run: | python -m pip install -U pip - python -m pip install -r ci/run_tests/requirements.txt - python ci/run_tests/run_tests.py --interface eth0 --use-sudo --pcap-test-args="-t xdp" + python -m pip install -r ci/run_tests/requirements.txt + python ci/run_tests/run_tests.py --interface eth0 --use-sudo --pcap-test-args="-t xdp" - name: Create Cobertura Report run: | From 0898be5c6a428bad9143725bda0b4976f79bb870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Mon, 8 Jul 2024 22:47:15 +0200 Subject: [PATCH 7/8] workflows: add Ccache for Fuzzer --- .github/workflows/build_and_test.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3b77c897f3..d988b406c2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -718,23 +718,40 @@ jobs: container: image: gcr.io/oss-fuzz-base/base-builder steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Install prerequisites run: | - apt-get update && apt-get install -y cmake autoconf flex bison + apt-get update && apt-get install -y autoconf bison ccache cmake flex cd .. cp -r PcapPlusPlus/ $SRC/PcapPlusPlus git clone --depth=1 https://github.com/the-tcpdump-group/libpcap.git $SRC/libpcap cd $SRC/PcapPlusPlus/ - - name: Compile fuzzer + + - name: Restore Ccache + id: ccache-restore + uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ matrix.engine }}-${{ matrix.sanitizer }}-ccache + + - name: Compile PcapPlusPlus Fuzzer application run: | export FUZZING_LANGUAGE=c export ARCHITECTURE=x86_64 export FUZZING_ENGINE=${{ matrix.engine }} export SANITIZER=${{ matrix.sanitizer }} $SRC/PcapPlusPlus/Tests/Fuzzers/ossfuzz.sh + - name: Check for regressions run: | export BINARY="$OUT/FuzzTarget" export SAMPLES="Tests/Fuzzers/RegressionTests/regression_samples" Tests/Fuzzers/RegressionTests/run_tests.sh + + - name: Save Ccache + uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} From 1c3235b59901e2eab6458c0fbbab7db43cac6bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20P=C3=A9ron?= Date: Tue, 9 Jul 2024 08:24:09 +0200 Subject: [PATCH 8/8] workflows: fix running macos cross binaries runs --- .github/workflows/build_and_test.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d988b406c2..f41055cef2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -269,27 +269,33 @@ jobs: path: ${{ env.CCACHE_DIR }} key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - macos-x86: + macos: runs-on: ${{ matrix.os-version }} strategy: matrix: include: - os-version: macos-12 + host-arch: x86_64 arch: x86_64 config-zstd: OFF - os-version: macos-12 + host-arch: x86_64 arch: arm64 config-zstd: OFF - os-version: macos-13 + host-arch: x86_64 arch: x86_64 config-zstd: OFF - os-version: macos-13 + host-arch: x86_64 arch: x86_64 config-zstd: ON - os-version: macos-13 + host-arch: x86_64 arch: arm64 config-zstd: OFF - os-version: macos-14 + host-arch: arm64 arch: arm64 config-zstd: OFF @@ -328,14 +334,14 @@ jobs: - name: Test PcapPlusPlus # We can't run cross compiled binaries - if: ${{ matrix.arch == 'x86_64' }} + if: ${{ matrix.host-arch == matrix.arch }} run: | python -m pip install -U pip python -m pip install -r ci/run_tests/requirements.txt python ci/run_tests/run_tests.py --interface en0 - name: Test Examples - if: ${{ matrix.arch == 'x86_64' }} + if: ${{ matrix.host-arch == matrix.arch }} run: | cd Tests/ExamplesTest python -m pip install -U pip @@ -355,7 +361,7 @@ jobs: cmake --build build_examples -j - name: Test Tutorials - if: ${{ matrix.arch == 'x86_64' }} + if: ${{ matrix.host-arch == matrix.arch }} run: cd build_examples/tutorials_bin && ./Tutorial-HelloWorld - name: Create Cobertura Report