diff --git a/.github/workflows/main.yaml b/.github/workflows/build-and-unit.yaml similarity index 59% rename from .github/workflows/main.yaml rename to .github/workflows/build-and-unit.yaml index 9ceb9a92..73d4f27b 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/build-and-unit.yaml @@ -16,9 +16,9 @@ # # SPDX-License-Identifier: Apache-2.0 -name: Build and Test +name: Check build and unit test -on: [push, pull_request, create] +on: [push, pull_request] jobs: build: @@ -61,11 +61,42 @@ jobs: pip install -e .[unit] - name: Run tests run: python -m pytest + coverage: + runs-on: ubuntu-latest + concurrency: + group: coverage-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[unit] - name: Run tests coverage run: python3 -m pytest --cov=astarte tests/ - # - name: Upload coverage to Codecov (Python 3.10) - # if: matrix.python-version == '3.10' - # uses: codecov/codecov-action@v3 - # with: - # token: ${{ secrets.CODECOV_TOKEN }} - # fail_ci_if_error: true + # If the workflow is triggered by a push upload coverage directly to codecov.io + - name: Upload coverage to codecov.io + if: ${{ github.event_name == 'push' }} + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + # If the workflow is triggered by a pull request upload coverage result as a github artifact + - name: Save PR number to file + if: ${{ github.event_name == 'pull_request' }} + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo "$PR_NUMBER" > ./pr_number + - name: Upload coverage artifact + if: ${{ github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: coverage + path: | + pr_number + .coverage diff --git a/.github/workflows/check-examples-grpc.yaml b/.github/workflows/check-examples-grpc.yaml index 46743928..e29191ad 100644 --- a/.github/workflows/check-examples-grpc.yaml +++ b/.github/workflows/check-examples-grpc.yaml @@ -16,9 +16,9 @@ # # SPDX-License-Identifier: Apache-2.0 -name: Check examples grpc +name: Check examples gRPC -on: [pull_request, create] +on: [pull_request] env: PB_REL: https://github.com/protocolbuffers/protobuf/releases diff --git a/.github/workflows/check-examples-mqtt.yaml b/.github/workflows/check-examples-mqtt.yaml index 0b055b38..9fbcb0bc 100644 --- a/.github/workflows/check-examples-mqtt.yaml +++ b/.github/workflows/check-examples-mqtt.yaml @@ -18,7 +18,7 @@ name: Check examples MQTT -on: [pull_request, create] +on: [pull_request] env: PB_REL: https://github.com/protocolbuffers/protobuf/releases diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index ad200d2e..5fce9457 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -16,9 +16,9 @@ # # SPDX-License-Identifier: Apache-2.0 -name: Code Quality Checks +name: Checks code quality -on: [push, pull_request, create] +on: [push, pull_request] jobs: static: diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-tests-grpc.yaml similarity index 62% rename from .github/workflows/e2e-test.yaml rename to .github/workflows/e2e-tests-grpc.yaml index 29c732a9..19cb7c07 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-tests-grpc.yaml @@ -18,90 +18,12 @@ name: End to End test -on: [pull_request, create] +on: [pull_request] env: PB_REL: https://github.com/protocolbuffers/protobuf/releases jobs: - e2e-test-mqtt: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - concurrency: - group: e2e-test-mqtt-${{ matrix.python-version }}-${{ github.ref }} - cancel-in-progress: true - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Create Astarte Cluster - id: astarte - uses: astarte-platform/astarte-cluster-action@v1 - with: - astarte_version: "1.1.1" - - name: Install interface - run: | - astartectl realm-management interfaces sync $GITHUB_WORKSPACE/e2etest/interfaces/*.json --non-interactive - astartectl realm-management interfaces ls - - name: Setup env variables common for all devices - run: | - echo "E2E_REALM=test" >> $GITHUB_ENV - APPENGINE_TOKEN=$(astartectl utils gen-jwt appengine) - echo "E2E_APPENGINE_TOKEN=$APPENGINE_TOKEN" >> $GITHUB_ENV - echo "E2E_APPENGINE_URL=https://api.autotest.astarte-platform.org/appengine" >> $GITHUB_ENV - echo "E2E_PAIRING_URL=https://api.autotest.astarte-platform.org/pairing" >> $GITHUB_ENV - - name: Setup env variables for device 1 - run: | - DEVICE_1_ID=$(astartectl utils device-id generate-random) - echo "E2E_DEVICE_1_ID=$DEVICE_1_ID" >> $GITHUB_ENV - CREDENTIALS_SECRET_1=$(astartectl pairing agent register --compact-output -- "$DEVICE_1_ID") - echo "E2E_CREDENTIALS_SECRET_1=$CREDENTIALS_SECRET_1" >> $GITHUB_ENV - - name: Setup env variables for device 2 - run: | - DEVICE_2_ID=$(astartectl utils device-id generate-random) - echo "E2E_DEVICE_2_ID=$DEVICE_2_ID" >> $GITHUB_ENV - CREDENTIALS_SECRET_2=$(astartectl pairing agent register --compact-output -- "$DEVICE_2_ID") - echo "E2E_CREDENTIALS_SECRET_2=$CREDENTIALS_SECRET_2" >> $GITHUB_ENV - - name: Setup env variables for device 3 - run: | - DEVICE_3_ID=$(astartectl utils device-id generate-random) - echo "E2E_DEVICE_3_ID=$DEVICE_3_ID" >> $GITHUB_ENV - CREDENTIALS_SECRET_3=$(astartectl pairing agent register --compact-output -- "$DEVICE_3_ID") - echo "E2E_CREDENTIALS_SECRET_3=$CREDENTIALS_SECRET_3" >> $GITHUB_ENV - - name: Install the Astarte device Python module - run: | - python3 -m pip install --upgrade pip - python3 -m pip install -e .[e2e] - - name: Run test base - uses: nick-fields/retry@v3 - with: - timeout_seconds: 60 - max_attempts: 2 - command: | - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - python3 e2etest/base/main.py --mock_data_n 1 - python3 e2etest/base/main.py --mock_data_n 2 - - name: Run test for persistency - uses: nick-fields/retry@v3 - with: - timeout_seconds: 30 - max_attempts: 2 - command: | - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - python3 e2etest/persistency/main.py - - name: Run test for reconnection - uses: nick-fields/retry@v3 - with: - timeout_seconds: 30 - max_attempts: 2 - command: | - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - python3 e2etest/reconnection/main.py e2e-test-grpc: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/e2e-tests-mqtt.yaml b/.github/workflows/e2e-tests-mqtt.yaml new file mode 100644 index 00000000..222eaa1e --- /dev/null +++ b/.github/workflows/e2e-tests-mqtt.yaml @@ -0,0 +1,104 @@ +# This file is part of Astarte. +# +# Copyright 2024 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +name: End to End test + +on: [pull_request] + +env: + PB_REL: https://github.com/protocolbuffers/protobuf/releases + +jobs: + e2e-test-mqtt: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + concurrency: + group: e2e-test-mqtt-${{ matrix.python-version }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Create Astarte Cluster + id: astarte + uses: astarte-platform/astarte-cluster-action@v1 + with: + astarte_version: "1.1.1" + - name: Install interface + run: | + astartectl realm-management interfaces sync $GITHUB_WORKSPACE/e2etest/interfaces/*.json --non-interactive + astartectl realm-management interfaces ls + - name: Setup env variables common for all devices + run: | + echo "E2E_REALM=test" >> $GITHUB_ENV + APPENGINE_TOKEN=$(astartectl utils gen-jwt appengine) + echo "E2E_APPENGINE_TOKEN=$APPENGINE_TOKEN" >> $GITHUB_ENV + echo "E2E_APPENGINE_URL=https://api.autotest.astarte-platform.org/appengine" >> $GITHUB_ENV + echo "E2E_PAIRING_URL=https://api.autotest.astarte-platform.org/pairing" >> $GITHUB_ENV + - name: Setup env variables for device 1 + run: | + DEVICE_1_ID=$(astartectl utils device-id generate-random) + echo "E2E_DEVICE_1_ID=$DEVICE_1_ID" >> $GITHUB_ENV + CREDENTIALS_SECRET_1=$(astartectl pairing agent register --compact-output -- "$DEVICE_1_ID") + echo "E2E_CREDENTIALS_SECRET_1=$CREDENTIALS_SECRET_1" >> $GITHUB_ENV + - name: Setup env variables for device 2 + run: | + DEVICE_2_ID=$(astartectl utils device-id generate-random) + echo "E2E_DEVICE_2_ID=$DEVICE_2_ID" >> $GITHUB_ENV + CREDENTIALS_SECRET_2=$(astartectl pairing agent register --compact-output -- "$DEVICE_2_ID") + echo "E2E_CREDENTIALS_SECRET_2=$CREDENTIALS_SECRET_2" >> $GITHUB_ENV + - name: Setup env variables for device 3 + run: | + DEVICE_3_ID=$(astartectl utils device-id generate-random) + echo "E2E_DEVICE_3_ID=$DEVICE_3_ID" >> $GITHUB_ENV + CREDENTIALS_SECRET_3=$(astartectl pairing agent register --compact-output -- "$DEVICE_3_ID") + echo "E2E_CREDENTIALS_SECRET_3=$CREDENTIALS_SECRET_3" >> $GITHUB_ENV + - name: Install the Astarte device Python module + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -e .[e2e] + - name: Run test base + uses: nick-fields/retry@v3 + with: + timeout_seconds: 60 + max_attempts: 2 + command: | + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + python3 e2etest/base/main.py --mock_data_n 1 + python3 e2etest/base/main.py --mock_data_n 2 + - name: Run test for persistency + uses: nick-fields/retry@v3 + with: + timeout_seconds: 30 + max_attempts: 2 + command: | + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + python3 e2etest/persistency/main.py + - name: Run test for reconnection + uses: nick-fields/retry@v3 + with: + timeout_seconds: 30 + max_attempts: 2 + command: | + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + python3 e2etest/reconnection/main.py diff --git a/.github/workflows/reuse-lint.yaml b/.github/workflows/reuse-lint.yaml index 8904fddb..4182fc03 100644 --- a/.github/workflows/reuse-lint.yaml +++ b/.github/workflows/reuse-lint.yaml @@ -4,7 +4,7 @@ name: REUSE Compliance Check -on: [push, pull_request, create] +on: [push, pull_request] jobs: check: diff --git a/.github/workflows/upload-coverage.yaml b/.github/workflows/upload-coverage.yaml new file mode 100644 index 00000000..6e47b32f --- /dev/null +++ b/.github/workflows/upload-coverage.yaml @@ -0,0 +1,59 @@ +# This file is part of Astarte. +# +# Copyright 2024 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +name: Upload code coverage + +on: + workflow_run: + workflows: ["build-and-unit"] + types: [completed] + +permissions: + contents: read + actions: read + +defaults: + run: + shell: bash + +jobs: + upload: + runs-on: ubuntu-latest + # Run only if originated from a PR + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} + - name: Download coverage artifact + uses: actions/download-artifact@v4 + with: + name: coverage + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + - name: Get PR number + run: | + echo "PR_NUMBER=$(cat ./pr_number)" >> "$GITHUB_ENV" + - name: Upload to codecov.io + uses: codecov/codecov-action@v4 + with: + token: ${{secrets.CODECOV_TOKEN}} + fail_ci_if_error: true + override_branch: ${{ github.event.workflow_run.head_branch }} + override_commit: ${{ github.event.workflow_run.head_sha }} + override_pr: ${{ env.PR_NUMBER }}