diff --git a/.github/actions/e2e-builder/action.yml b/.github/actions/e2e-builder/action.yml index 919a8c3f17..e7e6cceee7 100644 --- a/.github/actions/e2e-builder/action.yml +++ b/.github/actions/e2e-builder/action.yml @@ -103,12 +103,6 @@ runs: image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: global/builder - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/e2e-common/action.yml b/.github/actions/e2e-common/action.yml index b96e2921ea..ece3d7d46e 100644 --- a/.github/actions/e2e-common/action.yml +++ b/.github/actions/e2e-common/action.yml @@ -107,12 +107,6 @@ runs: image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: global/common - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/e2e-install-olm/action.yml b/.github/actions/e2e-install-olm/action.yml index a302a480e4..a6ef550e07 100644 --- a/.github/actions/e2e-install-olm/action.yml +++ b/.github/actions/e2e-install-olm/action.yml @@ -95,12 +95,6 @@ runs: catalog-source-name: ${{ steps.config-cluster.outputs.cluster-catalog-source-name }} catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: namespace/upgrade - - name: Run IT shell: bash run: | diff --git a/.github/actions/e2e-install-upgrade/action.yml b/.github/actions/e2e-install-upgrade/action.yml index ec7610b0e3..f5cadc83f8 100644 --- a/.github/actions/e2e-install-upgrade/action.yml +++ b/.github/actions/e2e-install-upgrade/action.yml @@ -95,12 +95,6 @@ runs: catalog-source-name: ${{ steps.config-cluster.outputs.cluster-catalog-source-name }} catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: namespace/upgrade - - name: Run IT shell: bash run: | diff --git a/.github/actions/e2e-install/action.yml b/.github/actions/e2e-install/action.yml index 68fee7459e..2b7ddcc05c 100644 --- a/.github/actions/e2e-install/action.yml +++ b/.github/actions/e2e-install/action.yml @@ -86,12 +86,6 @@ runs: image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: namespace/install - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/e2e-knative-yaks/action.yml b/.github/actions/e2e-knative-yaks/action.yml index 50281fe48c..016f98f838 100644 --- a/.github/actions/e2e-knative-yaks/action.yml +++ b/.github/actions/e2e-knative-yaks/action.yml @@ -108,12 +108,6 @@ runs: image-name: "docker.io/citrusframework/yaks" version: 0.19.1 - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: yaks/common - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/e2e-knative/action.yml b/.github/actions/e2e-knative/action.yml index f445d51bda..53d8a6261b 100644 --- a/.github/actions/e2e-knative/action.yml +++ b/.github/actions/e2e-knative/action.yml @@ -103,12 +103,6 @@ runs: image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: global/knative - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/e2e-native/action.yml b/.github/actions/e2e-native/action.yml index aed6049152..7f10490288 100644 --- a/.github/actions/e2e-native/action.yml +++ b/.github/actions/e2e-native/action.yml @@ -86,12 +86,6 @@ runs: image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: namespace/native - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/e2e-telemetry/action.yml b/.github/actions/e2e-telemetry/action.yml index 14f2f1bcb7..4ce1190d79 100644 --- a/.github/actions/e2e-telemetry/action.yml +++ b/.github/actions/e2e-telemetry/action.yml @@ -105,12 +105,6 @@ runs: otlp-collector-image-name: otel/opentelemetry-collector otlp-collector-image-version: 0.102.0 - - id: report-problematic - name: List Tests Marked As Problematic - uses: ./.github/actions/kamel-report-problematic - with: - test-suite: global/common - - id: run-it name: Run IT shell: bash diff --git a/.github/actions/kamel-report-problematic/action.yml b/.github/actions/kamel-report-problematic/action.yml deleted file mode 100644 index c981858aba..0000000000 --- a/.github/actions/kamel-report-problematic/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -# --------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- - -name: kamel-report-problematic -description: 'Reports the number of tests labelled as problematic in test suite' - -inputs: - test-suite: - description: "The test suite to be analysed" - required: true - -runs: - using: "composite" - steps: - - id: report-problematic - name: Report Tests Marked As Problematic - shell: bash - if: ${{ always() }} - run: | - ./.github/actions/kamel-report-problematic/report-problematic.sh \ - -t "${{ inputs.test-suite }}" diff --git a/.github/actions/kamel-report-problematic/report-problematic.sh b/.github/actions/kamel-report-problematic/report-problematic.sh deleted file mode 100755 index 03a6b17bde..0000000000 --- a/.github/actions/kamel-report-problematic/report-problematic.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# --------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- - -#### -# -# Find all test that are labelled as problematic -# -#### - -set -e - -while getopts ":t:" opt; do - case "${opt}" in - t) - TEST_SUITE=${OPTARG} - ;; - :) - echo "ERROR: Option -$OPTARG requires an argument" - exit 1 - ;; - \?) - echo "ERROR: Invalid option -$OPTARG" - exit 1 - ;; - esac -done -shift $((OPTIND-1)) - -if [ -z "${TEST_SUITE}" ]; then - echo "Error: ${0} -t " - exit 1 -fi - -TEST_DIR="./e2e/${TEST_SUITE}" - -if [ ! -d "${TEST_DIR}" ]; then - echo "No e2e directory available ... exiting" - exit 0 -fi - -PROBLEMATIC=() -while IFS= read -r -d '' f -do - - func="" - while IFS= read -r line - do - if [[ "${line}" =~ ^[\ ]+\*[\ ] ]]; then - # Ignore comments - continue - elif [[ "${line}" =~ ^func* ]]; then - func=$(echo "${line}" | sed -n "s/func \([a-zA-Z0-9]\+\).*/\1/p") - elif [[ "${line}" =~ CAMEL_K_TEST_SKIP_PROBLEMATIC ]]; then - PROBLEMATIC[${#PROBLEMATIC[*]}]="${f}::${func}" - fi - done < ${f} - -done < <(find "${TEST_DIR}" -name "*.go" -print0) - -if [ "${CAMEL_K_TEST_SKIP_PROBLEMATIC}" == "true" ]; then - if [ ${#PROBLEMATIC[*]} -gt 0 ]; then - echo "=== Problematic Tests (${#PROBLEMATIC[*]}) ===" - for prob in "${PROBLEMATIC[@]}" - do - echo " ${prob}" - done - echo "===" - else - echo "=== No Tests marked as Problematic ===" - fi -else - echo "=== All tests to be executed, including those marked as problematic (${#PROBLEMATIC[*]}) ===" -fi diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index aeb729e44f..6583552b2d 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -56,10 +56,6 @@ on: pre-built-kamel-image: description: 'Kamel image url for skipping building of kamel stages. Used for debugging' required: false - skip-problematic: - description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' - required: false - default: false test-filters: description: | Filter the tests in this test suite by assigning the test pattern to TEST_KNATIVE_RUN, @@ -90,7 +86,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index bdd2634b49..1df675dac5 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -59,10 +59,6 @@ on: pre-built-kamel-image: description: 'Kamel image url for skipping building of kamel stages. Used for debugging' required: false - skip-problematic: - description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' - required: false - default: false test-filters: description: | List of comma-separated key/value pairs to filter the tests in this test suite: @@ -91,7 +87,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" - name: Common tests @@ -114,7 +109,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" - name: Common advanced tests diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 2c27553591..bd8a998ac5 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -57,10 +57,6 @@ on: pre-built-kamel-image: description: 'Kamel image url for skipping building of kamel stages. Used for debugging' required: false - skip-problematic: - description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' - required: false - default: false test-filters: description: | List of comma-separated key/value pairs to filter the tests in this test suite: @@ -95,7 +91,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" @@ -119,7 +114,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" @@ -143,7 +137,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" diff --git a/.github/workflows/knative.yml b/.github/workflows/knative.yml index f70f203e68..098d8ba20f 100644 --- a/.github/workflows/knative.yml +++ b/.github/workflows/knative.yml @@ -55,10 +55,6 @@ on: pre-built-kamel-image: description: 'Kamel image url for skipping building of kamel stages. Used for debugging' required: false - skip-problematic: - description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' - required: false - default: false test-filters: description: | Filter the tests in this test suite by assigning the test pattern to TEST_KNATIVE_RUN, @@ -84,7 +80,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" diff --git a/.github/workflows/manual-exec-process-inputs.sh b/.github/workflows/manual-exec-process-inputs.sh index 26b89418e0..1e57427eca 100755 --- a/.github/workflows/manual-exec-process-inputs.sh +++ b/.github/workflows/manual-exec-process-inputs.sh @@ -32,9 +32,6 @@ while getopts ":i:p:q:t:" opt; do i) PRE_BUILT_IMAGE=${OPTARG} ;; - p) - SKIP_PROBLEMATIC=${OPTARG} - ;; q) LOG_LEVEL=${OPTARG} ;; @@ -61,13 +58,6 @@ if [ -n "${PRE_BUILT_IMAGE}" ]; then echo "DEBUG_USE_EXISTING_IMAGE=${PRE_BUILT_IMAGE}" >> $GITHUB_ENV fi -# -# Avoid problematic tests only if parameter set to true -# -if [ "${SKIP_PROBLEMATIC}" == "true" ]; then - echo "CAMEL_K_TEST_SKIP_PROBLEMATIC=true" >> $GITHUB_ENV -fi - # # Adds -run args to filter tests in test suites # diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 85362b6402..656f2ab945 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -60,10 +60,6 @@ on: pre-built-kamel-image: description: 'Kamel image url for skipping building of kamel stages. Used for debugging' required: false - skip-problematic: - description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' - required: false - default: false test-filters: description: | Filter the tests in this test suite by assigning the test pattern to TEST_NATIVE_RUN, @@ -89,7 +85,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" - name: Quarkus Native test diff --git a/.github/workflows/telemetry.yml b/.github/workflows/telemetry.yml index 1e31ce1bb0..331669ac56 100644 --- a/.github/workflows/telemetry.yml +++ b/.github/workflows/telemetry.yml @@ -55,10 +55,6 @@ on: pre-built-kamel-image: description: 'Kamel image url for skipping building of kamel stages. Used for debugging' required: false - skip-problematic: - description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' - required: false - default: false test-filters: description: | Filter the tests in this test suite by assigning the test pattern to TEST_TELEMETRY_RUN, @@ -85,7 +81,6 @@ jobs: run: | ./.github/workflows/manual-exec-process-inputs.sh \ -i "${{ github.event.inputs.pre-built-kamel-image }}" \ - -p "${{ github.event.inputs.skip-problematic }}" \ -q "${{ github.event.inputs.log-level }}" \ -t "${{ github.event.inputs.test-filters }}" diff --git a/docs/modules/ROOT/pages/contributing/e2e.adoc b/docs/modules/ROOT/pages/contributing/e2e.adoc index 68a0bc281c..9e054ce6c9 100644 --- a/docs/modules/ROOT/pages/contributing/e2e.adoc +++ b/docs/modules/ROOT/pages/contributing/e2e.adoc @@ -43,11 +43,7 @@ Each group tests a specific feature of Camel K. Typically any new test should be It's important to know that a subset of `common` named `test-smoke` is used as smoke test in the nightly release process. We want to keep this group of test as fast as possible. === Configure End To End tests runs with env vars -Some e2e test runs parameters can be configured usually with env vars. -Most of them are located at https://github.com/apache/camel-k/tree/main/e2e/support/test_support.go[e2e/support/test_support.go] in `init` and `kamelInstallWithContext` functions. -A list of the most commonly used: - -* `CAMEL_K_TEST_SKIP_PROBLEMATIC`: set it to `true` to skip tests that might fail. +For some e2e test runs parameters can be configured usually with env vars. Refer to the Camel K End-To-End test https://github.com/apache/camel-k/tree/main/e2e#environment-variables[environment-variables]. [[testing-operator]] == Testing Operator under development diff --git a/e2e/README.md b/e2e/README.md index a65207527c..e5ba93876a 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,6 +1,6 @@ # Camel K End-To-End tests -This directory contains the suite of test that are run on a CI to ensure the stability of the product and no regression are introduced at each PR. The full documentation can be found at https://camel.apache.org/camel-k/next/contributing/e2e.html +This directory contains the suite of tests that are run during a CI build to ensure the stability of the product by checking that no regression defects are introduced by a PR. The full documentation can be found at https://camel.apache.org/camel-k/next/contributing/e2e.html ## Environment variables @@ -8,7 +8,6 @@ You can set some environment variables to change the behavior of the E2E test su | Env | Default | Description | |-----------------------------------------|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| CAMEL_K_TEST_SKIP_PROBLEMATIC | false | Skips tests that are marked to be problematic (flaky) on certain environments (e.g. on OpenShift). | | CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE | false | Used to not remove the temporary test namespaces after the test run. Enables better analysis of resources after the test | | CAMEL_K_TEST_LOG_LEVEL | info | Logging level used to run the tests and used in Maven commands run by the operator (if level is `debug` the Maven commands use `-X` option). | | CAMEL_K_TEST_MAVEN_CLI_OPTIONS | {} | Maven CLI options used to run Camel K integrations during the tests. | diff --git a/e2e/advanced/operator_metrics_test.go b/e2e/advanced/operator_metrics_test.go index 5efa4c1638..0edc1680e7 100644 --- a/e2e/advanced/operator_metrics_test.go +++ b/e2e/advanced/operator_metrics_test.go @@ -27,7 +27,6 @@ import ( "context" "fmt" "math" - "os" "testing" "time" @@ -49,16 +48,10 @@ import ( * TODO * The duration_seconds tests keep randomly failing on OCP4 with slightly different duration values * May need to lessen the strict checking parameters -* -* Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. */ func TestMetrics(t *testing.T) { t.Parallel() - if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { - t.Skip("WARNING: Test marked as problematic ... skipping") - } - WithNewTestNamespace(t, func(ctx context.Context, g *WithT, ns string) { name := RandomizedSuffixName("java") operatorID := "camel-k-metrics" diff --git a/e2e/common/cli/dev_mode_test.go b/e2e/common/cli/dev_mode_test.go index 0c84b46767..29ad7977e4 100644 --- a/e2e/common/cli/dev_mode_test.go +++ b/e2e/common/cli/dev_mode_test.go @@ -42,12 +42,7 @@ func TestRunDevMode(t *testing.T) { * TODO * The changing of the yaml file constant from "string" to "magic" is not being * picked up when deploying on OCP4 and so the test is failing. - * - * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. */ - if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { - t.Skip("WARNING: Test marked as problematic ... skipping") - } t.Run("run yaml dev mode", func(t *testing.T) { ctx, cancel := context.WithCancel(ctx) diff --git a/e2e/common/cli/run_test.go b/e2e/common/cli/run_test.go index 4a1a05cd44..31a5bad675 100644 --- a/e2e/common/cli/run_test.go +++ b/e2e/common/cli/run_test.go @@ -184,13 +184,8 @@ func TestKamelCLIRun(t *testing.T) { /* * TODO * The dependency cannot be read by maven while building. See #3708 - * - * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. */ t.Run("Run with http dependency", func(t *testing.T) { - if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { - t.Skip("WARNING: Test marked as problematic ... skipping") - } // Requires a local integration platform in order to resolve the insecure registry // Install platform (use the installer to get staging if present) g.Expect(KamelInstallWithID(t, ctx, operatorID, ns, "--skip-operator-setup")).To(Succeed()) @@ -211,13 +206,8 @@ func TestKamelCLIRun(t *testing.T) { /* * TODO * The dependency cannot be read by maven while building. See #3708 - * - * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. */ t.Run("Run with http dependency using options", func(t *testing.T) { - if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { - t.Skip("WARNING: Test marked as problematic ... skipping") - } // Requires a local integration platform in order to resolve the insecure registry // Install platform (use the installer to get staging if present) g.Expect(KamelInstallWithID(t, ctx, operatorID, ns, "--skip-operator-setup")).To(Succeed())