From 3133e29f77be6f716da32cf389f2b032313e5040 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Tue, 5 Mar 2024 09:53:39 +0100 Subject: [PATCH 1/8] Fix assertion over message reload --- test/e2e/bots_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/e2e/bots_test.go b/test/e2e/bots_test.go index 5a68473b7..8da5ab437 100644 --- a/test/e2e/bots_test.go +++ b/test/e2e/bots_test.go @@ -1078,8 +1078,7 @@ func runBotTest(t *testing.T, limitMessagesNo := 2 if botDriver.Type().IsCloud() { - // There are 2 config reload requested after second cm update - limitMessagesNo = 2 * limitLastMessageAfterCloudReload + limitMessagesNo = limitLastMessageAfterCloudReload } err = botDriver.OnChannel().WaitForMessagePostedWithAttachment(botDriver.BotUserID(), botDriver.SecondChannel().ID(), limitMessagesNo, secondCMUpdate) require.NoError(t, err) From 5ab5f822ea71ad9a29bd9331bf28c3cdb043d8f7 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Fri, 8 Mar 2024 11:34:25 +0100 Subject: [PATCH 2/8] Test jobs --- .github/actions/cloud-slack-e2e/action.yaml | 24 +- .github/workflows/branch-build.yml | 384 ++++++++++---------- 2 files changed, 204 insertions(+), 204 deletions(-) diff --git a/.github/actions/cloud-slack-e2e/action.yaml b/.github/actions/cloud-slack-e2e/action.yaml index 16b499f55..04a53deb9 100644 --- a/.github/actions/cloud-slack-e2e/action.yaml +++ b/.github/actions/cloud-slack-e2e/action.yaml @@ -123,15 +123,15 @@ runs: echo "footer=${FOOTER}" >> $GITHUB_OUTPUT - - name: Slack Notification - uses: rtCamp/action-slack-notify@v2 - if: ${{ failure() }} - env: - SLACK_CHANNEL: 'botkube-cloud-ci-alerts' - SLACK_USERNAME: Botkube Cloud CI - SLACK_COLOR: 'red' - SLACK_TITLE: 'Message' - SLACK_MESSAGE: "Cloud Slack ${{ inputs.e2e_type }} E2E tests failed :scream:" - SLACK_ICON_EMOJI: ':this-is-fine-fire:' - SLACK_FOOTER: ${{ steps.footer.outputs.footer }} - SLACK_WEBHOOK: ${{ inputs.slack_alerts_webhook }} +# - name: Slack Notification +# uses: rtCamp/action-slack-notify@v2 +# if: ${{ failure() }} +# env: +# SLACK_CHANNEL: 'botkube-cloud-ci-alerts' +# SLACK_USERNAME: Botkube Cloud CI +# SLACK_COLOR: 'red' +# SLACK_TITLE: 'Message' +# SLACK_MESSAGE: "Cloud Slack ${{ inputs.e2e_type }} E2E tests failed :scream:" +# SLACK_ICON_EMOJI: ':this-is-fine-fire:' +# SLACK_FOOTER: ${{ steps.footer.outputs.footer }} +# SLACK_WEBHOOK: ${{ inputs.slack_alerts_webhook }} diff --git a/.github/workflows/branch-build.yml b/.github/workflows/branch-build.yml index 29f1ee18f..ac42253ae 100644 --- a/.github/workflows/branch-build.yml +++ b/.github/workflows/branch-build.yml @@ -16,201 +16,201 @@ env: GIT_USER: botkube-dev jobs: - extract-metadata: - if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch - runs-on: ubuntu-latest - outputs: - versions: ${{ steps.extract-version.outputs.versions }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Extract version - id: extract-version - run: | - IMAGE_VERSION=$(git rev-parse --short HEAD) - echo "versions={\"image-version\":[\"v9.99.9-dev\",\"0.0.0-${IMAGE_VERSION}\"]}" >> $GITHUB_OUTPUT - build: - if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch - needs: [extract-metadata] - strategy: - matrix: ${{ fromJson(needs.extract-metadata.outputs.versions) }} - runs-on: ubuntu-latest - env: - GO111MODULE: on - GOPATH: /home/runner/work/botkube - GOBIN: /home/runner/work/botkube/bin - DOCKER_CLI_EXPERIMENTAL: "enabled" - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Docker Login - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Install GoReleaser - uses: goreleaser/goreleaser-action@v5 - with: - install-only: true - version: latest - - name: Run GoReleaser - run: | - make release-snapshot - env: - ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }} - GORELEASER_CURRENT_TAG: ${{ matrix.image-version }} - IMAGE_TAG: ${{ matrix.image-version }} - - integration-tests: - if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch - name: Integration tests - runs-on: ubuntu-latest - needs: [ build ] - permissions: - contents: read - packages: read - - strategy: - # make the jobs independent - fail-fast: false - - matrix: - integration: - - slack - - discord - - teams - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup Go modules - uses: ./.github/actions/setup-go-mod-private - with: - access_token: ${{ secrets.E2E_TEST_GH_DEV_ACCOUNT_PAT }} - username: ${{ env.GIT_USER }} - - - name: Pub/Sub auth - uses: 'google-github-actions/auth@v1' - if: matrix.integration == 'teams' - with: - credentials_json: ${{ secrets.E2E_TEST_GCP_PUB_SUB_CREDENTIALS }} - - - name: Install Helm - uses: azure/setup-helm@v3 - with: - version: ${{ env.HELM_VERSION }} - - - name: Download k3d - run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash" - - - name: Create cluster to test ${{ matrix.integration }} - run: "k3d cluster create ${{ matrix.integration }}-test-cluster --wait --timeout=5m" - - - name: Install Botkube locally via helm - if: matrix.integration == 'discord' - env: - DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} - DISCORD_BOT_ID: ${{ secrets.DISCORD_BOT_ID }} - run: | - helm install botkube --namespace botkube ./helm/botkube --wait --create-namespace \ - -f ./helm/botkube/e2e-test-values.yaml \ - --set communications.default-group.discord.token="${DISCORD_BOT_TOKEN}" \ - --set communications.default-group.discord.botID="${DISCORD_BOT_ID}" \ - --set image.registry="${IMAGE_REGISTRY}" \ - --set image.repository="${IMAGE_REPOSITORY}" \ - --set image.tag="${IMAGE_TAG}" \ - - - name: Install GoReleaser - uses: goreleaser/goreleaser-action@v5 - with: - install-only: true - version: latest - - - name: Build all plugins into dist directory - env: - # we hardcode plugins version, so it's predictable in e2e tests - GORELEASER_CURRENT_TAG: "v0.0.0-latest" - OUTPUT_MODE: "binary" - SINGLE_PLATFORM: "true" - PLUGIN_TARGETS: "kubernetes,kubectl,cm-watcher,echo,helm" - run: | - make build-plugins - - - name: CLI Cache - if: matrix.integration != 'discord' - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - dist/botkube-cli_linux_amd64_v1/botkube - key: ${{ runner.os }}-botkube-cli - - - name: Build CLI - if: matrix.integration != 'discord' - run: make build-single-arch-cli - - - name: Add Botkube CLI to env - run: | - echo CONFIG_PROVIDER_BOTKUBE_CLI_BINARY_PATH="$PWD/dist/botkube-cli_linux_amd64_v1/botkube" >> $GITHUB_ENV - - - name: Run ${{ matrix.integration }} tests - env: - SLACK_TESTER_APP_TOKEN: ${{ secrets.SLACK_TESTER_APP_TOKEN }} - SLACK_CLOUD_TESTER_APP_TOKEN: ${{ secrets.SLACK_CLOUD_TESTER_APP_TOKEN }} - SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" - - DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }} - DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }} - DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" - - TEAMS_BOT_TESTER_APP_ID: ${{ secrets.TEAMS_BOT_TESTER_APP_ID }} - TEAMS_BOT_TESTER_APP_PASSWORD: ${{ secrets.TEAMS_BOT_TESTER_APP_PASSWORD }} - TEAMS_ORGANIZATION_TEAM_ID: ${{ secrets.TEAMS_ORGANIZATION_TEAM_ID }} - TEAMS_ORGANIZATION_TENANT_ID: ${{ secrets.TEAMS_ORGANIZATION_TENANT_ID }} - TEAMS_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" - - PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/plugin-dist - CONFIG_PROVIDER_API_KEY: ${{ secrets.CONFIG_PROVIDER_API_KEY }} - CONFIG_PROVIDER_ENDPOINT: ${{ secrets.CONFIG_PROVIDER_ENDPOINT }} - CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID: ${{ secrets.CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID }} - CONFIG_PROVIDER_IMAGE_REPOSITORY: ${{ env.IMAGE_REPOSITORY }} - CONFIG_PROVIDER_IMAGE_TAG: ${{ env.IMAGE_TAG }} - CONFIG_PROVIDER_HELM_REPO_DIRECTORY: ${{ github.workspace }}/helm - run: | - KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \ - make test-integration-${{ matrix.integration }} - - - name: Slack Notification - uses: rtCamp/action-slack-notify@v2 - if: ${{ failure() }} - env: - SLACK_USERNAME: Botkube Cloud CI - SLACK_COLOR: 'red' - SLACK_TITLE: 'Message' - SLACK_CHANNEL: 'botkube-cloud-ci-alerts' - SLACK_MESSAGE: 'Integration ${{ matrix.integration }} test failed :scream:' - SLACK_ICON_EMOJI: ':this-is-fine-fire:' - SLACK_WEBHOOK: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }} - SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..." +# extract-metadata: +# if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch +# runs-on: ubuntu-latest +# outputs: +# versions: ${{ steps.extract-version.outputs.versions }} +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# - name: Extract version +# id: extract-version +# run: | +# IMAGE_VERSION=$(git rev-parse --short HEAD) +# echo "versions={\"image-version\":[\"v9.99.9-dev\",\"0.0.0-${IMAGE_VERSION}\"]}" >> $GITHUB_OUTPUT +# build: +# if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch +# needs: [extract-metadata] +# strategy: +# matrix: ${{ fromJson(needs.extract-metadata.outputs.versions) }} +# runs-on: ubuntu-latest +# env: +# GO111MODULE: on +# GOPATH: /home/runner/work/botkube +# GOBIN: /home/runner/work/botkube/bin +# DOCKER_CLI_EXPERIMENTAL: "enabled" +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# - name: Setup Go +# uses: actions/setup-go@v5 +# with: +# go-version-file: 'go.mod' +# cache: true +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v3 +# - name: Docker Login +# uses: docker/login-action@v1 +# with: +# registry: ghcr.io +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} +# - name: Install GoReleaser +# uses: goreleaser/goreleaser-action@v5 +# with: +# install-only: true +# version: latest +# - name: Run GoReleaser +# run: | +# make release-snapshot +# env: +# ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }} +# GORELEASER_CURRENT_TAG: ${{ matrix.image-version }} +# IMAGE_TAG: ${{ matrix.image-version }} +# +# integration-tests: +# if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch +# name: Integration tests +# runs-on: ubuntu-latest +# needs: [ build ] +# permissions: +# contents: read +# packages: read +# +# strategy: +# # make the jobs independent +# fail-fast: false +# +# matrix: +# integration: +# - slack +# - discord +# - teams +# +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 +# with: +# persist-credentials: false +# +# - name: Setup Go modules +# uses: ./.github/actions/setup-go-mod-private +# with: +# access_token: ${{ secrets.E2E_TEST_GH_DEV_ACCOUNT_PAT }} +# username: ${{ env.GIT_USER }} +# +# - name: Pub/Sub auth +# uses: 'google-github-actions/auth@v1' +# if: matrix.integration == 'teams' +# with: +# credentials_json: ${{ secrets.E2E_TEST_GCP_PUB_SUB_CREDENTIALS }} +# +# - name: Install Helm +# uses: azure/setup-helm@v3 +# with: +# version: ${{ env.HELM_VERSION }} +# +# - name: Download k3d +# run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash" +# +# - name: Create cluster to test ${{ matrix.integration }} +# run: "k3d cluster create ${{ matrix.integration }}-test-cluster --wait --timeout=5m" +# +# - name: Install Botkube locally via helm +# if: matrix.integration == 'discord' +# env: +# DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} +# DISCORD_BOT_ID: ${{ secrets.DISCORD_BOT_ID }} +# run: | +# helm install botkube --namespace botkube ./helm/botkube --wait --create-namespace \ +# -f ./helm/botkube/e2e-test-values.yaml \ +# --set communications.default-group.discord.token="${DISCORD_BOT_TOKEN}" \ +# --set communications.default-group.discord.botID="${DISCORD_BOT_ID}" \ +# --set image.registry="${IMAGE_REGISTRY}" \ +# --set image.repository="${IMAGE_REPOSITORY}" \ +# --set image.tag="${IMAGE_TAG}" \ +# +# - name: Install GoReleaser +# uses: goreleaser/goreleaser-action@v5 +# with: +# install-only: true +# version: latest +# +# - name: Build all plugins into dist directory +# env: +# # we hardcode plugins version, so it's predictable in e2e tests +# GORELEASER_CURRENT_TAG: "v0.0.0-latest" +# OUTPUT_MODE: "binary" +# SINGLE_PLATFORM: "true" +# PLUGIN_TARGETS: "kubernetes,kubectl,cm-watcher,echo,helm" +# run: | +# make build-plugins +# +# - name: CLI Cache +# if: matrix.integration != 'discord' +# uses: actions/cache@v3 +# with: +# path: | +# ~/.cache/go-build +# ~/go/pkg/mod +# dist/botkube-cli_linux_amd64_v1/botkube +# key: ${{ runner.os }}-botkube-cli +# +# - name: Build CLI +# if: matrix.integration != 'discord' +# run: make build-single-arch-cli +# +# - name: Add Botkube CLI to env +# run: | +# echo CONFIG_PROVIDER_BOTKUBE_CLI_BINARY_PATH="$PWD/dist/botkube-cli_linux_amd64_v1/botkube" >> $GITHUB_ENV +# +# - name: Run ${{ matrix.integration }} tests +# env: +# SLACK_TESTER_APP_TOKEN: ${{ secrets.SLACK_TESTER_APP_TOKEN }} +# SLACK_CLOUD_TESTER_APP_TOKEN: ${{ secrets.SLACK_CLOUD_TESTER_APP_TOKEN }} +# SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" +# +# DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }} +# DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }} +# DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" +# +# TEAMS_BOT_TESTER_APP_ID: ${{ secrets.TEAMS_BOT_TESTER_APP_ID }} +# TEAMS_BOT_TESTER_APP_PASSWORD: ${{ secrets.TEAMS_BOT_TESTER_APP_PASSWORD }} +# TEAMS_ORGANIZATION_TEAM_ID: ${{ secrets.TEAMS_ORGANIZATION_TEAM_ID }} +# TEAMS_ORGANIZATION_TENANT_ID: ${{ secrets.TEAMS_ORGANIZATION_TENANT_ID }} +# TEAMS_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" +# +# PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/plugin-dist +# CONFIG_PROVIDER_API_KEY: ${{ secrets.CONFIG_PROVIDER_API_KEY }} +# CONFIG_PROVIDER_ENDPOINT: ${{ secrets.CONFIG_PROVIDER_ENDPOINT }} +# CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID: ${{ secrets.CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID }} +# CONFIG_PROVIDER_IMAGE_REPOSITORY: ${{ env.IMAGE_REPOSITORY }} +# CONFIG_PROVIDER_IMAGE_TAG: ${{ env.IMAGE_TAG }} +# CONFIG_PROVIDER_HELM_REPO_DIRECTORY: ${{ github.workspace }}/helm +# run: | +# KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \ +# make test-integration-${{ matrix.integration }} +# +# - name: Slack Notification +# uses: rtCamp/action-slack-notify@v2 +# if: ${{ failure() }} +# env: +# SLACK_USERNAME: Botkube Cloud CI +# SLACK_COLOR: 'red' +# SLACK_TITLE: 'Message' +# SLACK_CHANNEL: 'botkube-cloud-ci-alerts' +# SLACK_MESSAGE: 'Integration ${{ matrix.integration }} test failed :scream:' +# SLACK_ICON_EMOJI: ':this-is-fine-fire:' +# SLACK_WEBHOOK: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }} +# SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..." cli-migration-e2e: name: CLI Migration E2E tests runs-on: ubuntu-latest - needs: [ build ] +# needs: [ build ] permissions: contents: read packages: read @@ -290,7 +290,7 @@ jobs: cloud-slack-dev-e2e: name: Botkube Cloud Slack Dev E2E runs-on: ubuntu-latest - needs: [ build ] +# needs: [ build ] permissions: contents: read packages: read From b014446163f04fa07fb3520b663796e63e055d0a Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Fri, 8 Mar 2024 11:35:03 +0100 Subject: [PATCH 3/8] Fix uninstallation for Helm chart --- .../cloud_slack_dev_e2e_test.go | 21 ++++------- test/e2e/bots_test.go | 23 ++++++------ test/helmx/helm_helpers.go | 36 +++++++++++++++++++ test/helmx/redact_test.go | 1 - 4 files changed, 53 insertions(+), 28 deletions(-) diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index ca7e1fe41..35d23e5a5 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -90,7 +91,8 @@ func TestCloudSlackE2E(t *testing.T) { cfg.Slack.Tester.CloudBasedTestEnabled = false // override property used only in the Cloud Slack E2E tests authHeaderValue := "" - helmChartUninstalled := false + var botkubeDeploymentUninstalled atomic.Bool + botkubeDeploymentUninstalled.Store(true) // not yet installed gqlEndpoint := fmt.Sprintf("%s/%s", cfg.BotkubeCloud.APIBaseURL, cfg.BotkubeCloud.APIGraphQLEndpoint) if cfg.ScreenshotsDir != "" { @@ -319,18 +321,8 @@ func TestCloudSlackE2E(t *testing.T) { t.Log("Creating deployment...") deployment := gqlCli.MustCreateBasicDeploymentWithCloudSlack(t, channel.Name(), slackWorkspace.TeamID, channel.Name()) t.Cleanup(func() { - // We have a glitch on backend side and the logic below is a workaround for that. - // Tl;dr uninstalling Helm chart reports "DISCONNECTED" status, and deplyment deletion reports "DELETED" status. - // If we do these two things too quickly, we'll run into resource version mismatch in repository logic. - // Read more here: https://github.com/kubeshop/botkube-cloud/pull/486#issuecomment-1604333794 - - for !helmChartUninstalled { - t.Log("Waiting for Helm chart uninstallation, in order to proceed with deleting the first deployment...") - time.Sleep(1 * time.Second) - } - - t.Log("Helm chart uninstalled. Waiting a bit...") - time.Sleep(3 * time.Second) // ugly, but at least we will be pretty sure we won't run into the resource version mismatch + err := helmx.WaitForUninstallation(context.Background(), t, &botkubeDeploymentUninstalled) + assert.NoError(t, err) t.Log("Deleting first deployment...") gqlCli.MustDeleteDeployment(t, graphql.ID(deployment.ID)) @@ -343,6 +335,7 @@ func TestCloudSlackE2E(t *testing.T) { gqlCli.MustDeleteDeployment(t, graphql.ID(deployment2.ID)) }) + botkubeDeploymentUninstalled.Store(false) // about to be installed params := helmx.InstallChartParams{ RepoURL: "https://storage.googleapis.com/botkube-latest-main-charts", RepoName: "botkube", @@ -355,7 +348,7 @@ func TestCloudSlackE2E(t *testing.T) { t.Cleanup(func() { t.Log("Uninstalling Helm chart...") helmInstallCallback(t) - helmChartUninstalled = true + botkubeDeploymentUninstalled.Store(true) }) t.Log("Waiting for help message...") diff --git a/test/e2e/bots_test.go b/test/e2e/bots_test.go index 8da5ab437..23a29e393 100644 --- a/test/e2e/bots_test.go +++ b/test/e2e/bots_test.go @@ -10,10 +10,13 @@ import ( "regexp" "strconv" "strings" + "sync/atomic" "testing" "time" "unicode" + "botkube.io/botube/test/helmx" + "botkube.io/botube/test/botkubex" "botkube.io/botube/test/commplatform" "botkube.io/botube/test/diff" @@ -201,7 +204,9 @@ func runBotTest(t *testing.T, deployEnvSecondaryChannelIDName, deployEnvRbacChannelIDName string, ) { - botkubeDeploymentUninstalled := false + var botkubeDeploymentUninstalled atomic.Bool + botkubeDeploymentUninstalled.Store(true) // not yet installed + t.Logf("Creating API client with provided token for %s...", driverType) botDriver, err := newBotDriver(appCfg, driverType) require.NoError(t, err) @@ -259,22 +264,14 @@ func runBotTest(t *testing.T, gqlCli.MustCreateAlias(t, alias[0], alias[1], alias[2], deployment.ID) } t.Cleanup(func() { - // We have a glitch on backend side and the logic below is a workaround for that. - // Tl;dr uninstalling Helm chart reports "DISCONNECTED" status, and deployment deletion reports "DELETED" status. - // If we do these two things too quickly, we'll run into resource version mismatch in repository logic. - // Read more here: https://github.com/kubeshop/botkube-cloud/pull/486#issuecomment-1604333794 - for !botkubeDeploymentUninstalled { - t.Log("Waiting for Helm chart uninstallation, in order to proceed with deleting Botkube Cloud instance...") - time.Sleep(1 * time.Second) - } - - t.Log("Helm chart uninstalled. Waiting a bit...") - time.Sleep(3 * time.Second) // ugly, but at least we will be pretty sure we won't run into the resource version mismatch + err := helmx.WaitForUninstallation(context.Background(), t, &botkubeDeploymentUninstalled) + assert.NoError(t, err) t.Log("Deleting Botkube Cloud instance...") gqlCli.MustDeleteDeployment(t, graphql.ID(deployment.ID)) }) + botkubeDeploymentUninstalled.Store(false) // about to be installed err = botkubex.Install(t, botkubex.InstallParams{ BinaryPath: appCfg.ConfigProvider.BotkubeCliBinaryPath, HelmRepoDirectory: appCfg.ConfigProvider.HelmRepoDirectory, @@ -291,7 +288,7 @@ func runBotTest(t *testing.T, t.Cleanup(func() { t.Log("Uninstalling Helm chart...") botkubex.Uninstall(t, appCfg.ConfigProvider.BotkubeCliBinaryPath) - botkubeDeploymentUninstalled = true + botkubeDeploymentUninstalled.Store(true) }) } diff --git a/test/helmx/helm_helpers.go b/test/helmx/helm_helpers.go index 9b6c59802..ea276cbb0 100644 --- a/test/helmx/helm_helpers.go +++ b/test/helmx/helm_helpers.go @@ -1,12 +1,17 @@ package helmx import ( + "context" "encoding/json" "fmt" "os/exec" "regexp" "strings" + "sync/atomic" "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" "github.com/stretchr/testify/require" ) @@ -108,3 +113,34 @@ func redactAPIKey(in []string) []string { } return dst } + +const ( + uninstallPollInterval = 1 * time.Second + uninstallTimeout = 30 * time.Second +) + +// WaitForUninstallation waits until a Helm chart is uninstalled, based on the atomic value. +// It's a workaround for the Helm chart uninstallation issue. +// We have a glitch on backend side and the logic below is a workaround for that. +// Tl;dr uninstalling Helm chart reports "DISCONNECTED" status, and deployment deletion reports "DELETED" status. +// If we do these two things too quickly, we'll run into resource version mismatch in repository logic. +// Read more here: https://github.com/kubeshop/botkube-cloud/pull/486#issuecomment-1604333794 +func WaitForUninstallation(ctx context.Context, t *testing.T, alreadyUninstalled *atomic.Bool) error { + t.Helper() + t.Log("Waiting for Helm chart uninstallation, in order to proceed with deleting Botkube Cloud instance...") + err := wait.PollUntilContextTimeout(ctx, uninstallPollInterval, uninstallTimeout, false, func(ctx context.Context) (done bool, err error) { + return alreadyUninstalled.Load(), nil + }) + waitInterrupted := wait.Interrupted(err) + if err != nil && !waitInterrupted { + return err + } + + if waitInterrupted { + t.Log("Force uninstalling the Helm chart after a wait timeout") + } + + t.Log("Waiting a bit more...") + time.Sleep(3 * time.Second) // ugly, but at least we will be pretty sure we won't run into the resource version mismatch + return nil +} diff --git a/test/helmx/redact_test.go b/test/helmx/redact_test.go index 14701cfbe..34064d8d1 100644 --- a/test/helmx/redact_test.go +++ b/test/helmx/redact_test.go @@ -7,7 +7,6 @@ import ( ) func TestRedactAPIKey(t *testing.T) { - tests := []struct { name string input []string From 89eb7b4941f14669196e82fc4dd449c92bc42e8e Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Fri, 8 Mar 2024 12:39:59 +0100 Subject: [PATCH 4/8] Add name for cluster dump artifacts --- .github/actions/cloud-slack-e2e/action.yaml | 2 ++ .github/actions/dump-cluster/action.yaml | 7 ++++++- .github/workflows/branch-build.yml | 2 ++ test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/cloud-slack-e2e/action.yaml b/.github/actions/cloud-slack-e2e/action.yaml index 04a53deb9..a935a8e26 100644 --- a/.github/actions/cloud-slack-e2e/action.yaml +++ b/.github/actions/cloud-slack-e2e/action.yaml @@ -110,6 +110,8 @@ runs: - name: Dump cluster state if: ${{ failure() }} uses: ./.github/actions/dump-cluster + with: + name: cloud-slack-e2e - name: Detect failed jobs if: ${{ failure() }} diff --git a/.github/actions/dump-cluster/action.yaml b/.github/actions/dump-cluster/action.yaml index 1c8144da9..3afe4c960 100644 --- a/.github/actions/dump-cluster/action.yaml +++ b/.github/actions/dump-cluster/action.yaml @@ -1,6 +1,11 @@ name: Dump cluster state description: "Creates an artifacts with cluster dump" +inputs: + name: + description: "Cluster name" + required: true + runs: using: "composite" steps: @@ -17,6 +22,6 @@ runs: - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: cluster_dump_${{github.sha}} + name: cluster_dump_${{github.sha}}_${{ inputs.name }} path: "output" retention-days: 5 # Default 90 days diff --git a/.github/workflows/branch-build.yml b/.github/workflows/branch-build.yml index ac42253ae..45c1dfdab 100644 --- a/.github/workflows/branch-build.yml +++ b/.github/workflows/branch-build.yml @@ -273,6 +273,8 @@ jobs: - name: Dump cluster state if: ${{ failure() }} uses: ./.github/actions/dump-cluster + with: + name: cli-migration-e2e - name: Slack Notification uses: rtCamp/action-slack-notify@v2 diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index 35d23e5a5..52d688444 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -301,6 +301,7 @@ func TestCloudSlackE2E(t *testing.T) { if !cfg.Slack.DisconnectWorkspaceAfterTests { return } + t.Log("Disconnecting Slack workspace...") gqlCli.MustDeleteSlackWorkspace(t, cfg.BotkubeCloud.TeamOrganizationID, slackWorkspace.ID) }) From b2e12d41e46a7a4615c2687fe83dba86541cd656 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Fri, 8 Mar 2024 13:09:07 +0100 Subject: [PATCH 5/8] Do not uninstall Botkube on test failure --- .../cloud_slack_dev_e2e_test.go | 9 +++++++-- test/e2e/bots_test.go | 19 +++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index 52d688444..f5b24f59c 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -347,8 +347,13 @@ func TestCloudSlackE2E(t *testing.T) { } helmInstallCallback := helmx.InstallChart(t, params) t.Cleanup(func() { - t.Log("Uninstalling Helm chart...") - helmInstallCallback(t) + if t.Failed() { + t.Log("Tests failed, keeping the Botkube instance installed for debugging purposes.") + } else { + t.Log("Uninstalling Helm chart...") + helmInstallCallback(t) + } + botkubeDeploymentUninstalled.Store(true) }) diff --git a/test/e2e/bots_test.go b/test/e2e/bots_test.go index 23a29e393..c0f3b34d2 100644 --- a/test/e2e/bots_test.go +++ b/test/e2e/bots_test.go @@ -286,8 +286,13 @@ func runBotTest(t *testing.T, }) require.NoError(t, err) t.Cleanup(func() { - t.Log("Uninstalling Helm chart...") - botkubex.Uninstall(t, appCfg.ConfigProvider.BotkubeCliBinaryPath) + if t.Failed() { + t.Log("Tests failed, keeping the Botkube instance installed for debugging purposes.") + } else { + t.Log("Uninstalling Helm chart...") + botkubex.Uninstall(t, appCfg.ConfigProvider.BotkubeCliBinaryPath) + } + botkubeDeploymentUninstalled.Store(true) }) } @@ -1696,14 +1701,8 @@ func waitForRestart(t *testing.T, tester commplatform.BotDriver, userID, channel } } - err := tester.OnChannel().WaitForMessagePosted(userID, channel, 2, assertFn) - if err != nil && tester.Type() == commplatform.TeamsBot { - // TODO(https://github.com/kubeshop/botkube-cloud/issues/854): for some reason, Teams restarts are not deterministic and sometimes it doesn't happen - // We should add fetching Agent logs to see why it happens. - t.Logf("⚠️ Teams communication platform didn't restart on time: %v", err) - } else { - assert.NoError(t, err) - } + err := tester.OnChannel().WaitForMessagePosted(userID, channel, 3, assertFn) + assert.NoError(t, err) tester.SetTimeout(originalTimeout) } From 60a6ef4221c316bbe1be70bd061aab0f69c24610 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Fri, 8 Mar 2024 14:58:44 +0100 Subject: [PATCH 6/8] Dump logs on failure and fix restarts in integration tests --- .github/workflows/branch-build.yml | 10 +++++++++- .github/workflows/pr-build.yaml | 6 ++++++ test/e2e/bots_test.go | 23 ++++++++++++----------- test/helmx/helm_helpers.go | 3 ++- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/.github/workflows/branch-build.yml b/.github/workflows/branch-build.yml index 45c1dfdab..9e1e29c1f 100644 --- a/.github/workflows/branch-build.yml +++ b/.github/workflows/branch-build.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - fix-e2e-tests repository_dispatch: types: [ trigger-e2e-tests ] @@ -193,7 +194,14 @@ jobs: # run: | # KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \ # make test-integration-${{ matrix.integration }} -# + +# TODO: Uncomment +# - name: Dump cluster state +# if: ${{ failure() }} +# uses: ./.github/actions/dump-cluster +# with: +# name: ${{ matrix.integration }} + # - name: Slack Notification # uses: rtCamp/action-slack-notify@v2 # if: ${{ failure() }} diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 63e7f0c00..215111ddf 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -304,3 +304,9 @@ jobs: run: | KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \ make test-integration-${{ matrix.integration }} + + - name: Dump cluster state + if: ${{ failure() }} + uses: ./.github/actions/dump-cluster + with: + name: ${{ matrix.integration }} diff --git a/test/e2e/bots_test.go b/test/e2e/bots_test.go index c0f3b34d2..f23373483 100644 --- a/test/e2e/bots_test.go +++ b/test/e2e/bots_test.go @@ -840,10 +840,13 @@ func runBotTest(t *testing.T, expectedMessage = fmt.Sprintf("%s\n%s", cmdHeader(command), expectedBody) botDriver.PostMessageToBot(t, botDriver.SecondChannel().Identifier(), command) - err = botDriver.WaitForMessagePosted(botDriver.BotUserID(), botDriver.SecondChannel().ID(), limitMessages(), botDriver.AssertEquals(expectedMessage)) require.NoError(t, err) + if botDriver.Type().IsCloud() { + waitForRestart(t, botDriver, botDriver.BotUserID(), botDriver.FirstChannel().ID(), appCfg.ClusterName) + } + cfgMapCli := k8sCli.CoreV1().ConfigMaps(appCfg.Deployment.Namespace) t.Log("Creating ConfigMap...") @@ -949,12 +952,12 @@ func runBotTest(t *testing.T, expectedMessage = fmt.Sprintf("%s\n%s", cmdHeader(command), expectedBody) botDriver.PostMessageToBot(t, botDriver.FirstChannel().Identifier(), command) + err = botDriver.WaitForMessagePosted(botDriver.BotUserID(), botDriver.FirstChannel().ID(), limitMessages(), botDriver.AssertEquals(expectedMessage)) + assert.NoError(t, err) if botDriver.Type().IsCloud() { waitForRestart(t, botDriver, botDriver.BotUserID(), botDriver.FirstChannel().ID(), appCfg.ClusterName) } - err = botDriver.WaitForMessagePosted(botDriver.BotUserID(), botDriver.FirstChannel().ID(), limitMessages(), botDriver.AssertEquals(expectedMessage)) - assert.NoError(t, err) t.Log("Getting notifier status from second channel...") command = "status notifications" @@ -1683,16 +1686,11 @@ func crashConfigMapSourcePlugin(t *testing.T, cfgMapCli corev1.ConfigMapInterfac } func waitForRestart(t *testing.T, tester commplatform.BotDriver, userID, channel, clusterName string) { - t.Log("Waiting for restart...") + t.Logf("Waiting for restart (timestamp: %s)...", time.Now().String()) originalTimeout := tester.Timeout() - tester.SetTimeout(90 * time.Second) - if tester.Type() == commplatform.TeamsBot { - tester.SetTimeout(120 * time.Second) - } - // 2, since time to time latest message becomes upgrade message right after begin message + tester.SetTimeout(120 * time.Second) expMsg := fmt.Sprintf("My watch begins for cluster '%s'! :crossed_swords:", clusterName) - assertFn := tester.AssertEquals(expMsg) if tester.Type() == commplatform.TeamsBot { // Teams sends JSON (Adaptive Card), so we cannot do equal assertion expMsg = fmt.Sprintf("My watch begins for cluster '%s'!", clusterName) @@ -1701,9 +1699,12 @@ func waitForRestart(t *testing.T, tester commplatform.BotDriver, userID, channel } } - err := tester.OnChannel().WaitForMessagePosted(userID, channel, 3, assertFn) + // 2, since from time to time latest message becomes upgrade message right after begin message + err := tester.OnChannel().WaitForMessagePosted(userID, channel, 2, assertFn) assert.NoError(t, err) + t.Logf("Detected a successful restart (timestamp: %s).", time.Now().String()) + tester.SetTimeout(originalTimeout) } diff --git a/test/helmx/helm_helpers.go b/test/helmx/helm_helpers.go index ea276cbb0..18c04169d 100644 --- a/test/helmx/helm_helpers.go +++ b/test/helmx/helm_helpers.go @@ -137,7 +137,8 @@ func WaitForUninstallation(ctx context.Context, t *testing.T, alreadyUninstalled } if waitInterrupted { - t.Log("Force uninstalling the Helm chart after a wait timeout") + t.Log("Helm chart uninstallation timed out. Proceeding with deleting other resources...") + return nil } t.Log("Waiting a bit more...") From ed348b8d86ce91be38ce594887bb248090988843 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Fri, 8 Mar 2024 15:53:23 +0100 Subject: [PATCH 7/8] Fix another issues in tests --- test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go | 3 ++- test/e2e/bots_test.go | 9 +++++++-- test/helmx/helm_helpers.go | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index f5b24f59c..8c560b26f 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -89,6 +89,7 @@ func TestCloudSlackE2E(t *testing.T) { require.NoError(t, err) cfg.Slack.Tester.CloudBasedTestEnabled = false // override property used only in the Cloud Slack E2E tests + cfg.Slack.Tester.RecentMessagesLimit = 4 // this is used effectively only for the Botkube restarts. There are two of them in a short time window so it shouldn't be higher than 5. authHeaderValue := "" var botkubeDeploymentUninstalled atomic.Bool @@ -471,9 +472,9 @@ func TestCloudSlackE2E(t *testing.T) { t.Run("Botkube Deployment -> Cloud sync", func(t *testing.T) { t.Log("Disabling notification...") tester.PostMessageToBot(t, channel.Identifier(), "disable notifications") + t.Log("Waiting for config reload message...") expectedReloadMsg := fmt.Sprintf(":arrows_counterclockwise: Configuration reload requested for cluster '%s'. Hold on a sec...", deployment.Name) - err = tester.OnChannel().WaitForMessagePostedRecentlyEqual(tester.BotUserID(), channel.ID(), expectedReloadMsg) require.NoError(t, err) diff --git a/test/e2e/bots_test.go b/test/e2e/bots_test.go index f23373483..39140179f 100644 --- a/test/e2e/bots_test.go +++ b/test/e2e/bots_test.go @@ -1686,7 +1686,7 @@ func crashConfigMapSourcePlugin(t *testing.T, cfgMapCli corev1.ConfigMapInterfac } func waitForRestart(t *testing.T, tester commplatform.BotDriver, userID, channel, clusterName string) { - t.Logf("Waiting for restart (timestamp: %s)...", time.Now().String()) + t.Logf("Waiting for restart (timestamp: %s)...", time.Now().Format(time.DateTime)) originalTimeout := tester.Timeout() tester.SetTimeout(120 * time.Second) @@ -1703,7 +1703,12 @@ func waitForRestart(t *testing.T, tester commplatform.BotDriver, userID, channel err := tester.OnChannel().WaitForMessagePosted(userID, channel, 2, assertFn) assert.NoError(t, err) - t.Logf("Detected a successful restart (timestamp: %s).", time.Now().String()) + t.Logf("Detected a successful restart (timestamp: %s).", time.Now().Format(time.DateTime)) + + t.Logf("Waiting a bit longer just to make sure Botkube connects to the Cloud Router...") + // Yes, it's ugly but "My watch begins..." doesn't really mean the Slack/Teams gRPC connection has been established. + // So we wait a bit longer to avoid a race condition. + time.Sleep(3 * time.Second) tester.SetTimeout(originalTimeout) } diff --git a/test/helmx/helm_helpers.go b/test/helmx/helm_helpers.go index 18c04169d..828764e58 100644 --- a/test/helmx/helm_helpers.go +++ b/test/helmx/helm_helpers.go @@ -137,7 +137,7 @@ func WaitForUninstallation(ctx context.Context, t *testing.T, alreadyUninstalled } if waitInterrupted { - t.Log("Helm chart uninstallation timed out. Proceeding with deleting other resources...") + t.Log("Waiting for Helm chart uninstallation timed out. Proceeding with deleting other resources...") return nil } From 1359dffd32e466dd4cf87bb24d1c61fceac18c7b Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Mon, 11 Mar 2024 10:35:53 +0100 Subject: [PATCH 8/8] Finalize PR --- .github/actions/cloud-slack-e2e/action.yaml | 24 +- .github/actions/dump-cluster/action.yaml | 2 +- .github/workflows/branch-build.yml | 400 +++++++++--------- .github/workflows/pr-build.yaml | 4 +- .../cloud_slack_dev_e2e_test.go | 4 +- 5 files changed, 216 insertions(+), 218 deletions(-) diff --git a/.github/actions/cloud-slack-e2e/action.yaml b/.github/actions/cloud-slack-e2e/action.yaml index a935a8e26..c6e37eb70 100644 --- a/.github/actions/cloud-slack-e2e/action.yaml +++ b/.github/actions/cloud-slack-e2e/action.yaml @@ -125,15 +125,15 @@ runs: echo "footer=${FOOTER}" >> $GITHUB_OUTPUT -# - name: Slack Notification -# uses: rtCamp/action-slack-notify@v2 -# if: ${{ failure() }} -# env: -# SLACK_CHANNEL: 'botkube-cloud-ci-alerts' -# SLACK_USERNAME: Botkube Cloud CI -# SLACK_COLOR: 'red' -# SLACK_TITLE: 'Message' -# SLACK_MESSAGE: "Cloud Slack ${{ inputs.e2e_type }} E2E tests failed :scream:" -# SLACK_ICON_EMOJI: ':this-is-fine-fire:' -# SLACK_FOOTER: ${{ steps.footer.outputs.footer }} -# SLACK_WEBHOOK: ${{ inputs.slack_alerts_webhook }} + - name: Slack Notification + uses: rtCamp/action-slack-notify@v2 + if: ${{ failure() }} + env: + SLACK_CHANNEL: 'botkube-cloud-ci-alerts' + SLACK_USERNAME: Botkube Cloud CI + SLACK_COLOR: 'red' + SLACK_TITLE: 'Message' + SLACK_MESSAGE: "Cloud Slack ${{ inputs.e2e_type }} E2E tests failed :scream:" + SLACK_ICON_EMOJI: ':this-is-fine-fire:' + SLACK_FOOTER: ${{ steps.footer.outputs.footer }} + SLACK_WEBHOOK: ${{ inputs.slack_alerts_webhook }} diff --git a/.github/actions/dump-cluster/action.yaml b/.github/actions/dump-cluster/action.yaml index 3afe4c960..bbee69a90 100644 --- a/.github/actions/dump-cluster/action.yaml +++ b/.github/actions/dump-cluster/action.yaml @@ -20,7 +20,7 @@ runs: echo "Dumping cluster info into ${LOGS_DIR}" kubectl cluster-info dump --all-namespaces --output-directory="${LOGS_DIR}" - name: Upload artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cluster_dump_${{github.sha}}_${{ inputs.name }} path: "output" diff --git a/.github/workflows/branch-build.yml b/.github/workflows/branch-build.yml index 9e1e29c1f..bb9f705c7 100644 --- a/.github/workflows/branch-build.yml +++ b/.github/workflows/branch-build.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - fix-e2e-tests repository_dispatch: types: [ trigger-e2e-tests ] @@ -17,208 +16,207 @@ env: GIT_USER: botkube-dev jobs: -# extract-metadata: -# if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch -# runs-on: ubuntu-latest -# outputs: -# versions: ${{ steps.extract-version.outputs.versions }} -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# - name: Extract version -# id: extract-version -# run: | -# IMAGE_VERSION=$(git rev-parse --short HEAD) -# echo "versions={\"image-version\":[\"v9.99.9-dev\",\"0.0.0-${IMAGE_VERSION}\"]}" >> $GITHUB_OUTPUT -# build: -# if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch -# needs: [extract-metadata] -# strategy: -# matrix: ${{ fromJson(needs.extract-metadata.outputs.versions) }} -# runs-on: ubuntu-latest -# env: -# GO111MODULE: on -# GOPATH: /home/runner/work/botkube -# GOBIN: /home/runner/work/botkube/bin -# DOCKER_CLI_EXPERIMENTAL: "enabled" -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# - name: Setup Go -# uses: actions/setup-go@v5 -# with: -# go-version-file: 'go.mod' -# cache: true -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v3 -# - name: Docker Login -# uses: docker/login-action@v1 -# with: -# registry: ghcr.io -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} -# - name: Install GoReleaser -# uses: goreleaser/goreleaser-action@v5 -# with: -# install-only: true -# version: latest -# - name: Run GoReleaser -# run: | -# make release-snapshot -# env: -# ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }} -# GORELEASER_CURRENT_TAG: ${{ matrix.image-version }} -# IMAGE_TAG: ${{ matrix.image-version }} -# -# integration-tests: -# if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch -# name: Integration tests -# runs-on: ubuntu-latest -# needs: [ build ] -# permissions: -# contents: read -# packages: read -# -# strategy: -# # make the jobs independent -# fail-fast: false -# -# matrix: -# integration: -# - slack -# - discord -# - teams -# -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# with: -# persist-credentials: false -# -# - name: Setup Go modules -# uses: ./.github/actions/setup-go-mod-private -# with: -# access_token: ${{ secrets.E2E_TEST_GH_DEV_ACCOUNT_PAT }} -# username: ${{ env.GIT_USER }} -# -# - name: Pub/Sub auth -# uses: 'google-github-actions/auth@v1' -# if: matrix.integration == 'teams' -# with: -# credentials_json: ${{ secrets.E2E_TEST_GCP_PUB_SUB_CREDENTIALS }} -# -# - name: Install Helm -# uses: azure/setup-helm@v3 -# with: -# version: ${{ env.HELM_VERSION }} -# -# - name: Download k3d -# run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash" -# -# - name: Create cluster to test ${{ matrix.integration }} -# run: "k3d cluster create ${{ matrix.integration }}-test-cluster --wait --timeout=5m" -# -# - name: Install Botkube locally via helm -# if: matrix.integration == 'discord' -# env: -# DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} -# DISCORD_BOT_ID: ${{ secrets.DISCORD_BOT_ID }} -# run: | -# helm install botkube --namespace botkube ./helm/botkube --wait --create-namespace \ -# -f ./helm/botkube/e2e-test-values.yaml \ -# --set communications.default-group.discord.token="${DISCORD_BOT_TOKEN}" \ -# --set communications.default-group.discord.botID="${DISCORD_BOT_ID}" \ -# --set image.registry="${IMAGE_REGISTRY}" \ -# --set image.repository="${IMAGE_REPOSITORY}" \ -# --set image.tag="${IMAGE_TAG}" \ -# -# - name: Install GoReleaser -# uses: goreleaser/goreleaser-action@v5 -# with: -# install-only: true -# version: latest -# -# - name: Build all plugins into dist directory -# env: -# # we hardcode plugins version, so it's predictable in e2e tests -# GORELEASER_CURRENT_TAG: "v0.0.0-latest" -# OUTPUT_MODE: "binary" -# SINGLE_PLATFORM: "true" -# PLUGIN_TARGETS: "kubernetes,kubectl,cm-watcher,echo,helm" -# run: | -# make build-plugins -# -# - name: CLI Cache -# if: matrix.integration != 'discord' -# uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/go-build -# ~/go/pkg/mod -# dist/botkube-cli_linux_amd64_v1/botkube -# key: ${{ runner.os }}-botkube-cli -# -# - name: Build CLI -# if: matrix.integration != 'discord' -# run: make build-single-arch-cli -# -# - name: Add Botkube CLI to env -# run: | -# echo CONFIG_PROVIDER_BOTKUBE_CLI_BINARY_PATH="$PWD/dist/botkube-cli_linux_amd64_v1/botkube" >> $GITHUB_ENV -# -# - name: Run ${{ matrix.integration }} tests -# env: -# SLACK_TESTER_APP_TOKEN: ${{ secrets.SLACK_TESTER_APP_TOKEN }} -# SLACK_CLOUD_TESTER_APP_TOKEN: ${{ secrets.SLACK_CLOUD_TESTER_APP_TOKEN }} -# SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" -# -# DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }} -# DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }} -# DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" -# -# TEAMS_BOT_TESTER_APP_ID: ${{ secrets.TEAMS_BOT_TESTER_APP_ID }} -# TEAMS_BOT_TESTER_APP_PASSWORD: ${{ secrets.TEAMS_BOT_TESTER_APP_PASSWORD }} -# TEAMS_ORGANIZATION_TEAM_ID: ${{ secrets.TEAMS_ORGANIZATION_TEAM_ID }} -# TEAMS_ORGANIZATION_TENANT_ID: ${{ secrets.TEAMS_ORGANIZATION_TENANT_ID }} -# TEAMS_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" -# -# PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/plugin-dist -# CONFIG_PROVIDER_API_KEY: ${{ secrets.CONFIG_PROVIDER_API_KEY }} -# CONFIG_PROVIDER_ENDPOINT: ${{ secrets.CONFIG_PROVIDER_ENDPOINT }} -# CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID: ${{ secrets.CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID }} -# CONFIG_PROVIDER_IMAGE_REPOSITORY: ${{ env.IMAGE_REPOSITORY }} -# CONFIG_PROVIDER_IMAGE_TAG: ${{ env.IMAGE_TAG }} -# CONFIG_PROVIDER_HELM_REPO_DIRECTORY: ${{ github.workspace }}/helm -# run: | -# KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \ -# make test-integration-${{ matrix.integration }} - -# TODO: Uncomment -# - name: Dump cluster state -# if: ${{ failure() }} -# uses: ./.github/actions/dump-cluster -# with: -# name: ${{ matrix.integration }} - -# - name: Slack Notification -# uses: rtCamp/action-slack-notify@v2 -# if: ${{ failure() }} -# env: -# SLACK_USERNAME: Botkube Cloud CI -# SLACK_COLOR: 'red' -# SLACK_TITLE: 'Message' -# SLACK_CHANNEL: 'botkube-cloud-ci-alerts' -# SLACK_MESSAGE: 'Integration ${{ matrix.integration }} test failed :scream:' -# SLACK_ICON_EMOJI: ':this-is-fine-fire:' -# SLACK_WEBHOOK: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }} -# SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..." + extract-metadata: + if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.extract-version.outputs.versions }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Extract version + id: extract-version + run: | + IMAGE_VERSION=$(git rev-parse --short HEAD) + echo "versions={\"image-version\":[\"v9.99.9-dev\",\"0.0.0-${IMAGE_VERSION}\"]}" >> $GITHUB_OUTPUT + build: + if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch + needs: [extract-metadata] + strategy: + matrix: ${{ fromJson(needs.extract-metadata.outputs.versions) }} + runs-on: ubuntu-latest + env: + GO111MODULE: on + GOPATH: /home/runner/work/botkube + GOBIN: /home/runner/work/botkube/bin + DOCKER_CLI_EXPERIMENTAL: "enabled" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Docker Login + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + install-only: true + version: latest + - name: Run GoReleaser + run: | + make release-snapshot + env: + ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }} + GORELEASER_CURRENT_TAG: ${{ matrix.image-version }} + IMAGE_TAG: ${{ matrix.image-version }} + + integration-tests: + if: github.event_name != 'repository_dispatch' # skip if triggered by repository_dispatch + name: Integration tests + runs-on: ubuntu-latest + needs: [ build ] + permissions: + contents: read + packages: read + + strategy: + # make the jobs independent + fail-fast: false + + matrix: + integration: + - slack + - discord + - teams + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup Go modules + uses: ./.github/actions/setup-go-mod-private + with: + access_token: ${{ secrets.E2E_TEST_GH_DEV_ACCOUNT_PAT }} + username: ${{ env.GIT_USER }} + + - name: Pub/Sub auth + uses: 'google-github-actions/auth@v1' + if: matrix.integration == 'teams' + with: + credentials_json: ${{ secrets.E2E_TEST_GCP_PUB_SUB_CREDENTIALS }} + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Download k3d + run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash" + + - name: Create cluster to test ${{ matrix.integration }} + run: "k3d cluster create ${{ matrix.integration }}-test-cluster --wait --timeout=5m" + + - name: Install Botkube locally via helm + if: matrix.integration == 'discord' + env: + DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} + DISCORD_BOT_ID: ${{ secrets.DISCORD_BOT_ID }} + run: | + helm install botkube --namespace botkube ./helm/botkube --wait --create-namespace \ + -f ./helm/botkube/e2e-test-values.yaml \ + --set communications.default-group.discord.token="${DISCORD_BOT_TOKEN}" \ + --set communications.default-group.discord.botID="${DISCORD_BOT_ID}" \ + --set image.registry="${IMAGE_REGISTRY}" \ + --set image.repository="${IMAGE_REPOSITORY}" \ + --set image.tag="${IMAGE_TAG}" \ + + - name: Install GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + install-only: true + version: latest + + - name: Build all plugins into dist directory + env: + # we hardcode plugins version, so it's predictable in e2e tests + GORELEASER_CURRENT_TAG: "v0.0.0-latest" + OUTPUT_MODE: "binary" + SINGLE_PLATFORM: "true" + PLUGIN_TARGETS: "kubernetes,kubectl,cm-watcher,echo,helm" + run: | + make build-plugins + + - name: CLI Cache + if: matrix.integration != 'discord' + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + dist/botkube-cli_linux_amd64_v1/botkube + key: ${{ runner.os }}-botkube-cli + + - name: Build CLI + if: matrix.integration != 'discord' + run: make build-single-arch-cli + + - name: Add Botkube CLI to env + run: | + echo CONFIG_PROVIDER_BOTKUBE_CLI_BINARY_PATH="$PWD/dist/botkube-cli_linux_amd64_v1/botkube" >> $GITHUB_ENV + + - name: Run ${{ matrix.integration }} tests + env: + SLACK_TESTER_APP_TOKEN: ${{ secrets.SLACK_TESTER_APP_TOKEN }} + SLACK_CLOUD_TESTER_APP_TOKEN: ${{ secrets.SLACK_CLOUD_TESTER_APP_TOKEN }} + SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" + + DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }} + DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }} + DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" + + TEAMS_BOT_TESTER_APP_ID: ${{ secrets.TEAMS_BOT_TESTER_APP_ID }} + TEAMS_BOT_TESTER_APP_PASSWORD: ${{ secrets.TEAMS_BOT_TESTER_APP_PASSWORD }} + TEAMS_ORGANIZATION_TEAM_ID: ${{ secrets.TEAMS_ORGANIZATION_TEAM_ID }} + TEAMS_ORGANIZATION_TENANT_ID: ${{ secrets.TEAMS_ORGANIZATION_TENANT_ID }} + TEAMS_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}" + + PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/plugin-dist + CONFIG_PROVIDER_API_KEY: ${{ secrets.CONFIG_PROVIDER_API_KEY }} + CONFIG_PROVIDER_ENDPOINT: ${{ secrets.CONFIG_PROVIDER_ENDPOINT }} + CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID: ${{ secrets.CONFIG_PROVIDER_SLACK_WORKSPACE_TEAM_ID }} + CONFIG_PROVIDER_IMAGE_REPOSITORY: ${{ env.IMAGE_REPOSITORY }} + CONFIG_PROVIDER_IMAGE_TAG: ${{ env.IMAGE_TAG }} + CONFIG_PROVIDER_HELM_REPO_DIRECTORY: ${{ github.workspace }}/helm + run: | + KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \ + make test-integration-${{ matrix.integration }} + + - name: Dump cluster state + if: ${{ failure() }} + uses: ./.github/actions/dump-cluster + with: + name: ${{ matrix.integration }} + + - name: Slack Notification + uses: rtCamp/action-slack-notify@v2 + if: ${{ failure() }} + env: + SLACK_USERNAME: Botkube Cloud CI + SLACK_COLOR: 'red' + SLACK_TITLE: 'Message' + SLACK_CHANNEL: 'botkube-cloud-ci-alerts' + SLACK_MESSAGE: 'Integration ${{ matrix.integration }} test failed :scream:' + SLACK_ICON_EMOJI: ':this-is-fine-fire:' + SLACK_WEBHOOK: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }} + SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..." cli-migration-e2e: name: CLI Migration E2E tests runs-on: ubuntu-latest -# needs: [ build ] + needs: [ build ] permissions: contents: read packages: read @@ -271,7 +269,7 @@ jobs: KUBECONFIG=$(k3d kubeconfig write cli-migration-e2e-cluster) make test-cli-migration-e2e - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: screenshots_dump_${{github.sha}} @@ -300,7 +298,7 @@ jobs: cloud-slack-dev-e2e: name: Botkube Cloud Slack Dev E2E runs-on: ubuntu-latest -# needs: [ build ] + needs: [ build ] permissions: contents: read packages: read diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 215111ddf..638c46671 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -69,7 +69,7 @@ jobs: make save-images - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: botkube-${{github.sha}} path: ${{ env.IMAGE_SAVE_LOAD_DIR }} @@ -91,7 +91,7 @@ jobs: persist-credentials: false - name: Download artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: botkube-${{github.sha}} path: ${{ env.IMAGE_SAVE_LOAD_DIR }} diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index 8c560b26f..d9395ab22 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -89,7 +89,7 @@ func TestCloudSlackE2E(t *testing.T) { require.NoError(t, err) cfg.Slack.Tester.CloudBasedTestEnabled = false // override property used only in the Cloud Slack E2E tests - cfg.Slack.Tester.RecentMessagesLimit = 4 // this is used effectively only for the Botkube restarts. There are two of them in a short time window so it shouldn't be higher than 5. + cfg.Slack.Tester.RecentMessagesLimit = 4 // this is used effectively only for the Botkube restarts. There are two of them in a short time window, so it shouldn't be higher than 5. authHeaderValue := "" var botkubeDeploymentUninstalled atomic.Bool @@ -239,7 +239,7 @@ func TestCloudSlackE2E(t *testing.T) { go router.Run() defer router.MustStop() - t.Log("Ensuring proper organizaton is selected") + t.Log("Ensuring proper organization is selected") botkubePage.MustWaitOpen() screenshotIfShould(t, cfg, botkubePage) botkubePage.MustElement("a.logo-link")