From 6bc40930e47ee1eb456b3dad96015c27d765d36f Mon Sep 17 00:00:00 2001 From: Julien Duchesne Date: Mon, 6 Nov 2023 22:45:47 -0500 Subject: [PATCH] CI: Use only free runners Instead of running on big agents, split up the work into multiple pipelines: - Basic tests - Other tests - Long tests This allows fanning out the work to a swarm of agents. Also, it should lead to less failures due to less parallelism --- .github/workflows/acc-tests.yml | 23 ++++++++++++++----- GNUmakefile | 2 +- .../grafana/resource_organization_test.go | 5 +++- .../grafana/resource_service_account_test.go | 5 +++- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/workflows/acc-tests.yml b/.github/workflows/acc-tests.yml index 3b5202fdd..18cd65f27 100644 --- a/.github/workflows/acc-tests.yml +++ b/.github/workflows/acc-tests.yml @@ -43,22 +43,25 @@ jobs: # OSS tests, run on all versions version: ['10.2.0', '10.1.5', '9.5.13', '8.5.27'] type: ['oss'] - runner: ['ubuntu-latest-16-cores'] + subset: ['basic', 'other', 'long'] include: # TLS proxy tests, run only on latest version - version: '10.2.0' type: 'tls' - runner: 'ubuntu-latest' # Smaller instance + subset: 'basic' # Sub-path tests. Runs tests on localhost:3000/grafana/ - version: '10.2.0' type: 'subpath' - runner: 'ubuntu-latest-16-cores' + subset: 'basic' + - version: '10.2.0' + type: 'subpath' + subset: 'other' # Enterprise tests, run only on latest version - version: '10.2.0' type: 'enterprise' - runner: 'ubuntu-latest' # Smaller instance + subset: 'all' name: ${{ matrix.version }} - ${{ matrix.type }} - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 @@ -74,7 +77,15 @@ jobs: with: repo_secrets: | GF_ENTERPRISE_LICENSE_TEXT=enterprise:license + - name: Cache Docker image + uses: ScribeMD/docker-cache@0.3.6 + with: + key: docker-${{ runner.os }}-${{ matrix.type == 'enterprise' && 'enterprise' || 'oss' }} - run: make testacc-${{ matrix.type }}-docker env: GRAFANA_VERSION: ${{ matrix.version }} - TESTARGS: ${{ matrix.type == 'tls' && '-run ".*_basic"' || '' }} # Run subset of tests for TLS proxy, it's slower + TESTARGS: >- + ${{ matrix.subset == 'all' && '-parallel 2' || '' }} + ${{ matrix.subset == 'basic' && '-run=".*_basic$" -short -parallel 2' || '' }} + ${{ matrix.subset == 'other' && '-skip=".*_basic$" -short -parallel 2' || '' }} + ${{ matrix.subset == 'long' && '-run=".*longtest$" -parallel 1' || '' }} diff --git a/GNUmakefile b/GNUmakefile index c8ca6b728..8acab0ee5 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,7 +1,7 @@ GRAFANA_VERSION ?= 10.1.5 testacc: - TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 120m -parallel 4 + TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 120m # Test OSS features testacc-oss: diff --git a/internal/resources/grafana/resource_organization_test.go b/internal/resources/grafana/resource_organization_test.go index 9f5f47183..25037ecaf 100644 --- a/internal/resources/grafana/resource_organization_test.go +++ b/internal/resources/grafana/resource_organization_test.go @@ -198,7 +198,10 @@ func TestAccOrganization_roleNoneUser(t *testing.T) { }) } -func TestAccOrganization_createManyUsers(t *testing.T) { +func TestAccOrganization_createManyUsers_longtest(t *testing.T) { + if testing.Short() { // Also named "longtest" to allow targeting with -run=.*longtest + t.Skip("skipping test in short mode") + } testutils.CheckOSSTestsEnabled(t) var org gapi.Org diff --git a/internal/resources/grafana/resource_service_account_test.go b/internal/resources/grafana/resource_service_account_test.go index d17d41baf..98c1690dc 100644 --- a/internal/resources/grafana/resource_service_account_test.go +++ b/internal/resources/grafana/resource_service_account_test.go @@ -86,7 +86,10 @@ func TestAccServiceAccount_NoneRole(t *testing.T) { }) } -func TestAccServiceAccount_many(t *testing.T) { +func TestAccServiceAccount_many_longtest(t *testing.T) { + if testing.Short() { // Also named "longtest" to allow targeting with -run=.*longtest + t.Skip("skipping test in short mode") + } testutils.CheckOSSTestsEnabled(t, ">=9.1.0") name := acctest.RandString(10)