CI: Use only free runners (#1131)

* Fix TLS + urls in resources * Use 0.0.0.0 like docker compose did * CI: Use only free runners Instead of running on big agents, split up the work into multiple pipelines: - Basic tests - Other tests - Long tests This allows fanning out the work to a swarm of agents. Also, it should lead to less failures due to less parallelism
grafana · Nov 7, 2023 · a8b847f · a8b847f
1 parent e44a20e
commit a8b847f
Show file tree

Hide file tree

Showing 19 changed files with 42 additions and 25 deletions.
diff --git a/.github/workflows/acc-tests.yml b/.github/workflows/acc-tests.yml
@@ -43,22 +43,25 @@ jobs:
         # OSS tests, run on all versions
         version: ['10.2.0', '10.1.5', '9.5.13', '8.5.27']
         type: ['oss']
-        runner: ['ubuntu-latest-16-cores']
+        subset: ['basic', 'other', 'long']
         include:
           # TLS proxy tests, run only on latest version
           - version: '10.2.0'
             type: 'tls'
-            runner: 'ubuntu-latest' # Smaller instance
+            subset: 'basic'
           # Sub-path tests. Runs tests on localhost:3000/grafana/
           - version: '10.2.0'
             type: 'subpath'
-            runner: 'ubuntu-latest-16-cores'
+            subset: 'basic'
+          - version: '10.2.0'
+            type: 'subpath'
+            subset: 'other'
           # Enterprise tests, run only on latest version
           - version: '10.2.0'
             type: 'enterprise'
-            runner: 'ubuntu-latest' # Smaller instance
-    name: ${{ matrix.version }} - ${{ matrix.type }}
-    runs-on: ${{ matrix.runner }}
+            subset: 'all'
+    name: ${{ matrix.version }} - ${{ matrix.type }} - ${{ matrix.subset }}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
@@ -74,7 +77,15 @@ jobs:
         with:
           repo_secrets: |
             GF_ENTERPRISE_LICENSE_TEXT=enterprise:license
+      - name: Cache Docker image
+        uses: ScribeMD/docker-cache@0.3.6
+        with:
+          key: docker-${{ runner.os }}-${{ matrix.type == 'enterprise' && 'enterprise' || 'oss' }}-${{ matrix.version }}
       - run: make testacc-${{ matrix.type }}-docker
         env:
           GRAFANA_VERSION: ${{ matrix.version }}
-          TESTARGS: ${{ matrix.type == 'tls' && '-run ".*_basic"' || '' }} # Run subset of tests for TLS proxy, it's slower
+          TESTARGS: >- 
+            ${{ matrix.subset == 'all' && '-parallel 2' || '' }}
+            ${{ matrix.subset == 'basic' && '-run=".*_basic" -short -parallel 2' || '' }}
+            ${{ matrix.subset == 'other' && '-skip=".*_basic" -short -parallel 2' || '' }}
+            ${{ matrix.subset == 'long' && '-run=".*longtest" -parallel 1' || '' }}
diff --git a/GNUmakefile b/GNUmakefile
@@ -1,7 +1,7 @@
 GRAFANA_VERSION ?= 10.1.5
 
 testacc:
-	TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 120m -parallel 4
+	TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 120m
 
 # Test OSS features
 testacc-oss:

diff --git a/internal/resources/grafana/data_source_dashboard_test.go b/internal/resources/grafana/data_source_dashboard_test.go
@@ -12,7 +12,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceDashboardBasicID(t *testing.T) {
+func TestAccDatasourceDashboard_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var dashboard gapi.Dashboard

diff --git a/internal/resources/grafana/data_source_data_source_test.go b/internal/resources/grafana/data_source_data_source_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceDatasource(t *testing.T) {
+func TestAccDatasourceDatasource_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var dataSource gapi.DataSource

diff --git a/internal/resources/grafana/data_source_folder_test.go b/internal/resources/grafana/data_source_folder_test.go
@@ -11,7 +11,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceFolder(t *testing.T) {
+func TestAccDatasourceFolder_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var folder goapi.Folder

diff --git a/internal/resources/grafana/data_source_folders_test.go b/internal/resources/grafana/data_source_folders_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceFolders(t *testing.T) {
+func TestAccDatasourceFolders_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var folderA goapi.Folder

diff --git a/internal/resources/grafana/data_source_library_panel_test.go b/internal/resources/grafana/data_source_library_panel_test.go
@@ -9,7 +9,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceLibraryPanel(t *testing.T) {
+func TestAccDatasourceLibraryPanel_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t, ">=8.0.0")
 
 	var panel gapi.LibraryPanel

diff --git a/internal/resources/grafana/data_source_organization_preferences_test.go b/internal/resources/grafana/data_source_organization_preferences_test.go
@@ -7,7 +7,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceOrganizationPreferences(t *testing.T) {
+func TestAccDatasourceOrganizationPreferences_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	checks := []resource.TestCheckFunc{

diff --git a/internal/resources/grafana/data_source_organization_test.go b/internal/resources/grafana/data_source_organization_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceOrganization(t *testing.T) {
+func TestAccDatasourceOrganization_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var organization gapi.Org

diff --git a/internal/resources/grafana/data_source_role_test.go b/internal/resources/grafana/data_source_role_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceRole(t *testing.T) {
+func TestAccDatasourceRole_basic(t *testing.T) {
 	testutils.CheckEnterpriseTestsEnabled(t)
 
 	var role gapi.Role

diff --git a/internal/resources/grafana/data_source_team_test.go b/internal/resources/grafana/data_source_team_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceTeam(t *testing.T) {
+func TestAccDatasourceTeam_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var team goapi.TeamDTO

diff --git a/internal/resources/grafana/data_source_user_test.go b/internal/resources/grafana/data_source_user_test.go
@@ -9,7 +9,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceUser(t *testing.T) {
+func TestAccDatasourceUser_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	var user gapi.User

diff --git a/internal/resources/grafana/data_source_users_test.go b/internal/resources/grafana/data_source_users_test.go
@@ -7,7 +7,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
 
-func TestAccDatasourceUsers(t *testing.T) {
+func TestAccDatasourceUsers_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	checks := []resource.TestCheckFunc{

diff --git a/internal/resources/grafana/resource_api_key_test.go b/internal/resources/grafana/resource_api_key_test.go
@@ -15,7 +15,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
 )
 
-func TestAccGrafanaAuthKey(t *testing.T) {
+func TestAccGrafanaAuthKey_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
 	testName := acctest.RandString(10)

diff --git a/internal/resources/grafana/resource_organization_test.go b/internal/resources/grafana/resource_organization_test.go
@@ -198,7 +198,10 @@ func TestAccOrganization_roleNoneUser(t *testing.T) {
 	})
 }
 
-func TestAccOrganization_createManyUsers(t *testing.T) {
+func TestAccOrganization_createManyUsers_longtest(t *testing.T) {
+	if testing.Short() { // Also named "longtest" to allow targeting with -run=.*longtest
+		t.Skip("skipping test in short mode")
+	}
 	testutils.CheckOSSTestsEnabled(t)
 
 	var org gapi.Org

diff --git a/internal/resources/grafana/resource_report_test.go b/internal/resources/grafana/resource_report_test.go
@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
 )
 
-func TestAccResourceReport(t *testing.T) {
+func TestAccResourceReport_basic(t *testing.T) {
 	testutils.CheckEnterpriseTestsEnabled(t)
 
 	var report gapi.Report

diff --git a/internal/resources/grafana/resource_role_test.go b/internal/resources/grafana/resource_role_test.go
@@ -14,7 +14,7 @@ import (
 	"github.com/grafana/terraform-provider-grafana/internal/testutils"
 )
 
-func TestAccRole(t *testing.T) {
+func TestAccRole_basic(t *testing.T) {
 	testutils.CheckEnterpriseTestsEnabled(t)
 
 	var role gapi.Role

diff --git a/internal/resources/grafana/resource_service_account_permission_test.go b/internal/resources/grafana/resource_service_account_permission_test.go
@@ -15,7 +15,7 @@ import (
 	"github.com/grafana/terraform-provider-grafana/internal/testutils"
 )
 
-func TestAccServiceAccountPermission(t *testing.T) {
+func TestAccServiceAccountPermission_basic(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t, ">=9.2.4")
 
 	name := acctest.RandString(10)

diff --git a/internal/resources/grafana/resource_service_account_test.go b/internal/resources/grafana/resource_service_account_test.go
@@ -86,7 +86,10 @@ func TestAccServiceAccount_NoneRole(t *testing.T) {
 	})
 }
 
-func TestAccServiceAccount_many(t *testing.T) {
+func TestAccServiceAccount_many_longtest(t *testing.T) {
+	if testing.Short() { // Also named "longtest" to allow targeting with -run=.*longtest
+		t.Skip("skipping test in short mode")
+	}
 	testutils.CheckOSSTestsEnabled(t, ">=9.1.0")
 
 	name := acctest.RandString(10)