feat: Introduce persistent store testing support (#254)

.golangci.yml

@@ -19,7 +19,6 @@ linters:
     - govet
     - ineffassign
     - lll
-    - megacheck
     - misspell
     - nakedret
     - nolintlint

Makefile

@@ -1,9 +1,8 @@
-
 GORELEASER_VERSION=v1.24.0
 GORELEASER_DOWNLOAD_URL=https://github.com/goreleaser/goreleaser/releases/download/v1.24.0/goreleaser_$(shell uname)_$(shell uname -m).tar.gz
 GORELEASER=./bin/goreleaser/goreleaser
 
-GOLANGCI_LINT_VERSION=v1.56.2
+GOLANGCI_LINT_VERSION=v1.60.1
 
 LINTER=./bin/golangci-lint
 LINTER_VERSION_FILE=./bin/.golangci-lint-version-$(GOLANGCI_LINT_VERSION)

docs/running.md

@@ -16,6 +16,7 @@ Options besides `-url`:
 * `-junit <FILEPATH>` - writes test results in JUnit XML format to the specified file
 * `-debug` - enables verbose logging of test actions for failed tests
 * `-debug-all` - enables verbose logging of test actions for all tests
+* `-enable-persistence-tests` - enables tests that require external persistence (e.g. a database like redis)
 * `-record-failures` - record failed test IDs to the given file. Recorded tests can be skipped by the next run of 
 the harness via `-skip-from`.
 * `-skip-from` - skips any test IDs recorded in the specified file. May be used in conjunction with `-record-failures`

docs/service_spec.md

@@ -102,6 +102,24 @@ v4 of the event schema originally required a `contextKeys` property on all featu
 
 This means that the SDK supports technology migrations, a feature which allows customers to migrate between data sources using well-defined migration stages.
 
+#### Capability `"persistent-data-store-{integration}"`
+
+This means the SDK is capable of interacting with external persistent data stores. Each capability defines the specific persistence technology the SDK is capable of supporting. The full list of supported integrations is as follows:
+
+- `consul`
+- `dynamodb`
+- `redis`
+
+The persistence store tests are unique in that they rely on external services to be available prior to running the test harness. Because of this, the test harness will not run these tests by default, even if the capabilities exist, without also provided the `-enable-persistence-tests` flag.
+
+These services can be easily provided through the use of docker. Example commands are shown below for your convenience.
+
+```shell
+docker run -p 8000:8000 amazon/dynamodb-local
+docker run -p 8500:8500 hashicorp/consul
+docker run -p 6379:6379 redis
+```
+
 #### Capability `"polling-gzip"`
 
 This means the SDK is requesting gzip compression support on polling payloads. The SDK is expected to set the `Accept-Encoding` header to `gzip` in addition to enabling this capability.

framework/harness/harness.go

@@ -108,6 +108,7 @@ func NewTestHarness(
 	testServiceBaseURL string,
 	testHarnessExternalHostname string,
 	testHarnessPort int,
+	testHarnessEnablePersistenceTests bool,
 	statusQueryTimeout time.Duration,
 	debugLogger framework.Logger,
 	startupOutput io.Writer,
@@ -129,6 +130,21 @@ func NewTestHarness(
 	if err != nil {
 		return nil, err
 	}
+
+	// If we aren't running persistence tests, remove the capabilities that would enable it.
+	if !testHarnessEnablePersistenceTests {
+		filteredCapabilities := make([]string, 0, len(testServiceInfo.Capabilities))
+		for _, c := range testServiceInfo.Capabilities {
+			if c == servicedef.CapabilityPersistentDataStoreRedis ||
+				c == servicedef.CapabilityPersistentDataStoreDynamoDB ||
+				c == servicedef.CapabilityPersistentDataStoreConsul {
+				debugLogger.Printf("Disabling capability %q because persistence tests are disabled", c)
+				continue
+			}
+			filteredCapabilities = append(filteredCapabilities, c)
+		}
+		testServiceInfo.Capabilities = filteredCapabilities
+	}
 	h.testServiceInfo = testServiceInfo
 
 	if err := startServer(testHarnessPort, http.HandlerFunc(h.serveHTTP)); err != nil {

framework/harness/test_service.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/launchdarkly/sdk-test-harness/v2/framework/helpers"
 	"github.com/launchdarkly/sdk-test-harness/v2/serviceinfo"
 
 	"github.com/launchdarkly/sdk-test-harness/v2/framework"
@@ -25,19 +26,19 @@ type TestServiceEntity struct {
 }
 
 func queryTestServiceInfo(url string, timeout time.Duration, output io.Writer) (serviceinfo.TestServiceInfo, error) {
-	fmt.Fprintf(output, "Connecting to test service at %s", url)
+	helpers.MustFprintf(output, "Connecting to test service at %s", url)
 
 	deadline := time.Now().Add(timeout)
 	for {
-		fmt.Fprintf(output, ".")
+		helpers.MustFprintf(output, ".")
 		respData, _, err := doRequest("GET", url, nil)
 		if err == nil {
-			fmt.Fprintln(output)
+			helpers.MustFprintln(output)
 			if respData == nil {
-				fmt.Fprintf(output, "Status query successful, but service provided no metadata\n")
+				helpers.MustFprintf(output, "Status query successful, but service provided no metadata\n")
 				return serviceinfo.Empty(), nil
 			}
-			fmt.Fprintf(output, "Status query returned metadata: %s\n", string(respData))
+			helpers.MustFprintf(output, "Status query returned metadata: %s\n", string(respData))
 			var base serviceinfo.TestServiceInfoBase
 			if err := json.Unmarshal(respData, &base); err != nil {
 				return serviceinfo.Empty(), fmt.Errorf("malformed status response from test service: %s", string(respData))

framework/helpers/channels_test.go

@@ -4,8 +4,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/launchdarkly/sdk-test-harness/v2/framework/opt"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/launchdarkly/sdk-test-harness/v2/framework/opt"
 )
 
 func TestNonBlockingSend(t *testing.T) {

framework/helpers/printers.go

@@ -0,0 +1,25 @@
+package helpers
+
+import (
+	"fmt"
+	"io"
+)
+
+// The refactoring that led to the creation of this file was to avoid the need
+// to check for errors every time we print something. But, why are we printing
+// things with Fprintf so much?
+//
+// It seems like we could be using the go logger instead to much greater
+// effect.
+
+func MustFprintln(w io.Writer, a ...any) {
+	if _, err := fmt.Fprintln(w, a...); err != nil {
+		panic(err)
+	}
+}
+
+func MustFprintf(w io.Writer, format string, a ...any) {
+	if _, err := fmt.Fprintf(w, format, a...); err != nil {
+		panic(err)
+	}
+}

framework/ldtest/errors_test.go

@@ -3,9 +3,10 @@ package ldtest
 import (
 	"testing"
 
-	"github.com/launchdarkly/sdk-test-harness/v2/framework/ldtest/internal"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/launchdarkly/sdk-test-harness/v2/framework/ldtest/internal"
 )
 
 func TestStacktrace(t *testing.T) {

framework/ldtest/filter.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"regexp"
 	"strings"
+
+	"github.com/launchdarkly/sdk-test-harness/v2/framework/helpers"
 )
 
 // Filter is an object that can determine whether to run a specific test or not.
@@ -106,14 +108,14 @@ func (l TestIDPatternList) AnyMatch(id TestID, includeParents bool) bool {
 
 func (r RegexFilters) Describe(out io.Writer, supportedCapabilities, allCapabilities []string) {
 	if r.MustMatch.IsDefined() || r.MustNotMatch.IsDefined() {
-		fmt.Fprintln(out, "Some tests will be skipped based on the filter criteria for this test run:")
+		helpers.MustFprintln(out, "Some tests will be skipped based on the filter criteria for this test run:")
 		if r.MustMatch.IsDefined() {
-			fmt.Fprintf(out, "  skip any not matching %s\n", r.MustMatch)
+			helpers.MustFprintf(out, "  skip any not matching %s\n", r.MustMatch)
 		}
 		if r.MustNotMatch.IsDefined() {
-			fmt.Fprintf(out, "  skip any matching %s\n", r.MustNotMatch)
+			helpers.MustFprintf(out, "  skip any matching %s\n", r.MustNotMatch)
 		}
-		fmt.Fprintln(out)
+		helpers.MustFprintln(out)
 	}
 
 	if len(supportedCapabilities) != 0 {
@@ -128,9 +130,12 @@ func (r RegexFilters) Describe(out io.Writer, supportedCapabilities, allCapabili
 			}
 		}
 		if len(missingCapabilities) > 0 {
-			fmt.Fprintln(out, "Some tests may be skipped because the test service does not support the following capabilities:")
-			fmt.Fprintf(out, "  %s\n", strings.Join(missingCapabilities, ", "))
-			fmt.Fprintln(out)
+			helpers.MustFprintln(
+				out,
+				"Some tests may be skipped because the test service does not support the following capabilities:",
+			)
+			helpers.MustFprintf(out, "  %s\n", strings.Join(missingCapabilities, ", "))
+			helpers.MustFprintln(out)
 		}
 	}
 }

framework/ldtest/test_logger.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/launchdarkly/sdk-test-harness/v2/framework"
+	"github.com/launchdarkly/sdk-test-harness/v2/framework/helpers"
 
 	"github.com/fatih/color"
 )
@@ -96,7 +97,7 @@ func (c ConsoleTestLogger) EndLog(results Results) error {
 	}
 
 	if len(results.NonCriticalFailures) != 0 {
-		fmt.Fprintln(os.Stderr)
+		helpers.MustFprintln(os.Stderr)
 		_, _ = consoleTestFailedNonCriticalColor.Fprintf(os.Stderr,
 			"NON-CRITICAL FAILURES (%d):\n", len(results.NonCriticalFailures))
 		for _, f := range results.NonCriticalFailures {
@@ -106,7 +107,7 @@ func (c ConsoleTestLogger) EndLog(results Results) error {
 	}
 
 	if !results.OK() {
-		fmt.Fprintln(os.Stderr)
+		helpers.MustFprintln(os.Stderr)
 		_, _ = consoleTestFailedColor.Fprintf(os.Stderr, "FAILED TESTS (%d):\n", len(results.Failures))
 		for _, f := range results.Failures {
 			_, _ = consoleTestFailedColor.Fprintf(os.Stderr, "  * %s\n", f.TestID)

go.mod

@@ -1,28 +1,66 @@
 module github.com/launchdarkly/sdk-test-harness/v2
 
-go 1.19
+go 1.22
+
+toolchain go1.23.1
 
 require (
-	github.com/fatih/color v1.13.0
+	github.com/aws/aws-sdk-go v1.55.5
+	github.com/aws/aws-sdk-go-v2/config v1.28.0
+	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.36.2
+	github.com/fatih/color v1.16.0
 	github.com/gorilla/mux v1.8.0
+	github.com/hashicorp/consul/api v1.29.5
 	github.com/launchdarkly/eventsource v1.6.2
 	github.com/launchdarkly/go-jsonstream/v3 v3.0.0
 	github.com/launchdarkly/go-sdk-common/v3 v3.1.0
 	github.com/launchdarkly/go-server-sdk-evaluation/v3 v3.0.0
 	github.com/launchdarkly/go-test-helpers/v2 v2.3.2
-	github.com/stretchr/testify v1.7.0
-	golang.org/x/exp v0.0.0-20220823124025-807a23277127
-	gopkg.in/yaml.v3 v3.0.0
+	github.com/redis/go-redis/v9 v9.6.1
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
+	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/launchdarkly/go-semver v1.0.2 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.9 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )