Merge pull request #69 from programmfabrik/n_prallel_runs

Implemented repeated parallel execution of the same test case

api_testsuite.go

@@ -173,7 +173,7 @@ func (ats *Suite) Run() bool {
 	success := true
 	for k, v := range ats.Tests {
 		child := r.NewChild(strconv.Itoa(k))
-		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, k, false, child, ats.loader)
+		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, k, 1, false, child, ats.loader)
 		child.Leave(sTestSuccess)
 		if !sTestSuccess {
 			success = false
@@ -207,7 +207,7 @@ type TestContainer struct {
 	Path     string
 }
 
-func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath string, k int, runParallel bool, r *report.ReportElement, rootLoader template.Loader) bool {
+func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath string, k, repeatNTimes int, runParallel bool, r *report.ReportElement, rootLoader template.Loader) bool {
 	//Init variables
 	// logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters)
 	loader := template.NewLoader(ats.datastore)
@@ -224,7 +224,7 @@ func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath strin
 	isParallelPathSpec := false
 	switch t := v.(type) {
 	case string:
-		isParallelPathSpec = util.IsParallelPathSpec([]byte(t))
+		isParallelPathSpec = util.IsParallelPathSpec(t)
 	}
 
 	//Get the Manifest with @ logic
@@ -245,20 +245,25 @@ func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath strin
 	if err == nil {
 		d := 1
 		if isParallelPathSpec || runParallel {
+			if repeatNTimes > 1 {
+				logrus.Debugf("run %s parallel: repeat %d times", filepath.Base(testFilePath), repeatNTimes)
+			}
 			d = len(testCases)
 		}
 
-		waitCh := make(chan bool, d)
-		succCh := make(chan bool, len(testCases))
+		waitCh := make(chan bool, repeatNTimes*d)
+		succCh := make(chan bool, repeatNTimes*len(testCases))
 
 		go func() {
-			for ki, v := range testCases {
-				waitCh <- true
-				go testGoRoutine(k, ki, v, ats, testFilePath, manifestDir, dir, r, loader, waitCh, succCh, isParallelPathSpec || runParallel)
+			for kn := 0; kn < repeatNTimes; kn++ {
+				for ki, v := range testCases {
+					waitCh <- true
+					go testGoRoutine(k, kn+ki*repeatNTimes, v, ats, testFilePath, manifestDir, dir, r, loader, waitCh, succCh, isParallelPathSpec || runParallel)
+				}
 			}
 		}()
 
-		for i := 0; i < len(testCases); i++ {
+		for i := 0; i < repeatNTimes*len(testCases); i++ {
 			select {
 			case succ := <-succCh:
 				if succ == false {
@@ -269,6 +274,15 @@ func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath strin
 	} else {
 		// We were not able unmarshal into array, so we try to unmarshal into raw message
 
+		// Get the (optional) number of repititions from the test path spec
+		parallelRepititions := 1
+		if isParallelPathSpec {
+			switch t := v.(type) {
+			case string:
+				parallelRepititions, _ = util.GetParallelPathSpec(t)
+			}
+		}
+
 		// Parse as template always
 		requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil)
 		if lErr != nil {
@@ -280,7 +294,7 @@ func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath strin
 		// If objects are different, we did have a Go template, recurse one level deep
 		if string(requestBytes) != string(testObj) {
 			return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir),
-				testFilePath, k, isParallelPathSpec, r, loader)
+				testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader)
 		}
 
 		// Its a JSON at this point, assign and proceed to parse
@@ -291,7 +305,7 @@ func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath strin
 		if err == nil {
 
 			//Check if is @ and if so load the test
-			if util.IsPathSpec(testObj) {
+			if util.IsPathSpec(string(testObj)) {
 				var sS string
 
 				err := util.Unmarshal(testObj, &sS)
@@ -301,7 +315,7 @@ func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath strin
 					return false
 				}
 
-				return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k, isParallelPathSpec, r, template.Loader{})
+				return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k, parallelRepititions, isParallelPathSpec, r, template.Loader{})
 			} else {
 				return ats.runSingleTest(TestContainer{CaseByte: testObj, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, k, runParallel)
 			}
@@ -394,7 +408,7 @@ func testGoRoutine(k, ki int, v json.RawMessage, ats *Suite, testFilePath, manif
 	success := false
 
 	//Check if is @ and if so load the test
-	switch util.IsPathSpec(v) {
+	switch util.IsPathSpec(string(v)) {
 	case true:
 		var sS string
 		err := util.Unmarshal(v, &sS)
@@ -404,7 +418,7 @@ func testGoRoutine(k, ki int, v json.RawMessage, ats *Suite, testFilePath, manif
 			success = false
 			break
 		}
-		success = ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k+ki, runParallel, r, loader)
+		success = ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k+ki, 1, runParallel, r, loader)
 	default:
 		success = ats.runSingleTest(TestContainer{CaseByte: v, Path: filepath.Join(manifestDir, dir)},
 			r, testFilePath, loader, ki, runParallel)

go.mod

@@ -1,31 +1,53 @@
 module github.com/programmfabrik/apitest
 
-go 1.13
+go 1.17
 
 require (
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/clbanning/mxj v1.8.4
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/mattn/go-sqlite3 v1.14.8
-	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.9
 	github.com/moul/http2curl v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/programmfabrik/go-test-utils v0.0.0-20191114143449-b8e16b04adb1
-	github.com/sergi/go-diff v1.0.0
+	github.com/sergi/go-diff v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/afero v1.6.0
-	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.2.1
-	github.com/spf13/viper v1.8.1
-	github.com/tidwall/gjson v1.8.1
+	github.com/spf13/viper v1.9.0
+	github.com/tidwall/gjson v1.11.0
 	github.com/tidwall/jsonc v0.3.2
+	golang.org/x/mod v0.5.1
+	golang.org/x/oauth2 v0.0.0-20211028175245-ba495a64dcb5
+)
+
+require (
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	golang.org/x/mod v0.5.0
-	golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 // indirect
+	golang.org/x/sys v0.0.0-20211102192858-4dd72447c267 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

pkg/lib/api/build_policies.go

@@ -40,7 +40,7 @@ func buildMultipart(request Request) (additionalHeaders map[string]string, body
 		if !ok {
 			return additionalHeaders, body, fmt.Errorf("pathSpec should be a string")
 		}
-		if !util.IsPathSpec([]byte(pathSpec)) {
+		if !util.IsPathSpec(pathSpec) {
 			return additionalHeaders, body, fmt.Errorf("pathSpec %s is not valid", pathSpec)
 		}
 

pkg/lib/report/report_test.go

@@ -128,7 +128,7 @@ func TestReportGetJUnitResult(t *testing.T) {
 
 	for k, v := range realX.Testsuites {
 		realX.Testsuites[k].Time = 0
-		for ik, _ := range v.Testcases {
+		for ik := range v.Testcases {
 			realX.Testsuites[k].Testcases[ik].Time = 0
 		}
 	}

pkg/lib/template/util.go

@@ -9,7 +9,7 @@ import (
 )
 
 func loadFileFromPathSpec(pathSpec, manifestDir string) (string, []byte, error) {
-	if !util.IsPathSpec([]byte(pathSpec)) {
+	if !util.IsPathSpec(pathSpec) {
 		return "", nil, fmt.Errorf("spec was expected to be path spec, got %s instead", pathSpec)
 	}
 

pkg/lib/util/file.go

@@ -18,12 +18,13 @@ var c = &http.Client{
 // OpenFileOrUrl opens either a local file or gives the resp.Body from a remote file
 func OpenFileOrUrl(path, rootDir string) (string, io.ReadCloser, error) {
 	if strings.HasPrefix(path, "@") {
-		path = string([]rune(path)[1:])
-	}
-
-	// p@ -> parallel tests
-	if strings.HasPrefix(path, "p@") {
-		path = string([]rune(path)[2:])
+		path = path[1:]
+	} else if strings.HasPrefix(path, "p@") {
+		// p@ -> parallel tests
+		path = path[2:]
+	} else if IsParallelPathSpec(path) {
+		// pN@ -> N parallel repetitions of tests
+		_, path = GetParallelPathSpec(path)
 	}
 
 	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {

pkg/lib/util/file_test.go

@@ -13,12 +13,147 @@ import (
 	"github.com/spf13/afero"
 )
 
-type testStruct struct {
+type testParallelPathSpecStruct struct {
+	pathSpec               string
+	expIsPath              bool
+	expIsParallel          bool
+	expPath                string
+	expParallelRepititions int
+}
+
+type testOpenFileStruct struct {
 	filename string
 	expError error
 	expHash  [16]byte
 }
 
+func TestGetParallelPathSpec(t *testing.T) {
+
+	tests := []testParallelPathSpecStruct{
+		{
+			pathSpec:      "\"",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "[]",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "{}",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "p",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "@",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "1@",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "x@",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "p@",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "@path",
+			expIsPath:     true,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "1@a",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "x@a",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "p1@",
+			expIsPath:     false,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:               "p1@path",
+			expIsPath:              true,
+			expIsParallel:          true,
+			expPath:                "path",
+			expParallelRepititions: 1,
+		},
+		{
+			pathSpec:               "p10@path",
+			expIsPath:              true,
+			expIsParallel:          true,
+			expPath:                "path",
+			expParallelRepititions: 10,
+		},
+		{
+			pathSpec:               "p01@path",
+			expIsPath:              true,
+			expIsParallel:          true,
+			expPath:                "path",
+			expParallelRepititions: 1,
+		},
+		{
+			pathSpec:      "@path",
+			expIsPath:     true,
+			expIsParallel: false,
+		},
+		{
+			pathSpec:      "@../path",
+			expIsPath:     true,
+			expIsParallel: false,
+		},
+	}
+
+	for _, v := range tests {
+		t.Run(fmt.Sprintf("pathSpec:'%s'", v.pathSpec), func(t *testing.T) {
+			isPathSpec := IsPathSpec(v.pathSpec)
+			isParallelPathSpec := IsParallelPathSpec(v.pathSpec)
+			if isPathSpec != v.expIsPath {
+				t.Errorf("IsPathSpec: Got %v != %v Exp", isPathSpec, v.expIsPath)
+			}
+			if isParallelPathSpec != v.expIsParallel {
+				t.Errorf("IsParallelPathSpec: Got %v != %v Exp", isParallelPathSpec, v.expIsParallel)
+			}
+
+			if v.expIsPath {
+				// the path must also be recognized as a path in case it has trailing quotes
+				if !IsPathSpec(fmt.Sprintf("\"%s\"", v.pathSpec)) {
+					t.Errorf("IsPathSpec (with trailing \"): Got %v != %v Exp", isPathSpec, v.expIsPath)
+				}
+			}
+
+			if v.expIsParallel {
+				parallelRepititions, path := GetParallelPathSpec(v.pathSpec)
+				if parallelRepititions != v.expParallelRepititions {
+					t.Errorf("ParallelRepititions: Got '%d' != '%d' Exp", parallelRepititions, v.expParallelRepititions)
+				}
+				if path != v.expPath {
+					t.Errorf("Path: Got '%s' != '%s' Exp", path, v.expPath)
+				}
+			}
+		})
+	}
+}
+
 func TestOpenFileOrUrl(t *testing.T) {
 	filesystem.Fs = afero.NewMemMapFs()
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -32,7 +167,7 @@ func TestOpenFileOrUrl(t *testing.T) {
 
 	afero.WriteFile(filesystem.Fs, "localExists", []byte("Hallo ich bin da!"), 0644)
 
-	tests := []testStruct{
+	tests := []testOpenFileStruct{
 		{
 			filename: "localExists",
 			expError: nil,