Fix for: Backticks in array of objects 72324

README.md

@@ -151,9 +151,6 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and *
 
         // We also support the external loading of a complete test:
         "@pathToTest.json"
-
-        // By prefixing it with a p the testtool runs the tests all in parallel. All parallel tests are then set to ContinueOnFailure !
-        "p@pathToTestsThatShouldRunInParallel.json"
     ]
 }
 ```
@@ -416,26 +413,6 @@ However that one will be stripped before parsing the template, which would be ju
 
 ** Unlike with delimiters, external tests/requests/responses don't inherit those removals, and need to be specified per file.
 
-## Run tests in parallel
-
-The tool is able to do run tests in parallel. You activate this mechanism by including a external testfile with `p@pathtofile.json`.
-The `p@` indicates to load that external file and run all tests in it in parallel.
-
-**All tests that are run in parallel are implicit set to ContinueOnFailure as otherwise the log and report would make no
-sense**
-
-```yaml
-{
-    "name": "Binary Comparison",
-    "request": {
-        "endpoint": "suggest",
-        "method": "GET"
-    },
-    // Path to binary file with @
-    "response": "@simple.bin"
-}
-```
-
 ## Binary data comparison
 
 The tool is able to do a comparison with a binary file. Here we take a MD5 hash of the file and and then later compare

api_testsuite.go

@@ -179,7 +179,7 @@ func (ats *Suite) Run() bool {
 	success := true
 	for k, v := range ats.Tests {
 		child := r.NewChild(strconv.Itoa(k))
-		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, k, 1, false, child, ats.loader)
+		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, k, child, ats.loader)
 		child.Leave(sTestSuccess)
 		if !sTestSuccess {
 			success = false
@@ -213,7 +213,7 @@ type TestContainer struct {
 	Path     string
 }
 
-func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, repeatNTimes int, runParallel bool, r *report.ReportElement, rootLoader template.Loader) bool {
+func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k int, r *report.ReportElement, rootLoader template.Loader) bool {
 	//Init variables
 	// logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters)
 	loader := template.NewLoader(ats.datastore)
@@ -227,12 +227,6 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re
 	loader.ServerURL = serverURL
 	loader.OAuthClient = ats.Config.OAuthClient
 
-	isParallelPathSpec := false
-	switch t := v.(type) {
-	case string:
-		isParallelPathSpec = util.IsParallelPathSpec(t)
-	}
-
 	//Get the Manifest with @ logic
 	fileh, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir)
 	dir := filepath.Dir(fileh)
@@ -245,98 +239,72 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re
 		return false
 	}
 
-	//Try to directly unmarshal the manifest into testcase array
+	// Parse as template always
+	testObj, err = loader.Render(testObj, filepath.Join(manifestDir, dir), nil)
+	if err != nil {
+		r.SaveToReportLog(err.Error())
+		logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, err))
+		return false
+	}
+
+	// Build list of test cases
 	var testCases []json.RawMessage
 	err = util.Unmarshal(testObj, &testCases)
-	if err == nil {
-		d := 1
-		if isParallelPathSpec || runParallel {
-			if repeatNTimes > 1 {
-				logrus.Debugf("run %s parallel: repeat %d times", filepath.Base(testFilePath), repeatNTimes)
-			}
-			d = len(testCases)
-		}
-
-		waitCh := make(chan bool, repeatNTimes*d)
-		succCh := make(chan bool, repeatNTimes*len(testCases))
-
-		go func() {
-			for kn := 0; kn < repeatNTimes; kn++ {
-				for ki, v := range testCases {
-					waitCh <- true
-					go testGoRoutine(k, kn+ki*repeatNTimes, v, ats, testFilePath, manifestDir, dir, r, loader, waitCh, succCh, isParallelPathSpec || runParallel)
-				}
-			}
-		}()
-
-		for i := 0; i < repeatNTimes*len(testCases); i++ {
-			select {
-			case succ := <-succCh:
-				if succ == false {
-					return false
-				}
-			}
-		}
-	} else {
-		// We were not able unmarshal into array, so we try to unmarshal into raw message
-
-		// Get the (optional) number of repititions from the test path spec
-		parallelRepititions := 1
-		if isParallelPathSpec {
-			switch t := v.(type) {
-			case string:
-				parallelRepititions, _ = util.GetParallelPathSpec(t)
-			}
-		}
-
-		// Parse as template always
-		requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil)
-		if lErr != nil {
-			r.SaveToReportLog(lErr.Error())
-			logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, lErr))
-			return false
-		}
-
-		// If objects are different, we did have a Go template, recurse one level deep
-		if string(requestBytes) != string(testObj) {
-			return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir),
-				testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader)
-		}
-
-		// Its a JSON at this point, assign and proceed to parse
-		testObj = requestBytes
-
+	if err != nil {
+		// Input could not be deserialized into list, try to deserialize into single object
 		var singleTest json.RawMessage
 		err = util.Unmarshal(testObj, &singleTest)
-		if err == nil {
-
-			//Check if is @ and if so load the test
-			if util.IsPathSpec(string(testObj)) {
-				var sS string
-
-				err := util.Unmarshal(testObj, &sS)
-				if err != nil {
-					r.SaveToReportLog(err.Error())
-					logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err))
-					return false
-				}
-
-				return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k, parallelRepititions, isParallelPathSpec, r, template.Loader{})
-			} else {
-				return ats.runSingleTest(TestContainer{CaseByte: testObj, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, k, runParallel)
-			}
-		} else {
+		if err != nil {
 			// Malformed json
 			r.SaveToReportLog(err.Error())
 			logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err))
 			return false
 		}
+
+		testCases = []json.RawMessage{singleTest}
+	}
+
+	// Execute test cases
+	for i, testCase := range testCases {
+		var success bool
+
+		// If testCase can be unmarshalled as string, we may have a
+		// reference to another test using @ notation at hand
+		var testCaseStr string
+		err = util.Unmarshal(testCase, &testCaseStr)
+		if err == nil && util.IsPathSpec(testCaseStr) {
+			// Recurse if the testCase points to another file using @ notation
+			success = ats.parseAndRunTest(
+				testCaseStr,
+				filepath.Join(manifestDir, dir),
+				testFilePath,
+				i,
+				r,
+				loader,
+			)
+		} else {
+			// Otherwise simply run the literal test case
+			success = ats.runLiteralTest(
+				TestContainer{
+					CaseByte: testCase,
+					Path:     filepath.Join(manifestDir, dir),
+				},
+				r,
+				testFilePath,
+				loader,
+				i,
+			)
+		}
+
+		if !success {
+			return false
+		}
 	}
 
 	return true
 }
 
-func (ats *Suite) runSingleTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int, isParallel bool) bool {
+func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int) bool {
 	r.SetName(testFilePath)
 
 	var test Case
@@ -357,9 +325,6 @@ func (ats *Suite) runSingleTest(tc TestContainer, r *report.ReportElement, testF
 	test.dataStore = ats.datastore
 	test.standardHeader = ats.StandardHeader
 	test.standardHeaderFromStore = ats.StandardHeaderFromStore
-	if isParallel {
-		test.ContinueOnFailure = true
-	}
 	if test.LogNetwork == nil {
 		test.LogNetwork = &ats.Config.LogNetwork
 	}
@@ -409,29 +374,3 @@ func (ats *Suite) loadManifest() ([]byte, error) {
 	ats.loader = loader
 	return b, err
 }
-
-func testGoRoutine(k, ki int, v json.RawMessage, ats *Suite, testFilePath, manifestDir, dir string, r *report.ReportElement, loader template.Loader, waitCh, succCh chan bool, runParallel bool) {
-	success := false
-
-	//Check if is @ and if so load the test
-	switch util.IsPathSpec(string(v)) {
-	case true:
-		var sS string
-		err := util.Unmarshal(v, &sS)
-		if err != nil {
-			r.SaveToReportLog(err.Error())
-			logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err))
-			success = false
-			break
-		}
-		success = ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k+ki, 1, runParallel, r, loader)
-	default:
-		success = ats.runSingleTest(TestContainer{CaseByte: v, Path: filepath.Join(manifestDir, dir)},
-			r, testFilePath, loader, ki, runParallel)
-	}
-
-	succCh <- success
-	if success {
-		<-waitCh
-	}
-}

pkg/lib/template/template_loader.go

@@ -38,6 +38,11 @@ type delimiters struct {
 
 var delimsRE = regexp.MustCompile(`(?m)^[\t ]*(?://|/\*)[\t ]*template-delims:[\t ]*([^\t ]+)[\t ]+([^\t\n ]+).*$`)
 
+// Loader loads and executes a manifest template.
+//
+// A manifest template is a customized version of Go's text/template, plus
+// custom template functions (which are initialized with the Loader's
+// properties, where applicable).
 type Loader struct {
 	datastore      *datastore.Datastore
 	HTTPServerHost string
@@ -50,6 +55,14 @@ func NewLoader(datastore *datastore.Datastore) Loader {
 	return Loader{datastore: datastore}
 }
 
+// Render loads and executes a manifest template.
+//
+// For a description of the manifest template, refer to Loader's docstring.
+//
+//   - tmplBytes is the manifest template, as loaded from disk.
+//   - rootDir is the path of the directory in which manifest resides.
+//   - ctx is the data passed on to the template's Execute function.
+//     Contrary to what convention may suggest, it is not a context.Context.
 func (loader *Loader) Render(
 	tmplBytes []byte,
 	rootDir string,

test/datastore/requests.json

@@ -1,29 +1,29 @@
 [
     {
-        "name": "add data thru request - 1",
+        "name": "add data thru {{ printf `data1.json` }} request - 1",
         "request": {
             "server_url": "http://localhost:9999",
-            "endpoint": "data1.json",
+            "endpoint": "{{ printf `data1.json` }}",
             "method": "GET"
         },
         "response": {
             "body": {"some": "data"}
         }
     },
     {
-        "name": "add data thru request - 2",
+        "name": "add data thru {{ printf `data2.json` }} request - 2",
         "request": {
             "server_url": "http://localhost:9999",
-            "endpoint": "data2.json",
+            "endpoint": "{{ printf `data2.json` }}",
             "method": "GET"
         }
     },
     {
-        "name": "add data thru request - 3",
+        "name": "add data thru {{ printf `data1.json` }} request - 3",
         "request": {
             "server_url": "http://localhost:9999",
-            "endpoint": "data1.json",
+            "endpoint": "{{ printf `data1.json` }}",
             "method": "GET"
         }
     }
-]
+]

test/filemarshal/dummy.txt

@@ -0,0 +1 @@
+{"foo": "{\n    \"dummy\": \"data\"\n}\n"}

test/filemarshal/dummy_template.json

@@ -0,0 +1,3 @@
+{
+    "dummy": {{ printf `"data"` }}
+}

test/filemarshal/manifest.json

@@ -0,0 +1,11 @@
+{
+    "http_server": {
+        "addr": ":9999",
+        "dir": "./",
+        "testmode": false
+    },
+    "name": "ensure a file_render to JSON marshal pipe works as expected",
+    "tests": [
+        "@requests.json"
+    ]
+}

test/filemarshal/requests.json

@@ -0,0 +1,29 @@
+[
+    {
+        "name": "file_render to marshal pipeline should pass",
+        "request": {
+            "server_url": "http://localhost:9999",
+            "endpoint": "dummy.txt",
+            "method": "GET"
+        },
+        "response": {
+            "body": {
+                "foo": {{ file_render "dummy_template.json" | marshal }}
+            }
+        }
+    },
+    {
+        "name": "file to marshal pipeline should fail",
+        "request": {
+            "server_url": "http://localhost:9999",
+            "endpoint": "dummy.txt",
+            "method": "GET"
+        },
+        "response": {
+            "body": {
+                "foo": {{ file "dummy_template.json" | marshal }}
+            }
+        },
+        "reverse_test_result": true
+    }
+]