From c38e421a3be4287770b509d0330871f26942e019 Mon Sep 17 00:00:00 2001 From: Martin Rode Date: Wed, 8 May 2024 13:48:49 +0200 Subject: [PATCH 01/11] test: break "datastore" test to show broken backticks --- test/datastore/requests.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/datastore/requests.json b/test/datastore/requests.json index a9ca334..de583cd 100644 --- a/test/datastore/requests.json +++ b/test/datastore/requests.json @@ -1,9 +1,9 @@ [ { - "name": "add data thru request - 1", + "name": "add data thru {{ printf `data1.json` }} request - 1", "request": { "server_url": "http://localhost:9999", - "endpoint": "data1.json", + "endpoint": "{{ printf `data1.json` }}", "method": "GET" }, "response": { @@ -11,19 +11,19 @@ } }, { - "name": "add data thru request - 2", + "name": "add data thru {{ printf `data2.json` }} request - 2", "request": { "server_url": "http://localhost:9999", - "endpoint": "data2.json", + "endpoint": "{{ printf `data2.json` }}", "method": "GET" } }, { - "name": "add data thru request - 3", + "name": "add data thru {{ printf `data1.json` }} request - 3", "request": { "server_url": "http://localhost:9999", - "endpoint": "data1.json", + "endpoint": "{{ printf `data1.json` }}", "method": "GET" } } -] +] \ No newline at end of file From d348fdf53288809ebe13f4dd3fad75ff5ad8724e Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Mon, 3 Jun 2024 15:05:36 +0200 Subject: [PATCH 02/11] Fix for broken backticks in manifest Having a manifest with Go template expressions, but where all expressions only use backticks, would lead to the Go template not being executed, since such a manifest was also by coincidence a perfectly valid JSON array that was executed right away. Fixed by always executing the Go template, regardless of whether the input already is valid JSON. --- api_testsuite.go | 92 ++++++++++++++--------------- pkg/lib/template/template_loader.go | 13 ++++ 2 files changed, 58 insertions(+), 47 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index 4c52182..2aa48f9 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -227,10 +227,19 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re loader.ServerURL = serverURL loader.OAuthClient = ats.Config.OAuthClient + // Get the (optional) number of repititions from the test path spec isParallelPathSpec := false - switch t := v.(type) { - case string: - isParallelPathSpec = util.IsParallelPathSpec(t) + parallelRepititions := 1 + if isParallelPathSpec { + switch t := v.(type) { + case string: + parallelRepititions, _ = util.GetParallelPathSpec(t) + + // FIXME - Shouldn't this be > 1 (also in util.IsParallelPathSpec)? + // If so, the declaration in L.6 can be removed and this can be turned + // into a declaration and moved to after the if block. + isParallelPathSpec = parallelRepititions > 0 + } } //Get the Manifest with @ logic @@ -245,6 +254,22 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re return false } + // Parse as template always + requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil) + if lErr != nil { + r.SaveToReportLog(lErr.Error()) + logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, lErr)) + return false + } + + // If objects are different, we did have a Go template, recurse one level deep + if string(requestBytes) != string(testObj) { + return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir), + testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader) + } + + testObj = requestBytes + //Try to directly unmarshal the manifest into testcase array var testCases []json.RawMessage err = util.Unmarshal(testObj, &testCases) @@ -280,57 +305,30 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re } else { // We were not able unmarshal into array, so we try to unmarshal into raw message - // Get the (optional) number of repititions from the test path spec - parallelRepititions := 1 - if isParallelPathSpec { - switch t := v.(type) { - case string: - parallelRepititions, _ = util.GetParallelPathSpec(t) - } - } - - // Parse as template always - requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil) - if lErr != nil { - r.SaveToReportLog(lErr.Error()) - logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, lErr)) - return false - } - - // If objects are different, we did have a Go template, recurse one level deep - if string(requestBytes) != string(testObj) { - return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir), - testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader) - } - - // Its a JSON at this point, assign and proceed to parse - testObj = requestBytes - var singleTest json.RawMessage err = util.Unmarshal(testObj, &singleTest) - if err == nil { - - //Check if is @ and if so load the test - if util.IsPathSpec(string(testObj)) { - var sS string - - err := util.Unmarshal(testObj, &sS) - if err != nil { - r.SaveToReportLog(err.Error()) - logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err)) - return false - } - - return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k, parallelRepititions, isParallelPathSpec, r, template.Loader{}) - } else { - return ats.runSingleTest(TestContainer{CaseByte: testObj, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, k, runParallel) - } - } else { + if err != nil { // Malformed json r.SaveToReportLog(err.Error()) logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err)) return false } + + //Check if is @ and if so load the test + if util.IsPathSpec(string(testObj)) { + var sS string + + err := util.Unmarshal(testObj, &sS) + if err != nil { + r.SaveToReportLog(err.Error()) + logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err)) + return false + } + + return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k, parallelRepititions, isParallelPathSpec, r, template.Loader{}) + } else { + return ats.runSingleTest(TestContainer{CaseByte: testObj, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, k, runParallel) + } } return true diff --git a/pkg/lib/template/template_loader.go b/pkg/lib/template/template_loader.go index b67f4b8..9fd41a5 100644 --- a/pkg/lib/template/template_loader.go +++ b/pkg/lib/template/template_loader.go @@ -38,6 +38,11 @@ type delimiters struct { var delimsRE = regexp.MustCompile(`(?m)^[\t ]*(?://|/\*)[\t ]*template-delims:[\t ]*([^\t ]+)[\t ]+([^\t\n ]+).*$`) +// Loader loads and executes a manifest template. +// +// A manifest template is a customized version of Go's text/template, plus +// custom template functions (which are initialized with the Loader's +// properties, where applicable). type Loader struct { datastore *datastore.Datastore HTTPServerHost string @@ -50,6 +55,14 @@ func NewLoader(datastore *datastore.Datastore) Loader { return Loader{datastore: datastore} } +// Render loads and executes a manifest template. +// +// For a description of the manifest template, refer to Loader's docstring. +// +// - tmplBytes is the manifest template, as loaded from disk. +// - rootDir is the path of the directory in which manifest resides. +// - ctx is the data passed on to the template's Execute function. +// Contrary to what convention may suggest, it is not a context.Context. func (loader *Loader) Render( tmplBytes []byte, rootDir string, From f2c79372c23a4086850b5e3cf8ffbb5ef1ec678e Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Mon, 3 Jun 2024 16:26:07 +0200 Subject: [PATCH 03/11] Adressing review comments --- api_testsuite.go | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index 2aa48f9..30ef522 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -230,16 +230,14 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re // Get the (optional) number of repititions from the test path spec isParallelPathSpec := false parallelRepititions := 1 - if isParallelPathSpec { - switch t := v.(type) { - case string: - parallelRepititions, _ = util.GetParallelPathSpec(t) - - // FIXME - Shouldn't this be > 1 (also in util.IsParallelPathSpec)? - // If so, the declaration in L.6 can be removed and this can be turned - // into a declaration and moved to after the if block. - isParallelPathSpec = parallelRepititions > 0 - } + switch t := v.(type) { + case string: + parallelRepititions, _ = util.GetParallelPathSpec(t) + + // FIXME - Shouldn't this be > 1 (also in util.IsParallelPathSpec)? + // If so, the declaration in L.6 can be removed and this can be turned + // into a declaration and moved to after the if block. + isParallelPathSpec = parallelRepititions > 0 } //Get the Manifest with @ logic @@ -255,21 +253,13 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re } // Parse as template always - requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil) - if lErr != nil { - r.SaveToReportLog(lErr.Error()) - logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, lErr)) + testObj, err = loader.Render(testObj, filepath.Join(manifestDir, dir), nil) + if err != nil { + r.SaveToReportLog(err.Error()) + logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, err)) return false } - // If objects are different, we did have a Go template, recurse one level deep - if string(requestBytes) != string(testObj) { - return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir), - testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader) - } - - testObj = requestBytes - //Try to directly unmarshal the manifest into testcase array var testCases []json.RawMessage err = util.Unmarshal(testObj, &testCases) From 056bf86bc449a3ae291ff5960dc227f299221b27 Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Tue, 4 Jun 2024 12:19:27 +0200 Subject: [PATCH 04/11] Fix for "make apitest" regressions due to backtick fix Contrary to what we've first assumed, the recursion after template evaluation is currently required for correctly running the datastore tests. But since parallelRepititions was incorrectly assumed to be never zero, the recursive call to parseAndRunTest would never actually run its tests. This partially reverts commit f2c79372c23a4086850b5e3cf8ffbb5ef1ec678e. --- api_testsuite.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index 30ef522..dc026f5 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -234,10 +234,11 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re case string: parallelRepititions, _ = util.GetParallelPathSpec(t) - // FIXME - Shouldn't this be > 1 (also in util.IsParallelPathSpec)? - // If so, the declaration in L.6 can be removed and this can be turned - // into a declaration and moved to after the if block. isParallelPathSpec = parallelRepititions > 0 + + if parallelRepititions < 1 { + parallelRepititions = 1 + } } //Get the Manifest with @ logic @@ -253,13 +254,21 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re } // Parse as template always - testObj, err = loader.Render(testObj, filepath.Join(manifestDir, dir), nil) - if err != nil { - r.SaveToReportLog(err.Error()) - logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, err)) + requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil) + if lErr != nil { + r.SaveToReportLog(lErr.Error()) + logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, lErr)) return false } + // If objects are different, we did have a Go template, recurse one level deep + if string(requestBytes) != string(testObj) { + return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir), + testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader) + } + + testObj = requestBytes + //Try to directly unmarshal the manifest into testcase array var testCases []json.RawMessage err = util.Unmarshal(testObj, &testCases) From 29c63c285271798ae6d6c4e529330f90500f493f Mon Sep 17 00:00:00 2001 From: Martin Rode Date: Tue, 4 Jun 2024 12:45:06 +0200 Subject: [PATCH 05/11] improved code path to set isParallelPathSpec --- api_testsuite.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index dc026f5..b5afe21 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -233,12 +233,10 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re switch t := v.(type) { case string: parallelRepititions, _ = util.GetParallelPathSpec(t) - - isParallelPathSpec = parallelRepititions > 0 - if parallelRepititions < 1 { parallelRepititions = 1 } + isParallelPathSpec = parallelRepititions > 1 } //Get the Manifest with @ logic From 367615d9047616c699726c63b5e763f2b61221f8 Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Tue, 4 Jun 2024 15:42:47 +0200 Subject: [PATCH 06/11] Test case for file/file_render to marshal pipelines --- test/filemarshal/dummy.txt | 1 + test/filemarshal/dummy_template.json | 3 +++ test/filemarshal/manifest.json | 11 +++++++++++ test/filemarshal/requests.json | 29 ++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 test/filemarshal/dummy.txt create mode 100644 test/filemarshal/dummy_template.json create mode 100644 test/filemarshal/manifest.json create mode 100644 test/filemarshal/requests.json diff --git a/test/filemarshal/dummy.txt b/test/filemarshal/dummy.txt new file mode 100644 index 0000000..3ee0ed4 --- /dev/null +++ b/test/filemarshal/dummy.txt @@ -0,0 +1 @@ +{"foo": "{\n \"dummy\": \"data\"\n}\n"} diff --git a/test/filemarshal/dummy_template.json b/test/filemarshal/dummy_template.json new file mode 100644 index 0000000..88b578d --- /dev/null +++ b/test/filemarshal/dummy_template.json @@ -0,0 +1,3 @@ +{ + "dummy": {{ printf `"data"` }} +} diff --git a/test/filemarshal/manifest.json b/test/filemarshal/manifest.json new file mode 100644 index 0000000..4e1d34f --- /dev/null +++ b/test/filemarshal/manifest.json @@ -0,0 +1,11 @@ +{ + "http_server": { + "addr": ":9999", + "dir": "./", + "testmode": false + }, + "name": "ensure a file_render to JSON marshal pipe works as expected", + "tests": [ + "@requests.json" + ] +} diff --git a/test/filemarshal/requests.json b/test/filemarshal/requests.json new file mode 100644 index 0000000..7a66e49 --- /dev/null +++ b/test/filemarshal/requests.json @@ -0,0 +1,29 @@ +[ + { + "name": "file_render to marshal pipeline should pass", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "dummy.txt", + "method": "GET" + }, + "response": { + "body": { + "foo": {{ file_render "dummy_template.json" | marshal }} + } + } + }, + { + "name": "file to marshal pipeline should fail", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "dummy.txt", + "method": "GET" + }, + "response": { + "body": { + "foo": {{ file "dummy_template.json" | marshal }} + } + }, + "reverse_test_result": true + } +] From 2ff5a58d9421f36d0f83d615e0022e0a3859fa8b Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Wed, 5 Jun 2024 09:49:06 +0200 Subject: [PATCH 07/11] Temporarily removing parallel test suite for refactoring --- .../parallel/p/check_collected_responses.json | 26 ------------------ test/parallel/p/manifest.json | 21 --------------- test/parallel/p/parallel.json | 27 ------------------- .../pN/check_collected_responses.json | 26 ------------------ test/parallel/pN/manifest.json | 21 --------------- test/parallel/pN/parallel.json | 24 ----------------- 6 files changed, 145 deletions(-) delete mode 100644 test/parallel/p/check_collected_responses.json delete mode 100644 test/parallel/p/manifest.json delete mode 100644 test/parallel/p/parallel.json delete mode 100644 test/parallel/pN/check_collected_responses.json delete mode 100644 test/parallel/pN/manifest.json delete mode 100644 test/parallel/pN/parallel.json diff --git a/test/parallel/p/check_collected_responses.json b/test/parallel/p/check_collected_responses.json deleted file mode 100644 index 4c33b94..0000000 --- a/test/parallel/p/check_collected_responses.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "name": "bounce-json: bounce collected responses from parallel runs: {{ datastore "responses" }}", - "request": { - "server_url": "http://localhost{{ datastore "local_port" }}", - "endpoint": "bounce-json", - "method": "POST", - "body": { - "responses": {{ datastore "responses" | marshal }} - } - }, - "response": { - "statuscode": 200, - "body": { - "body": { - "responses": [ - {{ range $idx, $n := N (datastore "n_parallel") }} - {{ if gt $idx 0 }}, {{ end }} - {{ $idx }} - {{ end }} - ] - } - } - } - } -] \ No newline at end of file diff --git a/test/parallel/p/manifest.json b/test/parallel/p/manifest.json deleted file mode 100644 index 9698f9f..0000000 --- a/test/parallel/p/manifest.json +++ /dev/null @@ -1,21 +0,0 @@ -{{ $local_port := ":9999" }} -{{ $n_parallel := 5 }} -{ - "http_server": { - "addr": "{{ $local_port }}", - "dir": "../_res", - "testmode": false - }, - "name": "parallel run of files", - "tests": [ - { - "name": "port {{ $local_port }}", - "store": { - "n_parallel": {{ $n_parallel }}, - "local_port": {{ $local_port | marshal }} - } - }, - "p@parallel.json", - "@check_collected_responses.json" - ] -} \ No newline at end of file diff --git a/test/parallel/p/parallel.json b/test/parallel/p/parallel.json deleted file mode 100644 index 28ecce5..0000000 --- a/test/parallel/p/parallel.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - {{ range $idx, $n := N (datastore "n_parallel") }} - {{ if gt $idx 0 }}, {{ end }} - { - "name": "bounce-json: bounce n={{ $idx }}", - "request": { - "server_url": "http://localhost{{ datastore "local_port" }}", - "endpoint": "bounce-json", - "method": "POST", - "body": { - "n": {{ $idx }} - } - }, - "response": { - "statuscode": 200, - "body": { - "body": { - "n": {{ $idx }} - } - } - }, - "store_response_qjson": { - "responses[]": "body.body.n" - } - } - {{ end }} -] \ No newline at end of file diff --git a/test/parallel/pN/check_collected_responses.json b/test/parallel/pN/check_collected_responses.json deleted file mode 100644 index 564d76a..0000000 --- a/test/parallel/pN/check_collected_responses.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "name": "bounce-json: bounce collected responses from N={{datastore "n_parallel"}} parallel runs: {{ datastore "responses" }}", - "request": { - "server_url": "http://localhost{{ datastore "local_port" }}", - "endpoint": "bounce-json", - "method": "POST", - "body": { - "responses": {{ datastore "responses" | marshal }} - } - }, - "response": { - "statuscode": 200, - "body": { - "body": { - "responses": [ - {{ range $idx, $n := N (datastore "n_parallel") }} - {{ if gt $idx 0 }}, {{ end }} - 1 - {{ end }} - ] - } - } - } - } -] \ No newline at end of file diff --git a/test/parallel/pN/manifest.json b/test/parallel/pN/manifest.json deleted file mode 100644 index fc292b1..0000000 --- a/test/parallel/pN/manifest.json +++ /dev/null @@ -1,21 +0,0 @@ -{{ $local_port := ":9999" }} -{{ $n_parallel := 5 }} -{ - "http_server": { - "addr": "{{ $local_port }}", - "dir": "../_res", - "testmode": false - }, - "name": "parallel run of N={{ $n_parallel }} repetitions", - "tests": [ - { - "name": "port {{ $local_port }}", - "store": { - "n_parallel": {{ $n_parallel }}, - "local_port": {{ $local_port | marshal }} - } - } - , "p{{ $n_parallel }}@parallel.json" - , "@check_collected_responses.json" - ] -} \ No newline at end of file diff --git a/test/parallel/pN/parallel.json b/test/parallel/pN/parallel.json deleted file mode 100644 index 75b9d71..0000000 --- a/test/parallel/pN/parallel.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "name": "bounce-json: bounce n=1", - "request": { - "server_url": "http://localhost{{ datastore "local_port" }}", - "endpoint": "bounce-json", - "method": "POST", - "body": { - "n": 1 - } - }, - "response": { - "statuscode": 200, - "body": { - "body": { - "n": 1 - } - } - }, - "store_response_qjson": { - "responses[]": "body.body.n" - } - } -] \ No newline at end of file From fcc196355b818488d80161f2aa0bb70cc2fcecf8 Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Wed, 5 Jun 2024 10:14:21 +0200 Subject: [PATCH 08/11] Simplified api_testsuite code, removed Parallel Test code for now --- api_testsuite.go | 131 +++++++++++------------------------------------ 1 file changed, 31 insertions(+), 100 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index b5afe21..7f7cd49 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -179,7 +179,7 @@ func (ats *Suite) Run() bool { success := true for k, v := range ats.Tests { child := r.NewChild(strconv.Itoa(k)) - sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, k, 1, false, child, ats.loader) + sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, k, child, ats.loader) child.Leave(sTestSuccess) if !sTestSuccess { success = false @@ -213,7 +213,7 @@ type TestContainer struct { Path string } -func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, repeatNTimes int, runParallel bool, r *report.ReportElement, rootLoader template.Loader) bool { +func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k int, r *report.ReportElement, rootLoader template.Loader) bool { //Init variables // logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters) loader := template.NewLoader(ats.datastore) @@ -227,18 +227,6 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re loader.ServerURL = serverURL loader.OAuthClient = ats.Config.OAuthClient - // Get the (optional) number of repititions from the test path spec - isParallelPathSpec := false - parallelRepititions := 1 - switch t := v.(type) { - case string: - parallelRepititions, _ = util.GetParallelPathSpec(t) - if parallelRepititions < 1 { - parallelRepititions = 1 - } - isParallelPathSpec = parallelRepititions > 1 - } - //Get the Manifest with @ logic fileh, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir) dir := filepath.Dir(fileh) @@ -252,56 +240,18 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re } // Parse as template always - requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil) - if lErr != nil { - r.SaveToReportLog(lErr.Error()) - logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, lErr)) + testObj, err = loader.Render(testObj, filepath.Join(manifestDir, dir), nil) + if err != nil { + r.SaveToReportLog(err.Error()) + logrus.Error(fmt.Errorf("can not render template (%s): %s", testFilePath, err)) return false } - // If objects are different, we did have a Go template, recurse one level deep - if string(requestBytes) != string(testObj) { - return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir), - testFilePath, k, parallelRepititions, isParallelPathSpec, r, loader) - } - - testObj = requestBytes - - //Try to directly unmarshal the manifest into testcase array + // Build list of test cases var testCases []json.RawMessage err = util.Unmarshal(testObj, &testCases) - if err == nil { - d := 1 - if isParallelPathSpec || runParallel { - if repeatNTimes > 1 { - logrus.Debugf("run %s parallel: repeat %d times", filepath.Base(testFilePath), repeatNTimes) - } - d = len(testCases) - } - - waitCh := make(chan bool, repeatNTimes*d) - succCh := make(chan bool, repeatNTimes*len(testCases)) - - go func() { - for kn := 0; kn < repeatNTimes; kn++ { - for ki, v := range testCases { - waitCh <- true - go testGoRoutine(k, kn+ki*repeatNTimes, v, ats, testFilePath, manifestDir, dir, r, loader, waitCh, succCh, isParallelPathSpec || runParallel) - } - } - }() - - for i := 0; i < repeatNTimes*len(testCases); i++ { - select { - case succ := <-succCh: - if succ == false { - return false - } - } - } - } else { - // We were not able unmarshal into array, so we try to unmarshal into raw message - + if err != nil { + // Input could not be deserialized into list, try to deserialize into single object var singleTest json.RawMessage err = util.Unmarshal(testObj, &singleTest) if err != nil { @@ -311,27 +261,37 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, re return false } - //Check if is @ and if so load the test - if util.IsPathSpec(string(testObj)) { - var sS string + testCases = []json.RawMessage{singleTest} + } - err := util.Unmarshal(testObj, &sS) - if err != nil { - r.SaveToReportLog(err.Error()) - logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err)) - return false - } + // Execute test cases + for i, testCase := range testCases { + var success bool - return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k, parallelRepititions, isParallelPathSpec, r, template.Loader{}) + if util.IsPathSpec(string(testCase)) { + // Recurse if the testCase points to another file using @ notation + success = ats.parseAndRunTest( + testCase, + filepath.Join(manifestDir, dir), + testFilePath, + i, + r, + loader, + ) } else { - return ats.runSingleTest(TestContainer{CaseByte: testObj, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, k, runParallel) + // Otherwise simply run the literal test case + success = ats.runLiteralTest(TestContainer{CaseByte: testCase, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, i) + } + + if !success { + return false } } return true } -func (ats *Suite) runSingleTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int, isParallel bool) bool { +func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int) bool { r.SetName(testFilePath) var test Case @@ -352,9 +312,6 @@ func (ats *Suite) runSingleTest(tc TestContainer, r *report.ReportElement, testF test.dataStore = ats.datastore test.standardHeader = ats.StandardHeader test.standardHeaderFromStore = ats.StandardHeaderFromStore - if isParallel { - test.ContinueOnFailure = true - } if test.LogNetwork == nil { test.LogNetwork = &ats.Config.LogNetwork } @@ -404,29 +361,3 @@ func (ats *Suite) loadManifest() ([]byte, error) { ats.loader = loader return b, err } - -func testGoRoutine(k, ki int, v json.RawMessage, ats *Suite, testFilePath, manifestDir, dir string, r *report.ReportElement, loader template.Loader, waitCh, succCh chan bool, runParallel bool) { - success := false - - //Check if is @ and if so load the test - switch util.IsPathSpec(string(v)) { - case true: - var sS string - err := util.Unmarshal(v, &sS) - if err != nil { - r.SaveToReportLog(err.Error()) - logrus.Error(fmt.Errorf("can not unmarshal (%s): %s", testFilePath, err)) - success = false - break - } - success = ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), testFilePath, k+ki, 1, runParallel, r, loader) - default: - success = ats.runSingleTest(TestContainer{CaseByte: v, Path: filepath.Join(manifestDir, dir)}, - r, testFilePath, loader, ki, runParallel) - } - - succCh <- success - if success { - <-waitCh - } -} From 8e366e8c98519f41b5c405d745abcebdadf52e36 Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Wed, 5 Jun 2024 11:00:40 +0200 Subject: [PATCH 09/11] Adding test case to reproduce "can not LoadManifestDataAsRawJson" regression --- test/pathspec/manifest.json | 11 +++++++++++ test/pathspec/sub/subtest_including.json | 3 +++ test/pathspec/subtest_included.json | 11 +++++++++++ 3 files changed, 25 insertions(+) create mode 100644 test/pathspec/manifest.json create mode 100644 test/pathspec/sub/subtest_including.json create mode 100644 test/pathspec/subtest_included.json diff --git a/test/pathspec/manifest.json b/test/pathspec/manifest.json new file mode 100644 index 0000000..30c06dd --- /dev/null +++ b/test/pathspec/manifest.json @@ -0,0 +1,11 @@ +{ + "http_server": { + "addr": ":9999", + "dir": "../_res", + "testmode": false + }, + "name": "pathspec (@ notation) tests", + "tests": [ + "@sub/subtest_including.json" + ] +} diff --git a/test/pathspec/sub/subtest_including.json b/test/pathspec/sub/subtest_including.json new file mode 100644 index 0000000..3530058 --- /dev/null +++ b/test/pathspec/sub/subtest_including.json @@ -0,0 +1,3 @@ +[ + "@../subtest_included.json" +] diff --git a/test/pathspec/subtest_included.json b/test/pathspec/subtest_included.json new file mode 100644 index 0000000..1631d49 --- /dev/null +++ b/test/pathspec/subtest_included.json @@ -0,0 +1,11 @@ +{ + "name": "an arbitrary test", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "assets/empty.json", + "method": "GET" + }, + "response": { + "statuscode": 200 + } +} From 4943cba9c195661293fdd503a7f70bff9dad3e52 Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Wed, 5 Jun 2024 11:20:14 +0200 Subject: [PATCH 10/11] Fix for "can not LoadManifestDataAsRawJson" regression --- api_testsuite.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index 7f7cd49..0a8d466 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -268,10 +268,14 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k int for i, testCase := range testCases { var success bool - if util.IsPathSpec(string(testCase)) { + // If testCase can be unmarshalled as string, we may have a + // reference to another test using @ notation at hand + var testCaseStr string + err = util.Unmarshal(testCase, &testCaseStr) + if err == nil && util.IsPathSpec(testCaseStr) { // Recurse if the testCase points to another file using @ notation success = ats.parseAndRunTest( - testCase, + testCaseStr, filepath.Join(manifestDir, dir), testFilePath, i, @@ -280,7 +284,16 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k int ) } else { // Otherwise simply run the literal test case - success = ats.runLiteralTest(TestContainer{CaseByte: testCase, Path: filepath.Join(manifestDir, dir)}, r, testFilePath, loader, i) + success = ats.runLiteralTest( + TestContainer{ + CaseByte: testCase, + Path: filepath.Join(manifestDir, dir), + }, + r, + testFilePath, + loader, + i, + ) } if !success { From 34fd32c0dd873253ed2772a62bd3e922dea2cd54 Mon Sep 17 00:00:00 2001 From: Lucas Hinderberger Date: Wed, 5 Jun 2024 16:15:40 +0200 Subject: [PATCH 11/11] Removing documentation about parallel tests for now --- README.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/README.md b/README.md index 84b989e..f083d4a 100644 --- a/README.md +++ b/README.md @@ -151,9 +151,6 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * // We also support the external loading of a complete test: "@pathToTest.json" - - // By prefixing it with a p the testtool runs the tests all in parallel. All parallel tests are then set to ContinueOnFailure ! - "p@pathToTestsThatShouldRunInParallel.json" ] } ``` @@ -416,26 +413,6 @@ However that one will be stripped before parsing the template, which would be ju ** Unlike with delimiters, external tests/requests/responses don't inherit those removals, and need to be specified per file. -## Run tests in parallel - -The tool is able to do run tests in parallel. You activate this mechanism by including a external testfile with `p@pathtofile.json`. -The `p@` indicates to load that external file and run all tests in it in parallel. - -**All tests that are run in parallel are implicit set to ContinueOnFailure as otherwise the log and report would make no -sense** - -```yaml -{ - "name": "Binary Comparison", - "request": { - "endpoint": "suggest", - "method": "GET" - }, - // Path to binary file with @ - "response": "@simple.bin" -} -``` - ## Binary data comparison The tool is able to do a comparison with a binary file. Here we take a MD5 hash of the file and and then later compare