diff --git a/README.md b/README.md index b7efc45..544b8c8 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,3 @@ ---- -menu: - main: - name: "apitest" - identifier: apitester - parent: "cli" ---- - # fylr apitest The fylr apitesting tool helps you to build automated apitests that can be run after every build to ensure a constant product quality. @@ -21,8 +13,7 @@ The report parametere of this config can be overwritten via a command line flag. ```yaml apitest: - server: "http://5.simon.pf-berlin.de" //The base url to the instance you want to fire the apitests against. Important: don’t add a trailing ‘/’ - db-name: "easy5-simon" //The name of your database. The tool uses this parameter to double check if you selected the right instance to prevent damage. + server: "http://5.simon.pf-berlin.de/api/v1" //The base url to the api you want to fire the apitests against. Important: don’t add a trailing ‘/’ report: //Configures the maschine report. For usage with jenkis or any other CI tool file: "apitest_report" //Filename of the report file. The file gets saved in the same directory of the fylr binary format: "json" //Format of the report. (Supported formats: json or junit) @@ -47,15 +38,17 @@ This starts the command with the following default settings: - `--directory testDirectory` or `-d testDirectory`: Defines which directory should be used for running the tests in it. The tool walks recursively trough all subdirectories and runs alls tests that have a "manifest.json" file in alphabetical order of the folder names. (Depth-First-Search) - `--single path/to/a/single/manifest.json` or `-s path/to/a/single/manifest.json`: Run only a single test. The path needs to point directly to the manifest file. (Not the directory containing it) -- `--no-requirements`: Do not run the requirements of the test suite. Useful for development of a test (e.g. To save time not doing a purge every time) ### Configure logging - `--verbosity 1` or `-v 1`: Set the verbosity of the logger. (Verbosity defines what kind of communication with the server should be shown) Can be realy helpfull to find out why a test fails. - - `-v -1` (default): dont't log any communication with the server - - `-v 0`: log the communication in case a test fails - - `-v 1`: log the communication if it's so defined in the manifest - - `-v 2`: log all server communication + - `-v -1` (default): Only normal test ouput + - `-v 0`: All from '-1' plus failed test responses + - `-v 1`: All from '-1' plus all responses + - `-v 2`: All from '1' plus all requests + + +You can also set the log verbosity per single testcase. The greater verbosity wins. #### Console logging @@ -101,12 +94,6 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * ```json { - //Defines how a single test (if defined in the single testcase) should authenticate against the easydb. The authentication is valid for the complete testsuite and only one session can be used. - "authentication": { - "login": "root", - "method": "easydb", // This is the method given in the easydb doc - "password": "admin" - }, //General info about the testuite. Try to explain your problem indepth here. So that someone who works on the test years from now knows what is happening "description": "search api tests for filename", //Testname. Should be the ticket number if the test is based on a ticket @@ -143,21 +130,13 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * "continue_on_failure": true, //Name to identify this single test. Is important for the log. Try to give an explaning name "name": "Testname", - //Defines how this single test should authenticate against the easydb. This authentication is valid only for the single test and overwrites the testuite setting - "authentication": { - "login": "root", - "method": "easydb", - "password": "admin", - // Store parts of the session repsonse - "store_response_qjson": { - "max_event_id": "body.current_max_event_id" - }, - }, // Store custom values to the datastore "store": { "key1": "value1", "key2": "value2" }, + // Specify a unique log level only for this single test. (If cli has a greate log verbosity than this, cli wins) + "log_verbosity": 2, //Defines what gets send to the server "request": { //What endpoint we want to target. You find all possible endpoints in the api documentation @@ -174,6 +153,12 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * "header1":"value", "header2":"value" }, + // With header_from_you set a header to the value of the datastore field + // In this example we set the "Content-Type" header to the value "application/json" + // As "application/json" is stored as string in the datastore on index "contentType" + "header_from_store": { + "Content-Type": "contentType" + } //All the content you want to send in the http body. Is a JSON Object "body":{ "flower":"rose", @@ -233,12 +218,15 @@ The custom storage is persistent throughout the **apitest** run, so all requirem The custom store uses a **string** as index and can store any type of data. -If an **key** ends in `[]`, the value is assumed to be an Array, and is appended. If no Array exists, an array is created. +**Array**: If an key ends in `[]`, the value is assumed to be an Array, and is appended. If no Array exists, an array is created. + +**Map**: If an key ends in `[key]`, the value is assumed to be an map, and writes the data into the map at that key. If no map exists, an map is created. ```django { "store": { - "eas_ids[]": 15 + "eas_ids[]": 15, + "mapStorage[keyIWantToStore]": "value" } } ``` diff --git a/api_testcase.go b/api_testcase.go index f15d03b..17b0de8 100644 --- a/api_testcase.go +++ b/api_testcase.go @@ -10,11 +10,10 @@ import ( "github.com/programmfabrik/fylr-apitest/lib/api" "github.com/programmfabrik/fylr-apitest/lib/compare" - "github.com/programmfabrik/fylr-apitest/lib/logging" - apitestLogging "github.com/programmfabrik/fylr-apitest/lib/logging" "github.com/programmfabrik/fylr-apitest/lib/report" "github.com/programmfabrik/fylr-apitest/lib/template" "github.com/programmfabrik/fylr-apitest/lib/util" + log "github.com/sirupsen/logrus" ) const ( @@ -24,34 +23,44 @@ const ( // Case defines the structure of our single testcase // It gets read in by our config reader at the moment the mainfest.json gets parsed type Case struct { - Name string `json:"name"` - RequestData *util.GenericJson `json:"request"` - ResponseData util.GenericJson `json:"response"` - ContinueOnFailure bool `json:"continue_on_failure"` - Authentication *api.SessionAuthentication `json:"authentication"` - Store map[string]interface{} `json:"store"` // init datastore before testrun - StoreResponse map[string]string `json:"store_response_qjson"` // store qjson parsed response in datastore + Name string `json:"name"` + RequestData *util.GenericJson `json:"request"` + ResponseData util.GenericJson `json:"response"` + ContinueOnFailure bool `json:"continue_on_failure"` + Store map[string]interface{} `json:"store"` // init datastore before testrun + StoreResponse map[string]string `json:"store_response_qjson"` // store qjson parsed response in datastore Timeout int `json:"timeout_ms"` Delay *int `json:"delay_ms"` BreakResponse []util.GenericJson `json:"break_response"` CollectResponse util.GenericJson `json:"collect_response"` + LogVerbosity *int `json:"log_verbosity"` loader template.Loader - session api.Session manifestDir string reporter *report.Report suiteIndex int index int + dataStore *api.Datastore + + standardHeader map[string]*string + standardHeaderFromStore map[string]string + + ServerURL string } func (testCase Case) runAPITestCase() (success bool) { + if testCase.LogVerbosity != nil && *testCase.LogVerbosity > FylrConfig.Apitest.LogVerbosity { + defer FylrConfig.SetLogVerbosity(FylrConfig.Apitest.LogVerbosity) + FylrConfig.SetLogVerbosity(*testCase.LogVerbosity) + } + r := testCase.reporter if testCase.Name != "" { - logging.Infof(" [%2d] '%s'", testCase.index, testCase.Name) + log.Infof(" [%2d] '%s'", testCase.index, testCase.Name) } else { - logging.Infof(" [%2d] ''", testCase.index) + log.Infof(" [%2d] ''", testCase.index) } r.NewChild(testCase.Name) @@ -59,11 +68,17 @@ func (testCase Case) runAPITestCase() (success bool) { start := time.Now() // Store standard data into datastore - err := testCase.session.Store.SetMap(testCase.Store) + if testCase.dataStore == nil && len(testCase.Store) > 0 { + err := fmt.Errorf("error setting datastore. Datastore is nil") + r.SaveToReportLog(fmt.Sprintf("Error during execution: %s", err)) + log.Errorf(" [%2d] %s", testCase.index, err) + return false + } + err := testCase.dataStore.SetMap(testCase.Store) if err != nil { err = fmt.Errorf("error setting datastore map:%s", err) r.SaveToReportLog(fmt.Sprintf("Error during execution: %s", err)) - logging.Errorf(" [%2d] %s", testCase.index, err) + log.Errorf(" [%2d] %s", testCase.index, err) return false } @@ -76,25 +91,21 @@ func (testCase Case) runAPITestCase() (success bool) { if err != nil { r.SaveToReportLog(fmt.Sprintf("Error during execution: %s", err)) - logging.Errorf(" [%2d] %s", testCase.index, err) + log.Errorf(" [%2d] %s", testCase.index, err) success = false } if !success { - logging.WarnWithFieldsF( - map[string]interface{}{"elapsed": elapsed.Seconds()}, - " [%2d] %s", testCase.index, "failure") + log.WithFields(log.Fields{"elapsed": elapsed.Seconds()}).Warnf(" [%2d] failure", testCase.index) } else { - logging.InfoWithFieldsF( - map[string]interface{}{"elapsed": elapsed.Seconds()}, - " [%2d] %s", testCase.index, "success") + log.WithFields(log.Fields{"elapsed": elapsed.Seconds()}).Infof(" [%2d] success", testCase.index) } r.LeaveChild(success) return } -// checkForBreak Response tests the given response for a so called break response. +// cheRckForBreak Response tests the given response for a so called break response. // If this break response is present it returns a true func (testCase Case) breakResponseIsPresent(request api.Request, response api.Response) (found bool, err error) { @@ -115,7 +126,7 @@ func (testCase Case) breakResponseIsPresent(request api.Request, response api.Re return false, fmt.Errorf("error matching break responses: %s", err) } - fmt.Println("breakResponseIsPresent: ", responsesMatch) + log.Tracef("breakResponseIsPresent: %v", responsesMatch) if responsesMatch.Equal { return true, nil @@ -172,7 +183,7 @@ func (testCase *Case) checkCollectResponse(request api.Request, response api.Res testCase.CollectResponse = leftResponses - logging.Debugf("Remaining CheckReponses: %s", testCase.CollectResponse) + log.Tracef("Remaining CheckReponses: %s", testCase.CollectResponse) return len(leftResponses), nil } @@ -185,23 +196,30 @@ func (testCase Case) executeRequest(counter int) ( apiResp api.Response, err error) { + // Store datastore + err = testCase.dataStore.SetMap(testCase.Store) + if err != nil { + err = fmt.Errorf("error setting datastore map:%s", err) + } + //Do Request req, err = testCase.loadRequest() if err != nil { err = fmt.Errorf("error loading request: %s", err) return } - apitestLogging.DebugWithVerbosityf( - apitestLogging.V1, - "request: %s", req.ToString(testCase.session)) - apiResp, err = testCase.session.SendRequest(req) + + //Log request on trace level (so only v2 will trigger this) + log.Tracef("[REQUEST]:\n%s", req.ToString()) + + apiResp, err = req.Send() if err != nil { err = fmt.Errorf("error sending request: %s", err) return } // Store in custom store - err = testCase.session.Store.SetWithQjson(apiResp, testCase.StoreResponse) + err = testCase.dataStore.SetWithQjson(apiResp, testCase.StoreResponse) if err != nil { err = fmt.Errorf("error store repsonse with qjson: %s", err) return @@ -217,9 +235,9 @@ func (testCase Case) executeRequest(counter int) ( } // Store in datastore -1 list if counter == 0 { - testCase.session.Store.AppendResponse(json) + testCase.dataStore.AppendResponse(json) } else { - testCase.session.Store.UpdateLastResponse(json) + testCase.dataStore.UpdateLastResponse(json) } } @@ -229,9 +247,7 @@ func (testCase Case) executeRequest(counter int) ( err = fmt.Errorf("error loading response: %s", err) return } - apitestLogging.DebugWithVerbosityf( - apitestLogging.V1, - "response: %s", apiResp.ToString()) + responsesMatch, err = testCase.responsesEqual(response, apiResp) if err != nil { err = fmt.Errorf("error matching responses: %s", err) @@ -241,13 +257,10 @@ func (testCase Case) executeRequest(counter int) ( return } -func LogReqResp(request api.Request, response api.Response, session api.Session) { - apitestLogging.DebugWithVerbosityf( - apitestLogging.V0, - "[Request] %s", request.ToString(session)) - apitestLogging.DebugWithVerbosityf( - apitestLogging.V0, - "[Response] %s", response.ToString()) +func LogResp(response api.Response) { + if FylrConfig.Apitest.LogVerbosity == 0 { + log.Debugf("[RESPONSE]:\n%s", response.ToString()) + } } func (testCase Case) run() (success bool, err error) { @@ -268,7 +281,12 @@ func (testCase Case) run() (success bool, err error) { //Poll repeats the request until the right response is found, or a timeout triggers for { responsesMatch, request, apiResponse, err = testCase.executeRequest(requestCounter) + if FylrConfig.Apitest.LogVerbosity >= 1 { + log.Debugf("[RESPONSE]:\n%s", apiResponse.ToString()) + } + if err != nil { + LogResp(apiResponse) return false, err } @@ -278,18 +296,18 @@ func (testCase Case) run() (success bool, err error) { breakPresent, err := testCase.breakResponseIsPresent(request, apiResponse) if err != nil { - LogReqResp(request, apiResponse, testCase.session) + LogResp(apiResponse) return false, fmt.Errorf("error checking for break response: %s", err) } if breakPresent { - LogReqResp(request, apiResponse, testCase.session) + LogResp(apiResponse) return false, fmt.Errorf("Break response found") } collectLeft, err := testCase.checkCollectResponse(request, apiResponse) if err != nil { - LogReqResp(request, apiResponse, testCase.session) + LogResp(apiResponse) return false, fmt.Errorf("error checking for continue response: %s", err) } @@ -301,7 +319,7 @@ func (testCase Case) run() (success bool, err error) { //break if timeout or we do not have a repeater if timedOut := time.Now().Sub(startTime) > (time.Duration(testCase.Timeout) * time.Millisecond); timedOut && testCase.Timeout != -1 { if timedOut && testCase.Timeout > 0 { - logging.Warnf("Pull Timeout '%dms' exceeded", testCase.Timeout) + log.Warnf("Pull Timeout '%dms' exceeded", testCase.Timeout) r.SaveToReportLogF("Pull Timeout '%dms' exceeded", testCase.Timeout) timedOutFlag = true } @@ -318,27 +336,25 @@ func (testCase Case) run() (success bool, err error) { } if !responsesMatch.Equal || timedOutFlag { - //logging.Warnf("responses did not match: Expected: %s \n Actual: %s",response.ToString(), apiResponse.ToString()) for _, v := range responsesMatch.Failures { - logging.Warnf("[%s] %s", v.Key, v.Message) + log.Infof("[%s] %s", v.Key, v.Message) r.SaveToReportLog(fmt.Sprintf("[%s] %s", v.Key, v.Message)) } - //logging.Warnf("responses did not match: Expected: %s \n Actual: %s",response.ToString(), apiResponse.ToString()) collectArray, ok := testCase.CollectResponse.(util.JsonArray) if ok { for _, v := range collectArray { jsonV, err := json.Marshal(v) if err != nil { - LogReqResp(request, apiResponse, testCase.session) + LogResp(apiResponse) return false, err } - logging.Warnf("Collect response not found: %s", jsonV) + log.Warnf("Collect response not found: %s", jsonV) r.SaveToReportLog(fmt.Sprintf("Collect response not found: %s", jsonV)) } } - LogReqResp(request, apiResponse, testCase.session) + LogResp(apiResponse) return false, nil } @@ -393,6 +409,27 @@ func (testCase Case) loadRequestSerialization() (spec api.Request, err error) { } err = cjson.Unmarshal(specBytes, &spec) spec.ManifestDir = testCase.manifestDir + spec.DataStore = testCase.dataStore + spec.ServerURL = testCase.ServerURL + + if len(spec.Headers) == 0 { + spec.Headers = make(map[string]*string, 0) + } + for k, v := range testCase.standardHeader { + if spec.Headers[k] == nil { + spec.Headers[k] = v + } + } + + if len(spec.HeaderFromStore) == 0 { + spec.HeaderFromStore = make(map[string]string, 0) + } + for k, v := range testCase.standardHeaderFromStore { + if spec.HeaderFromStore[k] == "" { + spec.HeaderFromStore[k] = v + } + } + return } diff --git a/api_testcase_test.go b/api_testcase_test.go index 5286c3f..4b12500 100644 --- a/api_testcase_test.go +++ b/api_testcase_test.go @@ -8,8 +8,8 @@ import ( "testing" "github.com/programmfabrik/fylr-apitest/lib/api" - "github.com/programmfabrik/fylr-apitest/lib/report" "github.com/programmfabrik/fylr-apitest/lib/filesystem" + "github.com/programmfabrik/fylr-apitest/lib/report" "github.com/spf13/afero" ) @@ -68,7 +68,8 @@ func TestCollectResponseShouldWork(t *testing.T) { t.Fatal(err) } test.reporter = r - test.session, _ = api.NewSession(ts.URL, &http.Client{}, &api.SessionAuthentication{}, api.NewStore()) + test.ServerURL = ts.URL + test.dataStore = api.NewStore() test.runAPITestCase() @@ -120,7 +121,8 @@ func TestCollectLoadExternalFile(t *testing.T) { t.Fatal(err) } test.reporter = r - test.session, _ = api.NewSession(ts.URL, &http.Client{}, &api.SessionAuthentication{}, api.NewStore()) + test.ServerURL = ts.URL + test.dataStore = api.NewStore() test.runAPITestCase() @@ -180,7 +182,8 @@ func TestCollectLoadExternalCollect(t *testing.T) { t.Fatal(err) } test.reporter = r - test.session, _ = api.NewSession(ts.URL, &http.Client{}, &api.SessionAuthentication{}, api.NewStore()) + test.ServerURL = ts.URL + test.dataStore = api.NewStore() test.runAPITestCase() @@ -279,7 +282,8 @@ func TestCollectEvents(t *testing.T) { t.Fatal(err) } test.reporter = r - test.session, _ = api.NewSession(ts.URL, &http.Client{}, &api.SessionAuthentication{}, api.NewStore()) + test.ServerURL = ts.URL + test.dataStore = api.NewStore() test.runAPITestCase() @@ -331,7 +335,8 @@ func TestCollectResponseShouldFail(t *testing.T) { t.Fatal(err) } test.reporter = r - test.session, _ = api.NewSession(ts.URL, &http.Client{}, &api.SessionAuthentication{}, api.NewStore()) + test.ServerURL = ts.URL + test.dataStore = api.NewStore() test.runAPITestCase() diff --git a/api_testsuite.go b/api_testsuite.go index 3e3f70f..bb82162 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net/http" "path/filepath" "time" @@ -12,52 +11,48 @@ import ( "github.com/programmfabrik/fylr-apitest/lib/api" "github.com/programmfabrik/fylr-apitest/lib/cjson" + "github.com/programmfabrik/fylr-apitest/lib/filesystem" "github.com/programmfabrik/fylr-apitest/lib/report" "github.com/programmfabrik/fylr-apitest/lib/template" - "github.com/programmfabrik/fylr-apitest/lib/filesystem" - "github.com/programmfabrik/fylr-apitest/lib/logging" + log "github.com/sirupsen/logrus" ) // Suite defines the structure of our apitest // We do read this in with the config loader type Suite struct { - Name string `json:"name"` - Description string `json:"description"` - Authentication *api.SessionAuthentication `json:"authentication"` - Tests []util.GenericJson `json:"tests"` - RequirePaths []string `json:"require"` - Store map[string]interface{} `json:"store"` - - Client *http.Client - Config TestToolConfig - datastore *api.Datastore - manifestDir string - manifestPath string - reporter *report.Report - index int - executeRequirements bool + Name string `json:"name"` + Description string `json:"description"` + Tests []util.GenericJson `json:"tests"` + Store map[string]interface{} `json:"store"` + + StandardHeader map[string]*string `yaml:"header" json:"header"` + StandardHeaderFromStore map[string]string `yaml:"header_from_store" json:"header_from_store"` + + Config TestToolConfig + datastore *api.Datastore + manifestDir string + manifestPath string + reporter *report.Report + index int + serverURL string } // NewTestSuite creates a new suite on which we execute our tests on // Normally this only gets call from within the apitest main command func NewTestSuite( - client *http.Client, config TestToolConfig, manifestPath string, r *report.Report, - executeRequirements bool, datastore *api.Datastore, index int, ) (suite Suite, err error) { suite = Suite{ - Client: client, - Config: config, - manifestDir: filepath.Dir(manifestPath), - manifestPath: manifestPath, - reporter: r, - executeRequirements: executeRequirements, - datastore: datastore, - index: index, + Config: config, + manifestDir: filepath.Dir(manifestPath), + manifestPath: manifestPath, + reporter: r, + datastore: datastore, + index: index, } manifest, err := suite.loadManifest() @@ -80,169 +75,153 @@ func NewTestSuite( func (ats Suite) Run() (success bool) { r := ats.reporter - logging.Infof("[%2d] '%s'", ats.index, ats.Name) + log.Infof("[%2d] '%s'", ats.index, ats.Name) r.NewChild(ats.Name) r.SetTestCount(len(ats.Tests)) start := time.Now() - success = ats.run() + + success = true + for k, v := range ats.Tests { + sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, k) + if !sTestSuccess { + success = false + } + } + elapsed := time.Since(start) r.LeaveChild(success) if success { - logging.InfoWithFieldsF(map[string]interface{}{"elapsed": elapsed.Seconds()}, "[%2d] success", ats.index) + log.WithFields(log.Fields{"elapsed": elapsed.Seconds()}).Infof("[%2d] success", ats.index) } else { - logging.WarnWithFieldsF(map[string]interface{}{"elapsed": elapsed.Seconds()}, "[%2d] failure", ats.index) + log.WithFields(log.Fields{"elapsed": elapsed.Seconds()}).Warnf("[%2d] failure", ats.index) } return } -func (ats Suite) run() (success bool) { - if ats.executeRequirements { - success = ats.runRequirements() - if !success { - return false - } - } - - return ats.runTestCases() -} - -func (ats Suite) runRequirements() (success bool) { - //new empty reporter to don't include requirements in Testreport - r := report.NewReport() - - if ats.RequirePaths == nil { - return true - } - - logging.Infof("[%2d] %s", ats.index, "run requirements") - for _, parentPath := range ats.RequirePaths { - suite, err := NewTestSuite( - ats.Client, - ats.Config, - filepath.Join(ats.manifestDir, parentPath), - r, - ats.executeRequirements, - ats.datastore, - ats.index+1, - ) - if err != nil { - r.SaveToReportLog(err.Error()) - logging.Errorf("[%2d] error loading parent suite: %s", ats.index, err) - return false - } - - pSuccess := suite.Run() - if !pSuccess { - logging.Warnf("[%2d] requirements: failure", ats.index) - return false - } - } - - logging.Infof("[%2d] requirements: success", ats.index) - return true +type TestContainer struct { + CaseByte json.RawMessage + Path string } -/* -Runs TestCases of the TestSuite -We have to create the session at this point, otherwise -it might get deleted by a previous post to session/purgeall. -*/ -func (ats Suite) runTestCases() (success bool) { - /*defer func() { - if err := recover(); err != nil { - logging.Error(err) - success = false - } - }()*/ - +func (ats Suite) parseAndRunTest(v util.GenericJson, manifestDir string, k int) bool { + //Init variables r := ats.reporter + loader := template.NewLoader(ats.datastore) - datastoreShare := api.NewStoreShare(ats.datastore) - - loader := template.NewLoader(datastoreShare) - suiteSession, err := api.NewSession( - ats.Config.ServerURL, - ats.Client, - ats.Authentication, - datastoreShare) + //Get the Manifest with @ logic + dir, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir) if err != nil { r.SaveToReportLog(err.Error()) - logging.Error(err) - + log.Error(fmt.Errorf("can not LoadManifestDataAsRawJson: %s", err)) return false } - for k, v := range ats.Tests { - tests := make([]Case, 0) + //Try to directly unmarshal the manifest into testcase array + var testCases []json.RawMessage + err = cjson.Unmarshal(testObj, &testCases) + if err == nil { + //Did work -> No go template - intFilepath, testObj, err := template.LoadManifestDataAsObject(v, ats.manifestDir, loader) - if err != nil { - r.SaveToReportLog(err.Error()) - logging.Error(err) - return false - } + var success bool + //Run single tests + for ki, v := range testCases { - testJson, err := json.Marshal(testObj) - if err != nil { - r.SaveToReportLog(err.Error()) - logging.Error(err) - return false - } + //Check if is @ and if so load the test + if rune(v[1]) == rune('@') { + var sS string - err = json.Unmarshal(testJson, &tests) - if err != nil { - singleTest := Case{} - err = json.Unmarshal(testJson, &singleTest) - if err != nil { - r.SaveToReportLog(err.Error()) - logging.Error(err) - return false + err := cjson.Unmarshal(v, &sS) + if err != nil { + r.SaveToReportLog(err.Error()) + log.Error(fmt.Errorf("can not unmarshal: %s", err)) + return false + } + + success = ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), k+ki) + } else { + success = ats.runSingleTest(TestContainer{CaseByte: v, Path: filepath.Join(manifestDir, dir)}, r, loader, ki) } - tests = append(tests, singleTest) + if !success { + return false + } } + } else { + //We were not able unmarshal into array, so we try to unmarshal into raw message + var singleTest json.RawMessage + err = cjson.Unmarshal(testObj, &singleTest) + if err == nil { + //Did work to unmarshal -> no go template + + //Check if is @ and if so load the test + if rune(testObj[1]) == rune('@') { + var sS string - for _, test := range tests { - test.loader = loader - test.manifestDir = filepath.Join(ats.manifestDir, intFilepath) - test.reporter = r - test.suiteIndex = ats.index - test.index = k - - if test.Authentication != nil { - test.session, err = api.NewSession( - ats.Config.ServerURL, - ats.Client, - test.Authentication, - datastoreShare, - ) + err := cjson.Unmarshal(testObj, &sS) if err != nil { r.SaveToReportLog(err.Error()) - logging.Error(err) - if test.ContinueOnFailure { - continue - } else { - return false - } + log.Error(fmt.Errorf("can not unmarshal: %s", err)) + return false } + + return ats.parseAndRunTest(sS, filepath.Join(manifestDir, dir), k) } else { - test.session = suiteSession + return ats.runSingleTest(TestContainer{CaseByte: testObj, Path: filepath.Join(manifestDir, dir)}, r, loader, k) + } + } else { + //Did not work -> Could be go template or a mallformed json + requestBytes, lErr := loader.Render(testObj, filepath.Join(manifestDir, dir), nil) + if lErr != nil { + return false } - success := test.runAPITestCase() - - if !success && !test.ContinueOnFailure { + //If the both objects are the same we did not have a template, but a mallformed json -> Call error + if string(requestBytes) == string(testObj) { + r.SaveToReportLog(err.Error()) + log.Error(fmt.Errorf("can not unmarshal json: %s", err)) return false } + + //We have a template -> One level deeper with rendered bytes + return ats.parseAndRunTest([]byte(requestBytes), filepath.Join(manifestDir, dir), k) + } } return true } +func (ats Suite) runSingleTest(tc TestContainer, r *report.Report, loader template.Loader, k int) (success bool) { + var test Case + jErr := cjson.Unmarshal(tc.CaseByte, &test) + if jErr != nil { + r.SaveToReportLog(jErr.Error()) + log.Error(fmt.Errorf("can not unmarshal single test %s", jErr)) + return false + } + + test.loader = loader + test.manifestDir = tc.Path + test.reporter = r + test.suiteIndex = ats.index + test.index = k + test.dataStore = ats.datastore + test.ServerURL = ats.Config.ServerURL + test.standardHeader = ats.StandardHeader + test.standardHeaderFromStore = ats.StandardHeaderFromStore + + success = test.runAPITestCase() + + if !success && !test.ContinueOnFailure { + return false + } + + return true +} + func (ats Suite) loadManifest() (res []byte, err error) { loader := template.NewLoader(ats.datastore) manifestFile, err := filesystem.Fs.Open(ats.manifestPath) diff --git a/config.go b/config.go index a05c7bc..a48d7f3 100644 --- a/config.go +++ b/config.go @@ -2,17 +2,14 @@ package main import ( "fmt" - "github.com/programmfabrik/fylr-apitest/lib/api" "github.com/programmfabrik/fylr-apitest/lib/filesystem" + log "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/spf13/viper" - "log" - "net/http" "os" "path/filepath" "time" - ) - +) type FylrConfigStruct struct { Fylr struct { @@ -24,16 +21,15 @@ type FylrConfigStruct struct { } Apitest struct { Server string `mapstructure:"server"` - DBName string `mapstructure:"db-name"` StoreInit map[string]interface{} `mapstructure:"store"` Report struct { File string `mapstructure:"file"` Format string `mapstructure:"format"` } `mapstructure:"report"` + LogVerbosity int } } - var FylrConfig FylrConfigStruct var startTime time.Time @@ -55,29 +51,30 @@ func LoadConfig(cfgFile string) { } -func GetStartTime() time.Time { - return startTime +func (config *FylrConfigStruct) SetLogVerbosity(verbosity int) { + FylrConfig.Apitest.LogVerbosity = verbosity + if verbosity >= 2 { + log.SetLevel(log.TraceLevel) + } else if verbosity >= 0 { + log.SetLevel(log.DebugLevel) + } else { + log.SetLevel(log.InfoLevel) + } } - // TestToolConfig gives us the basic testtool infos type TestToolConfig struct { ServerURL string - DataBaseName string rootDirectorys []string TestDirectories []string } // NewTestToolConfig is mostly used for testing purpose. We can setup our config with this function -func NewTestToolConfig(serverURL, dataBaseName string, rootDirectory []string) (config TestToolConfig, err error) { +func NewTestToolConfig(serverURL string, rootDirectory []string) (config TestToolConfig, err error) { config = TestToolConfig{ ServerURL: serverURL, - DataBaseName: dataBaseName, rootDirectorys: rootDirectory, } - if err = config.checkDataBase(); err != nil { - return config, fmt.Errorf("error checking database names: %s", err) - } err = config.extractTestDirectories() return config, err } @@ -105,18 +102,3 @@ func (config *TestToolConfig) extractTestDirectories() error { } return nil } - -func (config *TestToolConfig) checkDataBase() error { - session, err := api.NewSession(config.ServerURL, &http.Client{}, nil, nil) - if err != nil { - return err - } - resp, err := session.SendSettingsRequest() - if err != nil { - return fmt.Errorf("error sending settings request: %s", err) - } - if resp.DbName != config.DataBaseName { - return fmt.Errorf("db settings differ: %s != %s", resp.DbName, config.DataBaseName) - } - return nil -} diff --git a/config_test.go b/config_test.go index d20b1ff..dc571e6 100644 --- a/config_test.go +++ b/config_test.go @@ -1,14 +1,13 @@ package main import ( - "fmt" "net/http" "net/http/httptest" "path/filepath" "testing" "github.com/programmfabrik/fylr-apitest/lib/filesystem" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" "github.com/spf13/afero" ) @@ -50,32 +49,15 @@ func SetupFS() { } -func TestTestToolConfig_DBName(t *testing.T) { - SetupFS() - - //Wrong db name -> Expect error - _, err := NewTestToolConfig(server.URL+"/api/v1", "Fail", []string{"path"}) - test_utils.ExpectError(t, err, "NewTestToolConfig did not fail on wrong dbName") - - //Wrong db name -> Expect error - _, err = NewTestToolConfig("invalid", "sTest", []string{"path"}) - test_utils.ExpectError(t, err, "NewTestToolConfig did not fail on wrong server url") - - //Wrong db name -> Expect error - _, err = NewTestToolConfig(server.URL+"/api/v1", "sTest", []string{"path"}) - test_utils.CheckError(t, err, fmt.Sprintf("NewTestToolConfig failed with right dbName: %s", err)) - -} - func TestTestToolConfig_ExtractTestDirectories(t *testing.T) { SetupFS() //Invalid rootDirectory -> Expect error - _, err := NewTestToolConfig(server.URL+"/api/v1", "sTest", []string{"invalid"}) + _, err := NewTestToolConfig(server.URL+"/api/v1", []string{"invalid"}) test_utils.ExpectError(t, err, "NewTestToolConfig did not fail on invalid root directory") //Invalid rootDirectory -> Expect error - conf, err := NewTestToolConfig(server.URL+"/api/v1", "sTest", []string{"path"}) + conf, err := NewTestToolConfig(server.URL+"/api/v1", []string{"path"}) test_utils.CheckError(t, err, "NewTestToolConfig did fail on valid root directory") expectedResults := []string{ diff --git a/go.mod b/go.mod index 147a737..6f199ff 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/programmfabrik/fylr-apitest require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/programmfabrik/go-test-utils v0.0.0-20190123092434-88356e86fb35 github.com/sirupsen/logrus v1.3.0 github.com/spf13/afero v1.1.2 github.com/spf13/cobra v0.0.3 diff --git a/lib/api/build_policies.go b/lib/api/build_policies.go index d6eebc4..9fd8b35 100644 --- a/lib/api/build_policies.go +++ b/lib/api/build_policies.go @@ -13,7 +13,7 @@ import ( ) func buildMultipart(request Request) (additionalHeaders map[string]string, body io.Reader, err error) { - additionalHeaders = make(map[string]string) + additionalHeaders = make(map[string]string, 0) var buf = bytes.NewBuffer([]byte{}) w := multipart.NewWriter(buf) @@ -40,14 +40,14 @@ func buildMultipart(request Request) (additionalHeaders map[string]string, body return additionalHeaders, nil, err } } - w.Close() - + err = w.Close() body = bytes.NewBuffer(buf.Bytes()) - return additionalHeaders, body, nil + + return } func buildUrlencoded(request Request) (additionalHeaders map[string]string, body io.Reader, err error) { - additionalHeaders = make(map[string]string) + additionalHeaders = make(map[string]string, 0) additionalHeaders["Content-Type"] = "application/x-www-form-urlencoded" formParams := url.Values{} for key, value := range request.Body.(map[string]string) { @@ -59,7 +59,7 @@ func buildUrlencoded(request Request) (additionalHeaders map[string]string, body } func buildRegular(request Request) (additionalHeaders map[string]string, body io.Reader, err error) { - additionalHeaders = make(map[string]string) + additionalHeaders = make(map[string]string, 0) additionalHeaders["Content-Type"] = "application/json" bodyBytes, err := cjson.Marshal(request.Body) if err != nil { diff --git a/lib/api/build_policies_test.go b/lib/api/build_policies_test.go index 7bf8f1c..a0de780 100644 --- a/lib/api/build_policies_test.go +++ b/lib/api/build_policies_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/programmfabrik/fylr-apitest/lib/filesystem" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" "github.com/spf13/afero" ) @@ -27,7 +27,7 @@ func TestBuildMultipart(t *testing.T) { BodyType: "multipart", } - httpRequest, err := testRequest.buildHttpRequest("some_interface", "some_token") + httpRequest, err := testRequest.buildHttpRequest() test_utils.CheckError(t, err, "error building multipart request") testReader, err := httpRequest.MultipartReader() diff --git a/lib/api/datastore.go b/lib/api/datastore.go index 7155042..fd3938f 100644 --- a/lib/api/datastore.go +++ b/lib/api/datastore.go @@ -2,7 +2,7 @@ package api import ( "fmt" - "github.com/programmfabrik/fylr-apitest/lib/logging" + log "github.com/sirupsen/logrus" "regexp" "strconv" "strings" @@ -132,7 +132,7 @@ func (this *Datastore) Set(index string, value interface{}) error { this.storage[index] = value } - logging.Debugf("Set datastore[\"%s\"]: '%#v'", index, value) + log.Tracef("Set datastore[\"%s\"]=%#v", index, value) return nil } diff --git a/lib/api/datastore_test.go b/lib/api/datastore_test.go index 1227712..11f1ccc 100644 --- a/lib/api/datastore_test.go +++ b/lib/api/datastore_test.go @@ -3,7 +3,7 @@ package api import ( "testing" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" ) func TestDataStore_Get(t *testing.T) { diff --git a/lib/api/request.go b/lib/api/request.go index 887a5b4..7a5a743 100644 --- a/lib/api/request.go +++ b/lib/api/request.go @@ -3,25 +3,38 @@ package api import ( "fmt" "github.com/programmfabrik/fylr-apitest/lib/util" + "time" + "io" "net/http" "net/http/httputil" ) +var c http.Client + +func init() { + c = http.Client{ + Timeout: time.Minute * 5, + } +} + type Request struct { - Endpoint string `yaml:"endpoint" json:"endpoint"` - Method string `yaml:"method" json:"method"` - QueryParams map[string]interface{} `yaml:"query_params" json:"query_params"` - Headers map[string]string `yaml:"header" json:"header"` - BodyType string `yaml:"body_type" json:"body_type"` - Body util.GenericJson `yaml:"body" json:"body"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + ServerURL string `yaml:"serverurl" json:"serverurl"` + Method string `yaml:"method" json:"method"` + QueryParams map[string]interface{} `yaml:"query_params" json:"query_params"` + Headers map[string]*string `yaml:"header" json:"header"` + HeaderFromStore map[string]string `yaml:"header_from_store" json:"header_from_store"` + BodyType string `yaml:"body_type" json:"body_type"` + Body util.GenericJson `yaml:"body" json:"body"` buildPolicy func(Request) (additionalHeaders map[string]string, body io.Reader, err error) DoNotStore bool ManifestDir string + DataStore *Datastore } -func (request Request) buildHttpRequest(serverUrl string, token string) (res *http.Request, err error) { +func (request Request) buildHttpRequest() (res *http.Request, err error) { if request.buildPolicy == nil { //Set Build policy switch request.BodyType { @@ -34,7 +47,7 @@ func (request Request) buildHttpRequest(serverUrl string, token string) (res *ht } } //Render Request Url - requestUrl := fmt.Sprintf("%s/%s", serverUrl, request.Endpoint) + requestUrl := fmt.Sprintf("%s/%s", request.ServerURL, request.Endpoint) additionalHeaders, body, err := request.buildPolicy(request) if err != nil { @@ -55,19 +68,54 @@ func (request Request) buildHttpRequest(serverUrl string, token string) (res *ht } res.URL.RawQuery = q.Encode() + for headerName, datastoreKey := range request.HeaderFromStore { + if request.DataStore == nil { + return res, fmt.Errorf("can't get header_from_store as the datastore is nil") + } + + headersInt, err := request.DataStore.Get(datastoreKey) + if err != nil { + return nil, fmt.Errorf("could not get '%s' from Datastore: %s", datastoreKey, err) + } + + ownHeaders, ok := headersInt.([]interface{}) + if ok { + for _, val := range ownHeaders { + valString, ok := val.(string) + if ok { + res.Header.Add(headerName, valString) + } + } + continue + } + + ownHeader, ok := headersInt.(string) + if ok { + res.Header.Add(headerName, ownHeader) + } else { + return nil, fmt.Errorf("could not set header '%s' from Datastore: '%s' is not a string. Got value: '%v'", headerName, datastoreKey, headersInt) + } + } + for key, val := range request.Headers { - res.Header.Add(key, val) + if *val == "" { + //Unset header explicit + res.Header.Del(key) + } else { + //ADD header + res.Header.Add(key, *val) + } } - additionalHeaders["x-easydb-token"] = token for key, val := range additionalHeaders { res.Header.Add(key, val) } + return res, nil } -func (request Request) ToString(session Session) (res string) { - httpRequest, err := request.buildHttpRequest(session.serverUrl, session.token) +func (request Request) ToString() (res string) { + httpRequest, err := request.buildHttpRequest() if err != nil { return fmt.Sprintf("could not build httpRequest: %s", err) } @@ -78,9 +126,27 @@ func (request Request) ToString(session Session) (res string) { } else { dumpBody = true } - resBytes, err := httputil.DumpRequest(httpRequest, dumpBody) + resBytes, err := httputil.DumpRequestOut(httpRequest, dumpBody) if err != nil { return fmt.Sprintf("could not dump httpRequest: %s", err) } return string(resBytes) } + +func (request Request) Send() (response Response, err error) { + httpRequest, err := request.buildHttpRequest() + if err != nil { + return response, err + } + + httpResponse, err := c.Do(httpRequest) + if err != nil { + return response, err + } + + response, err = NewResponse(httpResponse.StatusCode, httpResponse.Header, httpResponse.Body) + if err != nil { + return response, fmt.Errorf("error constructing response from http response") + } + return response, err +} diff --git a/lib/api/request_test.go b/lib/api/request_test.go index 014ef0d..eece885 100644 --- a/lib/api/request_test.go +++ b/lib/api/request_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" ) func TestRequestBuildHttp(t *testing.T) { @@ -17,6 +17,7 @@ func TestRequestBuildHttp(t *testing.T) { QueryParams: map[string]interface{}{ "query_param": "value", }, + ServerURL: "serverUrl", } request.buildPolicy = func(request Request) (ah map[string]string, b io.Reader, err error) { ah = make(map[string]string) @@ -24,10 +25,9 @@ func TestRequestBuildHttp(t *testing.T) { b = strings.NewReader("mock_body") return ah, b, nil } - httpRequest, err := request.buildHttpRequest("serverUrl", "token") + httpRequest, err := request.buildHttpRequest() test_utils.CheckError(t, err, fmt.Sprintf("error building http-request: %s", err)) test_utils.AssertStringEquals(t, httpRequest.Header.Get("mock-header"), "application/mock") - test_utils.AssertStringEquals(t, httpRequest.Header.Get("x-easydb-token"), "token") assertBody, err := ioutil.ReadAll(httpRequest.Body) test_utils.CheckError(t, err, fmt.Sprintf("error reading http-request body: %s", err)) diff --git a/lib/api/response.go b/lib/api/response.go index 711022c..9258608 100644 --- a/lib/api/response.go +++ b/lib/api/response.go @@ -9,7 +9,6 @@ import ( "io/ioutil" ) - type Response struct { statusCode int headers map[string][]string @@ -93,5 +92,14 @@ func (response *Response) marshalBodyInto(target interface{}) (err error) { } func (response Response) ToString() (res string) { - return fmt.Sprintf("Statuscode: %d, Body: %s", response.statusCode, string(response.Body())) + headersString := "" + for k, v := range response.headers { + value := "" + for _, iv := range v { + value = fmt.Sprintf("%s %s", value, iv) + + } + headersString = fmt.Sprintf("%s\n%s:%s", headersString, k, value) + } + return fmt.Sprintf("%d\n%s\n\n%s", response.statusCode, headersString, string(response.Body())) } diff --git a/lib/api/response_test.go b/lib/api/response_test.go index b168bc2..e1223d3 100644 --- a/lib/api/response_test.go +++ b/lib/api/response_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" ) func TestResponse_ToGenericJson(t *testing.T) { @@ -72,7 +72,7 @@ func TestResponse_NewResponse(t *testing.T) { func TestResponse_String(t *testing.T) { response, err := NewResponse(200, nil, strings.NewReader("{\"foo\": \"bar\"}")) - assertString := "Statuscode: 200, Body: {\"foo\": \"bar\"}" + assertString := "200\n\n\n{\"foo\": \"bar\"}" test_utils.CheckError(t, err, "error constructing response") test_utils.AssertStringEquals(t, response.ToString(), assertString) } diff --git a/lib/api/session.go b/lib/api/session.go deleted file mode 100644 index 9ab5486..0000000 --- a/lib/api/session.go +++ /dev/null @@ -1,244 +0,0 @@ -package api - -import ( - "encoding/json" - "fmt" - "net/http" - - "github.com/programmfabrik/fylr-apitest/lib/logging" -) - -/* -Performs API Calls in the context of the TestSuite -*/ - -type Session struct { - Store *Datastore - client *http.Client - serverUrl string - token string - MaxEventId int -} - -type SessionAuthentication struct { - Login string `json:"login"` - Password string `json:"password"` - Method string `json:"method"` - StoreResponse map[string]string `json:"store_response_qjson"` // store qjson parsed response in datastore -} - -func NewSession( - serverUrl string, - client *http.Client, - auth *SessionAuthentication, - store *Datastore, -) (session Session, err error) { - - session.client = client - session.serverUrl = serverUrl - session.Store = store - - response, err := session.SendSessionRequest() - if err != nil { - return session, fmt.Errorf("error GET-ing session endpoint: %s", err) - } - - session.token = response.Token - if auth != nil { - if err = session.login(*auth); err != nil { - return session, fmt.Errorf("error logging session in: %s", err) - } - } - return session, nil -} - -// login gets you an authenticated session token -// The supported authentication methods are: -// “easydb” (default): authenticate easydb user -// login: ist the username -// password: is the users password -// “email”: authenticate an easydb user of type “email” -// login: its email -// password: collection UUID -// “anonymous”: a virtual user will be created -// login: empty -// password: empty -// “task”: authenticate easydb to perform a task -// login: email-adress -// password: authentication token for the process -// “collection”: authenticate an easydb user of type “collection” -// login: collection UUID -// password: collection secret -func (session *Session) login(auth SessionAuthentication) (err error) { - - response, err := session.SendSessionAuthenticateRequest(auth) - if err != nil { - return fmt.Errorf("error performing authentication request: %s", err) - } - - session.MaxEventId = response.CurMaxEventId - return nil -} - -func (session *Session) SendRequest(request Request) (response Response, err error) { - logging.DebugWithVerbosityf(logging.V2, "request: %s", request.ToString(*session)) - httpRequest, err := request.buildHttpRequest(session.serverUrl, session.token) - if err != nil { - return response, err - } - - httpResponse, err := session.client.Do(httpRequest) - if err != nil { - return response, err - } - - response, err = NewResponse(httpResponse.StatusCode, httpResponse.Header, httpResponse.Body) - if err != nil { - return response, fmt.Errorf("error constructing response from http response") - } - logging.DebugWithVerbosityf(logging.V2, "response: %s", response.ToString()) - - return response, err -} - -/* -Convenience methods from within the application -*/ - -//Body of GET /settings response -//https://docs.easydb.de/en/technical/api/settings/settings.html -type settingsBody struct { - Name string `json:"name"` - Api int `json:"api"` - ServerVersion int `json:"server_version"` - UserSchema int `json:"user-schema"` - Solution string `json:"solution"` - DbName string `json:"db-name"` - ExternalEasUrl string `json:"external_eas_url"` - //startup time left out - //server time left out -} - -func (session *Session) SendSettingsRequest() (res settingsBody, err error) { - request := Request{ - Endpoint: "settings", - Method: "GET", - buildPolicy: buildRegular, - DoNotStore: true, - } - resp, err := session.SendRequest(request) - if err != nil { - return res, err - } - - if resp.statusCode != 200 { - apiErr := apiError{} - if err = resp.marshalBodyInto(&apiErr); err != nil { - return res, err - } - apiErr.Statuscode = resp.statusCode - - return res, apiErr - } - - if err = resp.marshalBodyInto(&res); err != nil { - return res, err - } - return res, nil -} - -//Body of GET /session response -//https://docs.easydb.de/en/technical/types/session/session.html -type sessionBody struct { - Token string `json:"token"` - //... -} - -type apiError struct { - Statuscode int `json:"statuscode"` - Code string `json:"code"` -} - -func (apiErr apiError) Error() string { - jsonErr, err := json.Marshal(apiErr) - if err != nil { - return err.Error() - } - return string(jsonErr) -} - -func (session *Session) SendSessionRequest() (res sessionBody, err error) { - request := Request{ - Endpoint: "session", - Method: "GET", - buildPolicy: buildRegular, - DoNotStore: true, - } - resp, err := session.SendRequest(request) - if err != nil { - return res, err - } - - if resp.statusCode != 200 { - apiErr := apiError{} - if err = resp.marshalBodyInto(&apiErr); err != nil { - return res, err - } - apiErr.Statuscode = resp.statusCode - - return res, apiErr - } - - if err = resp.marshalBodyInto(&res); err != nil { - return res, err - } - return res, nil -} - -//Body of POST /session/authenticate response -//https://docs.easydb.de/en/technical/api/session/session.html -type sessionAuthenticateBody struct { - CurMaxEventId int `json:"current_max_event_id"` -} - -func (session *Session) SendSessionAuthenticateRequest(auth SessionAuthentication) (res sessionAuthenticateBody, err error) { - request := Request{ - Endpoint: "session/authenticate", - Method: "POST", - Headers: map[string]string{ - "token": session.token, - }, - Body: map[string]string{ - "login": auth.Login, - "password": auth.Password, - "method": auth.Method, - }, - buildPolicy: buildUrlencoded, - DoNotStore: true, - } - - resp, err := session.SendRequest(request) - if err != nil { - return res, err - } - if resp.statusCode != 200 { - apiErr := apiError{} - if err = resp.marshalBodyInto(&apiErr); err != nil { - return res, err - } - apiErr.Statuscode = resp.statusCode - - return res, apiErr - } - - err = session.Store.SetWithQjson(resp, auth.StoreResponse) - if err != nil { - err = fmt.Errorf("unable to store response of session %s", err) - return - } - - if err = resp.marshalBodyInto(&res); err != nil { - return res, err - } - return res, nil -} diff --git a/lib/api/session_test.go b/lib/api/session_test.go deleted file mode 100644 index 8bc69d4..0000000 --- a/lib/api/session_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package api - -import ( - "fmt" - "net/http" - "strings" - "testing" - - "github.com/programmfabrik/fylr-apitest/lib/test_utils" -) - -func TestNewSessionSucceeds(t *testing.T) { - server := test_utils.NewTestServer(test_utils.Routes{ - "/api/v1/session": func(w *http.ResponseWriter, r *http.Request) { - (*w).Write([]byte("{\"token\": \"mock\"}")) - }, - }) - client := server.Client() - defer server.Close() - session, err := NewSession(server.URL+"/api/v1", client, nil, &Datastore{}) - test_utils.CheckError(t, err, fmt.Sprintf("error creating session: %s", err)) - test_utils.AssertStringEquals(t, session.token, "mock") -} - -func TestNewSessionFails(t *testing.T) { - server := test_utils.NewTestServer(test_utils.Routes{ - "/api/v1/session": func(w *http.ResponseWriter, r *http.Request) { - (*w).WriteHeader(http.StatusInternalServerError) - (*w).Write([]byte("{\"code\": \"error.server.fail\"}")) - }, - }) - client := server.Client() - defer server.Close() - _, err := NewSession(server.URL+"/api/v1", client, nil, &Datastore{}) - if err == nil || !strings.Contains(err.Error(), "error.server.fail") { - t.Errorf("expected error to contain: 'error.server.fail', got %s", err) - } -} - -func TestSessionLoginSuccess(t *testing.T) { - server := test_utils.NewTestServer(test_utils.Routes{ - "/api/v1/session": func(w *http.ResponseWriter, r *http.Request) { - (*w).Write([]byte("{\"token\": \"mock\"}")) - }, - "/api/v1/session/authenticate": func(w *http.ResponseWriter, r *http.Request) { - (*w).Write([]byte("{\"current_max_event_id\": 2391}")) - }, - }) - client := server.Client() - defer server.Close() - auth := &SessionAuthentication{} - session, err := NewSession(server.URL+"/api/v1", client, auth, &Datastore{}) - test_utils.CheckError(t, err, fmt.Sprintf("error creating session: %s", err)) - test_utils.AssertStringEquals(t, session.token, "mock") - test_utils.AssertIntEquals(t, session.MaxEventId, 2391) -} - -func TestSessionLoginFailsNotAuthenticated(t *testing.T) { - server := test_utils.NewTestServer(test_utils.Routes{ - "/api/v1/session": func(w *http.ResponseWriter, r *http.Request) { - (*w).Write([]byte("{\"token\": \"mock\"}")) - }, - "/api/v1/session/authenticate": func(w *http.ResponseWriter, r *http.Request) { - (*w).WriteHeader(400) - (*w).Write([]byte("{\"code\": \"error.something.failed\"}")) - }, - }) - client := server.Client() - defer server.Close() - auth := &SessionAuthentication{} - _, err := NewSession(server.URL+"/api/v1", client, auth, &Datastore{}) - if err == nil || !strings.Contains(err.Error(), "error.something.failed") { - t.Errorf("expected error to contain: 'error.something.failed', got %s", err) - } -} - -func TestSessionSettingsRequest(t *testing.T) { - server := test_utils.NewTestServer(test_utils.Routes{ - "/api/v1/settings": func(w *http.ResponseWriter, r *http.Request) { - (*w).Write([]byte("{\"db-name\": \"mock\"}")) - }, - }) - client := server.Client() - defer server.Close() - session := Session{Store: nil, client: client, serverUrl: server.URL + "/api/v1"} - settingsResponse, err := session.SendSettingsRequest() - test_utils.CheckError(t, err, fmt.Sprintf("error sending settings request: %s", err)) - test_utils.AssertStringEquals(t, settingsResponse.DbName, "mock") -} diff --git a/lib/cjson/cjson_test.go b/lib/cjson/cjson_test.go index dca4dab..00b199a 100644 --- a/lib/cjson/cjson_test.go +++ b/lib/cjson/cjson_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/programmfabrik/fylr-apitest/lib/compare" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" "github.com/programmfabrik/fylr-apitest/lib/util" + "github.com/programmfabrik/go-test-utils" ) func init() { diff --git a/lib/compare/comparison_functions_test.go b/lib/compare/comparison_functions_test.go index 7f681d3..8db94a1 100644 --- a/lib/compare/comparison_functions_test.go +++ b/lib/compare/comparison_functions_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" "github.com/programmfabrik/fylr-apitest/lib/util" ) diff --git a/lib/logging/multilogger.go b/lib/logging/multilogger.go deleted file mode 100644 index fd169a2..0000000 --- a/lib/logging/multilogger.go +++ /dev/null @@ -1,236 +0,0 @@ -package logging - -import ( - "bytes" - "fmt" - "io/ioutil" - "log" - "os" - "github.com/sirupsen/logrus" -) - -type LogInfo map[string]interface{} - -/* -Application wide logging package -*/ - -func init() { - ConfigureLogging(false, "info") -} - -type multiLogger struct { - loggers []*logrus.Logger -} - -func newMultiLogger() *multiLogger { - return &multiLogger{} -} - -func (multi *multiLogger) addLogger(logger *logrus.Logger) { - multi.loggers = append(multi.loggers, logger) -} - -var multi *multiLogger - - - -type enumVerbosity int - -const ( - V_ enumVerbosity = -1 - V0 enumVerbosity = 0 - V1 enumVerbosity = 1 - V2 enumVerbosity = 2 -) - -var _verbosity enumVerbosity - -func InitApiTestLogging(v int) error { - switch v { - case 0: - _verbosity = V0 - case 1: - _verbosity = V1 - case 2: - _verbosity = V2 - case -1: - _verbosity = V_ - default: - return fmt.Errorf("verbosity must be -1, 0, 1 or 2") - } - if v >= 0 { - if err := SetLevel("debug"); err != nil { - return err - } - } - return nil -} - -func DebugWithVerbosity(verbosity enumVerbosity, msg string) { - if verbosity == _verbosity { - Debug(msg) - } -} - -func DebugWithVerbosityf(verbosity enumVerbosity, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - DebugWithVerbosity(verbosity, msg) -} - - -//Initializes Logging with global config -func ConfigureLogging( - consoleEnabled bool, - consoleLevel string) (err error) { - - multi = newMultiLogger() - - if consoleEnabled { - consoleLogger, err := newConsoleLogger(consoleLevel) - if err != nil { - return fmt.Errorf("error adding consoleLogger: %s", err) - } - multi.addLogger(consoleLogger) - log.Printf("Console: { enabled: %t; level: %s }\n", consoleEnabled, consoleLevel) - } - - return nil -} - -//Set Loglevel for active loggers -func SetLevel(lvl string) error { - logLevel, err := logrus.ParseLevel(lvl) - if err != nil { - return err - } - for _, logger := range multi.loggers { - logger.SetLevel(logLevel) - } - return nil -} - -//Set Output for active loggers; needed to capture logging in Unittests -func SetOutput(buf *bytes.Buffer) { - for _, logger := range multi.loggers { - logger.SetOutput(buf) - } -} - -func newConsoleLogger(lvl string) (*logrus.Logger, error) { - logLevel, err := logrus.ParseLevel(lvl) - if err != nil { - return nil, fmt.Errorf("error parsing loglevel: %s", err) - } - logger := logrus.New() - logger.Out = os.Stderr - logger.SetLevel(logLevel) - return logger, nil -} - -//Method to allow subpackages to intercept logging statements -func AddHook(hook logrus.Hook) { - logger := logrus.New() - logger.Out = ioutil.Discard - logger.AddHook(hook) - multi.addLogger(logger) -} - -//LOGRUS Package Methods -// includes Debug, Info, Warn, Error and the *f variants; NOT the *ln variants - -// Debug logs a message at level Debug on the standard logger. -func Debug(args ...interface{}) { - for _, logger := range multi.loggers { - logger.Debug(args...) - } -} - -// Info logs a message at level Info on the standard logger. -func Info(args ...interface{}) { - for _, logger := range multi.loggers { - logger.Info(args...) - } -} - -// Warn logs a message at level Warn on the standard logger. -func Warn(args ...interface{}) { - for _, logger := range multi.loggers { - logger.Warn(args...) - } -} - -// Error logs a message at level Error on the standard logger. -func Error(args ...interface{}) { - for _, logger := range multi.loggers { - logger.Error(args...) - } -} - -// Debugf logs a message at level Debug on the standard logger. -func Debugf(format string, args ...interface{}) { - for _, logger := range multi.loggers { - logger.Debugf(format, args...) - } -} - -// Infof logs a message at level Info on the standard logger. -func Infof(format string, args ...interface{}) { - for _, logger := range multi.loggers { - logger.Infof(format, args...) - } -} - -// Warnf logs a message at level Warn on the standard logger. -func Warnf(format string, args ...interface{}) { - for _, logger := range multi.loggers { - logger.Warnf(format, args...) - } -} - -// Errorf logs a message at level Error on the standard logger. -func Errorf(format string, args ...interface{}) { - for _, logger := range multi.loggers { - logger.Errorf(format, args...) - } -} - -func InfoWithFieldsF(fields map[string]interface{}, msg string, args ...interface{}) { - for _, logger := range multi.loggers { - entry := logrus.NewEntry(logger) - for key, val := range fields { - entry = entry.WithFields(logrus.Fields{key: val}) - } - entry.Infof(msg, args...) - } -} - -func DebugWithFieldsF(fields map[string]interface{}, msg string, args ...interface{}) { - for _, logger := range multi.loggers { - entry := logrus.NewEntry(logger) - for key, val := range fields { - entry = entry.WithFields(logrus.Fields{key: val}) - } - entry.Debugf(msg, args...) - } -} - -func ErrorWithFieldsF(fields map[string]interface{}, msg string, args ...interface{}) { - for _, logger := range multi.loggers { - entry := logrus.NewEntry(logger) - for key, val := range fields { - entry = entry.WithFields(logrus.Fields{key: val}) - } - entry.Errorf(msg, args...) - } -} - -func WarnWithFieldsF(fields map[string]interface{}, msg string, args ...interface{}) { - for _, logger := range multi.loggers { - entry := logrus.NewEntry(logger) - for key, val := range fields { - entry = entry.WithFields(logrus.Fields{key: val}) - } - entry.Warnf(msg, args...) - } -} diff --git a/lib/template/template_funcs_test.go b/lib/template/template_funcs_test.go index 507907e..8df07d6 100644 --- a/lib/template/template_funcs_test.go +++ b/lib/template/template_funcs_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" ) func Test_QJson_String(t *testing.T) { diff --git a/lib/template/template_loader.go b/lib/template/template_loader.go index 5b3825c..8df37b9 100644 --- a/lib/template/template_loader.go +++ b/lib/template/template_loader.go @@ -3,13 +3,13 @@ package template import ( "bytes" "fmt" + log "github.com/sirupsen/logrus" "regexp" "strings" "text/template" "github.com/programmfabrik/fylr-apitest/lib/cjson" "github.com/programmfabrik/fylr-apitest/lib/csv" - "github.com/programmfabrik/fylr-apitest/lib/logging" "github.com/programmfabrik/fylr-apitest/lib/util" "io/ioutil" @@ -88,7 +88,7 @@ func (loader *Loader) Render( err = fmt.Errorf("The given json was empty") return } - logging.Debug(fmt.Sprintf("[QJSON] JSON input: %s", json)) + log.Tracef("[QJSON] JSON input: %s", json) result = gjson.Get(json, path).Raw if len(result) == 0 { diff --git a/lib/template/template_loader_test.go b/lib/template/template_loader_test.go index db2527e..0653527 100644 --- a/lib/template/template_loader_test.go +++ b/lib/template/template_loader_test.go @@ -9,7 +9,7 @@ import ( "github.com/programmfabrik/fylr-apitest/lib/api" "github.com/programmfabrik/fylr-apitest/lib/filesystem" - "github.com/programmfabrik/fylr-apitest/lib/test_utils" + "github.com/programmfabrik/go-test-utils" "github.com/spf13/afero" ) diff --git a/lib/template/util.go b/lib/template/util.go index 93d168c..47b793d 100644 --- a/lib/template/util.go +++ b/lib/template/util.go @@ -1,6 +1,7 @@ package template import ( + "encoding/json" "fmt" "io/ioutil" @@ -46,6 +47,7 @@ func LoadManifestDataAsObject(data util.GenericJson, manifestDir string, loader if err = cjson.Unmarshal(requestBytes, &jsonObject); err != nil { if err = cjson.Unmarshal(requestBytes, &jsonArray); err == nil { + return filepath, jsonArray, nil } return "", res, fmt.Errorf("error unmarshalling: %s", err) @@ -60,3 +62,28 @@ func LoadManifestDataAsObject(data util.GenericJson, manifestDir string, loader return "", res, fmt.Errorf("specification needs to be string[@...] or jsonObject but is: %s", data) } } + +func LoadManifestDataAsRawJson(data util.GenericJson, manifestDir string) (filepath string, res json.RawMessage, err error) { + switch typedData := data.(type) { + case []byte: + err = res.UnmarshalJSON(typedData) + return "", res, nil + case string: + filepath, res, err := loadFileFromPathSpec(typedData, manifestDir) + if err != nil { + return "", res, fmt.Errorf("error loading fileFromPathSpec: %s", err) + } + return filepath, res, nil + case util.JsonObject, util.JsonArray: + jsonMar, err := json.Marshal(typedData) + if err != nil { + return "", res, fmt.Errorf("error marshaling: %s", err) + } + if err = cjson.Unmarshal(jsonMar, &res); err != nil { + return "", res, fmt.Errorf("error unmarshalling: %s", err) + } + return "", res, nil + default: + return "", res, fmt.Errorf("specification needs to be string[@...] or jsonObject but is: %s", data) + } +} diff --git a/lib/test_utils/assert.go b/lib/test_utils/assert.go deleted file mode 100644 index 04db347..0000000 --- a/lib/test_utils/assert.go +++ /dev/null @@ -1,184 +0,0 @@ -package test_utils - -import ( - "strings" - "testing" -) - -func AssertStringEquals(t *testing.T, have, want string) { - if have != want { - t.Errorf("Have '%s' != '%s' Want", have, want) - } -} - -func AssertIntEquals(t *testing.T, have, want int) { - if have != want { - t.Errorf("Have '%d' != '%d' Want", have, want) - } -} - -func AssertErrorEquals(t *testing.T, have, want error) { - if want == nil && have != nil { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } - if want != nil && have == nil { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } - - if want == nil && have == nil { - return - } - - if have.Error() != want.Error() { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } -} - -func AsserErrorEqualsAny(t *testing.T, have error, want []error) { - if want == nil && have != nil { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } - if want != nil && have == nil { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } - - if want == nil && have == nil { - return - } - for _, v := range want { - if have.Error() == v.Error() { - return - } - } - - t.Errorf("'%v' not in '%v'", have, want) -} - -func AssertErrorContains(t *testing.T, have, want error) { - if want == nil && have != nil { - t.Errorf("'%v' != '%v'", have, want) - return - } - if want != nil && have == nil { - t.Errorf("'%v' != '%v'", have, want) - return - } - - if want == nil && have == nil { - return - } - - if !strings.Contains(have.Error(), want.Error()) { - t.Errorf("'%v' was not found in '%v'", want, have) - return - } -} - -func CheckError(t *testing.T, err error, errorMessage string) { - if err != nil { - t.Error(errorMessage) - } -} - -func ExpectError(t *testing.T, err error, errorMessage string) { - if err == nil { - t.Error(errorMessage) - } -} - -func AssertStringContainsSubstringsInOrder(t *testing.T, body string, expectedStrings []string) { - cI := 0 - print := false - for _, v := range expectedStrings { - i := strings.Index(body, v) - if i < cI { - t.Errorf("Wrong order in string. '%s' is at the wrong position", v) - print = true - } else { - cI = i - } - } - - if print { - t.Log(body) - } -} - -func AssertStringContainsSubstringsNoOrder(t *testing.T, body string, expectedStrings []string) { - for _, v := range expectedStrings { - if !strings.Contains(body, v) { - t.Errorf("'%s' not found.", v) - } - } -} - -func AssertStringContainsNoneOfTheSubstrings(t *testing.T, body string, notExpectedLogEntries []string) { - print := false - for _, v := range notExpectedLogEntries { - if strings.Contains(body, v) { - t.Errorf("We did not expect '%s' but found it in '%s'", v, body) - print = true - } - - } - - if print { - t.Log(body) - } -} - -// AssertMapsEqual checks if two maps have the exact same content -// Attention: This function changes the value of the first map! -func AssertMapsEqual(t *testing.T, got, want map[string]interface{}) { - - for k, v := range want { - if got[k] != v { - t.Errorf("[%s] Got '%v' != '%v' Want", k, got[k], v) - } else { - delete(got, k) - } - } - - for k, v := range got { - t.Errorf("[%s] Got '%v' != '' Want", k, v) - - } -} -func AssertStringArraysEqualNoOrder(t *testing.T, have, want []string) { - wantInner := make([]string, len(want)) - copy(wantInner, want) - - if want == nil && have != nil { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } - if want != nil && have == nil { - t.Errorf("Have '%v' != '%v' Want", have, want) - return - } - - if want == nil && have == nil { - return - } - - for _, v := range have { - for ik, iv := range wantInner { - if v == iv { - wantInner = append(wantInner[:ik], wantInner[ik+1:]...) - break - } - } - } - - if len(wantInner) > 0 { - for _, v := range wantInner { - t.Errorf("'%s' not found", v) - } - } - -} diff --git a/lib/test_utils/server.go b/lib/test_utils/server.go deleted file mode 100644 index f3add54..0000000 --- a/lib/test_utils/server.go +++ /dev/null @@ -1,26 +0,0 @@ -package test_utils - -import ( - "net/http" - "net/http/httptest" -) - -type handle func(*http.ResponseWriter, *http.Request) -type Routes map[string]handle - -func handleWithRoute(w http.ResponseWriter, r *http.Request, routes Routes) { - path := r.URL.Path - handle, ok := routes[path] - if !ok { - w.WriteHeader(500) - return - } - handle(&w, r) -} - -func NewTestServer(routes Routes) *httptest.Server { - routingHandle := func(w http.ResponseWriter, r *http.Request) { - handleWithRoute(w, r, routes) - } - return httptest.NewServer(http.HandlerFunc(routingHandle)) -} diff --git a/lib/test_utils/test_utils.go b/lib/test_utils/test_utils.go index f9db7f9..1557597 100644 --- a/lib/test_utils/test_utils.go +++ b/lib/test_utils/test_utils.go @@ -3,12 +3,13 @@ package test_utils import ( "bytes" "fmt" + "github.com/programmfabrik/go-test-utils" "net/http" "regexp" "strings" ) -var TestServer = NewTestServer(Routes{ +var TestServer = test_utils.NewTestServer(test_utils.Routes{ "/api/v1/session": func(w *http.ResponseWriter, r *http.Request) { (*w).Write([]byte("{\"token\": \"mock\"}")) }, diff --git a/main.go b/main.go index a7aed8e..214240b 100644 --- a/main.go +++ b/main.go @@ -6,40 +6,32 @@ import ( "fmt" "github.com/programmfabrik/fylr-apitest/lib/api" - "github.com/programmfabrik/fylr-apitest/lib/logging" + log "github.com/sirupsen/logrus" + "github.com/programmfabrik/fylr-apitest/lib/report" "io/ioutil" - "log" - "net/http" + "path/filepath" - "os" "github.com/spf13/cobra" "github.com/spf13/viper" - + "os" ) var ( - reportFormat, reportFile string - verbosity int - noRequirements bool + reportFormat, reportFile string + verbosity int rootDirectorys, singleTests []string ) - func init() { //Configure all the flags that fylr-apitest offers TestCMD.PersistentFlags().StringVarP(&cfgFile, "config", "c", "./fylr.yml", "config file") - TestCMD.PersistentFlags().String("log-console-level", "info", "console loglevel") - TestCMD.PersistentFlags().Bool("log-console-enable", true, "Set to true to enable console logging") + TestCMD.PersistentFlags().StringSliceVarP( &rootDirectorys, "directory", "d", []string{"."}, "path to directory containing the tests.") - TestCMD.PersistentFlags().Bool( - "no-requirements", false, - "don't run requirements for the testsuite.") - TestCMD.PersistentFlags().StringSliceVarP( &singleTests, "single", "s", []string{}, "path to a single manifest. Runs only that specified testsuite") @@ -47,11 +39,10 @@ func init() { TestCMD.PersistentFlags().IntVarP( &verbosity, "verbosity", "v", -1, `in [-1, 0, 1, 2], defines logging of requests and responses of the programm --1: dont log requests or responses -0: log requests and responses in case of test failure -1: log requests and responses that are documented in the manifest -2: log all requests and responses -verbosities >= 0 automatically set the global loglevel to debug`) +-1: Only normal test ouput +0: All from '-1' & failed test responses +1: All from '-1' & all responses +2: All from '1' & all requests`) TestCMD.PersistentFlags().StringVar( &reportFile, "report-file", "", @@ -61,149 +52,126 @@ verbosities >= 0 automatically set the global loglevel to debug`) &reportFormat, "report-format", "", "Defines how the report statements should be saved. [junit/json]") - //Bind the flags to overwrite the yml config if they are set - viper.BindPFlag("log.console.enable", TestCMD.PersistentFlags().Lookup("log-console-enable")) - viper.BindPFlag("log.console.level", TestCMD.PersistentFlags().Lookup("log-console-level")) viper.BindPFlag("apitest.report.file", TestCMD.PersistentFlags().Lookup("report-file")) viper.BindPFlag("apitest.report.format", TestCMD.PersistentFlags().Lookup("report-format")) } - var TestCMD = &cobra.Command{ Args: cobra.MaximumNArgs(0), PersistentPreRun: setup, - Use: "fylr apitest", - Short: "flyr Apitester lets you define API tests on the go", - Long: `A fast and flexible API testing tool. Helping you to define API tests on the go`, - Run: func(cmd *cobra.Command, args []string) { + Use: "fylr apitest", + Short: "flyr Apitester lets you define API tests on the go", + Long: `A fast and flexible API testing tool. Helping you to define API tests on the go`, + Run: runApiTests, +} + +func main() { + err := TestCMD.Execute() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} - noRequirements = (cmd.Flag("no-requirements").Value.String() == "true") +var cfgFile string - //Check if paths are valid +func setup(ccmd *cobra.Command, args []string) { + //Load yml config + LoadConfig(cfgFile) - for _, rootDirectory := range rootDirectorys { - if _, err := os.Stat(rootDirectory); rootDirectory != "." && os.IsNotExist(err) { - logging.Errorf("The path '%s' for the test folders is not valid", rootDirectory) - os.Exit(1) - } + //Set log verbosity + FylrConfig.SetLogVerbosity(verbosity) +} + +func runApiTests(cmd *cobra.Command, args []string) { + + //Check if paths are valid + + for _, rootDirectory := range rootDirectorys { + if _, err := os.Stat(rootDirectory); rootDirectory != "." && os.IsNotExist(err) { + log.Fatalf("The path '%s' for the test folders is not valid", rootDirectory) } - for _, singleTest := range singleTests { - if _, err := os.Stat(singleTest); singleTest != "" && os.IsNotExist(err) { - logging.Errorf("The path '%s' for the single test is not valid", singleTest) - os.Exit(1) - } + } + for _, singleTest := range singleTests { + if _, err := os.Stat(singleTest); singleTest != "" && os.IsNotExist(err) { + log.Fatalf("The path '%s' for the single test is not valid", singleTest) } + } - if err := logging.InitApiTestLogging(verbosity); err != nil { - logging.Errorf("Could not configure verbosity of apitest logging") - os.Exit(1) - } + serverUrl := FylrConfig.Apitest.Server + reportFormat = FylrConfig.Apitest.Report.Format + reportFile = FylrConfig.Apitest.Report.File + + //Save the config into TestToolConfig + testToolConfig, err := NewTestToolConfig(serverUrl, rootDirectorys) + if err != nil { + log.Fatal(err) + } - serverUrl := FylrConfig.Apitest.Server - dbName := FylrConfig.Apitest.DBName - reportFormat = FylrConfig.Apitest.Report.Format - reportFile = FylrConfig.Apitest.Report.File + datastore := api.NewStore() + for k, v := range FylrConfig.Apitest.StoreInit { + datastore.Set(k, v) + log.Debugf("Add Init value for datastore Key: '%s', Value: '%v'", k, v) + } - //Save the config into TestToolConfig - testToolConfig, err := NewTestToolConfig(serverUrl, dbName, rootDirectorys) + //Actually run the tests + //Run test function + runSingleTest := func(manifestPath string, r *report.Report) { + suite, err := NewTestSuite( + testToolConfig, + manifestPath, + r, + datastore, + 0, + ) if err != nil { - logging.Error(err) - os.Exit(1) + log.Fatal(err) } - datastore := api.NewStore() - for k, v := range FylrConfig.Apitest.StoreInit { - datastore.Set(k, v) - logging.Infof("Add Init value for datastore Key: '%s', Value: '%v'", k, v) - } + suite.Run() + } - //Actually run the tests - //Run test function - runSingleTest := func(manifestPath string, r *report.Report) { - suite, err := NewTestSuite( - &http.Client{}, - testToolConfig, - manifestPath, - r, - !noRequirements, - datastore, - 0, - ) - if err != nil { - logging.Error(err) - os.Exit(1) - } - - suite.Run() - } + r := report.NewReport() - r := report.NewReport() - - //Decide if run only one test - if len(singleTests) > 0 { - for _, singleTest := range singleTests { - absManifestPath, _ := filepath.Abs(singleTest) - runSingleTest(absManifestPath, r) - } - } else { - for _, singlerootDirectory := range testToolConfig.TestDirectories { - manifestPath := filepath.Join(singlerootDirectory, "manifest.json") - absManifestPath, _ := filepath.Abs(manifestPath) - runSingleTest(absManifestPath, r) - } + //Decide if run only one test + if len(singleTests) > 0 { + for _, singleTest := range singleTests { + absManifestPath, _ := filepath.Abs(singleTest) + runSingleTest(absManifestPath, r) } - - //Create report - if reportFile != "" { - var parsingFunction func(baseResult *report.ReportElement) []byte - switch reportFormat { - case "junit": - parsingFunction = report.ParseJUnitResult - case "json": - parsingFunction = report.ParseJSONResult - default: - logging.Errorf( - "Given report format '%s' not supported. Saving report '%s' as json", - reportFormat, - reportFile) - - parsingFunction = report.ParseJSONResult - } - - err = ioutil.WriteFile(reportFile, r.GetTestResult(parsingFunction), 0644) - if err != nil { - fmt.Println("Could not save into file: ", err) - } + } else { + for _, singlerootDirectory := range testToolConfig.TestDirectories { + manifestPath := filepath.Join(singlerootDirectory, "manifest.json") + absManifestPath, _ := filepath.Abs(manifestPath) + runSingleTest(absManifestPath, r) } + } - if r.DidFail() { - os.Exit(1) + //Create report + if reportFile != "" { + var parsingFunction func(baseResult *report.ReportElement) []byte + switch reportFormat { + case "junit": + parsingFunction = report.ParseJUnitResult + case "json": + parsingFunction = report.ParseJSONResult + default: + log.Errorf( + "Given report format '%s' not supported. Saving report '%s' as json", + reportFormat, + reportFile) + + parsingFunction = report.ParseJSONResult } - }, -} - -func main() { - err := TestCMD.Execute() - if err != nil { - fmt.Println(err) - os.Exit(1) + err = ioutil.WriteFile(reportFile, r.GetTestResult(parsingFunction), 0644) + if err != nil { + fmt.Println("Could not save into file: ", err) + } } -} - -var cfgFile string - -func setup(ccmd *cobra.Command, args []string) { - //Load yml config - LoadConfig(cfgFile) - - //Setup logging - if err := logging.ConfigureLogging( - viper.GetBool("log.console.enable"), - viper.GetString("log.console.level")); err != nil{ - log.Fatalf("error configuring logging: %s", err) + if r.DidFail() { + os.Exit(1) } } -