Skip to content

Commit

Permalink
Merge pull request #9 from programmfabrik/develop
Browse files Browse the repository at this point in the history 
Release fylr-apitest v0.82
  • Loading branch information
@simonfrey
simonfrey authored May 7, 2019
2 parents e6c69d8 + a465ad6 commit 6d22c95
Show file tree
Hide file tree
Showing 19 changed files with 565 additions and 351 deletions.
16 changes: 11 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,12 @@ This starts the command with the following default settings:

### Configure logging

- `--verbosity 1` or `-v 1`: Set the verbosity of the logger. (Verbosity defines what kind of communication with the server should be shown) Can be realy helpfull to find out why a test fails.
- `-v -1` (default): Only normal test ouput
- `-v 0`: All from '-1' plus failed test responses
- `-v 1`: All from '-1' plus all responses
- `-v 2`: All from '1' plus all requests
Per default request and response of a request will be logged on test failure. If you want to see more information you
can configure the tool with additional log flags

- `--log-network`: Log all network traffic
- `--log-datastore`: Log datastore operations
- `--log-verbose`: `--log-network`, `--log-datastore` and a few additional trace informations


You can also set the log verbosity per single testcase. The greater verbosity wins.
Expand Down Expand Up @@ -257,6 +258,11 @@ The method `store_respsonse_qjson` takes only **string** as value. This qjson-st

The data from the custom store is retrieved using the `datastore <key>`Template function. `key`must be used in any store method before it is requested. If the key is unset, the datastore function returns an empty **string**. Use the special key `-` to return the entire datastore.

Slices allow the backwards index access. If you have a slice of length 3 and access it at index `-1` you get the last
element in the slice (original index `2`)

If you access an invalid index for datastore `map[index]` or `slice[]` you get an empty string. No error is thrown.

### Get Data from Sequential Store

To get the data from the sequential store an integer number has to be given to the datastore function as **string**. So `datastore "0"` would be a valid request. This would return the response from first test of the current manifest. `datastore "-1"` returns the last response from the current manifest. `datastore "-2"` returns second to last from the current manifest. If the index is wrong the function returns an error.
Expand Down
80 changes: 54 additions & 26 deletions api_testcase.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/programmfabrik/fylr-apitest/lib/datastore"
"time"

"github.com/programmfabrik/fylr-apitest/lib/cjson"
Expand Down Expand Up @@ -35,14 +36,15 @@ type Case struct {
Delay *int `json:"delay_ms"`
BreakResponse []util.GenericJson `json:"break_response"`
CollectResponse util.GenericJson `json:"collect_response"`
LogVerbosity *int `json:"log_verbosity"`

loader template.Loader
manifestDir string
reporter *report.Report
suiteIndex int
index int
dataStore *api.Datastore
dataStore *datastore.Datastore
logNetwork bool
logVerbose bool

standardHeader map[string]*string
standardHeaderFromStore map[string]string
Expand All @@ -56,18 +58,12 @@ type CaseResponse struct {
}

func (testCase Case) runAPITestCase() (success bool) {
if testCase.LogVerbosity != nil && *testCase.LogVerbosity > FylrConfig.Apitest.LogVerbosity {
defer FylrConfig.SetLogVerbosity(FylrConfig.Apitest.LogVerbosity)
FylrConfig.SetLogVerbosity(*testCase.LogVerbosity)
}

r := testCase.reporter

if testCase.Name != "" {
log.Infof(" [%2d] '%s'", testCase.index, testCase.Name)
} else {
log.Infof(" [%2d] '<no name>'", testCase.index)
if testCase.Name == "" {
testCase.Name = "<no name>"
}
log.Infof(" [%2d] '%s'", testCase.index, testCase.Name)

//Marshal external datastructure in our internal used reponseData
jsonResponse, err := json.Marshal(testCase.TypedResponseData)
Expand All @@ -94,13 +90,15 @@ func (testCase Case) runAPITestCase() (success bool) {
err := fmt.Errorf("error setting datastore. Datastore is nil")
r.SaveToReportLog(fmt.Sprintf("Error during execution: %s", err))
log.Errorf(" [%2d] %s", testCase.index, err)

return false
}
err = testCase.dataStore.SetMap(testCase.Store)
if err != nil {
err = fmt.Errorf("error setting datastore map:%s", err)
r.SaveToReportLog(fmt.Sprintf("Error during execution: %s", err))
log.Errorf(" [%2d] %s", testCase.index, err)

return false
}

Expand All @@ -123,7 +121,6 @@ func (testCase Case) runAPITestCase() (success bool) {
log.WithFields(log.Fields{"elapsed": elapsed.Seconds()}).Infof(" [%2d] success", testCase.index)
}

r.LeaveChild(success)
return
}

Expand All @@ -148,7 +145,9 @@ func (testCase Case) breakResponseIsPresent(request api.Request, response api.Re
return false, fmt.Errorf("error matching break responses: %s", err)
}

log.Tracef("breakResponseIsPresent: %v", responsesMatch)
if testCase.logVerbose {
log.Tracef("breakResponseIsPresent: %v", responsesMatch)
}

if responsesMatch.Equal {
return true, nil
Expand Down Expand Up @@ -205,7 +204,10 @@ func (testCase *Case) checkCollectResponse(request api.Request, response api.Res

testCase.CollectResponse = leftResponses

log.Tracef("Remaining CheckReponses: %s", testCase.CollectResponse)
if testCase.logVerbose {
log.Tracef("Remaining CheckReponses: %s", testCase.CollectResponse)
}

return len(leftResponses), nil
}

Expand All @@ -232,17 +234,28 @@ func (testCase Case) executeRequest(counter int) (
}

//Log request on trace level (so only v2 will trigger this)
log.Tracef("[REQUEST]:\n%s", req.ToString())
if testCase.logNetwork {
log.Tracef("[REQUEST]:\n%s", req.ToString())
}

apiResp, err = req.Send()
if err != nil {
testCase.LogReq(req)
err = fmt.Errorf("error sending request: %s", err)
return
}

apiRespJson, err := apiResp.ToJsonString()
if err != nil {
testCase.LogReq(req)
err = fmt.Errorf("error getting json from response: %s", err)
return
}

// Store in custom store
err = testCase.dataStore.SetWithQjson(apiResp, testCase.StoreResponse)
err = testCase.dataStore.SetWithQjson(apiRespJson, testCase.StoreResponse)
if err != nil {
testCase.LogReq(req)
err = fmt.Errorf("error store repsonse with qjson: %s", err)
return
}
Expand All @@ -252,6 +265,7 @@ func (testCase Case) executeRequest(counter int) (

json, err = apiResp.ToJsonString()
if err != nil {
testCase.LogReq(req)
err = fmt.Errorf("error prepareing response for datastore: %s", err)
return
}
Expand All @@ -266,22 +280,30 @@ func (testCase Case) executeRequest(counter int) (
//Compare Responses
response, err := testCase.loadResponse()
if err != nil {
testCase.LogReq(req)
err = fmt.Errorf("error loading response: %s", err)
return
}

responsesMatch, err = testCase.responsesEqual(response, apiResp)
if err != nil {
testCase.LogReq(req)
err = fmt.Errorf("error matching responses: %s", err)
return
}

return
}

func LogResp(response api.Response) {
if FylrConfig.Apitest.LogVerbosity == 0 {
log.Debugf("[RESPONSE]:\n%s", response.ToString())
func (testCase Case) LogResp(response api.Response) {
if !testCase.logNetwork && !testCase.ContinueOnFailure {
log.Debugf("[RESPONSE]:\n%s\n", response.ToString())
}
}

func (testCase Case) LogReq(request api.Request) {
if !testCase.logNetwork && !testCase.ContinueOnFailure {
log.Debugf("[REQUEST]:\n%s\n", request.ToString())
}
}

Expand Down Expand Up @@ -310,12 +332,13 @@ func (testCase Case) run() (success bool, err error) {
}

responsesMatch, request, apiResponse, err = testCase.executeRequest(requestCounter)
if FylrConfig.Apitest.LogVerbosity >= 1 {
if testCase.logNetwork {
log.Debugf("[RESPONSE]:\n%s", apiResponse.ToString())
}

if err != nil {
LogResp(apiResponse)
testCase.LogReq(request)
testCase.LogResp(apiResponse)
return false, err
}

Expand All @@ -325,18 +348,21 @@ func (testCase Case) run() (success bool, err error) {

breakPresent, err := testCase.breakResponseIsPresent(request, apiResponse)
if err != nil {
LogResp(apiResponse)
testCase.LogReq(request)
testCase.LogResp(apiResponse)
return false, fmt.Errorf("error checking for break response: %s", err)
}

if breakPresent {
LogResp(apiResponse)
testCase.LogReq(request)
testCase.LogResp(apiResponse)
return false, fmt.Errorf("Break response found")
}

collectLeft, err := testCase.checkCollectResponse(request, apiResponse)
if err != nil {
LogResp(apiResponse)
testCase.LogReq(request)
testCase.LogResp(apiResponse)
return false, fmt.Errorf("error checking for continue response: %s", err)
}

Expand Down Expand Up @@ -369,15 +395,17 @@ func (testCase Case) run() (success bool, err error) {
for _, v := range collectArray {
jsonV, err := json.Marshal(v)
if err != nil {
LogResp(apiResponse)
testCase.LogReq(request)
testCase.LogResp(apiResponse)
return false, err
}
log.Warnf("Collect response not found: %s", jsonV)
r.SaveToReportLog(fmt.Sprintf("Collect response not found: %s", jsonV))
}
}

LogResp(apiResponse)
testCase.LogReq(request)
testCase.LogResp(apiResponse)
return false, nil
}

Expand Down
75 changes: 65 additions & 10 deletions api_testcase_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package main
import (
"encoding/json"
"fmt"
"github.com/programmfabrik/fylr-apitest/lib/datastore"
"github.com/tidwall/gjson"
"net/http"
"net/http/httptest"
"testing"

"github.com/programmfabrik/fylr-apitest/lib/api"
"github.com/programmfabrik/fylr-apitest/lib/filesystem"
"github.com/programmfabrik/fylr-apitest/lib/report"
"github.com/spf13/afero"
Expand Down Expand Up @@ -76,7 +76,7 @@ func TestCollectResponseShouldWork(t *testing.T) {
}
test.reporter = r
test.ServerURL = ts.URL
test.dataStore = api.NewStore()
test.dataStore = datastore.NewStore(false)

test.runAPITestCase()

Expand Down Expand Up @@ -129,7 +129,7 @@ func TestCollectLoadExternalFile(t *testing.T) {
}
test.reporter = r
test.ServerURL = ts.URL
test.dataStore = api.NewStore()
test.dataStore = datastore.NewStore(false)

test.runAPITestCase()

Expand Down Expand Up @@ -190,7 +190,7 @@ func TestCollectLoadExternalCollect(t *testing.T) {
}
test.reporter = r
test.ServerURL = ts.URL
test.dataStore = api.NewStore()
test.dataStore = datastore.NewStore(false)

test.runAPITestCase()

Expand Down Expand Up @@ -290,7 +290,7 @@ func TestCollectEvents(t *testing.T) {
}
test.reporter = r
test.ServerURL = ts.URL
test.dataStore = api.NewStore()
test.dataStore = datastore.NewStore(false)

test.runAPITestCase()

Expand Down Expand Up @@ -343,9 +343,10 @@ func TestCollectResponseShouldFail(t *testing.T) {
}
test.reporter = r
test.ServerURL = ts.URL
test.dataStore = api.NewStore()
test.dataStore = datastore.NewStore(false)

test.runAPITestCase()
r.NewChild(test.Name)
r.LeaveChild(test.runAPITestCase())

r.GetTestResult(report.ParseJSONResult)

Expand Down Expand Up @@ -406,15 +407,69 @@ func TestHeaderFromDatastoreWithMap(t *testing.T) {
test.reporter = r
test.ServerURL = ts.URL

test.dataStore = api.NewStore()
test.dataStore = datastore.NewStore(false)
test.dataStore.Set("hallo[du]", "du index")
test.dataStore.Set("hallo[sie]", "sie index")

test.runAPITestCase()

r.GetTestResult(report.ParseJSONResult)
if !r.DidFail() {
t.Errorf("Did not fail but it should")
if r.DidFail() {
t.Errorf("Did fail but it should not")
}

}

func TestHeaderFromDatastoreWithSlice(t *testing.T) {

ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, `{"Auth": "%s"}`, r.Header.Get("AuthHeader"))
}))
defer ts.Close()

testManifest := []byte(`
{
"name": "CollectTest",
"request":{
"endpoint": "suggest",
"method": "GET",
"header_from_store":{
"authHeader":"hallo[3]"
}
},
"response":{
"body": {
"Auth": "es index"
}
}
}
`)

filesystem.Fs = afero.NewMemMapFs()
afero.WriteFile(filesystem.Fs, "manifest.json", []byte(testManifest), 644)

r := report.NewReport()

var test Case
err := json.Unmarshal(testManifest, &test)
if err != nil {
t.Fatal(err)
}
test.reporter = r
test.ServerURL = ts.URL

test.dataStore = datastore.NewStore(false)
test.dataStore.Set("hallo[]", "du index")
test.dataStore.Set("hallo[]", "sie index")
test.dataStore.Set("hallo[]", "er index")
test.dataStore.Set("hallo[]", "es index")
test.dataStore.Set("hallo[]", "mama index")

test.runAPITestCase()

r.GetTestResult(report.ParseJSONResult)
if r.DidFail() {
t.Errorf("Did fail but it should not")
}

}
Loading

0 comments on commit 6d22c95

Please sign in to comment.