template function "file_xml2json" (#74)

* new implementations of marshalling xml into json - use simpler json output without "#seq": - new template function "file_xml2json" - new response format type "xml2" see #64950 * added new testcase for new template exiftool_result.xml: compare result of exiftool request with local xml file; see #64950 * updated README for new xml to json functions * Update README.md * changed response debug output if xml2 format is used: output formatted json instead of xml Co-authored-by: Philipp Hempel <philipp.hempel@programmfabrik.de>
programmfabrik · Jun 17, 2022 · ba8c6d3 · ba8c6d3
1 parent 25b6041
commit ba8c6d3
Show file tree

Hide file tree

Showing 15 changed files with 358 additions and 17 deletions.
diff --git a/README.md b/README.md
@@ -466,15 +466,17 @@ For comparing a binary file, simply point the response to the binary file:
 
 ## XML Data comparison
 
-If the response format is specified as `"type": "xml"`, we internally marshal that XML into json. (With github.com/clbanning/mxj `NewMapXmlSeq()`).
+If the response format is specified as `"type": "xml"` or `"type": "xml2"`, we internally marshal that XML into json using [github.com/clbanning/mxj](https://github.com/clbanning/mxj).
 
-On that json you can work as you are used to with the json syntax. For seeing how the convert json looks you can use the `--log-verbose` command line flag
+The format `"xml"` uses `NewMapXmlSeq()`, whereas the format `"xml2"` uses `NewMapXml()`, which provides a simpler json format (see also template [`file_xml2json`](#file_xml2json-path)).
+
+On that json you can work as you are used to with the json syntax. For seeing how the converted json looks you can use the `--log-verbose` command line flag
 
 ## CSV Data comparison
 
 If the response format is specified as `"type": "csv"`, we internally marshal that CSV into json.
 
-On that json you can work as you are used to with the json syntax. For seeing how the convert json looks you can use the `--log-verbose` command line flag
+On that json you can work as you are used to with the json syntax. For seeing how the converted json looks you can use the `--log-verbose` command line flag
 
 You can also specify the delimiter (`comma`) for the CSV format (default: `,`):
 
@@ -1790,6 +1792,61 @@ int64,string
 [map[name:simon] map[name:martin] map[name:roman] map[name:klaus] map[name:sebastian]]
 ```
 
+## `file_xml2json [path]`
+
+Helper function to parse an XML file and convert it into json
+- `@path`: string; a path to the xml file that should be loaded. The path is either relative to the manifest or a weburl
+
+This function uses the function `NewMapXml()` from [github.com/clbanning/mxj](https://github.com/clbanning/mxj).
+
+### Example
+
+Content of XML file `some/path/example.xml`:
+
+```xml
+<objects xmlns="https://schema.easydb.de/EASYDB/1.0/objects/">
+    <obj>
+        <_standard>
+            <de-DE>Beispiel Objekt</de-DE>
+            <en-US>Example Object</en-US>
+        </_standard>
+        <_system_object_id>123</_system_object_id>
+        <_id>45</_id>
+        <name type="text_oneline"
+            column-api-id="263">Example</name>
+    </obj>
+</objects>
+```
+
+The call
+
+```django
+{{ file_xml2json "some/path/example.xml" }}
+```
+
+would result in
+
+```json
+{
+    "objects": {
+        "-xmlns": "https://schema.easydb.de/EASYDB/1.0/objects/",
+        "obj": {
+            "_id": "45",
+            "_standard": {
+                "de-DE": "Beispiel Objekt",
+                "en-US": "Example Object"
+            },
+            "_system_object_id": "123",
+            "name": {
+                "#text": "Example",
+                "-column-api-id": "263",
+                "-type": "text_oneline"
+            }
+        }
+    }
+}
+```
+
 ## `file_sqlite [path] [statement]`
 
 Helper function to return the result of an SQL statement from a sqlite3 file.

diff --git a/pkg/lib/api/response.go b/pkg/lib/api/response.go
@@ -9,12 +9,10 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"regexp"
 	"strings"
 	"time"
 	"unicode/utf8"
 
-	"github.com/clbanning/mxj"
 	"github.com/pkg/errors"
 
 	"github.com/programmfabrik/apitest/pkg/lib/csv"
@@ -67,7 +65,7 @@ type ResponseSerialization struct {
 
 type ResponseFormat struct {
 	IgnoreBody bool   `json:"-"`    // if true, do not try to parse the body (since it is not expected in the response)
-	Type       string `json:"type"` // default "json", allowed: "csv", "json", "xml", "binary"
+	Type       string `json:"type"` // default "json", allowed: "csv", "json", "xml", "xml2", "binary"
 	CSV        struct {
 		Comma string `json:"comma,omitempty"`
 	} `json:"csv,omitempty"`
@@ -148,16 +146,8 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm
 	}
 
 	switch responseFormat.Type {
-	case "xml":
-		xmlDeclarationRegex := regexp.MustCompile(`<\?xml.*?\?>`)
-		replacedXML := xmlDeclarationRegex.ReplaceAll(resp.Body, []byte{})
-
-		mv, err := mxj.NewMapXmlSeq(replacedXML)
-		if err != nil {
-			return res, errors.Wrap(err, "Could not parse xml")
-		}
-
-		bodyData, err = mv.JsonIndent("", " ")
+	case "xml", "xml2":
+		bodyData, err = util.Xml2Json(resp.Body, responseFormat.Type)
 		if err != nil {
 			return res, errors.Wrap(err, "Could not marshal xml to json")
 		}
@@ -359,7 +349,7 @@ func (response Response) ToString() string {
 
 	body := resp.Body
 	switch resp.Format.Type {
-	case "xml", "csv":
+	case "xml", "xml2", "csv":
 		if utf8.Valid(body) {
 			bodyString, err = resp.ServerResponseToJsonString(true)
 			if err != nil {

diff --git a/pkg/lib/template/template_loader.go b/pkg/lib/template/template_loader.go
@@ -172,6 +172,19 @@ func (loader *Loader) Render(
 
 			return data, nil
 		},
+		"file_xml2json": func(path string) (string, error) {
+			fileBytes, err := fileReadInternal(path, rootDir)
+			if err != nil {
+				return "", err
+			}
+
+			bytes, err := util.Xml2Json(fileBytes, "xml2")
+			if err != nil {
+				return "", errors.Wrap(err, "Could not marshal xml to json")
+			}
+
+			return string(bytes), nil
+		},
 		"file_path": func(path string) string {
 			return util.LocalPath(path, rootDir)
 		},

diff --git a/pkg/lib/util/util.go b/pkg/lib/util/util.go
@@ -3,7 +3,11 @@ package util
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"strconv"
+
+	"github.com/clbanning/mxj"
+	"github.com/pkg/errors"
 )
 
 func Max(x, y int) int {
@@ -38,3 +42,36 @@ func GetStringFromInterface(queryParam interface{}) (string, error) {
 		return string(jsonVal), err
 	}
 }
+
+// Xml2Json parses the raw xml data and converts it into a json string
+// there are 2 formats for the result json:
+// - "xml": use mxj.NewMapXmlSeq (more complex format including #seq)
+// - "xml2": use mxj.NewMapXmlSeq (simpler format)
+func Xml2Json(rawXml []byte, format string) ([]byte, error) {
+	var (
+		mv  mxj.Map
+		err error
+	)
+
+	xmlDeclarationRegex := regexp.MustCompile(`<\?xml.*?\?>`)
+	replacedXML := xmlDeclarationRegex.ReplaceAll(rawXml, []byte{})
+
+	switch format {
+	case "xml":
+		mv, err = mxj.NewMapXmlSeq(replacedXML)
+	case "xml2":
+		mv, err = mxj.NewMapXml(replacedXML)
+	default:
+		return []byte{}, errors.Errorf("Unknown format %s", format)
+	}
+
+	if err != nil {
+		return []byte{}, errors.Wrap(err, "Could not parse xml")
+	}
+
+	json, err := mv.JsonIndent("", " ")
+	if err != nil {
+		return []byte{}, errors.Wrap(err, "Could not convert to json")
+	}
+	return json, nil
+}
diff --git a/test/xml/berlin.jpg b/test/xml/berlin.jpg
diff --git a/test/xml/check_local_file_against_response.json b/test/xml/check_local_file_against_response.json
@@ -0,0 +1,16 @@
+{
+    "name": "Get existing XML file",
+    "request": {
+        "server_url": "{{ datastore "req_base_url" }}",
+        "endpoint": "bounce-json",
+        "method": "POST",
+        "body": {{ file_xml2json "sample.xml" }}
+    },
+    "response": {
+        "statuscode": 200,
+        "body": {
+            "header": {},
+            "body": {{ file "result_xml2.json" }}
+        }
+    }
+}
diff --git a/test/xml/check_response_against_local_file.json b/test/xml/check_response_against_local_file.json
@@ -0,0 +1,15 @@
+{
+    "name": "Get existing XML file",
+    "request": {
+        "server_url": "{{ datastore "req_base_url" }}",
+        "endpoint": "bounce-json",
+        "method": "POST",
+        "body": {{ file "result_xml2.json" }}
+    },
+    "response": {
+        "statuscode": 200,
+        "body": {
+            "body": {{ file_xml2json "sample.xml" }}
+        }
+    }
+}
diff --git a/test/xml/check_response_format_xml.json b/test/xml/check_response_format_xml.json
@@ -0,0 +1,19 @@
+{
+    "name": "bounce xml file, use response format \"xml\"",
+    "request": {
+        "server_url": "{{ datastore "req_base_url" }}",
+        "endpoint": "bounce",
+        "method": "POST",
+        "body": {
+            "file": "@sample.xml"
+        },
+        "body_type": "multipart"
+    },
+    "response": {
+        "format": {
+            // uses mxj.NewMapXmlSeq() from https://github.com/clbanning/mxj
+            "type": "xml"
+        },
+        "body": {{ file "result_xml.json" }}
+    }
+}
diff --git a/test/xml/check_response_format_xml2.json b/test/xml/check_response_format_xml2.json
@@ -0,0 +1,20 @@
+{
+    "name": "bounce xml file, use response format \"xml2\"",
+    "request": {
+        "server_url": "{{ datastore "req_base_url" }}",
+        "endpoint": "bounce",
+        "method": "POST",
+        "body": {
+            "file": "@sample.xml"
+        },
+        "body_type": "multipart"
+    },
+    "response": {
+        "format": {
+            // new simpler format without #seq
+            // uses mxj.NewMapXml from https://github.com/clbanning/mxj
+            "type": "xml2"
+        },
+        "body": {{ file "result_xml2.json" }}
+    }
+}
diff --git a/test/xml/compare_exiftool_with_xml.json b/test/xml/compare_exiftool_with_xml.json
@@ -0,0 +1,25 @@
+{
+    "name": "preprocess asset berlin.jpg with exiftool in xml format, compare \"exiftool_result.xml\"",
+    "request": {
+        // load static file
+        "server_url": "{{ datastore "req_base_url" }}",
+        "endpoint": "berlin.jpg",
+        "method": "GET"
+    },
+    "response": {
+        "format": {
+            "pre_process": {
+                "cmd": {
+                    "name": "exiftool",
+                    "args": [
+                        "-X",
+                        "-l",
+                        "-"
+                    ]
+                }
+            },
+            "type": "xml2"
+        },
+        "body": {{ file_xml2json "exiftool_result.xml" }}
+    }
+}
diff --git a/test/xml/exiftool_result.xml b/test/xml/exiftool_result.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdf:RDF>
+    <rdf:Description rdf:about="-">
+        <File:FileType rdf:parseType="Resource">
+            <et:desc>File Type</et:desc>
+            <et:prt>JPEG</et:prt>
+        </File:FileType>
+        <File:FileTypeExtension rdf:parseType="Resource">
+            <et:desc>File Type Extension</et:desc>
+            <et:prt>jpg</et:prt>
+            <et:val>JPG</et:val>
+        </File:FileTypeExtension>
+        <File:MIMEType rdf:parseType="Resource">
+            <et:desc>MIME Type</et:desc>
+            <et:prt>image/jpeg</et:prt>
+        </File:MIMEType>
+        <File:ImageWidth rdf:parseType="Resource">
+            <et:desc>Image Width</et:desc>
+            <et:prt>1920</et:prt>
+        </File:ImageWidth>
+        <File:ImageHeight rdf:parseType="Resource">
+            <et:desc>Image Height</et:desc>
+            <et:prt>1280</et:prt>
+        </File:ImageHeight>
+        <JFIF:XResolution rdf:parseType="Resource">
+            <et:desc>X Resolution</et:desc>
+            <et:prt>1</et:prt>
+        </JFIF:XResolution>
+        <JFIF:YResolution rdf:parseType="Resource">
+            <et:desc>Y Resolution</et:desc>
+            <et:prt>1</et:prt>
+        </JFIF:YResolution>
+        <Composite:ImageSize rdf:parseType="Resource">
+            <et:desc>Image Size</et:desc>
+            <et:prt>1920x1280</et:prt>
+            <et:val>1920 1280</et:val>
+        </Composite:ImageSize>
+    </rdf:Description>
+</rdf:RDF>
diff --git a/test/xml/manifest.json b/test/xml/manifest.json
@@ -0,0 +1,19 @@
+{{ $local_port:=":9999"}}
+{
+    "http_server": {
+        "addr": "{{ $local_port }}",
+        "dir": ".",
+        "testmode": false
+    },
+    "store": {
+        "req_base_url": "http://localhost{{ $local_port }}"
+    },
+    "name": "XML tests",
+    "tests": [
+        "@check_local_file_against_response.json"
+        , "@check_response_against_local_file.json"
+        , "@check_response_format_xml.json"
+        , "@check_response_format_xml2.json"
+        , "@compare_exiftool_with_xml.json"
+    ]
+}