fix nested RESOURCE attribute processing

Signed-off-by: Doug Davis <duglin@gmail.com>

Makefile

@@ -181,26 +181,31 @@ prof: server qtest
 	@go tool pprof -top -cum cpu.prof | sed -n '0,/flat/p;/xreg/p' | more
 	@rm -f cpu.prof mem.prof tests.test
 
-testdev:
+devimage:
 	@# See the misc/Dockerfile-dev for more info
-	@echo "# Make sure mysql isn't running"
-	-docker rm -f mysql > /dev/null 2>&1
 	@echo
 	@echo "# Build the dev image"
 	@misc/errOutput docker build -t duglin/xreg-dev --no-cache \
 		--build-arg GIT_COMMIT=$(GIT_COMMIT) -f misc/Dockerfile-dev .
+
+testdev: devimage
+	@# See the misc/Dockerfile-dev for more info
+	@echo
+	@echo "# Make sure mysql isn't running"
+	-docker rm -f mysql > /dev/null 2>&1
 	@echo
 	@echo "## Build, test and run the server all within the dev image"
 	docker run -ti -v /var/run/docker.sock:/var/run/docker.sock \
 		-e VERIFY=--verify --network host duglin/xreg-dev make clean all
 	@echo "## Done! Exit the dev image testing"
 
 clean:
+	@echo
 	@echo "# Cleaning"
 	@rm -f cpu.prof mem.prof
-	@rm -f server xr
+	@rm -f server xr xrconform
 	@rm -f .test .image .push
 	@go clean -cache -testcache
 	@-! which k3d > /dev/null || k3d cluster delete xreg > /dev/null 2>&1
 	@-docker rm -f mysql mysql-client > /dev/null 2>&1
-	@docker system prune -f > /dev/null
+	@-docker system prune -f -a --volumes > /dev/null

README.md

@@ -116,7 +116,7 @@ TODOs:
 - add model tests for typemap - just that we can set via full model updates
 - support the DB vanishing for a while
 - create an UpdateDefaultVersion func in resource.go to move it from http logic
-- support ?export
+- support ?export  /export
 - test xref more
 - support readonly - remove resource.readonly
 - remove "readonly" attribute from model Resources, add to Resource
@@ -138,3 +138,4 @@ TODOs:
 - add "compatibility" to resources
 - multi-delete must ignore all attributes except id and epoch
 - fix init.sql, it's too slow due to latest xref stuff in commit 9c583e7
+- add support for inline=endpoints.*

misc/Dockerfile-dev

@@ -21,5 +21,8 @@ RUN apk add make git bash docker
 WORKDIR /go/src/github.com/duglin/xreg-github
 COPY . ./
 
+# Erase any built files from the COPY so we have a clean env
+RUN make clean
+
 # Pull down all of the go packages just so we don't need to do it later
 RUN go get ./...

registry/group.go

@@ -255,6 +255,7 @@ func (g *Group) UpsertResourceWithObject(rType string, id string, vID string, ob
 					fmt.Errorf("Key %q in attribute %q doesn't "+
 						"appear to be of type %q", verID, plural, singular)
 			}
+
 			_, _, err := r.UpsertVersionWithObject(verID, verObj, addType)
 			if err != nil {
 				return nil, false, err

registry/httpStuff.go

@@ -373,15 +373,32 @@ func (pw *PageWriter) Done() {
 	}
 
 	checked := ""
+	inlines := ""
+	inlineCount := 0
+
+	if pw.Info.IsInlineSet(NewPPP("model").DB()) {
+		checked = " checked"
+	}
+	if len(pw.Info.Parts) == 0 {
+		inlines += fmt.Sprintf(`
+    <div class=inlines>
+      <input id=inline%d type='checkbox' value='model'`+checked+`/>model
+    </div>`, inlineCount)
+		inlineCount++
+	}
+	checked = ""
+
 	if pw.Info.IsInlineSet("*") {
 		checked = " checked"
 	}
-	inlines := `
+	inlines += fmt.Sprintf(`
     <div class=inlines>
-      <input id=inline0 type='checkbox' value='*'` + checked + `/>* (all)
-    </div>`
+      <input id=inline%d type='checkbox' value='*'`+checked+`/>* (all - except 'model')
+    </div>`, inlineCount)
+	inlineCount++
+
 	pp, _ := PropPathFromPath(pw.Info.Abstract)
-	for i, inline := range inlineOptions {
+	for _, inline := range inlineOptions {
 		checked = ""
 		pInline := MustPropPathFromUI(inline)
 		fullInline := pp.Append(pInline).DB()
@@ -391,7 +408,8 @@ func (pw *PageWriter) Done() {
 		inlines += fmt.Sprintf(`
     <div class=inlines>
       <input id=inline%d type='checkbox' value='%s'%s/>%s
-    </div>`, i+1, inline, checked, inline)
+    </div>`, inlineCount, inline, checked, inline)
+		inlineCount++
 	}
 
 	tmp := pw.Info.BaseURL
@@ -1055,6 +1073,12 @@ func HTTPPutPost(info *RequestInfo) error {
 	if len(info.Parts) == 0 {
 		// PUT /
 
+		err = ConvertRegistryContents(IncomingObj, info.Registry.Model)
+		if err != nil {
+			info.StatusCode = http.StatusBadRequest
+			return err
+		}
+
 		addType := ADD_UPDATE
 		if method == "PATCH" {
 			addType = ADD_PATCH
@@ -1089,6 +1113,12 @@ func HTTPPutPost(info *RequestInfo) error {
 		}
 
 		for id, obj := range objMap {
+			err = ConvertGroupContents(obj, info.GroupModel)
+			if err != nil {
+				info.StatusCode = http.StatusBadRequest
+				return err
+			}
+
 			g, _, err := info.Registry.UpsertGroupWithObject(info.GroupType,
 				id, obj, addType, info.HasNested)
 			if err != nil {
@@ -1113,6 +1143,12 @@ func HTTPPutPost(info *RequestInfo) error {
 			addType = ADD_PATCH
 		}
 
+		err = ConvertGroupContents(IncomingObj, info.GroupModel)
+		if err != nil {
+			info.StatusCode = http.StatusBadRequest
+			return err
+		}
+
 		group, isNew, err := info.Registry.UpsertGroupWithObject(info.GroupType,
 			info.GroupUID, IncomingObj, addType, info.HasNested)
 		if err != nil {
@@ -1223,6 +1259,12 @@ func HTTPPutPost(info *RequestInfo) error {
 				"not %q", info.ResourceModel.Singular, resourceUID, propsID)
 		}
 
+		err = ConvertResourceContents(IncomingObj, info.ResourceModel)
+		if err != nil {
+			info.StatusCode = http.StatusBadRequest
+			return err
+		}
+
 		if resource != nil {
 			// version, err = resource.GetDefault()
 
@@ -1275,6 +1317,12 @@ func HTTPPutPost(info *RequestInfo) error {
 			}
 		}
 
+		err = ConvertVersionContents(IncomingObj, info.ResourceModel)
+		if err != nil {
+			info.StatusCode = http.StatusBadRequest
+			return err
+		}
+
 		if resource == nil {
 			// Implicitly create the resource
 			resource, isNew, err = group.UpsertResourceWithObject(
@@ -1314,7 +1362,7 @@ func HTTPPutPost(info *RequestInfo) error {
 		}
 
 		for _, obj := range objMap {
-			err = ConvertResourceContents(obj, info.ResourceModel)
+			err = ConvertVersionContents(obj, info.ResourceModel)
 			if err != nil {
 				info.StatusCode = http.StatusBadRequest
 				return err
@@ -1409,6 +1457,12 @@ func HTTPPutPost(info *RequestInfo) error {
 			}
 		}
 
+		err = ConvertVersionContents(IncomingObj, info.ResourceModel)
+		if err != nil {
+			info.StatusCode = http.StatusBadRequest
+			return err
+		}
+
 		if resource == nil {
 			// Implicitly create the resource
 			resource, err = group.AddResourceWithObject(info.ResourceType,
@@ -2068,11 +2122,6 @@ func ExtractIncomingObject(info *RequestInfo, body []byte) (Object, error) {
 			info.StatusCode = http.StatusBadRequest
 			return nil, err
 		}
-
-		err = ConvertResourceContents(IncomingObj, info.ResourceModel)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	// xReg metadata are in headers, so move them into IncomingObj. We'll
@@ -2198,13 +2247,101 @@ func ExtractIncomingObject(info *RequestInfo, body []byte) (Object, error) {
 	return IncomingObj, nil
 }
 
+// Find all Groups/Resources/Versions in 'obj' and make sure their
+// RESOURCE attribute is converted into a byte array
+func ConvertRegistryContents(obj map[string]any, m *Model) error {
+	for _, gm := range m.Groups {
+		val, ok := obj[gm.Plural]
+		if ok && !IsNil(val) {
+			gColl, ok := val.(map[string]any)
+			if !ok {
+				return fmt.Errorf("%q should be a map", gm.Plural)
+			}
+			for _, val := range gColl {
+				g, ok := val.(map[string]any)
+				if !ok {
+					return fmt.Errorf("%q should be a map of %q, not %T",
+						gm.Plural, gm.Singular, val)
+				}
+				// Traverse into each Group
+				if err := ConvertGroupContents(g, gm); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// Find all Resources/Versions in 'obj' and make sure their
+// RESOURCE attribute is converted into a byte array
+func ConvertGroupContents(obj map[string]any, gm *GroupModel) error {
+	for _, rm := range gm.Resources {
+		val, ok := obj[rm.Plural]
+		if ok && !IsNil(val) {
+			rColl, ok := val.(map[string]any)
+			if !ok {
+				return fmt.Errorf("%q should be a map", rm.Plural)
+			}
+			for _, val := range rColl {
+				r, ok := val.(map[string]any)
+				if !ok {
+					return fmt.Errorf("%s should be a map of %q, not %T",
+						rm.Plural, rm.Singular, val)
+				}
+				// Traverse into each Resource
+				if err := ConvertResourceContents(r, rm); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// Convert the RESOURCE attribute from JSON to a byte array, if present.
+// Then do any Versions in the "versions" collection, if there.
 func ConvertResourceContents(obj map[string]any, rm *ResourceModel) error {
 	if rm == nil || rm.GetHasDocument() == false {
 		return nil
 	}
 
+	// Process the local RESOURCE attribute before we look for "versions"
+	if err := ConvertVersionContents(obj, rm); err != nil {
+		return err
+	}
+
+	val, ok := obj["versions"]
+	if ok && !IsNil(val) {
+		vColl, ok := val.(map[string]any)
+		if !ok {
+			return fmt.Errorf("%q should be a map", "versions")
+		}
+		for _, val := range vColl {
+			v, ok := val.(map[string]any)
+			if !ok {
+				return fmt.Errorf("%q should be a map of %q, not %T",
+					"versions", "version", val)
+			}
+			// Traverse into each Version
+			if err := ConvertVersionContents(v, rm); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Convert the RESOURCE attribute from JSON to a byte array, if present.
+func ConvertVersionContents(obj map[string]any, rm *ResourceModel) error {
+	if rm == nil || rm.GetHasDocument() == false {
+		return nil
+	}
+
 	var err error
 	data, ok := obj[rm.Singular]
+
 	if ok {
 
 		// Special case of: RESOURCE:null
@@ -2214,6 +2351,11 @@ func ConvertResourceContents(obj map[string]any, rm *ResourceModel) error {
 			return nil
 		}
 
+		// Keep bytes as bytes - possibly already converted (eg was in body)
+		if reflect.ValueOf(data).Type().String() == "[]uint8" {
+			return nil
+		}
+
 		// Get the raw bytes of the "rm.Singular" json attribute
 		buf := []byte(nil)
 		switch reflect.ValueOf(data).Kind() {