From 31ed123abd640d3102f9bc157f8ebeba9480b1a2 Mon Sep 17 00:00:00 2001 From: quobix Date: Tue, 30 Jul 2024 11:18:11 -0400 Subject: [PATCH 01/13] Added a back tick for consistent rendering --- index/extract_refs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index/extract_refs.go b/index/extract_refs.go index 708eafb5..b49c9a97 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -12,8 +12,8 @@ import ( "strings" "github.com/pb33f/libopenapi/utils" - "slices" "gopkg.in/yaml.v3" + "slices" ) // ExtractRefs will return a deduplicated slice of references for every unique ref found in the document. @@ -662,7 +662,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) indexError := &IndexingError{ - Err: fmt.Errorf("component '%s' does not exist in the specification", ref.Definition), + Err: fmt.Errorf("component `%s` does not exist in the specification", ref.Definition), Node: ref.Node, Path: path, KeyNode: ref.KeyNode, From 70f406b6cff5d98756da54c9ff8d0fa2eb6b5537 Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 11:01:23 -0400 Subject: [PATCH 02/13] Added `NodeMap` to all low level objects. this sync map tracks all the nodes that apply to this object. --- bundler/bundler_test.go | 2 +- datamodel/high/base/schema_proxy.go | 8 ++ datamodel/high/node_builder.go | 2 +- datamodel/high/v3/document.go | 5 + datamodel/high/v3/document_test.go | 1 + datamodel/low/base/contact.go | 6 +- datamodel/low/base/context.go | 30 +++++ datamodel/low/base/discriminator.go | 14 ++ datamodel/low/base/example.go | 21 ++- datamodel/low/base/external_doc.go | 4 +- datamodel/low/base/info.go | 2 + datamodel/low/base/license.go | 3 + datamodel/low/base/schema.go | 80 ++++++++++- datamodel/low/base/schema_proxy.go | 21 +++ datamodel/low/base/security_requirement.go | 7 +- datamodel/low/base/tag.go | 3 + datamodel/low/base/xml.go | 2 + datamodel/low/node_map.go | 148 +++++++++++++++++++++ datamodel/low/node_map_test.go | 111 ++++++++++++++++ datamodel/low/v3/callback.go | 7 +- datamodel/low/v3/components.go | 44 ++++-- datamodel/low/v3/create_document.go | 12 +- datamodel/low/v3/document.go | 2 + datamodel/low/v3/encoding.go | 6 + datamodel/low/v3/header.go | 14 +- datamodel/low/v3/link.go | 11 ++ datamodel/low/v3/media_type.go | 18 +++ datamodel/low/v3/oauth_flows.go | 14 +- datamodel/low/v3/operation.go | 22 +++ datamodel/low/v3/parameter.go | 15 +++ datamodel/low/v3/path_item.go | 6 +- datamodel/low/v3/paths.go | 3 + datamodel/low/v3/request_body.go | 7 + datamodel/low/v3/response.go | 15 +++ datamodel/low/v3/responses.go | 8 ++ datamodel/low/v3/security_scheme.go | 3 + datamodel/low/v3/server.go | 18 ++- datamodel/low/v3/server_variable.go | 14 ++ datamodel/spec_info.go | 6 +- document_test.go | 2 +- go.mod | 1 + index/find_component_test.go | 2 +- index/rolodex.go | 5 + index/rolodex_test.go | 1 + utils/utils.go | 26 +++- utils/utils_test.go | 4 +- 46 files changed, 712 insertions(+), 44 deletions(-) create mode 100644 datamodel/low/base/context.go create mode 100644 datamodel/low/node_map.go create mode 100644 datamodel/low/node_map_test.go diff --git a/bundler/bundler_test.go b/bundler/bundler_test.go index 7ebbdf5f..c2b366ec 100644 --- a/bundler/bundler_test.go +++ b/bundler/bundler_test.go @@ -150,7 +150,7 @@ components: assert.ErrorIs(t, unwrap[0], ErrInvalidModel) unwrapNext := utils.UnwrapErrors(unwrap[1]) require.Len(t, unwrapNext, 2) - assert.Equal(t, "component 'bork' does not exist in the specification", unwrapNext[0].Error()) + assert.Equal(t, "component `bork` does not exist in the specification", unwrapNext[0].Error()) assert.Equal(t, "cannot resolve reference `bork`, it's missing: $bork [5:7]", unwrapNext[1].Error()) logEntries := strings.Split(byteBuf.String(), "\n") diff --git a/datamodel/high/base/schema_proxy.go b/datamodel/high/base/schema_proxy.go index 24874e03..188e7347 100644 --- a/datamodel/high/base/schema_proxy.go +++ b/datamodel/high/base/schema_proxy.go @@ -72,6 +72,14 @@ func CreateSchemaProxyRef(ref string) *SchemaProxy { return &SchemaProxy{refStr: ref, lock: &sync.Mutex{}} } +// GetValueNode returns the value node of the SchemaProxy. +func (sp *SchemaProxy) GetValueNode() *yaml.Node { + if sp.schema != nil { + return sp.schema.ValueNode + } + return nil +} + // Schema will create a new Schema instance using NewSchema from the low-level SchemaProxy backing this high-level one. // If there is a problem building the Schema, then this method will return nil. Use GetBuildError to gain access // to that building error. diff --git a/datamodel/high/node_builder.go b/datamodel/high/node_builder.go index a3d1f0ee..25b938ee 100644 --- a/datamodel/high/node_builder.go +++ b/datamodel/high/node_builder.go @@ -443,7 +443,7 @@ func (n *NodeBuilder) AddYAMLNode(parent *yaml.Node, entry *nodes.NodeEntry) *ya lr := lut.(low.IsReferenced) ut := reflect.ValueOf(lr) if !ut.IsNil() { - if lut.(low.IsReferenced).IsReference() { + if lr != nil && lr.IsReference() { if !n.Resolve { valueNode = n.renderReference(lut.(low.IsReferenced)) break diff --git a/datamodel/high/v3/document.go b/datamodel/high/v3/document.go index 1a049504..df6337f7 100644 --- a/datamodel/high/v3/document.go +++ b/datamodel/high/v3/document.go @@ -155,6 +155,11 @@ func (d *Document) GoLow() *low.Document { return d.low } +// GoLowUntyped returns the low-level Document that was used to create the high level one, however, it's untyped. +func (d *Document) GoLowUntyped() any { + return d.low +} + // Render will return a YAML representation of the Document object as a byte slice. func (d *Document) Render() ([]byte, error) { return yaml.Marshal(d) diff --git a/datamodel/high/v3/document_test.go b/datamodel/high/v3/document_test.go index f29ee57b..ea72ff40 100644 --- a/datamodel/high/v3/document_test.go +++ b/datamodel/high/v3/document_test.go @@ -87,6 +87,7 @@ func TestNewDocument_Info(t *testing.T) { assert.Equal(t, "1.2", highDoc.Info.Version) assert.Equal(t, "https://pb33f.io/schema", highDoc.JsonSchemaDialect) + assert.NotNil(t, highDoc.GoLowUntyped()) wentLow := highDoc.GoLow() assert.Equal(t, 1, wentLow.Version.ValueNode.Line) assert.Equal(t, 3, wentLow.Info.Value.Title.KeyNode.Line) diff --git a/datamodel/low/base/contact.go b/datamodel/low/base/contact.go index be08f1a1..2f044db1 100644 --- a/datamodel/low/base/contact.go +++ b/datamodel/low/base/contact.go @@ -23,14 +23,14 @@ type Contact struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } -// Build is not implemented for Contact (there is nothing to build). -func (c *Contact) Build(_ context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { +func (c *Contact) Build(ctx context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { c.KeyNode = keyNode c.RootNode = root c.Reference = new(low.Reference) - // not implemented. + c.Nodes = low.ExtractNodes(ctx, root) return nil } diff --git a/datamodel/low/base/context.go b/datamodel/low/base/context.go new file mode 100644 index 00000000..f8981169 --- /dev/null +++ b/datamodel/low/base/context.go @@ -0,0 +1,30 @@ +// Copyright 2023-2024 Princess Beef Heavy Industries, LLC / Dave Shanley +// https://pb33f.io + +package base + +import ( + "golang.org/x/net/context" + "sync" +) + +// ModelContext is a struct that holds various persistent data structures for the model +// that passes through the entire model building process. +type ModelContext struct { + SchemaCache *sync.Map +} + +// GetModelContext will return the ModelContext from a context.Context object +// if it is available, otherwise it will return nil. +func GetModelContext(ctx context.Context) *ModelContext { + if ctx == nil { + return nil + } + if ctx.Value("modelCtx") == nil { + return nil + } + if c, ok := ctx.Value("modelCtx").(*ModelContext); ok { + return c + } + return nil +} diff --git a/datamodel/low/base/discriminator.go b/datamodel/low/base/discriminator.go index 533a2db8..082b2cee 100644 --- a/datamodel/low/base/discriminator.go +++ b/datamodel/low/base/discriminator.go @@ -5,6 +5,7 @@ package base import ( "crypto/sha256" + "gopkg.in/yaml.v3" "strings" "github.com/pb33f/libopenapi/datamodel/low" @@ -23,7 +24,20 @@ import ( type Discriminator struct { PropertyName low.NodeReference[string] Mapping low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[string]]] + KeyNode *yaml.Node + RootNode *yaml.Node low.Reference + low.NodeMap +} + +// GetRootNode will return the root yaml node of the Discriminator object +func (d *Discriminator) GetRootNode() *yaml.Node { + return d.RootNode +} + +// GetKeyNode will return the key yaml node of the Discriminator object +func (d *Discriminator) GetKeyNode() *yaml.Node { + return d.KeyNode } // FindMappingValue will return a ValueReference containing the string mapping value diff --git a/datamodel/low/base/example.go b/datamodel/low/base/example.go index 5efcfc9e..a84ccb9b 100644 --- a/datamodel/low/base/example.go +++ b/datamodel/low/base/example.go @@ -28,6 +28,7 @@ type Example struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindExtension returns a ValueReference containing the extension value, if found. @@ -67,12 +68,13 @@ func (ex *Example) Hash() [32]byte { } // Build extracts extensions and example value -func (ex *Example) Build(_ context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { +func (ex *Example) Build(ctx context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { ex.KeyNode = keyNode root = utils.NodeAlias(root) ex.RootNode = root utils.CheckForMergeNodes(root) ex.Reference = new(low.Reference) + ex.Nodes = low.ExtractNodes(ctx, root) ex.Extensions = low.ExtractExtensions(root) _, ln, vn := utils.FindKeyNodeFull(ValueLabel, root.Content) @@ -82,6 +84,23 @@ func (ex *Example) Build(_ context.Context, keyNode, root *yaml.Node, _ *index.S KeyNode: ln, ValueNode: vn, } + + // extract nodes for all value nodes down the tree. + expChildNodes := low.ExtractNodesRecursive(ctx, vn) + expChildNodes.Range(func(k, v interface{}) bool { + if arr, ko := v.([]*yaml.Node); ko { + if ext, ok := ex.Nodes.Load(k); ok { + if extArr, kk := ext.([]*yaml.Node); kk { + ex.Nodes.Store(k, append(extArr, arr...)) + } else { + ex.Nodes.Store(k, arr) + } + } else { + ex.Nodes.Store(k, arr) + } + } + return true + }) return nil } return nil diff --git a/datamodel/low/base/external_doc.go b/datamodel/low/base/external_doc.go index 615d71ea..d1fba4b2 100644 --- a/datamodel/low/base/external_doc.go +++ b/datamodel/low/base/external_doc.go @@ -28,6 +28,7 @@ type ExternalDoc struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindExtension returns a ValueReference containing the extension value, if found. @@ -46,12 +47,13 @@ func (ex *ExternalDoc) GetKeyNode() *yaml.Node { } // Build will extract extensions from the ExternalDoc instance. -func (ex *ExternalDoc) Build(_ context.Context, keyNode, root *yaml.Node, idx *index.SpecIndex) error { +func (ex *ExternalDoc) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index.SpecIndex) error { ex.KeyNode = keyNode root = utils.NodeAlias(root) ex.RootNode = root utils.CheckForMergeNodes(root) ex.Reference = new(low.Reference) + ex.Nodes = low.ExtractNodes(ctx, root) ex.Extensions = low.ExtractExtensions(root) return nil } diff --git a/datamodel/low/base/info.go b/datamodel/low/base/info.go index ccfe46f7..cc4715b2 100644 --- a/datamodel/low/base/info.go +++ b/datamodel/low/base/info.go @@ -35,6 +35,7 @@ type Info struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindExtension attempts to locate an extension with the supplied key @@ -64,6 +65,7 @@ func (i *Info) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index.S i.RootNode = root utils.CheckForMergeNodes(root) i.Reference = new(low.Reference) + i.Nodes = low.ExtractNodes(ctx, root) i.Extensions = low.ExtractExtensions(root) // extract contact diff --git a/datamodel/low/base/license.go b/datamodel/low/base/license.go index c50bd7cb..0499c52d 100644 --- a/datamodel/low/base/license.go +++ b/datamodel/low/base/license.go @@ -25,6 +25,7 @@ type License struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // Build out a license, complain if both a URL and identifier are present as they are mutually exclusive @@ -34,6 +35,8 @@ func (l *License) Build(ctx context.Context, keyNode, root *yaml.Node, idx *inde l.RootNode = root utils.CheckForMergeNodes(root) l.Reference = new(low.Reference) + no := low.ExtractNodes(ctx, root) + l.Nodes = no if l.URL.Value != "" && l.Identifier.Value != "" { return fmt.Errorf("license cannot have both a URL and an identifier, they are mutually exclusive") } diff --git a/datamodel/low/base/schema.go b/datamodel/low/base/schema.go index df006596..9b4c21b8 100644 --- a/datamodel/low/base/schema.go +++ b/datamodel/low/base/schema.go @@ -144,6 +144,7 @@ type Schema struct { Index *index.SpecIndex RootNode *yaml.Node *low.Reference + low.NodeMap } // Hash will calculate a SHA256 hash from the values of the schema, This allows equality checking against @@ -470,8 +471,11 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) s.Reference = new(low.Reference) + no := low.ExtractNodes(ctx, root) + s.Nodes = no s.Index = idx s.RootNode = root + if h, _, _ := utils.IsNodeRefValue(root); h { ref, _, err, fctx := low.LocateRefNodeWithContext(ctx, root, idx) if ref != nil { @@ -497,6 +501,20 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde s.extractExtensions(root) + // if the schema has required values, extract the nodes for them. + if s.Required.Value != nil { + for _, r := range s.Required.Value { + s.AddNode(r.ValueNode.Line, r.ValueNode) + } + } + + // same thing with enums + if s.Enum.Value != nil { + for _, e := range s.Enum.Value { + s.AddNode(e.ValueNode.Line, e.ValueNode) + } + } + // determine schema type, singular (3.0) or multiple (3.1), use a variable value _, typeLabel, typeValue := utils.FindKeyNodeFullTop(TypeLabel, root.Content) if typeValue != nil { @@ -630,6 +648,24 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde _, expLabel, expNode := utils.FindKeyNodeFullTop(ExampleLabel, root.Content) if expNode != nil { s.Example = low.NodeReference[*yaml.Node]{Value: expNode, KeyNode: expLabel, ValueNode: expNode} + + // extract nodes for all value nodes down the tree. + expChildNodes := low.ExtractNodesRecursive(ctx, expNode) + // map to the local schema + expChildNodes.Range(func(k, v interface{}) bool { + if arr, ko := v.([]*yaml.Node); ko { + if ext, ok := s.Nodes.Load(k); ok { + if extArr, kk := ext.([]*yaml.Node); kk { + s.Nodes.Store(k, append(extArr, arr...)) + } else { + s.Nodes.Store(k, arr) + } + } else { + s.Nodes.Store(k, arr) + } + } + return true + }) } // handle examples if set.(3.1) @@ -645,6 +681,23 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde ValueNode: expArrNode, KeyNode: expArrLabel, } + // extract nodes for all value nodes down the tree. + expChildNodes := low.ExtractNodesRecursive(ctx, expArrNode) + // map to the local schema + expChildNodes.Range(func(k, v interface{}) bool { + if arr, ko := v.([]*yaml.Node); ko { + if ext, ok := s.Nodes.Load(k); ok { + if extArr, kk := ext.([]*yaml.Node); kk { + s.Nodes.Store(k, append(extArr, arr...)) + } else { + s.Nodes.Store(k, arr) + } + } else { + s.Nodes.Store(k, arr) + } + } + return true + }) } } @@ -674,7 +727,23 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde if discNode != nil { var discriminator Discriminator _ = low.BuildModel(discNode, &discriminator) + discriminator.KeyNode = discLabel + discriminator.RootNode = discNode + discriminator.Nodes = low.ExtractNodes(ctx, discNode) s.Discriminator = low.NodeReference[*Discriminator]{Value: &discriminator, KeyNode: discLabel, ValueNode: discNode} + // add discriminator nodes, because there is no build method. + dn := low.ExtractNodesRecursive(ctx, discNode) + dn.Range(func(key, val any) bool { + if n, ok := val.([]*yaml.Node); ok { + for _, g := range n { + discriminator.AddNode(key.(int), g) + } + } + if n, ok := val.(*yaml.Node); ok { + discriminator.AddNode(key.(int), n) + } + return true + }) } // handle externalDocs if set. @@ -683,6 +752,7 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde var exDoc ExternalDoc _ = low.BuildModel(extDocNode, &exDoc) _ = exDoc.Build(ctx, extDocLabel, extDocNode, idx) // throws no errors, can't check for one. + exDoc.Nodes = low.ExtractNodes(ctx, extDocNode) s.ExternalDocs = low.NodeReference[*ExternalDoc]{Value: &exDoc, KeyNode: extDocLabel, ValueNode: extDocNode} } @@ -693,11 +763,12 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde _ = low.BuildModel(xmlNode, &xml) // extract extensions if set. _ = xml.Build(xmlNode, idx) // returns no errors, can't check for one. + xml.Nodes = low.ExtractNodes(ctx, xmlNode) s.XML = low.NodeReference[*XML]{Value: &xml, KeyNode: xmlLabel, ValueNode: xmlNode} } // handle properties - props, err := buildPropertyMap(ctx, root, idx, PropertiesLabel) + props, err := buildPropertyMap(ctx, s, root, idx, PropertiesLabel) if err != nil { return err } @@ -706,7 +777,7 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde } // handle dependent schemas - props, err = buildPropertyMap(ctx, root, idx, DependentSchemasLabel) + props, err = buildPropertyMap(ctx, s, root, idx, DependentSchemasLabel) if err != nil { return err } @@ -715,7 +786,7 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde } // handle pattern properties - props, err = buildPropertyMap(ctx, root, idx, PatternPropertiesLabel) + props, err = buildPropertyMap(ctx, s, root, idx, PatternPropertiesLabel) if err != nil { return err } @@ -1013,7 +1084,7 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde return nil } -func buildPropertyMap(ctx context.Context, root *yaml.Node, idx *index.SpecIndex, label string) (*low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[*SchemaProxy]]], error) { +func buildPropertyMap(ctx context.Context, parent *Schema, root *yaml.Node, idx *index.SpecIndex, label string) (*low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[*SchemaProxy]]], error) { _, propLabel, propsNode := utils.FindKeyNodeFullTop(label, root.Content) if propsNode != nil { propertyMap := orderedmap.New[low.KeyReference[string], low.ValueReference[*SchemaProxy]]() @@ -1021,6 +1092,7 @@ func buildPropertyMap(ctx context.Context, root *yaml.Node, idx *index.SpecIndex for i, prop := range propsNode.Content { if i%2 == 0 { currentProp = prop + parent.Nodes.Store(prop.Line, prop) continue } diff --git a/datamodel/low/base/schema_proxy.go b/datamodel/low/base/schema_proxy.go index 9adeeb6c..54e5f54c 100644 --- a/datamodel/low/base/schema_proxy.go +++ b/datamodel/low/base/schema_proxy.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "log/slog" + "sync" "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/index" @@ -54,6 +55,7 @@ type SchemaProxy struct { rendered *Schema buildError error ctx context.Context + *low.NodeMap } // Build will prepare the SchemaProxy for rendering, it does not build the Schema, only sets up internal state. @@ -66,6 +68,8 @@ func (sp *SchemaProxy) Build(ctx context.Context, key, value *yaml.Node, idx *in if rf, _, r := utils.IsNodeRefValue(value); rf { sp.SetReference(r, value) } + var m sync.Map + sp.NodeMap = &low.NodeMap{Nodes: &m} return nil } @@ -93,6 +97,14 @@ func (sp *SchemaProxy) Schema() *Schema { } schema.ParentProxy = sp // https://github.com/pb33f/libopenapi/issues/29 sp.rendered = schema + + // for all the nodes added, copy them over to the schema + if sp.NodeMap != nil { + sp.NodeMap.Nodes.Range(func(key, value any) bool { + schema.AddNode(key.(int), value.(*yaml.Node)) + return true + }) + } return schema } @@ -158,3 +170,12 @@ func (sp *SchemaProxy) Hash() [32]byte { // hash reference value only, do not resolve! return sha256.Sum256([]byte(sp.GetReference())) } + +// AddNode stores nodes in the underlying schema if rendered, otherwise holds in the proxy until build. +func (sp *SchemaProxy) AddNode(key int, node *yaml.Node) { + if sp.rendered != nil { + sp.rendered.AddNode(key, node) + } else { + sp.Nodes.Store(key, node) + } +} diff --git a/datamodel/low/base/security_requirement.go b/datamodel/low/base/security_requirement.go index 9d4de32f..c87beb98 100644 --- a/datamodel/low/base/security_requirement.go +++ b/datamodel/low/base/security_requirement.go @@ -31,15 +31,18 @@ type SecurityRequirement struct { RootNode *yaml.Node ContainsEmptyRequirement bool // if a requirement is empty (this means it's optional) *low.Reference + low.NodeMap } // Build will extract security requirements from the node (the structure is odd, to be honest) -func (s *SecurityRequirement) Build(_ context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { +func (s *SecurityRequirement) Build(ctx context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { + s.KeyNode = keyNode root = utils.NodeAlias(root) s.RootNode = root utils.CheckForMergeNodes(root) s.Reference = new(low.Reference) + s.Nodes = low.ExtractNodes(ctx, root) var labelNode *yaml.Node valueMap := orderedmap.New[low.KeyReference[string], low.ValueReference[[]low.ValueReference[string]]]() var arr []low.ValueReference[string] @@ -57,6 +60,7 @@ func (s *SecurityRequirement) Build(_ context.Context, keyNode, root *yaml.Node, Value: root.Content[i].Content[j].Value, ValueNode: root.Content[i].Content[j], }) + s.Nodes.Store(root.Content[i].Content[j].Line, root.Content[i].Content[j]) } valueMap.Set( low.KeyReference[string]{ @@ -76,6 +80,7 @@ func (s *SecurityRequirement) Build(_ context.Context, keyNode, root *yaml.Node, Value: valueMap, ValueNode: root, } + return nil } diff --git a/datamodel/low/base/tag.go b/datamodel/low/base/tag.go index b0642404..a86842f0 100644 --- a/datamodel/low/base/tag.go +++ b/datamodel/low/base/tag.go @@ -29,6 +29,7 @@ type Tag struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindExtension returns a ValueReference containing the extension value, if found. @@ -53,7 +54,9 @@ func (t *Tag) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index.Sp t.RootNode = root utils.CheckForMergeNodes(root) t.Reference = new(low.Reference) + t.Nodes = low.ExtractNodes(ctx, root) t.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, t.Extensions, t.Nodes) // extract externalDocs extDocs, err := low.ExtractObject[*ExternalDoc](ctx, ExternalDocsLabel, root, idx) diff --git a/datamodel/low/base/xml.go b/datamodel/low/base/xml.go index e7c75672..1b7ed874 100644 --- a/datamodel/low/base/xml.go +++ b/datamodel/low/base/xml.go @@ -30,6 +30,7 @@ type XML struct { Extensions *orderedmap.Map[low.KeyReference[string], low.ValueReference[*yaml.Node]] RootNode *yaml.Node *low.Reference + low.NodeMap } // Build will extract extensions from the XML instance. @@ -38,6 +39,7 @@ func (x *XML) Build(root *yaml.Node, _ *index.SpecIndex) error { utils.CheckForMergeNodes(root) x.RootNode = root x.Reference = new(low.Reference) + x.Nodes = low.ExtractNodes(nil, root) x.Extensions = low.ExtractExtensions(root) return nil } diff --git a/datamodel/low/node_map.go b/datamodel/low/node_map.go new file mode 100644 index 00000000..ac5111a4 --- /dev/null +++ b/datamodel/low/node_map.go @@ -0,0 +1,148 @@ +// Copyright 2023-2024 Princess Beef Heavy Industries, LLC / Dave Shanley +// https://pb33f.io +// MIT License + +package low + +import ( + "context" + "github.com/pb33f/libopenapi/orderedmap" + "gopkg.in/yaml.v3" + "sync" +) + +// HasNodes is an interface that defines a method to get a map of nodes +type HasNodes interface { + GetNodes() map[int][]*yaml.Node +} + +// AddNodes is an interface that defined a method to add nodes. +type AddNodes interface { + AddNode(key int, node *yaml.Node) +} + +// NodeMap represents a map of yaml nodes +type NodeMap struct { + + // Nodes is a sync map of nodes for this object, and the key is the line number of the node + // a line can contain many nodes (in JSON), so the value is a slice of *yaml.Node + Nodes *sync.Map `yaml:"-" json:"-"` +} + +// AddNode will add a node to the NodeMap +func (nm *NodeMap) AddNode(key int, node *yaml.Node) { + if existing, ok := nm.Nodes.Load(key); ok { + if ext, ko := existing.(*yaml.Node); ko { + nm.Nodes.Store(key, []*yaml.Node{ext, node}) + } + if ext, ko := existing.([]*yaml.Node); ko { + ext = append(ext, node) + nm.Nodes.Store(key, ext) + } + } else { + nm.Nodes.Store(key, []*yaml.Node{node}) + } +} + +// GetNodes will return the map of nodes +func (nm *NodeMap) GetNodes() map[int][]*yaml.Node { + composed := make(map[int][]*yaml.Node) + nm.Nodes.Range(func(key, value interface{}) bool { + if v, ok := value.([]*yaml.Node); ok { + composed[key.(int)] = v + } + if v, ok := value.(*yaml.Node); ok { + composed[key.(int)] = []*yaml.Node{v} + } + + return true + }) + if len(composed) <= 0 { + composed[0] = []*yaml.Node{} // return an empty slice if there are no nodes + } + return composed +} + +// ExtractNodes will iterate over a *yaml.Node and extract all nodes with a line number into a map +func (nm *NodeMap) ExtractNodes(node *yaml.Node, recurse bool) { + if node == nil { + return + } + // if the node has content, iterate over it and extract every top level line number + if node.Content != nil { + for i := 0; i < len(node.Content); i++ { + if node.Content[i].Line != 0 && len(node.Content[i].Content) <= 0 { + nm.AddNode(node.Content[i].Line, node.Content[i]) + } + if node.Content[i].Line != 0 && len(node.Content[i].Content) > 0 { + if recurse { + nm.AddNode(node.Content[i].Line, node.Content[i]) + nm.ExtractNodes(node.Content[i], recurse) + } + } + } + } +} + +// ContainsLine will return true if the NodeMap contains a node with the supplied line number +func (nm *NodeMap) ContainsLine(line int, recurse bool) bool { + if _, ok := nm.Nodes.Load(line); ok { + return true + } + return false +} + +// ExtractNodes will extract all nodes from a yaml.Node and return them in a map +func ExtractNodes(_ context.Context, root *yaml.Node) *sync.Map { + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + nm.ExtractNodes(root, false) + return nm.Nodes +} + +// ExtractNodesRecursive will extract all nodes from a yaml.Node and return them in a map, just like ExtractNodes +// however, this version will dive-down the tree and extract all nodes from all child nodes as well until the tree +// is done. +func ExtractNodesRecursive(_ context.Context, root *yaml.Node) *sync.Map { + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + nm.ExtractNodes(root, true) + return nm.Nodes +} + +// ExtractExtensionNodes will extract all extension nodes from a map of extensions, recursively. +func ExtractExtensionNodes(_ context.Context, + extensionMap *orderedmap.Map[KeyReference[string], + ValueReference[*yaml.Node]], nodeMap *sync.Map) { + + // range over the extension map and extract all nodes + for extPairs := extensionMap.First(); extPairs != nil; extPairs = extPairs.Next() { + k := extPairs.Key() + v := extPairs.Value() + + results := []*yaml.Node{k.KeyNode} + var newNodeMap sync.Map + nm := &NodeMap{Nodes: &newNodeMap} + if len(v.ValueNode.Content) > 0 { + nm.ExtractNodes(v.ValueNode, true) + nm.Nodes.Range(func(key, value interface{}) bool { + for _, n := range value.([]*yaml.Node) { + results = append(results, n) + } + return true + }) + } else { + results = append(results, v.ValueNode) + } + if k.KeyNode.Line == v.ValueNode.Line { + nodeMap.Store(k.KeyNode.Line, results) + } else { + nodeMap.Store(k.KeyNode.Line, results[0]) + gingo := results[1:] + for _, y := range gingo { + nodeMap.Store(y.Line, []*yaml.Node{y}) + } + //nodeMap.Store(v.ValueNode.Line, gingo) + } + } +} diff --git a/datamodel/low/node_map_test.go b/datamodel/low/node_map_test.go new file mode 100644 index 00000000..bdca201c --- /dev/null +++ b/datamodel/low/node_map_test.go @@ -0,0 +1,111 @@ +// Copyright 2023-2024 Princess Beef Heavy Industries, LLC / Dave Shanley +// https://pb33f.io + +package low + +import ( + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "sync" + "testing" +) + +func Test_NodeMapExtractNodes(t *testing.T) { + + yml := `one: hello +two: there +three: nice one +four: + shoes: yes + socks: of course +` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + nm.ExtractNodes(root.Content[0], false) + testTheThing(t, nm) + +} + +func testTheThing(t *testing.T, nm *NodeMap) { + count := 0 + nm.Nodes.Range(func(key, value interface{}) bool { + count++ + return true + }) + + assert.Equal(t, 4, count) + + nodes := nm.GetNodes() + + assert.Equal(t, 2, len(nodes[1])) + assert.Equal(t, 2, len(nodes[2])) + assert.Equal(t, 2, len(nodes[3])) + assert.Equal(t, 1, len(nodes[4])) + assert.Equal(t, "one", nodes[1][0].Value) + assert.Equal(t, "hello", nodes[1][1].Value) + assert.Equal(t, "two", nodes[2][0].Value) + assert.Equal(t, "there", nodes[2][1].Value) + assert.Equal(t, "three", nodes[3][0].Value) + assert.Equal(t, "nice one", nodes[3][1].Value) + assert.Equal(t, "four", nodes[4][0].Value) +} + +func testTheThingUnmarshalled(t *testing.T, nm *sync.Map) { + n := &NodeMap{Nodes: nm} + nodes := n.GetNodes() + + assert.Equal(t, 2, len(nodes[1])) + assert.Equal(t, 2, len(nodes[2])) + assert.Equal(t, 2, len(nodes[3])) + assert.Equal(t, 1, len(nodes[4])) + assert.Equal(t, "one", nodes[1][0].Value) + assert.Equal(t, "hello", nodes[1][1].Value) + assert.Equal(t, "two", nodes[2][0].Value) + assert.Equal(t, "there", nodes[2][1].Value) + assert.Equal(t, "three", nodes[3][0].Value) + assert.Equal(t, "nice one", nodes[3][1].Value) + assert.Equal(t, "four", nodes[4][0].Value) +} + +func TestExtractNodes(t *testing.T) { + + yml := `one: hello +two: there +three: nice one +four: + shoes: yes + socks: of course +` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + + nm := ExtractNodes(nil, root.Content[0]) + + count := 0 + nm.Range(func(key, value interface{}) bool { + count++ + return true + }) + + assert.Equal(t, 4, count) + testTheThingUnmarshalled(t, nm) + +} + +func TestExtractNodes_Nil(t *testing.T) { + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + nm.ExtractNodes(nil, false) + + count := 0 + nm.Nodes.Range(func(key, value interface{}) bool { + count++ + return true + }) + + assert.Equal(t, 0, count) +} diff --git a/datamodel/low/v3/callback.go b/datamodel/low/v3/callback.go index 51f91d03..6ccc8c54 100644 --- a/datamodel/low/v3/callback.go +++ b/datamodel/low/v3/callback.go @@ -29,6 +29,7 @@ type Callback struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetExtensions returns all Callback extensions and satisfies the low.HasExtensions interface. @@ -58,14 +59,18 @@ func (cb *Callback) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in cb.RootNode = root utils.CheckForMergeNodes(root) cb.Reference = new(low.Reference) + cb.Nodes = low.ExtractNodes(ctx, root) cb.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, cb.Extensions, cb.Nodes) expressions, err := extractPathItemsMap(ctx, root, idx) if err != nil { return err } cb.Expression = expressions - + for xp := expressions.First(); xp != nil; xp = xp.Next() { + cb.Nodes.Store(xp.Key().KeyNode.Line, xp.Key().KeyNode) + } return nil } diff --git a/datamodel/low/v3/components.go b/datamodel/low/v3/components.go index 76489349..75a7c424 100644 --- a/datamodel/low/v3/components.go +++ b/datamodel/low/v3/components.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "fmt" + "reflect" "strings" "sync" @@ -38,6 +39,7 @@ type Components struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } type componentBuildResult[T any] struct { @@ -139,7 +141,9 @@ func (co *Components) Build(ctx context.Context, root *yaml.Node, idx *index.Spe root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) co.Reference = new(low.Reference) + co.Nodes = low.ExtractNodes(ctx, root) co.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, co.Extensions, co.Nodes) co.RootNode = root co.KeyNode = root var reterr error @@ -156,55 +160,55 @@ func (co *Components) Build(ctx context.Context, root *yaml.Node, idx *index.Spe } go func() { - schemas, err := extractComponentValues[*base.SchemaProxy](ctx, SchemasLabel, root, idx) + schemas, err := extractComponentValues[*base.SchemaProxy](ctx, SchemasLabel, root, idx, co) captureError(err) co.Schemas = schemas wg.Done() }() go func() { - parameters, err := extractComponentValues[*Parameter](ctx, ParametersLabel, root, idx) + parameters, err := extractComponentValues[*Parameter](ctx, ParametersLabel, root, idx, co) captureError(err) co.Parameters = parameters wg.Done() }() go func() { - responses, err := extractComponentValues[*Response](ctx, ResponsesLabel, root, idx) + responses, err := extractComponentValues[*Response](ctx, ResponsesLabel, root, idx, co) captureError(err) co.Responses = responses wg.Done() }() go func() { - examples, err := extractComponentValues[*base.Example](ctx, base.ExamplesLabel, root, idx) + examples, err := extractComponentValues[*base.Example](ctx, base.ExamplesLabel, root, idx, co) captureError(err) co.Examples = examples wg.Done() }() go func() { - requestBodies, err := extractComponentValues[*RequestBody](ctx, RequestBodiesLabel, root, idx) + requestBodies, err := extractComponentValues[*RequestBody](ctx, RequestBodiesLabel, root, idx, co) captureError(err) co.RequestBodies = requestBodies wg.Done() }() go func() { - headers, err := extractComponentValues[*Header](ctx, HeadersLabel, root, idx) + headers, err := extractComponentValues[*Header](ctx, HeadersLabel, root, idx, co) captureError(err) co.Headers = headers wg.Done() }() go func() { - securitySchemes, err := extractComponentValues[*SecurityScheme](ctx, SecuritySchemesLabel, root, idx) + securitySchemes, err := extractComponentValues[*SecurityScheme](ctx, SecuritySchemesLabel, root, idx, co) captureError(err) co.SecuritySchemes = securitySchemes wg.Done() }() go func() { - links, err := extractComponentValues[*Link](ctx, LinksLabel, root, idx) + links, err := extractComponentValues[*Link](ctx, LinksLabel, root, idx, co) captureError(err) co.Links = links wg.Done() }() go func() { - callbacks, err := extractComponentValues[*Callback](ctx, CallbacksLabel, root, idx) + callbacks, err := extractComponentValues[*Callback](ctx, CallbacksLabel, root, idx, co) captureError(err) co.Callbacks = callbacks wg.Done() @@ -217,12 +221,13 @@ func (co *Components) Build(ctx context.Context, root *yaml.Node, idx *index.Spe // extractComponentValues converts all the YAML nodes of a component type to // low level model. // Process each node in parallel. -func extractComponentValues[T low.Buildable[N], N any](ctx context.Context, label string, root *yaml.Node, idx *index.SpecIndex) (low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[T]]], error) { +func extractComponentValues[T low.Buildable[N], N any](ctx context.Context, label string, root *yaml.Node, idx *index.SpecIndex, co *Components) (low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[T]]], error) { var emptyResult low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[T]]] _, nodeLabel, nodeValue := utils.FindKeyNodeFullTop(label, root.Content) if nodeValue == nil { return emptyResult, nil } + co.Nodes.Store(nodeLabel.Line, nodeLabel) componentValues := orderedmap.New[low.KeyReference[string], low.ValueReference[T]]() if utils.IsNodeArray(nodeValue) { return emptyResult, fmt.Errorf("node is array, cannot be used in components: line %d, column %d", nodeValue.Line, nodeValue.Column) @@ -298,6 +303,21 @@ func extractComponentValues[T low.Buildable[N], N any](ctx context.Context, labe if err != nil { return componentBuildResult[T]{}, err } + + nType := reflect.TypeOf(n) + nValue := reflect.ValueOf(n) + + // Check if the type implements low.HasKeyNode + hasKeyNodeType := reflect.TypeOf((*low.HasKeyNode)(nil)).Elem() + if nType.Implements(hasKeyNodeType) { + r := nValue.Interface() + if h, ok := r.(low.HasKeyNode); ok { + if k, ko := r.(low.AddNodes); ko { + k.AddNode(h.GetKeyNode().Line, h.GetKeyNode()) + } + } + + } return componentBuildResult[T]{ key: low.KeyReference[string]{ KeyNode: currentLabel, @@ -315,6 +335,10 @@ func extractComponentValues[T low.Buildable[N], N any](ctx context.Context, labe return emptyResult, err } + //for rt := componentValues.First(); rt != nil; rt = rt.Next() { + // + //} + results := low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[T]]]{ KeyNode: nodeLabel, ValueNode: nodeValue, diff --git a/datamodel/low/v3/create_document.go b/datamodel/low/v3/create_document.go index 3ccd6495..f9718730 100644 --- a/datamodel/low/v3/create_document.go +++ b/datamodel/low/v3/create_document.go @@ -36,7 +36,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur } version = low.NodeReference[string]{Value: versionNode.Value, KeyNode: labelNode, ValueNode: versionNode} doc := Document{Version: version} - + doc.Nodes = low.ExtractNodes(nil, info.RootNode.Content[0]) // create an index config and shadow the document configuration. idxConfig := index.CreateClosedAPIIndexConfig() idxConfig.SpecInfo = info @@ -130,7 +130,12 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur doc.Index = rolodex.GetRootIndex() var wg sync.WaitGroup + var cacheMap sync.Map + modelContext := base.ModelContext{SchemaCache: &cacheMap} + ctx := context.WithValue(context.Background(), "modelCtx", &modelContext) + doc.Extensions = low.ExtractExtensions(info.RootNode.Content[0]) + low.ExtractExtensionNodes(ctx, doc.Extensions, doc.Nodes) // if set, extract jsonSchemaDialect (3.1) _, dialectLabel, dialectNode := utils.FindKeyNodeFull(JSONSchemaDialectLabel, info.RootNode.Content) @@ -161,8 +166,6 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur extractWebhooks, } - ctx := context.Background() - wg.Add(len(extractionFuncs)) if config.Logger != nil { config.Logger.Debug("running extractions") @@ -310,6 +313,9 @@ func extractWebhooks(ctx context.Context, info *datamodel.SpecInfo, doc *Documen KeyNode: hooksL, ValueNode: hooksN, } + for xj := hooks.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } return nil } diff --git a/datamodel/low/v3/document.go b/datamodel/low/v3/document.go index 8aff44be..56de3c22 100644 --- a/datamodel/low/v3/document.go +++ b/datamodel/low/v3/document.go @@ -86,6 +86,8 @@ type Document struct { // Rolodex is a reference to the rolodex used when creating this document. Rolodex *index.Rolodex + + low.NodeMap } // FindSecurityRequirement will attempt to locate a security requirement string from a supplied name. diff --git a/datamodel/low/v3/encoding.go b/datamodel/low/v3/encoding.go index 838358f1..ad75874d 100644 --- a/datamodel/low/v3/encoding.go +++ b/datamodel/low/v3/encoding.go @@ -27,6 +27,7 @@ type Encoding struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindHeader attempts to locate a Header with the supplied name @@ -67,6 +68,7 @@ func (en *Encoding) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in root = utils.NodeAlias(root) en.RootNode = root utils.CheckForMergeNodes(root) + en.Nodes = low.ExtractNodes(ctx, root) en.Reference = new(low.Reference) headers, hL, hN, err := low.ExtractMap[*Header](ctx, HeadersLabel, root, idx) if err != nil { @@ -78,6 +80,10 @@ func (en *Encoding) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: hL, ValueNode: hN, } + en.Nodes.Store(hL.Line, hL) + for xj := headers.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } return nil } diff --git a/datamodel/low/v3/header.go b/datamodel/low/v3/header.go index ab39bb76..34718ada 100644 --- a/datamodel/low/v3/header.go +++ b/datamodel/low/v3/header.go @@ -35,6 +35,7 @@ type Header struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindExtension will attempt to locate an extension with the supplied name @@ -104,8 +105,9 @@ func (h *Header) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index h.RootNode = root utils.CheckForMergeNodes(root) h.Reference = new(low.Reference) + h.Nodes = low.ExtractNodes(ctx, root) h.Extensions = low.ExtractExtensions(root) - + low.ExtractExtensionNodes(ctx, h.Extensions, h.Nodes) // handle example if set. _, expLabel, expNode := utils.FindKeyNodeFull(base.ExampleLabel, root.Content) if expNode != nil { @@ -114,6 +116,12 @@ func (h *Header) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index ValueNode: expNode, KeyNode: expLabel, } + h.Nodes.Store(expLabel.Line, expLabel) + m := low.ExtractNodes(ctx, expNode) + m.Range(func(key, value any) bool { + h.Nodes.Store(key, value) + return true + }) } // handle examples if set. @@ -127,6 +135,7 @@ func (h *Header) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index KeyNode: expsL, ValueNode: expsN, } + h.Nodes.Store(expsL.Line, expsL) } // handle schema @@ -148,6 +157,9 @@ func (h *Header) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index KeyNode: cL, ValueNode: cN, } + if cL != nil { + h.Nodes.Store(cL.Line, cL) + } return nil } diff --git a/datamodel/low/v3/link.go b/datamodel/low/v3/link.go index cb8fe5b3..1114a8f7 100644 --- a/datamodel/low/v3/link.go +++ b/datamodel/low/v3/link.go @@ -38,6 +38,7 @@ type Link struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetExtensions returns all Link extensions and satisfies the low.HasExtensions interface. @@ -72,7 +73,17 @@ func (l *Link) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index.S l.RootNode = root utils.CheckForMergeNodes(root) l.Reference = new(low.Reference) + l.Nodes = low.ExtractNodes(ctx, root) l.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, l.Extensions, l.Nodes) + + // extract parameter nodes. + if l.Parameters.Value != nil && l.Parameters.Value.Len() > 0 { + for fk := l.Parameters.Value.First(); fk != nil; fk = fk.Next() { + l.Nodes.Store(fk.Key().KeyNode.Line, fk.Key().KeyNode) + } + } + // extract server. ser, sErr := low.ExtractObject[*Server](ctx, ServerLabel, root, idx) if sErr != nil { diff --git a/datamodel/low/v3/media_type.go b/datamodel/low/v3/media_type.go index 49016443..51a79693 100644 --- a/datamodel/low/v3/media_type.go +++ b/datamodel/low/v3/media_type.go @@ -30,6 +30,7 @@ type MediaType struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetExtensions returns all MediaType extensions and satisfies the low.HasExtensions interface. @@ -74,12 +75,20 @@ func (mt *MediaType) Build(ctx context.Context, keyNode, root *yaml.Node, idx *i mt.RootNode = root utils.CheckForMergeNodes(root) mt.Reference = new(low.Reference) + mt.Nodes = low.ExtractNodes(ctx, root) mt.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, mt.Extensions, mt.Nodes) // handle example if set. _, expLabel, expNode := utils.FindKeyNodeFullTop(base.ExampleLabel, root.Content) if expNode != nil { mt.Example = low.NodeReference[*yaml.Node]{Value: expNode, KeyNode: expLabel, ValueNode: expNode} + mt.Nodes.Store(expLabel.Line, expLabel) + m := low.ExtractNodesRecursive(ctx, expNode) + m.Range(func(key, value any) bool { + mt.Nodes.Store(key, value) + return true + }) } // handle schema @@ -97,11 +106,16 @@ func (mt *MediaType) Build(ctx context.Context, keyNode, root *yaml.Node, idx *i return eErr } if exps != nil && slices.Contains(root.Content, expsL) { + mt.Nodes.Store(expsL.Line, expsL) + for xj := exps.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } mt.Examples = low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[*base.Example]]]{ Value: exps, KeyNode: expsL, ValueNode: expsN, } + } // handle encoding @@ -115,6 +129,10 @@ func (mt *MediaType) Build(ctx context.Context, keyNode, root *yaml.Node, idx *i KeyNode: encsL, ValueNode: encsN, } + mt.Nodes.Store(encsL.Line, encsL) + for xj := encs.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } return nil } diff --git a/datamodel/low/v3/oauth_flows.go b/datamodel/low/v3/oauth_flows.go index d8ed5ca4..537a7598 100644 --- a/datamodel/low/v3/oauth_flows.go +++ b/datamodel/low/v3/oauth_flows.go @@ -27,6 +27,7 @@ type OAuthFlows struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetExtensions returns all OAuthFlows extensions and satisfies the low.HasExtensions interface. @@ -56,6 +57,7 @@ func (o *OAuthFlows) Build(ctx context.Context, keyNode, root *yaml.Node, idx *i o.RootNode = root utils.CheckForMergeNodes(root) o.Reference = new(low.Reference) + o.Nodes = low.ExtractNodes(ctx, root) o.Extensions = low.ExtractExtensions(root) v, vErr := low.ExtractObject[*OAuthFlow](ctx, ImplicitLabel, root, idx) @@ -113,6 +115,7 @@ type OAuthFlow struct { Extensions *orderedmap.Map[low.KeyReference[string], low.ValueReference[*yaml.Node]] RootNode *yaml.Node *low.Reference + low.NodeMap } // GetExtensions returns all OAuthFlow extensions and satisfies the low.HasExtensions interface. @@ -136,9 +139,18 @@ func (o *OAuthFlow) GetRootNode() *yaml.Node { } // Build will extract extensions from the node. -func (o *OAuthFlow) Build(_ context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { +func (o *OAuthFlow) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { o.Reference = new(low.Reference) + o.Nodes = low.ExtractNodes(ctx, root) o.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, o.Extensions, o.Nodes) + + if o.Scopes.Value != nil && o.Scopes.Value.Len() > 0 { + for fk := o.Scopes.Value.First(); fk != nil; fk = fk.Next() { + o.Nodes.Store(fk.Key().KeyNode.Line, fk.Key().KeyNode) + } + } + o.RootNode = root return nil } diff --git a/datamodel/low/v3/operation.go b/datamodel/low/v3/operation.go index b823f24e..22eee70d 100644 --- a/datamodel/low/v3/operation.go +++ b/datamodel/low/v3/operation.go @@ -40,6 +40,7 @@ type Operation struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // FindCallback will attempt to locate a Callback instance by the supplied name. @@ -77,7 +78,9 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) o.Reference = new(low.Reference) + o.Nodes = low.ExtractNodes(ctx, root) o.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, o.Extensions, o.Nodes) // extract externalDocs extDocs, dErr := low.ExtractObject[*base.ExternalDoc](ctx, base.ExternalDocsLabel, root, idx) @@ -97,6 +100,7 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: ln, ValueNode: vn, } + o.Nodes.Store(ln.Line, ln) } // extract request body @@ -106,6 +110,17 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in } o.RequestBody = rBody + // extract tags, but only extract nodes, the model has already been built + k, v := utils.FindKeyNode(TagsLabel, root.Content) + if k != nil && v != nil { + o.Nodes.Store(k.Line, k) + nm := low.ExtractNodesRecursive(ctx, v) + nm.Range(func(key, value interface{}) bool { + o.Nodes.Store(key, value) + return true + }) + } + // extract responses respBody, respErr := low.ExtractObject[*Responses](ctx, ResponsesLabel, root, idx) if respErr != nil { @@ -124,6 +139,10 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: cbL, ValueNode: cbN, } + o.Nodes.Store(cbL.Line, cbL) + for xj := callbacks.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } // extract security @@ -139,6 +158,7 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: sln, ValueNode: svn, } + o.Nodes.Store(sln.Line, sln) } // if security is set, but no requirements are defined. @@ -149,6 +169,7 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: sln, ValueNode: svn, } + o.Nodes.Store(sln.Line, svn) } // extract servers @@ -162,6 +183,7 @@ func (o *Operation) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: sl, ValueNode: sn, } + o.Nodes.Store(sl.Line, sl) } return nil } diff --git a/datamodel/low/v3/parameter.go b/datamodel/low/v3/parameter.go index ed16e6ce..e11d3965 100644 --- a/datamodel/low/v3/parameter.go +++ b/datamodel/low/v3/parameter.go @@ -40,6 +40,7 @@ type Parameter struct { Content low.NodeReference[*orderedmap.Map[low.KeyReference[string], low.ValueReference[*MediaType]]] Extensions *orderedmap.Map[low.KeyReference[string], low.ValueReference[*yaml.Node]] *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the Parameter object. @@ -79,12 +80,15 @@ func (p *Parameter) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in p.RootNode = root utils.CheckForMergeNodes(root) p.Reference = new(low.Reference) + p.Nodes = low.ExtractNodes(ctx, root) p.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, p.Extensions, p.Nodes) // handle example if set. _, expLabel, expNode := utils.FindKeyNodeFullTop(base.ExampleLabel, root.Content) if expNode != nil { p.Example = low.NodeReference[*yaml.Node]{Value: expNode, KeyNode: expLabel, ValueNode: expNode} + p.Nodes.Store(expLabel.Line, expLabel) } // handle schema @@ -108,6 +112,10 @@ func (p *Parameter) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: expsL, ValueNode: expsN, } + p.Nodes.Store(expsL.Line, expsL) + for xj := exps.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } // handle content, if set. @@ -120,6 +128,13 @@ func (p *Parameter) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in KeyNode: cL, ValueNode: cN, } + if cL != nil { + p.Nodes.Store(cL.Line, cL) + for xj := con.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } + } + return nil } diff --git a/datamodel/low/v3/path_item.go b/datamodel/low/v3/path_item.go index 27b7d00c..dc2dce08 100644 --- a/datamodel/low/v3/path_item.go +++ b/datamodel/low/v3/path_item.go @@ -42,6 +42,7 @@ type PathItem struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // Hash will return a consistent SHA256 Hash of the PathItem object @@ -121,13 +122,14 @@ func (p *PathItem) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind p.RootNode = root utils.CheckForMergeNodes(root) p.Reference = new(low.Reference) + p.Nodes = low.ExtractNodes(ctx, root) p.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, p.Extensions, p.Nodes) skip := false var currentNode *yaml.Node var wg sync.WaitGroup var errors []error - var ops []low.NodeReference[*Operation] // extract parameters @@ -141,6 +143,7 @@ func (p *PathItem) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind KeyNode: ln, ValueNode: vn, } + p.Nodes.Store(ln.Line, ln) } _, ln, vn = utils.FindKeyNodeFullTop(ServersLabel, root.Content) @@ -163,6 +166,7 @@ func (p *PathItem) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind KeyNode: ln, ValueNode: vn, } + p.Nodes.Store(ln.Line, ln) } } diff --git a/datamodel/low/v3/paths.go b/datamodel/low/v3/paths.go index d06398ad..3b3c76a2 100644 --- a/datamodel/low/v3/paths.go +++ b/datamodel/low/v3/paths.go @@ -30,6 +30,7 @@ type Paths struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the Paths object. @@ -82,7 +83,9 @@ func (p *Paths) Build(ctx context.Context, keyNode, root *yaml.Node, idx *index. p.RootNode = root utils.CheckForMergeNodes(root) p.Reference = new(low.Reference) + p.Nodes = low.ExtractNodes(ctx, root) p.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, p.Extensions, p.Nodes) pathsMap, err := extractPathItemsMap(ctx, root, idx) if err != nil { diff --git a/datamodel/low/v3/request_body.go b/datamodel/low/v3/request_body.go index d3ff14bd..bb45d19e 100644 --- a/datamodel/low/v3/request_body.go +++ b/datamodel/low/v3/request_body.go @@ -26,6 +26,7 @@ type RequestBody struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the RequestBody object. @@ -60,7 +61,9 @@ func (rb *RequestBody) Build(ctx context.Context, keyNode, root *yaml.Node, idx rb.RootNode = root utils.CheckForMergeNodes(root) rb.Reference = new(low.Reference) + rb.Nodes = low.ExtractNodes(ctx, root) rb.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, rb.Extensions, rb.Nodes) // handle content, if set. con, cL, cN, cErr := low.ExtractMap[*MediaType](ctx, ContentLabel, root, idx) @@ -73,6 +76,10 @@ func (rb *RequestBody) Build(ctx context.Context, keyNode, root *yaml.Node, idx KeyNode: cL, ValueNode: cN, } + rb.Nodes.Store(cL.Line, cL) + for xj := con.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } return nil } diff --git a/datamodel/low/v3/response.go b/datamodel/low/v3/response.go index 5693f64f..197b3419 100644 --- a/datamodel/low/v3/response.go +++ b/datamodel/low/v3/response.go @@ -30,6 +30,7 @@ type Response struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the Response object. @@ -74,7 +75,9 @@ func (r *Response) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind r.RootNode = root utils.CheckForMergeNodes(root) r.Reference = new(low.Reference) + r.Nodes = low.ExtractNodes(ctx, root) r.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, r.Extensions, r.Nodes) // extract headers headers, lN, kN, err := low.ExtractMapExtensions[*Header](ctx, HeadersLabel, root, idx, true) @@ -87,6 +90,10 @@ func (r *Response) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind KeyNode: lN, ValueNode: kN, } + r.Nodes.Store(lN.Line, lN) + for xj := headers.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } con, clN, cN, cErr := low.ExtractMap[*MediaType](ctx, ContentLabel, root, idx) @@ -99,6 +106,10 @@ func (r *Response) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind KeyNode: clN, ValueNode: cN, } + r.Nodes.Store(clN.Line, clN) + for xj := con.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } // handle links if set @@ -112,6 +123,10 @@ func (r *Response) Build(ctx context.Context, keyNode, root *yaml.Node, idx *ind KeyNode: linkLabel, ValueNode: linkValue, } + r.Nodes.Store(linkLabel.Line, linkLabel) + for xj := links.First(); xj != nil; xj = xj.Next() { + xj.Value().Value.Nodes.Store(xj.Key().KeyNode.Line, xj.Key().KeyNode) + } } return nil } diff --git a/datamodel/low/v3/responses.go b/datamodel/low/v3/responses.go index 8c8cc32b..4ce0fb0d 100644 --- a/datamodel/low/v3/responses.go +++ b/datamodel/low/v3/responses.go @@ -41,6 +41,7 @@ type Responses struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the Responses object. @@ -64,7 +65,9 @@ func (r *Responses) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in root = utils.NodeAlias(root) r.RootNode = root r.Reference = new(low.Reference) + r.Nodes = low.ExtractNodes(ctx, root) r.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, r.Extensions, r.Nodes) utils.CheckForMergeNodes(root) if utils.IsNodeMap(root) { codes, err := low.ExtractMapNoLookup[*Response](ctx, root, idx) @@ -73,12 +76,17 @@ func (r *Responses) Build(ctx context.Context, keyNode, root *yaml.Node, idx *in } if codes != nil { r.Codes = codes + for codePairs := codes.First(); codePairs != nil; codePairs = codePairs.Next() { + code := codePairs.Key() + r.Nodes.Store(code.KeyNode.Line, code.KeyNode) + } } def := r.getDefault() if def != nil { // default is bundled into codes, pull it out r.Default = *def + r.Nodes.Store(def.KeyNode.Line, def.KeyNode) // remove default from codes r.deleteCode(DefaultLabel) } diff --git a/datamodel/low/v3/security_scheme.go b/datamodel/low/v3/security_scheme.go index d8c53f15..67993207 100644 --- a/datamodel/low/v3/security_scheme.go +++ b/datamodel/low/v3/security_scheme.go @@ -38,6 +38,7 @@ type SecurityScheme struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the SecurityScheme object. @@ -67,7 +68,9 @@ func (ss *SecurityScheme) Build(ctx context.Context, keyNode, root *yaml.Node, i ss.RootNode = root utils.CheckForMergeNodes(root) ss.Reference = new(low.Reference) + ss.Nodes = low.ExtractNodes(ctx, root) ss.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, ss.Extensions, ss.Nodes) oa, oaErr := low.ExtractObject[*OAuthFlows](ctx, OAuthFlowsLabel, root, idx) if oaErr != nil { diff --git a/datamodel/low/v3/server.go b/datamodel/low/v3/server.go index 4abe70eb..458b3d06 100644 --- a/datamodel/low/v3/server.go +++ b/datamodel/low/v3/server.go @@ -25,6 +25,7 @@ type Server struct { KeyNode *yaml.Node RootNode *yaml.Node *low.Reference + low.NodeMap } // GetRootNode returns the root yaml node of the Server object. @@ -43,13 +44,16 @@ func (s *Server) FindVariable(serverVar string) *low.ValueReference[*ServerVaria } // Build will extract server variables from the supplied node. -func (s *Server) Build(_ context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { +func (s *Server) Build(ctx context.Context, keyNode, root *yaml.Node, _ *index.SpecIndex) error { s.KeyNode = keyNode root = utils.NodeAlias(root) s.RootNode = root utils.CheckForMergeNodes(root) s.Reference = new(low.Reference) + s.Nodes = low.ExtractNodes(ctx, root) s.Extensions = low.ExtractExtensions(root) + low.ExtractExtensionNodes(ctx, s.Extensions, s.Nodes) + kn, vars := utils.FindKeyNode(VariablesLabel, root.Content) if vars == nil { return nil @@ -57,20 +61,26 @@ func (s *Server) Build(_ context.Context, keyNode, root *yaml.Node, _ *index.Spe variablesMap := orderedmap.New[low.KeyReference[string], low.ValueReference[*ServerVariable]]() if utils.IsNodeMap(vars) { var currentNode string - var keyNode *yaml.Node + var localKeyNode *yaml.Node for i, varNode := range vars.Content { if i%2 == 0 { currentNode = varNode.Value - keyNode = varNode + localKeyNode = varNode continue } variable := ServerVariable{} variable.Reference = new(low.Reference) _ = low.BuildModel(varNode, &variable) + variable.Nodes = low.ExtractNodesRecursive(ctx, varNode) + if localKeyNode != nil { + variable.Nodes.Store(localKeyNode.Line, localKeyNode) + } + variable.RootNode = varNode + variable.KeyNode = localKeyNode variablesMap.Set( low.KeyReference[string]{ Value: currentNode, - KeyNode: keyNode, + KeyNode: localKeyNode, }, low.ValueReference[*ServerVariable]{ ValueNode: varNode, diff --git a/datamodel/low/v3/server_variable.go b/datamodel/low/v3/server_variable.go index e538a3ca..d85e907f 100644 --- a/datamodel/low/v3/server_variable.go +++ b/datamodel/low/v3/server_variable.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" + "gopkg.in/yaml.v3" "sort" "strings" ) @@ -19,7 +20,20 @@ type ServerVariable struct { Enum []low.NodeReference[string] Default low.NodeReference[string] Description low.NodeReference[string] + KeyNode *yaml.Node + RootNode *yaml.Node *low.Reference + low.NodeMap +} + +// GetRootNode returns the root yaml node of the ServerVariable object. +func (s *ServerVariable) GetRootNode() *yaml.Node { + return s.RootNode +} + +// GetKeyNode returns the key yaml node of the ServerVariable object. +func (s *ServerVariable) GetKeyNode() *yaml.Node { + return s.RootNode } // Hash will return a consistent SHA256 Hash of the ServerVariable object diff --git a/datamodel/spec_info.go b/datamodel/spec_info.go index 904cbd5a..28f429af 100644 --- a/datamodel/spec_info.go +++ b/datamodel/spec_info.go @@ -22,6 +22,7 @@ const ( // used by the library, this contains the top of the document tree that every single low model is based off. type SpecInfo struct { SpecType string `json:"type"` + NumLines int `json:"numLines"` Version string `json:"version"` VersionNumeric float32 `json:"versionNumeric"` SpecFormat string `json:"format"` @@ -62,7 +63,8 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro // set original bytes specInfo.SpecBytes = &spec - runes := []rune(strings.TrimSpace(string(spec))) + stringSpec := string(spec) + runes := []rune(strings.TrimSpace(stringSpec)) if len(runes) <= 0 { return specInfo, errors.New("there is nothing in the spec, it's empty - so there is nothing to be done") } @@ -73,6 +75,8 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro specInfo.SpecFileType = YAMLFileType } + specInfo.NumLines = strings.Count(stringSpec, "\n") + 1 + err := yaml.Unmarshal(spec, &parsedSpec) if err != nil { return nil, fmt.Errorf("unable to parse specification: %s", err.Error()) diff --git a/document_test.go b/document_test.go index e45feb9d..2c116d58 100644 --- a/document_test.go +++ b/document_test.go @@ -365,7 +365,7 @@ func TestDocument_RenderAndReload_WithErrors(t *testing.T) { _, _, _, errors := doc.RenderAndReload() assert.Len(t, errors, 2) - assert.Equal(t, errors[0].Error(), "component '#/components/schemas/Pet' does not exist in the specification") + assert.Equal(t, errors[0].Error(), "component `#/components/schemas/Pet` does not exist in the specification") } func TestDocument_Render(t *testing.T) { diff --git a/go.mod b/go.mod index eaab11e5..d7a4a48b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/vmware-labs/yaml-jsonpath v0.3.2 github.com/wk8/go-ordered-map/v2 v2.1.8 + golang.org/x/net v0.0.0-20220225172249-27dd8689420f gopkg.in/yaml.v3 v3.0.1 ) diff --git a/index/find_component_test.go b/index/find_component_test.go index 919c675b..a6f085ec 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -274,7 +274,7 @@ paths: index := NewSpecIndexWithConfig(&rootNode, c) assert.Len(t, index.GetReferenceIndexErrors(), 1) - assert.Equal(t, "component '#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[0].Error()) + assert.Equal(t, "component `#/paths/~1pet~1%$petId%7D/get/parameters` does not exist in the specification", index.GetReferenceIndexErrors()[0].Error()) } func TestSpecIndex_LocateRemoteDocsWithEscapedCharacters(t *testing.T) { diff --git a/index/rolodex.go b/index/rolodex.go index d08bed92..8e1f8c54 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -121,6 +121,11 @@ func (r *Rolodex) GetRootIndex() *SpecIndex { return r.rootIndex } +// GetConfig returns the index configuration of the rolodex. +func (r *Rolodex) GetConfig() *SpecIndexConfig { + return r.indexConfig +} + // GetRootNode returns the root index of the rolodex (the entry point, the main document) func (r *Rolodex) GetRootNode() *yaml.Node { return r.rootNode diff --git a/index/rolodex_test.go b/index/rolodex_test.go index 5cb20b1f..c24e0c7b 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -31,6 +31,7 @@ func TestRolodex_NewRolodex(t *testing.T) { assert.Nil(t, rolo.GetRootIndex()) assert.Len(t, rolo.GetIndexes(), 0) assert.Len(t, rolo.GetCaughtErrors(), 0) + assert.NotNil(t, rolo.GetConfig()) } func TestRolodex_NoFS(t *testing.T) { diff --git a/utils/utils.go b/utils/utils.go index e9244b8b..d4b3ec8e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -613,6 +613,12 @@ func IsJSON(testString string) bool { } // IsYAML will tell you if a string is YAML or not. +var ( + yamlKeyValuePattern = regexp.MustCompile(`(?m)^\s*[a-zA-Z0-9_-]+\s*:\s*.+$`) + yamlListPattern = regexp.MustCompile(`(?m)^\s*-\s+.+$`) + yamlHeaderPattern = regexp.MustCompile(`(?m)^---\s*$`) +) + func IsYAML(testString string) bool { if testString == "" { return false @@ -620,13 +626,21 @@ func IsYAML(testString string) bool { if IsJSON(testString) { return false } - var n interface{} - err := yaml.Unmarshal([]byte(testString), &n) - if err != nil { - return false + + // Trim leading and trailing whitespace + s := strings.TrimSpace(testString) + + // Fast checks for common YAML features + if strings.Contains(s, ": ") || strings.Contains(s, "- ") || strings.Contains(s, "\n- ") { + return true + } + + // Regular expressions for more robust detection + if yamlKeyValuePattern.MatchString(s) || yamlListPattern.MatchString(s) || yamlHeaderPattern.MatchString(s) { + return true } - _, err = yaml.Marshal(n) - return err == nil + + return false } // ConvertYAMLtoJSON will do exactly what you think it will. It will deserialize YAML into serialized JSON. diff --git a/utils/utils_test.go b/utils/utils_test.go index 116c4885..ce98dc3d 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -672,10 +672,10 @@ func TestIsJSON(t *testing.T) { func TestIsYAML(t *testing.T) { assert.True(t, IsYAML("hello:\n there:\n my-name: is quobix")) - assert.True(t, IsYAML("potato shoes")) + assert.False(t, IsYAML("potato shoes")) assert.False(t, IsYAML("{'hello':'there'}")) assert.False(t, IsYAML("")) - assert.False(t, IsYAML("8908: hello: yeah: \n12309812: :123")) + assert.True(t, IsYAML("8908: hello: yeah: \n12309812: :123")) } func TestConvertYAMLtoJSON(t *testing.T) { From 4949d03e5c5cda766fc097f55ceac5b339b8bc0b Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 19:48:00 -0400 Subject: [PATCH 03/13] bumped coverage --- datamodel/low/node_map.go | 6 +- datamodel/low/node_map_test.go | 120 +++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/datamodel/low/node_map.go b/datamodel/low/node_map.go index ac5111a4..efc32b09 100644 --- a/datamodel/low/node_map.go +++ b/datamodel/low/node_map.go @@ -85,7 +85,7 @@ func (nm *NodeMap) ExtractNodes(node *yaml.Node, recurse bool) { } // ContainsLine will return true if the NodeMap contains a node with the supplied line number -func (nm *NodeMap) ContainsLine(line int, recurse bool) bool { +func (nm *NodeMap) ContainsLine(line int) bool { if _, ok := nm.Nodes.Load(line); ok { return true } @@ -138,11 +138,9 @@ func ExtractExtensionNodes(_ context.Context, nodeMap.Store(k.KeyNode.Line, results) } else { nodeMap.Store(k.KeyNode.Line, results[0]) - gingo := results[1:] - for _, y := range gingo { + for _, y := range results[1:] { nodeMap.Store(y.Line, []*yaml.Node{y}) } - //nodeMap.Store(v.ValueNode.Line, gingo) } } } diff --git a/datamodel/low/node_map_test.go b/datamodel/low/node_map_test.go index bdca201c..07ab8bd5 100644 --- a/datamodel/low/node_map_test.go +++ b/datamodel/low/node_map_test.go @@ -96,6 +96,32 @@ four: } +func TestExtractNodesRecursive(t *testing.T) { + + yml := `one: hello +two: there +three: nice one +four: + shoes: yes + socks: of course +` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + + nm := ExtractNodesRecursive(nil, root.Content[0]) + + count := 0 + nm.Range(func(key, value interface{}) bool { + count++ + return true + }) + + assert.Equal(t, 6, count) + testTheThingUnmarshalled(t, nm) + +} + func TestExtractNodes_Nil(t *testing.T) { var syncMap sync.Map nm := &NodeMap{Nodes: &syncMap} @@ -109,3 +135,97 @@ func TestExtractNodes_Nil(t *testing.T) { assert.Equal(t, 0, count) } + +func Test_NodeMapExtractNodes_SingleNode(t *testing.T) { + + yml := `one: hello +two: there +three: nice one +four: + shoes: yes + socks: of course +` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + + syncMap.Store(1, root.Content[0]) + + nm.ExtractNodes(root.Content[0], false) + +} + +func Test_NodeMapGetNodes_SingleNode(t *testing.T) { + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + + syncMap.Store(1, &yaml.Node{}) + ex := nm.GetNodes() + assert.Equal(t, 1, len(ex)) + +} + +func Test_NodeMapContainsLine(t *testing.T) { + + yml := `one: hello +two: there +three: nice one +four: + shoes: yes + socks: of course +` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + + nm.ExtractNodes(root.Content[0], true) + assert.True(t, nm.ContainsLine(1)) + assert.True(t, nm.ContainsLine(2)) + assert.True(t, nm.ContainsLine(3)) + assert.True(t, nm.ContainsLine(4)) + assert.True(t, nm.ContainsLine(5)) + assert.True(t, nm.ContainsLine(6)) + assert.False(t, nm.ContainsLine(7)) +} + +func Test_NodeMapGetNodes_EmptyNode(t *testing.T) { + var syncMap sync.Map + nm := &NodeMap{Nodes: &syncMap} + + ex := nm.GetNodes() + assert.Equal(t, 1, len(ex)) + +} + +func TestExtractExtensionNodes(t *testing.T) { + + yml := `openapi: 3.1 +chack: spack +x-fresh: nice +x-cakes: yes +x-socks: of course +x-rice: + yes: no + no: yes` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + + extensions := ExtractExtensions(root.Content[0]) + var syncMap sync.Map + ExtractExtensionNodes(nil, extensions, &syncMap) + + count := 0 + syncMap.Range(func(key, value interface{}) bool { + count++ + return true + }) + + assert.Equal(t, 6, count) + +} From c80e928e8b96f6e73c52674db32d8111d38e6c7c Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 20:16:50 -0400 Subject: [PATCH 04/13] added more coverage --- datamodel/low/base/schema_test.go | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/datamodel/low/base/schema_test.go b/datamodel/low/base/schema_test.go index ce375c29..278a943c 100644 --- a/datamodel/low/base/schema_test.go +++ b/datamodel/low/base/schema_test.go @@ -1919,3 +1919,49 @@ components: assert.Equal(t, "schema build failed: reference '#/' cannot be found at line 2, col 9", e.Error()) } + +func TestExtractSchema_CheckExampleNodesExtracted(t *testing.T) { + + yml := `schema: + type: object + example: + ping: pong + jing: + jong: jang + examples: + - tang: bang + - bom: jog` + + var iNode yaml.Node + mErr := yaml.Unmarshal([]byte(yml), &iNode) + assert.NoError(t, mErr) + + config := index.CreateOpenAPIIndexConfig() + config.SpecInfo = &datamodel.SpecInfo{ + VersionNumeric: 3.0, + } + idx := index.NewSpecIndexWithConfig(&iNode, config) + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + ctx := context.WithValue(context.Background(), index.CurrentPathKey, "test") + + res, e := ExtractSchema(ctx, idxNode.Content[0], idx) + if res != nil { + sch := res.Value.Schema() + assert.NotNil(t, sch.Nodes) + assert.NoError(t, e) + + n, _ := sch.Nodes.Load(4) + assert.NotNil(t, n.([]*yaml.Node)[1]) + assert.Equal(t, "ping", n.([]*yaml.Node)[0].Value) + assert.Equal(t, "pong", n.([]*yaml.Node)[1].Value) + + n, _ = sch.Nodes.Load(8) + assert.NotNil(t, n.([]*yaml.Node)[0]) + assert.Equal(t, "tang", n.([]*yaml.Node)[1].Value) + assert.Equal(t, "bang", n.([]*yaml.Node)[2].Value) + + } else { + t.Fail() + } +} From 631e4a8478f1a47e931b1fdb0ecc1484ac4b01b3 Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 20:44:12 -0400 Subject: [PATCH 05/13] Improving coverage --- datamodel/low/base/schema.go | 16 ++-------------- datamodel/low/base/schema_test.go | 3 ++- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/datamodel/low/base/schema.go b/datamodel/low/base/schema.go index 9b4c21b8..3abddba2 100644 --- a/datamodel/low/base/schema.go +++ b/datamodel/low/base/schema.go @@ -654,13 +654,7 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde // map to the local schema expChildNodes.Range(func(k, v interface{}) bool { if arr, ko := v.([]*yaml.Node); ko { - if ext, ok := s.Nodes.Load(k); ok { - if extArr, kk := ext.([]*yaml.Node); kk { - s.Nodes.Store(k, append(extArr, arr...)) - } else { - s.Nodes.Store(k, arr) - } - } else { + if _, ok := s.Nodes.Load(k); !ok { s.Nodes.Store(k, arr) } } @@ -686,13 +680,7 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde // map to the local schema expChildNodes.Range(func(k, v interface{}) bool { if arr, ko := v.([]*yaml.Node); ko { - if ext, ok := s.Nodes.Load(k); ok { - if extArr, kk := ext.([]*yaml.Node); kk { - s.Nodes.Store(k, append(extArr, arr...)) - } else { - s.Nodes.Store(k, arr) - } - } else { + if _, ok := s.Nodes.Load(k); !ok { s.Nodes.Store(k, arr) } } diff --git a/datamodel/low/base/schema_test.go b/datamodel/low/base/schema_test.go index 278a943c..210ab35f 100644 --- a/datamodel/low/base/schema_test.go +++ b/datamodel/low/base/schema_test.go @@ -1930,7 +1930,8 @@ func TestExtractSchema_CheckExampleNodesExtracted(t *testing.T) { jong: jang examples: - tang: bang - - bom: jog` + - bom: jog + ding: dong` var iNode yaml.Node mErr := yaml.Unmarshal([]byte(yml), &iNode) From a37f50c51514a22c3ea1fb4aa374ef940c39ad16 Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 20:55:04 -0400 Subject: [PATCH 06/13] bumped coverage --- datamodel/low/base/context_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 datamodel/low/base/context_test.go diff --git a/datamodel/low/base/context_test.go b/datamodel/low/base/context_test.go new file mode 100644 index 00000000..a86faf5f --- /dev/null +++ b/datamodel/low/base/context_test.go @@ -0,0 +1,23 @@ +// Copyright 2023-2024 Princess Beef Heavy Industries, LLC / Dave Shanley +// https://pb33f.io + +package base + +import ( + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" + "testing" +) + +func TestGetModelContext(t *testing.T) { + + assert.Nil(t, GetModelContext(nil)) + assert.Nil(t, GetModelContext(context.Background())) + + ctx := context.WithValue(context.Background(), "modelCtx", &ModelContext{}) + assert.NotNil(t, GetModelContext(ctx)) + + ctx = context.WithValue(context.Background(), "modelCtx", "wrong") + assert.Nil(t, GetModelContext(ctx)) + +} From 6be80a5d07723d45171c08a76295fe49ea6ffcec Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 21:41:28 -0400 Subject: [PATCH 07/13] bumped coverage --- datamodel/low/base/schema_proxy_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/datamodel/low/base/schema_proxy_test.go b/datamodel/low/base/schema_proxy_test.go index 1227241e..32070cbd 100644 --- a/datamodel/low/base/schema_proxy_test.go +++ b/datamodel/low/base/schema_proxy_test.go @@ -169,3 +169,27 @@ func TestSchemaProxy_Build_HashFail(t *testing.T) { v := sp.Hash() assert.Equal(t, [32]byte{}, v) } + +func TestSchemaProxy_AddNodePassthrough(t *testing.T) { + yml := `type: int +description: cakes` + + sch := SchemaProxy{} + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + + err := sch.Build(context.Background(), nil, idxNode.Content[0], nil) + assert.NoError(t, err) + + n, f := sch.Nodes.Load(3) + assert.False(t, f) + assert.Nil(t, n) + + sch.Nodes.Store(3, &yaml.Node{Value: "cake"}) + s := sch.Schema() + + n, f = s.Nodes.Load(3) + assert.True(t, f) + assert.NotNil(t, n) + +} From d09ad8e050f7dea345d88dd650a3b19a0048e7b3 Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 4 Aug 2024 21:47:47 -0400 Subject: [PATCH 08/13] bumped coverage --- datamodel/low/base/schema_proxy_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datamodel/low/base/schema_proxy_test.go b/datamodel/low/base/schema_proxy_test.go index 32070cbd..0eec16fa 100644 --- a/datamodel/low/base/schema_proxy_test.go +++ b/datamodel/low/base/schema_proxy_test.go @@ -185,11 +185,16 @@ description: cakes` assert.False(t, f) assert.Nil(t, n) - sch.Nodes.Store(3, &yaml.Node{Value: "cake"}) + sch.AddNode(3, &yaml.Node{Value: "3"}) s := sch.Schema() + sch.AddNode(4, &yaml.Node{Value: "4"}) n, f = s.Nodes.Load(3) assert.True(t, f) assert.NotNil(t, n) + n, f = s.Nodes.Load(4) + assert.True(t, f) + assert.NotNil(t, n) + } From 14ae35229717882f9ef5ddaeb350c0f00f6a6e7f Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 5 Aug 2024 08:11:06 -0400 Subject: [PATCH 09/13] more coverage bumps --- datamodel/high/base/schema_proxy_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/datamodel/high/base/schema_proxy_test.go b/datamodel/high/base/schema_proxy_test.go index 51ebac1c..0c604a33 100644 --- a/datamodel/high/base/schema_proxy_test.go +++ b/datamodel/high/base/schema_proxy_test.go @@ -69,6 +69,20 @@ func TestCreateSchemaProxy(t *testing.T) { sp := CreateSchemaProxy(&Schema{Description: "iAmASchema"}) assert.Equal(t, "iAmASchema", sp.rendered.Description) assert.False(t, sp.IsReference()) + assert.Nil(t, sp.GetValueNode()) +} + +func TestCreateSchemaProxy_NoNilValue(t *testing.T) { + sp := CreateSchemaProxy(&Schema{Description: "iAmASchema"}) + sp.Schema() + + // jerry rig the test. + nodeRef := low.NodeReference[*lowbase.SchemaProxy]{} + nodeRef.ValueNode = &yaml.Node{} + sp.schema = &nodeRef + + assert.Equal(t, "iAmASchema", sp.rendered.Description) + assert.NotNil(t, sp.GetValueNode()) } func TestCreateSchemaProxyRef(t *testing.T) { From 7bb16cdc0440891534adc994070e001e08d0c9a9 Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 5 Aug 2024 08:25:09 -0400 Subject: [PATCH 10/13] ye olde code bump --- datamodel/low/base/discriminator_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datamodel/low/base/discriminator_test.go b/datamodel/low/base/discriminator_test.go index f6c145c0..22fea0f0 100644 --- a/datamodel/low/base/discriminator_test.go +++ b/datamodel/low/base/discriminator_test.go @@ -48,4 +48,7 @@ propertyName: freshCakes` _ = low.BuildModel(rNode.Content[0], &rDoc) assert.Equal(t, lDoc.Hash(), rDoc.Hash()) + assert.NotNil(t, lDoc.GetRootNode()) + assert.NotNil(t, lDoc.GetKeyNode()) + } From 4a72248424a48207c4c6382f337322836f5d6375 Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 5 Aug 2024 08:31:41 -0400 Subject: [PATCH 11/13] fixed ye olde tests --- datamodel/low/base/discriminator_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datamodel/low/base/discriminator_test.go b/datamodel/low/base/discriminator_test.go index 22fea0f0..1d4f629d 100644 --- a/datamodel/low/base/discriminator_test.go +++ b/datamodel/low/base/discriminator_test.go @@ -46,6 +46,8 @@ propertyName: freshCakes` var rDoc Discriminator _ = low.BuildModel(lNode.Content[0], &lDoc) _ = low.BuildModel(rNode.Content[0], &rDoc) + lDoc.RootNode = &lNode + lDoc.KeyNode = &rNode assert.Equal(t, lDoc.Hash(), rDoc.Hash()) assert.NotNil(t, lDoc.GetRootNode()) From 527d3cdc8127c1dc72223f8439063d1a00722509 Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 5 Aug 2024 08:43:28 -0400 Subject: [PATCH 12/13] more ye olde time coverage for ye tests. --- datamodel/low/base/example.go | 10 +--------- datamodel/low/base/schema.go | 3 --- datamodel/low/v3/server_test.go | 6 ++++++ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/datamodel/low/base/example.go b/datamodel/low/base/example.go index a84ccb9b..2ddf5418 100644 --- a/datamodel/low/base/example.go +++ b/datamodel/low/base/example.go @@ -89,15 +89,7 @@ func (ex *Example) Build(ctx context.Context, keyNode, root *yaml.Node, _ *index expChildNodes := low.ExtractNodesRecursive(ctx, vn) expChildNodes.Range(func(k, v interface{}) bool { if arr, ko := v.([]*yaml.Node); ko { - if ext, ok := ex.Nodes.Load(k); ok { - if extArr, kk := ext.([]*yaml.Node); kk { - ex.Nodes.Store(k, append(extArr, arr...)) - } else { - ex.Nodes.Store(k, arr) - } - } else { - ex.Nodes.Store(k, arr) - } + ex.Nodes.Store(k, arr) } return true }) diff --git a/datamodel/low/base/schema.go b/datamodel/low/base/schema.go index 3abddba2..e1fdba54 100644 --- a/datamodel/low/base/schema.go +++ b/datamodel/low/base/schema.go @@ -727,9 +727,6 @@ func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecInde discriminator.AddNode(key.(int), g) } } - if n, ok := val.(*yaml.Node); ok { - discriminator.AddNode(key.(int), n) - } return true }) } diff --git a/datamodel/low/v3/server_test.go b/datamodel/low/v3/server_test.go index fc133944..abfe2668 100644 --- a/datamodel/low/v3/server_test.go +++ b/datamodel/low/v3/server_test.go @@ -50,6 +50,12 @@ variables: low.GenerateHashString(s.Value)) assert.Equal(t, 1, orderedmap.Len(n.GetExtensions())) + + // check nodes on variables + for k := n.Variables.Value.First(); k != nil; k = k.Next() { + assert.NotNil(t, k.Value().Value.GetKeyNode()) + assert.NotNil(t, k.Value().Value.GetRootNode()) + } } func TestServer_Build_NoVars(t *testing.T) { From 82f580042d7472e43178f0969d380729a5a556f0 Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 5 Aug 2024 09:14:07 -0400 Subject: [PATCH 13/13] another tiny cov bumpy --- utils/utils_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/utils_test.go b/utils/utils_test.go index ce98dc3d..de315fcd 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -676,6 +676,7 @@ func TestIsYAML(t *testing.T) { assert.False(t, IsYAML("{'hello':'there'}")) assert.False(t, IsYAML("")) assert.True(t, IsYAML("8908: hello: yeah: \n12309812: :123")) + assert.True(t, IsYAML("---")) } func TestConvertYAMLtoJSON(t *testing.T) {