diff --git a/go.mod b/go.mod index ba8ef189..a1de8338 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/stretchr/testify v1.8.0 github.com/vmware-labs/yaml-jsonpath v0.3.2 + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb golang.org/x/sync v0.1.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 67087946..37cea8bc 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -91,8 +93,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/index/extract_refs.go b/index/extract_refs.go index 8346eb76..637c3b54 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -6,9 +6,11 @@ package index import ( "errors" "fmt" + "strings" + "github.com/pb33f/libopenapi/utils" + "golang.org/x/exp/slices" "gopkg.in/yaml.v3" - "strings" ) // ExtractRefs will return a deduplicated slice of references for every unique ref found in the document. @@ -40,21 +42,26 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, // check if we're dealing with an inline schema definition, that isn't part of an array // (which means it's being used as a value in an array, and it's not a label) // https://github.com/pb33f/libopenapi/issues/76 - if i%2 == 0 && n.Value == "schema" && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { + schemaContainingNodes := []string{"schema", "items", "additionalProperties", "contains", "not", "unevaluatedItems", "unevaluatedProperties"} + if i%2 == 0 && slices.Contains(schemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { + ref := &Reference{ + Node: node.Content[i+1], + Path: fmt.Sprintf("$.%s.%s", strings.Join(seenPath, "."), n.Value), + } + isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1]) if isRef { // record this reference - ref := &Reference{ - Node: node.Content[i+1], - Path: fmt.Sprintf("$.%s.schema", strings.Join(seenPath, ".")), - } index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) continue } - ref := &Reference{ - Node: node.Content[i+1], - Path: fmt.Sprintf("$.%s.schema", strings.Join(seenPath, ".")), + + if n.Value == "additionalProperties" || n.Value == "unevaluatedProperties" { + if utils.IsNodeBoolValue(node.Content[i+1]) { + continue + } } + index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) // check if the schema is an object or an array, @@ -67,14 +74,10 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, } } - // Perform the same check for all properties in an inline schema definition + // Perform the same check for all maps of schemas like properties and patternProperties // https://github.com/pb33f/libopenapi/issues/76 - if i%2 == 0 && n.Value == "properties" && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { - isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1]) - if isRef { - continue - } - + mapOfSchemaContainingNodes := []string{"properties", "patternProperties"} + if i%2 == 0 && slices.Contains(mapOfSchemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { // for each property add it to our schema definitions label := "" for h, prop := range node.Content[i+1].Content { @@ -83,16 +86,51 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, label = prop.Value continue } - ref := &Reference{ Node: prop, - Path: fmt.Sprintf("$.%s.properties.%s", strings.Join(seenPath, "."), label), + Path: fmt.Sprintf("$.%s.%s.%s", strings.Join(seenPath, "."), n.Value, label), + } + + isRef, _, _ := utils.IsNodeRefValue(prop) + if isRef { + // record this reference + index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) + continue + } + + index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) + + // check if the schema is an object or an array, + // and if so, add it to the list of inline schema object definitions. + k, v := utils.FindKeyNodeTop("type", prop.Content) + if k != nil && v != nil { + if v.Value == "object" || v.Value == "array" { + index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref) + } + } + } + } + + // Perform the same check for all arrays of schemas like allOf, anyOf, oneOf + arrayOfSchemaContainingNodes := []string{"allOf", "anyOf", "oneOf", "prefixItems"} + if i%2 == 0 && slices.Contains(arrayOfSchemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { + // for each element in the array, add it to our schema definitions + for h, element := range node.Content[i+1].Content { + ref := &Reference{ + Node: element, + Path: fmt.Sprintf("$.%s.%s[%d]", strings.Join(seenPath, "."), n.Value, h), + } + + isRef, _, _ := utils.IsNodeRefValue(element) + if isRef { // record this reference + index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) + continue } index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) // check if the schema is an object or an array, // and if so, add it to the list of inline schema object definitions. - k, v := utils.FindKeyNodeTop("type", node.Content[i+1].Content) + k, v := utils.FindKeyNodeTop("type", element.Content) if k != nil && v != nil { if v.Value == "object" || v.Value == "array" { index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref) @@ -342,7 +380,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, if i < len(node.Content)-1 { next := node.Content[i+1] - if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) { + if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) && len(seenPath) > 0 { seenPath = seenPath[:len(seenPath)-1] } } @@ -366,7 +404,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference { var found []*Reference - //run this async because when things get recursive, it can take a while + // run this async because when things get recursive, it can take a while c := make(chan bool) locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) { @@ -419,7 +457,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc for r := range refsToCheck { // expand our index of all mapped refs go locate(refsToCheck[r], r, mappedRefsInSequence) - //locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing. + // locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing. } completedRefs := 0 diff --git a/index/extract_refs_test.go b/index/extract_refs_test.go index 805e87e6..afdc4927 100644 --- a/index/extract_refs_test.go +++ b/index/extract_refs_test.go @@ -4,13 +4,13 @@ package index import ( + "testing" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" - "testing" ) func TestSpecIndex_ExtractRefs_CheckDescriptionNotMap(t *testing.T) { - yml := `openapi: 3.1.0 info: description: This is a description @@ -37,7 +37,6 @@ paths: } func TestSpecIndex_ExtractRefs_CheckPropertiesForInlineSchema(t *testing.T) { - yml := `openapi: 3.1.0 servers: - url: http://localhost:8080 @@ -54,14 +53,63 @@ paths: properties: test: type: array - items: true + items: + type: object + prefixItems: + - $ref: '#/components/schemas/Test' + additionalProperties: false + unevaluatedProperties: false +components: + schemas: + Test: + type: object + additionalProperties: + type: string + contains: + type: string + not: + type: number + unevaluatedProperties: + type: boolean + patternProperties: + ^S_: + type: string + ^I_: + type: integer + prefixItems: + - type: string + AllOf: + allOf: + - type: object + properties: + test: + type: string + - type: object + properties: + test2: + type: string + AnyOf: + anyOf: + - type: object + properties: + test: + type: string + - type: object + properties: + test2: + type: string + OneOf: + oneOf: + - type: string + - type: number + - type: boolean ` var rootNode yaml.Node _ = yaml.Unmarshal([]byte(yml), &rootNode) c := CreateOpenAPIIndexConfig() idx := NewSpecIndexWithConfig(&rootNode, c) - assert.Len(t, idx.allInlineSchemaDefinitions, 2) - assert.Len(t, idx.allInlineSchemaObjectDefinitions, 1) + assert.Len(t, idx.allInlineSchemaDefinitions, 21) + assert.Len(t, idx.allInlineSchemaObjectDefinitions, 7) } // https://github.com/pb33f/libopenapi/issues/112 diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 340b2cc5..9061f9eb 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -44,7 +44,7 @@ func TestSpecIndex_ExtractRefsStripe(t *testing.T) { assert.Len(t, index.GetPolyAllOfReferences(), 0) assert.Len(t, index.GetPolyOneOfReferences(), 275) assert.Len(t, index.GetPolyAnyOfReferences(), 553) - assert.Len(t, index.GetAllReferenceSchemas(), 696) + assert.Len(t, index.GetAllReferenceSchemas(), 1972) assert.NotNil(t, index.GetRootServersNode()) assert.Len(t, index.GetAllRootServers(), 1) @@ -283,7 +283,7 @@ func TestSpecIndex_BurgerShop(t *testing.T) { assert.Equal(t, 6, index.GetPathCount()) assert.Equal(t, 6, len(index.GetAllComponentSchemas())) - assert.Equal(t, 47, len(index.GetAllSchemas())) + assert.Equal(t, 56, len(index.GetAllSchemas())) assert.Equal(t, 34, len(index.GetAllSequencedReferences())) assert.NotNil(t, index.GetSchemasNode()) @@ -1096,10 +1096,10 @@ func ExampleNewSpecIndex() { // 246 paths // 402 operations // 537 component schemas - // 696 reference schemas - // 9798 inline schemas - // 711 inline schemas that are objects or arrays - // 11031 total schemas + // 1972 reference schemas + // 11749 inline schemas + // 2612 inline schemas that are objects or arrays + // 14258 total schemas // 1516 enums // 828 polymorphic references }