Skip to content

Commit

Permalink
fix: improves the ability to get all inline schemas and identify inli…
Browse files Browse the repository at this point in the history 
…ne object schemas (#154)

* fix: improves the ability to get all inline schemas and identify inline object schemas

* fix

* fix

* fix

* fix
  • Loading branch information
@TristanSpeakEasy
TristanSpeakEasy committed Aug 22, 2023
1 parent 976ca09 commit 74c7a3f
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 35 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.20
require (
github.com/stretchr/testify v1.8.0
github.com/vmware-labs/yaml-jsonpath v0.3.2
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
golang.org/x/sync v0.1.0
gopkg.in/yaml.v3 v3.0.1
)
Expand Down
4 changes: 3 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA=
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
Expand Down Expand Up @@ -91,8 +93,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
82 changes: 60 additions & 22 deletions index/extract_refs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ package index
import (
"errors"
"fmt"
"strings"

"github.com/pb33f/libopenapi/utils"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v3"
"strings"
)

// ExtractRefs will return a deduplicated slice of references for every unique ref found in the document.
Expand Down Expand Up @@ -40,21 +42,26 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
// check if we're dealing with an inline schema definition, that isn't part of an array
// (which means it's being used as a value in an array, and it's not a label)
// https://github.com/pb33f/libopenapi/issues/76
if i%2 == 0 && n.Value == "schema" && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
schemaContainingNodes := []string{"schema", "items", "additionalProperties", "contains", "not", "unevaluatedItems", "unevaluatedProperties"}
if i%2 == 0 && slices.Contains(schemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
ref := &Reference{
Node: node.Content[i+1],
Path: fmt.Sprintf("$.%s.%s", strings.Join(seenPath, "."), n.Value),
}

isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1])
if isRef {
// record this reference
ref := &Reference{
Node: node.Content[i+1],
Path: fmt.Sprintf("$.%s.schema", strings.Join(seenPath, ".")),
}
index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref)
continue
}
ref := &Reference{
Node: node.Content[i+1],
Path: fmt.Sprintf("$.%s.schema", strings.Join(seenPath, ".")),

if n.Value == "additionalProperties" || n.Value == "unevaluatedProperties" {
if utils.IsNodeBoolValue(node.Content[i+1]) {
continue
}
}

index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref)

// check if the schema is an object or an array,
Expand All @@ -67,14 +74,10 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
}
}

// Perform the same check for all properties in an inline schema definition
// Perform the same check for all maps of schemas like properties and patternProperties
// https://github.com/pb33f/libopenapi/issues/76
if i%2 == 0 && n.Value == "properties" && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1])
if isRef {
continue
}

mapOfSchemaContainingNodes := []string{"properties", "patternProperties"}
if i%2 == 0 && slices.Contains(mapOfSchemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
// for each property add it to our schema definitions
label := ""
for h, prop := range node.Content[i+1].Content {
Expand All @@ -83,16 +86,51 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
label = prop.Value
continue
}

ref := &Reference{
Node: prop,
Path: fmt.Sprintf("$.%s.properties.%s", strings.Join(seenPath, "."), label),
Path: fmt.Sprintf("$.%s.%s.%s", strings.Join(seenPath, "."), n.Value, label),
}

isRef, _, _ := utils.IsNodeRefValue(prop)
if isRef {
// record this reference
index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref)
continue
}

index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref)

// check if the schema is an object or an array,
// and if so, add it to the list of inline schema object definitions.
k, v := utils.FindKeyNodeTop("type", prop.Content)
if k != nil && v != nil {
if v.Value == "object" || v.Value == "array" {
index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref)
}
}
}
}

// Perform the same check for all arrays of schemas like allOf, anyOf, oneOf
arrayOfSchemaContainingNodes := []string{"allOf", "anyOf", "oneOf", "prefixItems"}
if i%2 == 0 && slices.Contains(arrayOfSchemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
// for each element in the array, add it to our schema definitions
for h, element := range node.Content[i+1].Content {
ref := &Reference{
Node: element,
Path: fmt.Sprintf("$.%s.%s[%d]", strings.Join(seenPath, "."), n.Value, h),
}

isRef, _, _ := utils.IsNodeRefValue(element)
if isRef { // record this reference
index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref)
continue
}
index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref)

// check if the schema is an object or an array,
// and if so, add it to the list of inline schema object definitions.
k, v := utils.FindKeyNodeTop("type", node.Content[i+1].Content)
k, v := utils.FindKeyNodeTop("type", element.Content)
if k != nil && v != nil {
if v.Value == "object" || v.Value == "array" {
index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref)
Expand Down Expand Up @@ -342,7 +380,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
if i < len(node.Content)-1 {
next := node.Content[i+1]

if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) {
if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) && len(seenPath) > 0 {
seenPath = seenPath[:len(seenPath)-1]
}
}
Expand All @@ -366,7 +404,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference {
var found []*Reference

//run this async because when things get recursive, it can take a while
// run this async because when things get recursive, it can take a while
c := make(chan bool)

locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
Expand Down Expand Up @@ -419,7 +457,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
for r := range refsToCheck {
// expand our index of all mapped refs
go locate(refsToCheck[r], r, mappedRefsInSequence)
//locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
// locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
}

completedRefs := 0
Expand Down
60 changes: 54 additions & 6 deletions index/extract_refs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
package index

import (
"testing"

"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
"testing"
)

func TestSpecIndex_ExtractRefs_CheckDescriptionNotMap(t *testing.T) {

yml := `openapi: 3.1.0
info:
description: This is a description
Expand All @@ -37,7 +37,6 @@ paths:
}

func TestSpecIndex_ExtractRefs_CheckPropertiesForInlineSchema(t *testing.T) {

yml := `openapi: 3.1.0
servers:
- url: http://localhost:8080
Expand All @@ -54,14 +53,63 @@ paths:
properties:
test:
type: array
items: true
items:
type: object
prefixItems:
- $ref: '#/components/schemas/Test'
additionalProperties: false
unevaluatedProperties: false
components:
schemas:
Test:
type: object
additionalProperties:
type: string
contains:
type: string
not:
type: number
unevaluatedProperties:
type: boolean
patternProperties:
^S_:
type: string
^I_:
type: integer
prefixItems:
- type: string
AllOf:
allOf:
- type: object
properties:
test:
type: string
- type: object
properties:
test2:
type: string
AnyOf:
anyOf:
- type: object
properties:
test:
type: string
- type: object
properties:
test2:
type: string
OneOf:
oneOf:
- type: string
- type: number
- type: boolean
`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &rootNode)
c := CreateOpenAPIIndexConfig()
idx := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, idx.allInlineSchemaDefinitions, 2)
assert.Len(t, idx.allInlineSchemaObjectDefinitions, 1)
assert.Len(t, idx.allInlineSchemaDefinitions, 21)
assert.Len(t, idx.allInlineSchemaObjectDefinitions, 7)
}

// https://github.com/pb33f/libopenapi/issues/112
Expand Down
12 changes: 6 additions & 6 deletions index/spec_index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestSpecIndex_ExtractRefsStripe(t *testing.T) {
assert.Len(t, index.GetPolyAllOfReferences(), 0)
assert.Len(t, index.GetPolyOneOfReferences(), 275)
assert.Len(t, index.GetPolyAnyOfReferences(), 553)
assert.Len(t, index.GetAllReferenceSchemas(), 696)
assert.Len(t, index.GetAllReferenceSchemas(), 1972)
assert.NotNil(t, index.GetRootServersNode())
assert.Len(t, index.GetAllRootServers(), 1)

Expand Down Expand Up @@ -283,7 +283,7 @@ func TestSpecIndex_BurgerShop(t *testing.T) {
assert.Equal(t, 6, index.GetPathCount())

assert.Equal(t, 6, len(index.GetAllComponentSchemas()))
assert.Equal(t, 47, len(index.GetAllSchemas()))
assert.Equal(t, 56, len(index.GetAllSchemas()))

assert.Equal(t, 34, len(index.GetAllSequencedReferences()))
assert.NotNil(t, index.GetSchemasNode())
Expand Down Expand Up @@ -1096,10 +1096,10 @@ func ExampleNewSpecIndex() {
// 246 paths
// 402 operations
// 537 component schemas
// 696 reference schemas
// 9798 inline schemas
// 711 inline schemas that are objects or arrays
// 11031 total schemas
// 1972 reference schemas
// 11749 inline schemas
// 2612 inline schemas that are objects or arrays
// 14258 total schemas
// 1516 enums
// 828 polymorphic references
}
Expand Down

0 comments on commit 74c7a3f

Please sign in to comment.