Skip to content

Commit

Permalink
encoding/jsonschema: verify result is concrete in external tests
Browse files Browse the repository at this point in the history 
The result of unifying a JSON Schema with a concrete instance should
itself be concrete, but that's not necessarily the case.

Verify this by checking for concreteness in the test.

The test statistics before were:

```
v2:
	schema extract (pass / total): 971 / 1637 = 59.3%
	tests (pass / total): 3032 / 7175 = 42.3%
	tests on extracted schemas (pass / total): 3032 / 3542 = 85.6%

v3:
	schema extract (pass / total): 971 / 1637 = 59.3%
	tests (pass / total): 3014 / 7175 = 42.0%
	tests on extracted schemas (pass / total): 3014 / 3542 = 85.1%
```

With this change applied they are:

```
v2:
	schema extract (pass / total): 971 / 1637 = 59.3%
	tests (pass / total): 3081 / 7175 = 42.9%
	tests on extracted schemas (pass / total): 3081 / 3542 = 87.0%

v3:
	schema extract (pass / total): 971 / 1637 = 59.3%
	tests (pass / total): 3063 / 7175 = 42.7%
	tests on extracted schemas (pass / total): 3063 / 3542 = 86.5%
```

That is, overall, 49 more tests pass on both v2 and v3. This is
(almost?) entirely down to the fact that with out concreteness checking
enabled, we won't fail a test that is checking for a required field that
isn't present.

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: I6c2a8cceb454826aa02c32525070d093160fec9e
  • Loading branch information
@rogpeppe
rogpeppe committed Sep 3, 2024
1 parent e2685dd commit cf6b11d
Show file tree
Hide file tree
Showing 30 changed files with 294 additions and 490 deletions.
2 changes: 1 addition & 1 deletion encoding/jsonschema/external_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func runExternalSchemaTests(t *testing.T, m *cuetdtest.M, filename string, s *ex

instValue := ctx.BuildExpr(instAST)
qt.Assert(t, qt.IsNil(instValue.Err()))
err = instValue.Unify(schemaValue).Err()
err = instValue.Unify(schemaValue).Validate(cue.Concrete(true))
if test.Valid {
if err != nil {
testFailed(t, m, &test.Skip, test, errors.Details(err, nil))
Expand Down
36 changes: 6 additions & 30 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/allOf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,14 @@
"data": {
"foo": "baz"
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "mismatch first",
"data": {
"bar": 2
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "wrong type",
Expand Down Expand Up @@ -118,46 +110,30 @@
"foo": "quux",
"baz": null
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "mismatch first allOf",
"data": {
"bar": 2,
"baz": null
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "mismatch second allOf",
"data": {
"foo": "quux",
"bar": 2
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "mismatch both",
"data": {
"bar": 2
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
}
]
},
Expand Down
12 changes: 10 additions & 2 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/anyOf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,22 @@
"data": {
"bar": 2
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {bar:2} | {bar:2,foo!:string}\n",
"v3": "incomplete value {bar:2} | {bar:2,foo!:string}\n"
}
},
{
"description": "second anyOf valid (complex)",
"data": {
"foo": "baz"
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n",
"v3": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n"
}
},
{
"description": "both anyOf valid (complex)",
Expand Down
12 changes: 2 additions & 10 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/enum.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,20 +160,12 @@
"data": {
"foo": "foo"
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "missing all properties is invalid",
"data": {},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
}
]
},
Expand Down
8 changes: 2 additions & 6 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/items.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,11 +417,7 @@
}
]
],
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "fewer items is valid",
Expand All @@ -439,7 +435,7 @@
],
"valid": true,
"skip": {
"v2": "incompatible list lengths (2 and 3)\n",
"v2": "incompatible list lengths (2 and 3)\n0: incompatible list lengths (1 and 2)\n1: incompatible list lengths (1 and 2)\n",
"v3": "incompatible list lengths (2 and 3):\n generated.cue:2:1\n"
}
}
Expand Down
54 changes: 33 additions & 21 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/oneOf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,14 +192,22 @@
"data": {
"bar": 2
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {bar:2} | {bar:2,foo!:string}\n",
"v3": "incomplete value {bar:2} | {bar:2,foo!:string}\n"
}
},
{
"description": "second oneOf valid (complex)",
"data": {
"foo": "baz"
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n",
"v3": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n"
}
},
{
"description": "both oneOf valid (complex)",
Expand Down Expand Up @@ -277,27 +285,31 @@
"data": {
"bar": 2
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "first valid - valid",
"data": {
"foo": 1,
"bar": 2
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {foo:1,bar:2} | {foo:1,bar:2,baz!:_}\n",
"v3": "incomplete value {foo:1,bar:2} | {foo:1,bar:2,baz!:_}\n"
}
},
{
"description": "second valid - valid",
"data": {
"foo": 1,
"baz": 3
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {foo:1,baz:3,bar!:_} | {foo:1,baz:3}\n",
"v3": "incomplete value {foo:1,baz:3,bar!:_} | {foo:1,baz:3}\n"
}
},
{
"description": "both valid - invalid",
Expand Down Expand Up @@ -344,37 +356,37 @@
"data": {
"bar": 8
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {bar:8,baz?:_} | {bar:8,foo!:_}\n",
"v3": "incomplete value {bar:8,baz?:_} | {bar:8,foo!:_}\n"
}
},
{
"description": "second oneOf valid",
"data": {
"foo": "foo"
},
"valid": true
"valid": true,
"skip": {
"v2": "incomplete value {foo:\"foo\",bar!:_,baz?:_} | {foo:\"foo\"}\n",
"v3": "incomplete value {foo:\"foo\",bar!:_,baz?:_} | {foo:\"foo\"}\n"
}
},
{
"description": "both oneOf valid",
"data": {
"foo": "foo",
"bar": 8
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "neither oneOf valid",
"data": {
"baz": "quux"
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
}
]
},
Expand Down
36 changes: 6 additions & 30 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/required.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,7 @@
"data": {
"bar": 1
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "ignores arrays",
Expand Down Expand Up @@ -112,11 +108,7 @@
"foo\nbar": "1",
"foo\"bar": "1"
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
}
]
},
Expand Down Expand Up @@ -145,22 +137,14 @@
{
"description": "none of the properties mentioned",
"data": {},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "__proto__ present",
"data": {
"__proto__": "foo"
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "toString present",
Expand All @@ -169,11 +153,7 @@
"length": 37
}
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "constructor present",
Expand All @@ -182,11 +162,7 @@
"length": 37
}
},
"valid": false,
"skip": {
"v2": "unexpected success",
"v3": "unexpected success"
}
"valid": false
},
{
"description": "all present",
Expand Down
Loading

0 comments on commit cf6b11d

Please sign in to comment.