encoding/jsonschema: add external test suite

This adds test data acquired from
github.com/json-schema-org/JSON-Schema-Test-Suite and uses it to test
CUE's encoding/jsonschema package.

Clearly, there are many tests and schemas that fail, so we need to avoid
those failures causing the CUE CI tests to fail. We could maintain an
auxilliary data structure to record which tests are expected to
pass/fail, but that would be hard to understand, because each failure is
only understandable in the context of the test data that produced it.

So instead, we add the auxilliary data directly inside the JSON data
directly, using the `CUE_UPDATE` convention to enable updating that
information. Specifically each schema and each test can be associated
with a `skip` field that causes a test failure on that schema or that
test to be ignored, and describes the reason for the failure.

The `vendor-external` script can be used to pull in updated tests from
the external repository.

The `teststats.go` program can be used to show information on which
tests pass and fail.

At the time of writing, the summary stats are as follows:

```
schema extract (pass / total): 971 / 1637 = 59.3%
tests (pass / total): 3032 / 7175 = 42.3%
tests on extracted schemas (pass / total): 3032 / 3542 = 85.6%
```

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: I653201803df8a9165671bd79929e12f37e549258
Dispatch-Trailer: {"type":"trybot","CL":1200255,"patchset":2,"ref":"refs/changes/55/1200255/2","targetBranch":"master"}

encoding/jsonschema/external_test.go

@@ -0,0 +1,184 @@
+package jsonschema_test
+
+import (
+	"bytes"
+	stdjson "encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/go-quicktest/qt"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/format"
+	"cuelang.org/go/encoding/json"
+	"cuelang.org/go/encoding/jsonschema"
+	"cuelang.org/go/encoding/jsonschema/internal/externaltest"
+	"cuelang.org/go/internal/cuetest"
+)
+
+// TestExternal runs the externally defined JSON Schema test suite,
+// as defined in https://github.com/json-schema-org/JSON-Schema-Test-Suite.
+func TestExternal(t *testing.T) {
+	tests, err := externaltest.ReadTestDir("testdata/external")
+	qt.Assert(t, qt.IsNil(err))
+
+	t.Logf("read %d test files", len(tests))
+	// Group the tests under a single subtest so that we can use
+	// t.Parallel and still guarantee that all tests have completed
+	// by the end.
+	t.Run("tests", func(t *testing.T) {
+		// Run tests in deterministic order so we get some consistency between runs.
+		for _, filename := range sortedKeys(tests) {
+			schemas := tests[filename]
+			t.Run(strings.ReplaceAll(filename, "/", "__"), func(t *testing.T) {
+				for _, s := range schemas {
+					t.Run(strings.ReplaceAll(s.Description, "/", "__"), func(t *testing.T) {
+						runExternalSchemaTests(t, filename, s)
+					})
+				}
+			})
+		}
+	})
+	if !cuetest.UpdateGoldenFiles {
+		return
+	}
+	for filename, schemas := range tests {
+		filename = filepath.Join("testdata/external", filename)
+		data, err := stdjson.MarshalIndent(schemas, "", "\t")
+		qt.Assert(t, qt.IsNil(err))
+		data = append(data, '\n')
+		oldData, err := os.ReadFile(filename)
+		qt.Assert(t, qt.IsNil(err))
+		if bytes.Equal(oldData, data) {
+			continue
+		}
+		err = os.WriteFile(filename, data, 0o666)
+		qt.Assert(t, qt.IsNil(err))
+	}
+}
+
+func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schema) {
+	t.Logf("file %v", path.Join("testdata/external", filename))
+	ctx := cuecontext.New()
+	jsonSchemaAST, err := json.Extract("schema.json", s.Schema)
+	qt.Assert(t, qt.IsNil(err))
+	jsonSchemav := ctx.BuildExpr(jsonSchemaAST)
+	qt.Assert(t, qt.IsNil(jsonSchemav.Err()))
+	versStr, _, _ := strings.Cut(strings.TrimPrefix(filename, "tests/"), "/")
+	vers, ok := extVersionToVersion[versStr]
+	if !ok {
+		t.Fatalf("unknown JSON schema version for file %q", filename)
+	}
+	if vers == jsonschema.VersionUnknown {
+		t.Skipf("skipping test for unknown schema version %v", versStr)
+	}
+	schemaAST, extractErr := jsonschema.Extract(jsonSchemav, &jsonschema.Config{
+		Strict:         true,
+		DefaultVersion: vers,
+	})
+	var schemav cue.Value
+	if extractErr == nil {
+		// Round-trip via bytes because that's what will usually happen
+		// to the generated schema.
+		b, err := format.Node(schemaAST, format.Simplify())
+		qt.Assert(t, qt.IsNil(err))
+		schemav = ctx.CompileBytes(b, cue.Filename("generated.cue"))
+		if err := schemav.Err(); err != nil {
+			extractErr = fmt.Errorf("cannot compile resulting schema: %v", errors.Details(err, nil))
+		}
+	}
+
+	if extractErr != nil {
+		if cuetest.UpdateGoldenFiles {
+			s.Skip = fmt.Sprintf("extract error: %v", extractErr)
+			for _, t := range s.Tests {
+				t.Skip = "could not compile schema"
+			}
+			return
+		}
+		if s.Skip != "" {
+			t.SkipNow()
+		}
+		t.Fatalf("extract error: %v", extractErr)
+	}
+	if s.Skip != "" {
+		t.Errorf("unexpected test success on skipped test")
+	}
+
+	for _, test := range s.Tests {
+		t.Run("", func(t *testing.T) {
+			t.Logf("description: %s", test.Description)
+			instAST, err := json.Extract("instance.json", test.Data)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			qt.Assert(t, qt.IsNil(err), qt.Commentf("test data: %q; details: %v", test.Data, errors.Details(err, nil)))
+
+			instv := ctx.BuildExpr(instAST)
+			qt.Assert(t, qt.IsNil(instv.Err()))
+			err = instv.Unify(schemav).Err()
+			if test.Valid {
+				if cuetest.UpdateGoldenFiles {
+					if err == nil {
+						test.Skip = ""
+					} else {
+						test.Skip = errors.Details(err, nil)
+					}
+					return
+				}
+				if err != nil {
+					if test.Skip != "" {
+						t.SkipNow()
+					}
+					t.Fatalf("error: %v", errors.Details(err, nil))
+				} else if test.Skip != "" {
+					t.Error("unexpectedly more correct behavior (test success) on skipped test")
+				}
+			} else {
+				if cuetest.UpdateGoldenFiles {
+					if err != nil {
+						test.Skip = ""
+					} else {
+						test.Skip = "unexpected success"
+					}
+					return
+				}
+				if err == nil {
+					if test.Skip != "" {
+						t.SkipNow()
+					}
+					t.Fatal("unexpected success")
+				} else if test.Skip != "" {
+					t.Error("unexpectedly more correct behavior (test failure) on skipped test")
+				}
+			}
+		})
+	}
+}
+
+var extVersionToVersion = map[string]jsonschema.Version{
+	"draft3":       jsonschema.VersionUnknown,
+	"draft4":       jsonschema.VersionDraft4,
+	"draft6":       jsonschema.VersionDraft6,
+	"draft7":       jsonschema.VersionDraft7,
+	"draft2019-09": jsonschema.VersionDraft2019_09,
+	"draft2020-12": jsonschema.VersionDraft2020_12,
+	"draft-next":   jsonschema.VersionUnknown,
+}
+
+func sortedKeys[T any](m map[string]T) []string {
+	ks := make([]string, 0, len(m))
+	for k := range m {
+		ks = append(ks, k)
+	}
+	sort.Strings(ks)
+	return ks
+}

encoding/jsonschema/internal/externaltest/tests.go

@@ -0,0 +1,65 @@
+package externaltest
+
+import (
+	"bytes"
+	stdjson "encoding/json"
+	"os"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/interpreter/embed"
+	"cuelang.org/go/cue/load"
+)
+
+type Schema struct {
+	Description string             `json:"description"`
+	Comment     string             `json:"comment,omitempty"`
+	Schema      stdjson.RawMessage `json:"schema"`
+	Skip        string             `json:"skip,omitempty"`
+	Tests       []*Test            `json:"tests"`
+}
+
+type Test struct {
+	Description string             `json:"description"`
+	Comment     string             `json:"comment,omitempty"`
+	Data        stdjson.RawMessage `json:"data"`
+	Valid       bool               `json:"valid"`
+	Skip        string             `json:"skip,omitempty"`
+}
+
+func ReadTestDir(dir string) (tests map[string][]*Schema, err error) {
+	os.Setenv("CUE_EXPERIMENT", "embed")
+	inst := load.Instances([]string{"."}, &load.Config{
+		Dir: dir,
+	})[0]
+	if err != nil {
+		return nil, err
+	}
+	ctx := cuecontext.New(cuecontext.Interpreter(embed.New()))
+	instVal := ctx.BuildInstance(inst)
+	if err := instVal.Err(); err != nil {
+		return nil, err
+	}
+	val := instVal.LookupPath(cue.MakePath(cue.Str("allTests")))
+	if err := val.Err(); err != nil {
+		return nil, err
+	}
+	if err := val.Decode(&tests); err != nil {
+		return nil, err
+	}
+	// Fix up the raw JSON data to avoid running into some decode issues.
+	for _, schemas := range tests {
+		for _, schema := range schemas {
+			for _, test := range schema.Tests {
+				if len(test.Data) == 0 {
+					// See https://github.com/cue-lang/cue/issues/3397
+					test.Data = []byte("null")
+					continue
+				}
+				// See https://github.com/cue-lang/cue/issues/3398
+				test.Data = bytes.ReplaceAll(test.Data, []byte("\ufeff"), []byte(`\ufeff`))
+			}
+		}
+	}
+	return tests, nil
+}

encoding/jsonschema/testdata/external/config.cue

@@ -0,0 +1,37 @@
+@extern(embed)
+
+package external
+
+allTests: _	@embed(glob=tests/*/*.json)
+allTests: _	@embed(glob=tests/*/*/*.json)
+allTests: _	@embed(glob=tests/*/*/*/*.json)
+
+allTests: [_]: [... #Schema]
+#Schema: {
+	description!: string
+	comment?: string
+	specification?: _
+	schema!: _
+	tests!: [... #Test]
+
+	// skip is not part of the orginal test data, but
+	// is inserted by our test logic (when CUE_UPDATE=1)
+	// to indicate which tests are passing and which
+	// are failing. The text indicates some reason as to
+	// why the schema is skipped.
+	skip?: string
+}
+
+#Test: {
+	description!: string
+	comment?: string
+	data!: _
+	valid!: bool
+
+	// skip is not part of the orginal test data, but
+	// is inserted by our test logic (when CUE_UPDATE=1)
+	// to indicate which tests are passing and which
+	// are failing. The text indicates some reason as to
+	// why the test is skipped.
+	skip?: string
+}