From a9d97076b53feeba64179cf7d9fdc241e127ce88 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:26:12 -0700 Subject: [PATCH 1/3] Ensure a null PublicKey is supported on assembly refs (#108928) Co-authored-by: Steve Harter --- src/coreclr/md/compiler/importhelper.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/coreclr/md/compiler/importhelper.cpp b/src/coreclr/md/compiler/importhelper.cpp index d1b2ac55fdc6d..f1dd230eeb4db 100644 --- a/src/coreclr/md/compiler/importhelper.cpp +++ b/src/coreclr/md/compiler/importhelper.cpp @@ -3174,6 +3174,8 @@ ImportHelper::CreateAssemblyRefFromAssembly( mdAssemblyRef tkAssemRef; HRESULT hr = S_OK; StrongNameToken token; + const void *pbToken = NULL; + ULONG cbToken = 0; ULONG i; // Set output to Nil. @@ -3193,6 +3195,9 @@ ImportHelper::CreateAssemblyRefFromAssembly( IfFailGo(StrongNameTokenFromPublicKey((BYTE*)pbPublicKey, cbPublicKey, &token)); + + pbToken = &token; + cbToken = StrongNameToken::SIZEOF_TOKEN; } else _ASSERTE(!IsAfPublicKey(dwFlags)); @@ -3209,8 +3214,8 @@ ImportHelper::CreateAssemblyRefFromAssembly( continue; // See if the AssemblyRef already exists in the emit scope. - hr = FindAssemblyRef(pMiniMdEmit, szName, szLocale, &token, - StrongNameToken::SIZEOF_TOKEN, usMajorVersion, usMinorVersion, + hr = FindAssemblyRef(pMiniMdEmit, szName, szLocale, pbToken, + cbToken, usMajorVersion, usMinorVersion, usBuildNumber, usRevisionNumber, dwFlags, &tkAssemRef); if (hr == CLDB_E_RECORD_NOTFOUND) From 92b4ddd3cb195fa55f2ea859ac213777cb5e44e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:28:40 -0700 Subject: [PATCH 2/3] [release/9.0] Fix handling of appending keywords to boolean schemas. (#108248) * Fix handling of appending keywords to boolean schemas. * Add a few comments documenting boolean schemas. --------- Co-authored-by: Eirik Tsarpalis --- .../src/System/Text/Json/Schema/JsonSchema.cs | 27 +++++++++++++++++++ .../Text/Json/Schema/JsonSchemaExporter.cs | 1 + .../JsonSchemaExporterTests.TestTypes.cs | 17 ++++++++++++ .../Serialization/JsonSchemaExporterTests.cs | 1 + 4 files changed, 46 insertions(+) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchema.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchema.cs index d261b374b390a..0948acc19bb92 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchema.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchema.cs @@ -34,6 +34,14 @@ public JsonSchema() { } public bool IsTrue => _trueOrFalse is true; public bool IsFalse => _trueOrFalse is false; + + /// + /// Per the JSON schema core specification section 4.3 + /// (https://json-schema.org/draft/2020-12/json-schema-core#name-json-schema-documents) + /// A JSON schema must either be an object or a boolean. + /// We represent false and true schemas using this flag. + /// It is not possible to specify keywords in boolean schemas. + /// private readonly bool? _trueOrFalse; public string? Ref { get => _ref; set { VerifyMutable(); _ref = value; } } @@ -95,6 +103,7 @@ public int KeywordCount { if (_trueOrFalse != null) { + // Boolean schemas admit no keywords return 0; } @@ -129,6 +138,7 @@ public void MakeNullable() { if (_trueOrFalse != null) { + // boolean schemas do not admit type keywords. return; } @@ -260,6 +270,23 @@ JsonNode CompleteSchema(JsonNode schema) } } + /// + /// If the schema is boolean, replaces it with a semantically + /// equivalent object schema that allows appending keywords. + /// + public static void EnsureMutable(ref JsonSchema schema) + { + switch (schema._trueOrFalse) + { + case false: + schema = new JsonSchema { Not = True }; + break; + case true: + schema = new JsonSchema(); + break; + } + } + private static ReadOnlySpan s_schemaValues => [ // NB the order of these values influences order of types in the rendered schema diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs index 063bd95147673..9690b2b308def 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs @@ -240,6 +240,7 @@ private static JsonSchema MapJsonSchemaCore( if (property.AssociatedParameter is { HasDefaultValue: true } parameterInfo) { + JsonSchema.EnsureMutable(ref propertySchema); propertySchema.DefaultValue = JsonSerializer.SerializeToNode(parameterInfo.DefaultValue, property.JsonTypeInfo); propertySchema.HasDefaultValue = true; } diff --git a/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs b/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs index 12ee630b378fb..0f247cab034bc 100644 --- a/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs +++ b/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs @@ -1034,6 +1034,18 @@ public static IEnumerable GetTestDataCore() } """); + yield return new TestData( + Value: new(value: null), + AdditionalValues: [new(true), new(42), new(""), new(new object()), new(Array.Empty())], + ExpectedJsonSchema: """ + { + "type": ["object","null"], + "properties": { + "Value": { "default": null } + } + } + """); + // Collection types yield return new TestData([1, 2, 3], ExpectedJsonSchema: """{"type":["array","null"],"items":{"type":"integer"}}"""); yield return new TestData>([false, true, false], ExpectedJsonSchema: """{"type":["array","null"],"items":{"type":"boolean"}}"""); @@ -1441,6 +1453,11 @@ public class ClassWithJsonPointerEscapablePropertyNames public PocoWithRecursiveMembers Value { get; set; } } + public class ClassWithOptionalObjectParameter(object? value = null) + { + public object? Value { get; } = value; + } + public readonly struct StructDictionary(IEnumerable> values) : IReadOnlyDictionary where TKey : notnull diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs index a7b0775361de9..6946ea661b561 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs @@ -107,6 +107,7 @@ public sealed partial class JsonSchemaExporterTests_SourceGen() [JsonSerializable(typeof(PocoCombiningPolymorphicTypeAndDerivedTypes))] [JsonSerializable(typeof(ClassWithComponentModelAttributes))] [JsonSerializable(typeof(ClassWithJsonPointerEscapablePropertyNames))] + [JsonSerializable(typeof(ClassWithOptionalObjectParameter))] // Collection types [JsonSerializable(typeof(int[]))] [JsonSerializable(typeof(List))] From 83d152c4b5ff2f50a6d01a20fcc22be07da05ab3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:29:03 -0700 Subject: [PATCH 3/3] Make the schema generator reuse algorithm more aggressive to reduce generated schema size. (#108800) Co-authored-by: Eirik Tsarpalis --- .../Text/Json/Schema/JsonSchemaExporter.cs | 77 +++++++++---------- .../Json/Schema/JsonSchemaExporterContext.cs | 2 +- .../JsonSchemaExporterTests.TestTypes.cs | 64 +++++++++++++++ .../tests/Common/JsonSchemaExporterTests.cs | 8 ++ .../Serialization/JsonSchemaExporterTests.cs | 3 + 5 files changed, 113 insertions(+), 41 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs index 9690b2b308def..0e8fa55dc6493 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Runtime.InteropServices; using System.Text.Json.Nodes; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; @@ -76,9 +77,12 @@ private static JsonSchema MapJsonSchemaCore( { Debug.Assert(typeInfo.IsConfigured); - if (cacheResult && state.TryPushType(typeInfo, propertyInfo, out string? existingJsonPointer)) + JsonSchemaExporterContext exporterContext = state.CreateContext(typeInfo, propertyInfo, parentPolymorphicTypeInfo); + + if (cacheResult && typeInfo.Kind is not JsonTypeInfoKind.None && + state.TryGetExistingJsonPointer(exporterContext, out string? existingJsonPointer)) { - // We're generating the schema of a recursive type, return a reference pointing to the outermost schema. + // The schema context has already been generated in the schema document, return a reference to it. return CompleteSchema(ref state, new JsonSchema { Ref = existingJsonPointer }); } @@ -364,17 +368,12 @@ JsonSchema CompleteSchema(ref GenerationState state, JsonSchema schema) { schema.MakeNullable(); } - - if (cacheResult) - { - state.PopGeneratedType(); - } } if (state.ExporterOptions.TransformSchemaNode != null) { // Prime the schema for invocation by the JsonNode transformer. - schema.ExporterContext = state.CreateContext(typeInfo, propertyInfo, parentPolymorphicTypeInfo); + schema.ExporterContext = exporterContext; } return schema; @@ -409,7 +408,7 @@ private static bool IsPolymorphicTypeThatSpecifiesItselfAsDerivedType(JsonTypeIn private readonly ref struct GenerationState(JsonSerializerOptions options, JsonSchemaExporterOptions exporterOptions) { private readonly List _currentPath = []; - private readonly List<(JsonTypeInfo typeInfo, JsonPropertyInfo? propertyInfo, int depth)> _generationStack = []; + private readonly Dictionary<(JsonTypeInfo, JsonPropertyInfo?), string[]> _generated = new(); public int CurrentDepth => _currentPath.Count; public JsonSerializerOptions Options { get; } = options; @@ -432,77 +431,75 @@ public void PopSchemaNode() } /// - /// Pushes the current type/property to the generation stack or returns a JSON pointer if the type is recursive. + /// Registers the current schema node generation context; if it has already been generated return a JSON pointer to its location. /// - public bool TryPushType(JsonTypeInfo typeInfo, JsonPropertyInfo? propertyInfo, [NotNullWhen(true)] out string? existingJsonPointer) + public bool TryGetExistingJsonPointer(in JsonSchemaExporterContext context, [NotNullWhen(true)] out string? existingJsonPointer) { - foreach ((JsonTypeInfo otherTypeInfo, JsonPropertyInfo? otherPropertyInfo, int depth) in _generationStack) + (JsonTypeInfo TypeInfo, JsonPropertyInfo? PropertyInfo) key = (context.TypeInfo, context.PropertyInfo); +#if NET + ref string[]? pathToSchema = ref CollectionsMarshal.GetValueRefOrAddDefault(_generated, key, out bool exists); +#else + bool exists = _generated.TryGetValue(key, out string[]? pathToSchema); +#endif + if (exists) { - if (typeInfo == otherTypeInfo && propertyInfo == otherPropertyInfo) - { - existingJsonPointer = FormatJsonPointer(_currentPath, depth); - return true; - } + existingJsonPointer = FormatJsonPointer(pathToSchema); + return true; } - - _generationStack.Add((typeInfo, propertyInfo, CurrentDepth)); +#if NET + pathToSchema = context._path; +#else + _generated[key] = context._path; +#endif existingJsonPointer = null; return false; } - public void PopGeneratedType() - { - Debug.Assert(_generationStack.Count > 0); - _generationStack.RemoveAt(_generationStack.Count - 1); - } - public JsonSchemaExporterContext CreateContext(JsonTypeInfo typeInfo, JsonPropertyInfo? propertyInfo, JsonTypeInfo? baseTypeInfo) { - return new JsonSchemaExporterContext(typeInfo, propertyInfo, baseTypeInfo, _currentPath.ToArray()); + return new JsonSchemaExporterContext(typeInfo, propertyInfo, baseTypeInfo, [.. _currentPath]); } - private static string FormatJsonPointer(List currentPathList, int depth) + private static string FormatJsonPointer(ReadOnlySpan path) { - Debug.Assert(0 <= depth && depth < currentPathList.Count); - - if (depth == 0) + if (path.IsEmpty) { return "#"; } - using ValueStringBuilder sb = new(initialCapacity: depth * 10); + using ValueStringBuilder sb = new(initialCapacity: path.Length * 10); sb.Append('#'); - for (int i = 0; i < depth; i++) + foreach (string segment in path) { - ReadOnlySpan segment = currentPathList[i].AsSpan(); + ReadOnlySpan span = segment.AsSpan(); sb.Append('/'); do { // Per RFC 6901 the characters '~' and '/' must be escaped. - int pos = segment.IndexOfAny('~', '/'); + int pos = span.IndexOfAny('~', '/'); if (pos < 0) { - sb.Append(segment); + sb.Append(span); break; } - sb.Append(segment.Slice(0, pos)); + sb.Append(span.Slice(0, pos)); - if (segment[pos] == '~') + if (span[pos] == '~') { sb.Append("~0"); } else { - Debug.Assert(segment[pos] == '/'); + Debug.Assert(span[pos] == '/'); sb.Append("~1"); } - segment = segment.Slice(pos + 1); + span = span.Slice(pos + 1); } - while (!segment.IsEmpty); + while (!span.IsEmpty); } return sb.ToString(); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporterContext.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporterContext.cs index fc9a0c0be97dd..3ea21d9b3e3b4 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporterContext.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporterContext.cs @@ -10,7 +10,7 @@ namespace System.Text.Json.Schema /// public readonly struct JsonSchemaExporterContext { - private readonly string[] _path; + internal readonly string[] _path; internal JsonSchemaExporterContext( JsonTypeInfo typeInfo, diff --git a/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs b/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs index 0f247cab034bc..e128d6e6e474c 100644 --- a/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs +++ b/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs @@ -469,6 +469,62 @@ public static IEnumerable GetTestDataCore() """, Options: new() { TreatNullObliviousAsNonNullable = true }); + SimpleRecord recordValue = new(42, "str", true, 3.14); + yield return new TestData( + Value: new() { Value1 = recordValue, Value2 = recordValue, ArrayValue = [recordValue], ListValue = [recordValue] }, + ExpectedJsonSchema: """ + { + "type": ["object","null"], + "properties": { + "Value1": { + "type": "object", + "properties": { + "X": { "type": "integer" }, + "Y": { "type": "string" }, + "Z": { "type": "boolean" }, + "W": { "type": "number" } + }, + "required": ["X", "Y", "Z", "W"] + }, + /* The same type on a different property is repeated to + account for potential metadata resolved from attributes. */ + "Value2": { + "type": "object", + "properties": { + "X": { "type": "integer" }, + "Y": { "type": "string" }, + "Z": { "type": "boolean" }, + "W": { "type": "number" } + }, + "required": ["X", "Y", "Z", "W"] + }, + /* This collection element is the first occurrence + of the type without contextual metadata. */ + "ListValue": { + "type": "array", + "items": { + "type": ["object","null"], + "properties": { + "X": { "type": "integer" }, + "Y": { "type": "string" }, + "Z": { "type": "boolean" }, + "W": { "type": "number" } + }, + "required": ["X", "Y", "Z", "W"] + } + }, + /* This collection element is the second occurrence + of the type which points to the first occurrence. */ + "ArrayValue": { + "type": "array", + "items": { + "$ref": "#/properties/ListValue/items" + } + } + } + } + """); + yield return new TestData( Value: new() { X = 42 }, ExpectedJsonSchema: """ @@ -1226,6 +1282,14 @@ public class PocoWithRecursiveDictionaryValue public Dictionary Children { get; init; } = new(); } + public class PocoWithNonRecursiveDuplicateOccurrences + { + public SimpleRecord Value1 { get; set; } + public SimpleRecord Value2 { get; set; } + public List ListValue { get; set; } + public SimpleRecord[] ArrayValue { get; set; } + } + [Description("The type description")] public class PocoWithDescription { diff --git a/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.cs b/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.cs index 314e3f68817fe..5bc3c54224666 100644 --- a/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.cs +++ b/src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.cs @@ -9,6 +9,7 @@ using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; using System.Text.Json.Serialization.Tests; +using System.Xml.Linq; using Json.Schema; using Xunit; using Xunit.Sdk; @@ -90,6 +91,13 @@ public void UnsupportedType_ReturnsExpectedSchema(Type type) Assert.Equal(""""{"$comment":"Unsupported .NET type","not":true}"""", schema.ToJsonString()); } + [Fact] + public void CanGenerateXElementSchema() + { + JsonNode schema = Serializer.DefaultOptions.GetJsonSchemaAsNode(typeof(XElement)); + Assert.True(schema.ToJsonString().Length < 100_000); + } + [Fact] public void TypeWithDisallowUnmappedMembers_AdditionalPropertiesFailValidation() { diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs index 6946ea661b561..01f3b7747fedf 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs @@ -9,6 +9,7 @@ using System.Text.Json.Nodes; using System.Text.Json.Schema.Tests; using System.Text.Json.Serialization; +using System.Xml.Linq; namespace System.Text.Json.SourceGeneration.Tests { @@ -88,6 +89,7 @@ public sealed partial class JsonSchemaExporterTests_SourceGen() [JsonSerializable(typeof(PocoWithRecursiveMembers))] [JsonSerializable(typeof(PocoWithRecursiveCollectionElement))] [JsonSerializable(typeof(PocoWithRecursiveDictionaryValue))] + [JsonSerializable(typeof(PocoWithNonRecursiveDuplicateOccurrences))] [JsonSerializable(typeof(PocoWithDescription))] [JsonSerializable(typeof(PocoWithCustomConverter))] [JsonSerializable(typeof(PocoWithCustomPropertyConverter))] @@ -125,6 +127,7 @@ public sealed partial class JsonSchemaExporterTests_SourceGen() [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(Hashtable))] [JsonSerializable(typeof(StructDictionary))] + [JsonSerializable(typeof(XElement))] public partial class TestTypesContext : JsonSerializerContext; } }