Skip to content

Commit

Permalink
Make the schema generator reuse algorithm more aggressive to reduce g…
Browse files Browse the repository at this point in the history 
…enerated schema size. (#108800)

Co-authored-by: Eirik Tsarpalis <eirik.tsarpalis@gmail.com>
  • Loading branch information
@github-actions @eiriktsarpalis
github-actions[bot] and eiriktsarpalis authored Oct 18, 2024
1 parent 92b4ddd commit 83d152c
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 41 deletions.
77 changes: 37 additions & 40 deletions src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
Expand Down Expand Up @@ -76,9 +77,12 @@ private static JsonSchema MapJsonSchemaCore(
{
Debug.Assert(typeInfo.IsConfigured);

if (cacheResult && state.TryPushType(typeInfo, propertyInfo, out string? existingJsonPointer))
JsonSchemaExporterContext exporterContext = state.CreateContext(typeInfo, propertyInfo, parentPolymorphicTypeInfo);

if (cacheResult && typeInfo.Kind is not JsonTypeInfoKind.None &&
state.TryGetExistingJsonPointer(exporterContext, out string? existingJsonPointer))
{
// We're generating the schema of a recursive type, return a reference pointing to the outermost schema.
// The schema context has already been generated in the schema document, return a reference to it.
return CompleteSchema(ref state, new JsonSchema { Ref = existingJsonPointer });
}

Expand Down Expand Up @@ -364,17 +368,12 @@ JsonSchema CompleteSchema(ref GenerationState state, JsonSchema schema)
{
schema.MakeNullable();
}

if (cacheResult)
{
state.PopGeneratedType();
}
}

if (state.ExporterOptions.TransformSchemaNode != null)
{
// Prime the schema for invocation by the JsonNode transformer.
schema.ExporterContext = state.CreateContext(typeInfo, propertyInfo, parentPolymorphicTypeInfo);
schema.ExporterContext = exporterContext;
}

return schema;
Expand Down Expand Up @@ -409,7 +408,7 @@ private static bool IsPolymorphicTypeThatSpecifiesItselfAsDerivedType(JsonTypeIn
private readonly ref struct GenerationState(JsonSerializerOptions options, JsonSchemaExporterOptions exporterOptions)
{
private readonly List<string> _currentPath = [];
private readonly List<(JsonTypeInfo typeInfo, JsonPropertyInfo? propertyInfo, int depth)> _generationStack = [];
private readonly Dictionary<(JsonTypeInfo, JsonPropertyInfo?), string[]> _generated = new();

public int CurrentDepth => _currentPath.Count;
public JsonSerializerOptions Options { get; } = options;
Expand All @@ -432,77 +431,75 @@ public void PopSchemaNode()
}

/// <summary>
/// Pushes the current type/property to the generation stack or returns a JSON pointer if the type is recursive.
/// Registers the current schema node generation context; if it has already been generated return a JSON pointer to its location.
/// </summary>
public bool TryPushType(JsonTypeInfo typeInfo, JsonPropertyInfo? propertyInfo, [NotNullWhen(true)] out string? existingJsonPointer)
public bool TryGetExistingJsonPointer(in JsonSchemaExporterContext context, [NotNullWhen(true)] out string? existingJsonPointer)
{
foreach ((JsonTypeInfo otherTypeInfo, JsonPropertyInfo? otherPropertyInfo, int depth) in _generationStack)
(JsonTypeInfo TypeInfo, JsonPropertyInfo? PropertyInfo) key = (context.TypeInfo, context.PropertyInfo);
#if NET
ref string[]? pathToSchema = ref CollectionsMarshal.GetValueRefOrAddDefault(_generated, key, out bool exists);
#else
bool exists = _generated.TryGetValue(key, out string[]? pathToSchema);
#endif
if (exists)
{
if (typeInfo == otherTypeInfo && propertyInfo == otherPropertyInfo)
{
existingJsonPointer = FormatJsonPointer(_currentPath, depth);
return true;
}
existingJsonPointer = FormatJsonPointer(pathToSchema);
return true;
}

_generationStack.Add((typeInfo, propertyInfo, CurrentDepth));
#if NET
pathToSchema = context._path;
#else
_generated[key] = context._path;
#endif
existingJsonPointer = null;
return false;
}

public void PopGeneratedType()
{
Debug.Assert(_generationStack.Count > 0);
_generationStack.RemoveAt(_generationStack.Count - 1);
}

public JsonSchemaExporterContext CreateContext(JsonTypeInfo typeInfo, JsonPropertyInfo? propertyInfo, JsonTypeInfo? baseTypeInfo)
{
return new JsonSchemaExporterContext(typeInfo, propertyInfo, baseTypeInfo, _currentPath.ToArray());
return new JsonSchemaExporterContext(typeInfo, propertyInfo, baseTypeInfo, [.. _currentPath]);
}

private static string FormatJsonPointer(List<string> currentPathList, int depth)
private static string FormatJsonPointer(ReadOnlySpan<string> path)
{
Debug.Assert(0 <= depth && depth < currentPathList.Count);

if (depth == 0)
if (path.IsEmpty)
{
return "#";
}

using ValueStringBuilder sb = new(initialCapacity: depth * 10);
using ValueStringBuilder sb = new(initialCapacity: path.Length * 10);
sb.Append('#');

for (int i = 0; i < depth; i++)
foreach (string segment in path)
{
ReadOnlySpan<char> segment = currentPathList[i].AsSpan();
ReadOnlySpan<char> span = segment.AsSpan();
sb.Append('/');

do
{
// Per RFC 6901 the characters '~' and '/' must be escaped.
int pos = segment.IndexOfAny('~', '/');
int pos = span.IndexOfAny('~', '/');
if (pos < 0)
{
sb.Append(segment);
sb.Append(span);
break;
}

sb.Append(segment.Slice(0, pos));
sb.Append(span.Slice(0, pos));

if (segment[pos] == '~')
if (span[pos] == '~')
{
sb.Append("~0");
}
else
{
Debug.Assert(segment[pos] == '/');
Debug.Assert(span[pos] == '/');
sb.Append("~1");
}

segment = segment.Slice(pos + 1);
span = span.Slice(pos + 1);
}
while (!segment.IsEmpty);
while (!span.IsEmpty);
}

return sb.ToString();
Expand Down
2 changes: 1 addition & 1 deletion src/libraries/System.Text.Json/src/System/Text/Json/Schema/JsonSchemaExporterContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace System.Text.Json.Schema
/// </summary>
public readonly struct JsonSchemaExporterContext
{
private readonly string[] _path;
internal readonly string[] _path;

internal JsonSchemaExporterContext(
JsonTypeInfo typeInfo,
Expand Down
64 changes: 64 additions & 0 deletions src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.TestTypes.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,62 @@ public static IEnumerable<ITestData> GetTestDataCore()
""",
Options: new() { TreatNullObliviousAsNonNullable = true });

SimpleRecord recordValue = new(42, "str", true, 3.14);
yield return new TestData<PocoWithNonRecursiveDuplicateOccurrences>(
Value: new() { Value1 = recordValue, Value2 = recordValue, ArrayValue = [recordValue], ListValue = [recordValue] },
ExpectedJsonSchema: """
{
"type": ["object","null"],
"properties": {
"Value1": {
"type": "object",
"properties": {
"X": { "type": "integer" },
"Y": { "type": "string" },
"Z": { "type": "boolean" },
"W": { "type": "number" }
},
"required": ["X", "Y", "Z", "W"]
},
/* The same type on a different property is repeated to
account for potential metadata resolved from attributes. */
"Value2": {
"type": "object",
"properties": {
"X": { "type": "integer" },
"Y": { "type": "string" },
"Z": { "type": "boolean" },
"W": { "type": "number" }
},
"required": ["X", "Y", "Z", "W"]
},
/* This collection element is the first occurrence
of the type without contextual metadata. */
"ListValue": {
"type": "array",
"items": {
"type": ["object","null"],
"properties": {
"X": { "type": "integer" },
"Y": { "type": "string" },
"Z": { "type": "boolean" },
"W": { "type": "number" }
},
"required": ["X", "Y", "Z", "W"]
}
},
/* This collection element is the second occurrence
of the type which points to the first occurrence. */
"ArrayValue": {
"type": "array",
"items": {
"$ref": "#/properties/ListValue/items"
}
}
}
}
""");

yield return new TestData<PocoWithDescription>(
Value: new() { X = 42 },
ExpectedJsonSchema: """
Expand Down Expand Up @@ -1226,6 +1282,14 @@ public class PocoWithRecursiveDictionaryValue
public Dictionary<string, PocoWithRecursiveDictionaryValue> Children { get; init; } = new();
}

public class PocoWithNonRecursiveDuplicateOccurrences
{
public SimpleRecord Value1 { get; set; }
public SimpleRecord Value2 { get; set; }
public List<SimpleRecord> ListValue { get; set; }
public SimpleRecord[] ArrayValue { get; set; }
}

[Description("The type description")]
public class PocoWithDescription
{
Expand Down
8 changes: 8 additions & 0 deletions src/libraries/System.Text.Json/tests/Common/JsonSchemaExporterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using System.Text.Json.Serialization.Tests;
using System.Xml.Linq;
using Json.Schema;
using Xunit;
using Xunit.Sdk;
Expand Down Expand Up @@ -90,6 +91,13 @@ public void UnsupportedType_ReturnsExpectedSchema(Type type)
Assert.Equal(""""{"$comment":"Unsupported .NET type","not":true}"""", schema.ToJsonString());
}

[Fact]
public void CanGenerateXElementSchema()
{
JsonNode schema = Serializer.DefaultOptions.GetJsonSchemaAsNode(typeof(XElement));
Assert.True(schema.ToJsonString().Length < 100_000);
}

[Fact]
public void TypeWithDisallowUnmappedMembers_AdditionalPropertiesFailValidation()
{
Expand Down
3 changes: 3 additions & 0 deletions ...on/tests/System.Text.Json.SourceGeneration.Tests/Serialization/JsonSchemaExporterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Text.Json.Nodes;
using System.Text.Json.Schema.Tests;
using System.Text.Json.Serialization;
using System.Xml.Linq;

namespace System.Text.Json.SourceGeneration.Tests
{
Expand Down Expand Up @@ -88,6 +89,7 @@ public sealed partial class JsonSchemaExporterTests_SourceGen()
[JsonSerializable(typeof(PocoWithRecursiveMembers))]
[JsonSerializable(typeof(PocoWithRecursiveCollectionElement))]
[JsonSerializable(typeof(PocoWithRecursiveDictionaryValue))]
[JsonSerializable(typeof(PocoWithNonRecursiveDuplicateOccurrences))]
[JsonSerializable(typeof(PocoWithDescription))]
[JsonSerializable(typeof(PocoWithCustomConverter))]
[JsonSerializable(typeof(PocoWithCustomPropertyConverter))]
Expand Down Expand Up @@ -125,6 +127,7 @@ public sealed partial class JsonSchemaExporterTests_SourceGen()
[JsonSerializable(typeof(Dictionary<string, object>))]
[JsonSerializable(typeof(Hashtable))]
[JsonSerializable(typeof(StructDictionary<string, int>))]
[JsonSerializable(typeof(XElement))]
public partial class TestTypesContext : JsonSerializerContext;
}
}

0 comments on commit 83d152c

Please sign in to comment.