From 43c1299acb505171417103a625e4b4c575b410b8 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 5 Sep 2024 22:58:42 +1200 Subject: [PATCH] [Backport main] Add support for codegen overrides (#1185) * Add support for codegen overrides * Implement ability to override property type mappings Signed-off-by: Thomas Farr * Ensure generated code is up-to-date Signed-off-by: Thomas Farr * Fix license header Signed-off-by: Thomas Farr * Fix tests Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr (cherry picked from commit 9c5d9c1f0ba20bdc85d0c5d48706ae02a33901a1) * spotlessApply Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- .../client/codegen/CodeGenerator.java | 3 +- .../codegen/model/ShapeRenderingContext.java | 9 +- .../client/codegen/model/SpecTransformer.java | 16 +++- .../opensearch/client/codegen/model/Type.java | 16 ++-- .../model/TypeParameterDefinition.java | 9 +- .../codegen/model/TypeParameterDiamond.java | 9 +- .../client/codegen/model/Types.java | 1 + .../codegen/model/overrides/Overrides.java | 60 +++++++++++++ .../model/overrides/PropertyOverride.java | 69 +++++++++++++++ .../model/overrides/SchemaOverride.java | 74 ++++++++++++++++ .../client/codegen/openapi/OpenApiSchema.java | 9 +- .../codegen/renderer/JavaCodeFormatter.java | 9 +- .../renderer/TemplateGlobalContext.java | 9 +- .../codegen/renderer/TemplateLoader.java | 9 +- .../codegen/renderer/TemplateRenderer.java | 9 +- .../renderer/TemplateValueFormatter.java | 9 +- .../client/codegen/utils/MapBuilderBase.java | 45 ++++++++++ .../codegen/utils/ObjectBuilderBase.java | 88 +++++++++++++++++-- .../templates/Type/directSerializer.mustache | 7 +- 19 files changed, 394 insertions(+), 66 deletions(-) create mode 100644 java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/Overrides.java create mode 100644 java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/PropertyOverride.java create mode 100644 java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/SchemaOverride.java create mode 100644 java-codegen/src/main/java/org/opensearch/client/codegen/utils/MapBuilderBase.java diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/CodeGenerator.java b/java-codegen/src/main/java/org/opensearch/client/codegen/CodeGenerator.java index 9d59d803e1..86c3b198fd 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/CodeGenerator.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/CodeGenerator.java @@ -35,6 +35,7 @@ import org.opensearch.client.codegen.model.OperationGroupMatcher; import org.opensearch.client.codegen.model.ShapeRenderingContext; import org.opensearch.client.codegen.model.SpecTransformer; +import org.opensearch.client.codegen.model.overrides.Overrides; import org.opensearch.client.codegen.openapi.OpenApiSpecification; public class CodeGenerator { @@ -121,7 +122,7 @@ public static void main(String[] args) { private static Namespace parseSpec(URI location) throws ApiSpecificationParseException { var spec = OpenApiSpecification.retrieve(location); - var transformer = new SpecTransformer(OPERATION_MATCHER); + var transformer = new SpecTransformer(OPERATION_MATCHER, Overrides.OVERRIDES); transformer.visit(spec); return transformer.getRoot(); } diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/ShapeRenderingContext.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/ShapeRenderingContext.java index 097dee806c..72a163ea3b 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/model/ShapeRenderingContext.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/ShapeRenderingContext.java @@ -78,13 +78,12 @@ public static final class Builder extends ObjectBuilderBase visitedSchemas = new HashSet<>(); @Nonnull private final Map schemaToType = new ConcurrentHashMap<>(); - public SpecTransformer(@Nonnull OperationGroupMatcher matcher) { + public SpecTransformer(@Nonnull OperationGroupMatcher matcher, @Nonnull Overrides overrides) { this.matcher = Objects.requireNonNull(matcher, "matcher must not be null"); + this.overrides = Objects.requireNonNull(overrides, "overrides must not be null"); } @Nonnull @@ -285,9 +290,12 @@ private void visitInto(OpenApiSchema schema, ObjectShape shape) { final var additionalProperties = new ArrayList(); final var required = collectObjectProperties(schema, properties, additionalProperties); - properties.forEach( - (k, v) -> shape.addBodyField(new Field(k, mapType(v), required.contains(k), v.getDescription().orElse(null), null)) - ); + final var overrides = this.overrides.getSchema(schema.getPointer()); + + properties.forEach((k, v) -> { + var type = overrides.flatMap(so -> so.getProperty(k)).flatMap(PropertyOverride::getMappedType).orElseGet(() -> mapType(v)); + shape.addBodyField(new Field(k, type, required.contains(k), v.getDescription().orElse(null), null)); + }); if (!additionalProperties.isEmpty()) { var valueSchema = additionalProperties.size() == 1 ? additionalProperties.get(0) : OpenApiSchema.ANONYMOUS_UNTYPED; diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/Type.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/Type.java index 18f5eee2d1..b8e3585151 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/model/Type.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/Type.java @@ -39,7 +39,8 @@ public class Type { "float", "Float", "double", - "Double" + "Double", + "Number" ); public static Builder builder() { @@ -148,6 +149,10 @@ public boolean isPrimitive() { return PRIMITIVES.contains(name); } + public boolean isNumber() { + return "Number".equals(name); + } + public boolean isEnum() { return isEnum; } @@ -218,13 +223,12 @@ public static final class Builder extends ObjectBuilderBase { private Type[] typeParams; private boolean isEnum; - private Builder() { - super(Type::new); - } + private Builder() {} + @Nonnull @Override - protected @Nonnull Builder self() { - return this; + protected Type construct() { + return new Type(this); } @Nonnull diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/TypeParameterDefinition.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/TypeParameterDefinition.java index 1fb3e9d656..76935c0273 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/model/TypeParameterDefinition.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/TypeParameterDefinition.java @@ -47,13 +47,12 @@ public static final class Builder extends ObjectBuilderBase { private Either params; - private Builder() { - super(TypeParameterDiamond::new); - } + private Builder() {} + @Nonnull @Override - protected @Nonnull Builder self() { - return this; + protected TypeParameterDiamond construct() { + return new TypeParameterDiamond(this); } @Nonnull diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/Types.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/Types.java index a0c3af44b0..be82acb55f 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/model/Types.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/Types.java @@ -67,6 +67,7 @@ public static final class Lang { public static final Type Float = Type.builder().withPackage(PACKAGE).withName("Float").build(); public static final Type Double = Type.builder().withPackage(PACKAGE).withName("Double").build(); public static final Type Object = Type.builder().withPackage(PACKAGE).withName("Object").build(); + public static final Type Number = Type.builder().withPackage(PACKAGE).withName("Number").build(); } public static final class Util { diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/Overrides.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/Overrides.java new file mode 100644 index 0000000000..e4c9759ca2 --- /dev/null +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/Overrides.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.codegen.model.overrides; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.opensearch.client.codegen.openapi.JsonPointer; +import org.opensearch.client.codegen.utils.ObjectBuilder; +import org.opensearch.client.codegen.utils.ObjectBuilderBase; + +public class Overrides { + public static final Overrides OVERRIDES = builder().build(); + + @Nonnull + private final Map schemas; + + private Overrides(Builder builder) { + this.schemas = builder.schemas != null ? Collections.unmodifiableMap(builder.schemas) : Collections.emptyMap(); + } + + @Nonnull + public Optional getSchema(@Nonnull JsonPointer pointer) { + return Optional.ofNullable(schemas.get(pointer)); + } + + @Nonnull + public static Builder builder() { + return new Builder(); + } + + public static final class Builder extends ObjectBuilderBase { + @Nullable + private Map schemas; + + private Builder() {} + + @Nonnull + @Override + protected Overrides construct() { + return new Overrides(this); + } + + @Nonnull + public Builder withSchemas(@Nonnull Function>> fn) { + this.schemas = Objects.requireNonNull(fn, "fn must not be null").apply(SchemaOverride.mapBuilder()).build(); + return this; + } + } +} diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/PropertyOverride.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/PropertyOverride.java new file mode 100644 index 0000000000..c7e01d10a8 --- /dev/null +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/PropertyOverride.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.codegen.model.overrides; + +import java.util.Optional; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.opensearch.client.codegen.model.Type; +import org.opensearch.client.codegen.utils.MapBuilderBase; +import org.opensearch.client.codegen.utils.ObjectBuilderBase; + +public final class PropertyOverride { + @Nullable + private final Type mappedType; + + private PropertyOverride(Builder builder) { + this.mappedType = builder.mappedType; + } + + @Nonnull + public Optional getMappedType() { + return Optional.ofNullable(mappedType); + } + + @Nonnull + public static Builder builder() { + return new Builder(); + } + + @Nonnull + public static MapBuilder mapBuilder() { + return new MapBuilder(); + } + + public static final class Builder extends ObjectBuilderBase { + @Nullable + private Type mappedType; + + private Builder() {} + + @Nonnull + @Override + protected PropertyOverride construct() { + return new PropertyOverride(this); + } + + @Nonnull + public Builder withMappedType(@Nullable Type mappedType) { + this.mappedType = mappedType; + return this; + } + } + + public static final class MapBuilder extends MapBuilderBase { + private MapBuilder() {} + + @Nonnull + @Override + protected Builder valueBuilder() { + return builder(); + } + } +} diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/SchemaOverride.java b/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/SchemaOverride.java new file mode 100644 index 0000000000..d8359b11ff --- /dev/null +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/model/overrides/SchemaOverride.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.codegen.model.overrides; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.opensearch.client.codegen.openapi.JsonPointer; +import org.opensearch.client.codegen.utils.MapBuilderBase; +import org.opensearch.client.codegen.utils.ObjectBuilder; +import org.opensearch.client.codegen.utils.ObjectBuilderBase; + +public final class SchemaOverride { + @Nonnull + private final Map properties; + + private SchemaOverride(Builder builder) { + this.properties = builder.properties != null ? Collections.unmodifiableMap(builder.properties) : Collections.emptyMap(); + } + + @Nonnull + public Optional getProperty(@Nonnull String key) { + return Optional.ofNullable(properties.get(key)); + } + + @Nonnull + public static Builder builder() { + return new Builder(); + } + + @Nonnull + public static MapBuilder mapBuilder() { + return new MapBuilder(); + } + + public static final class Builder extends ObjectBuilderBase { + @Nullable + private Map properties; + + private Builder() {} + + @Nonnull + @Override + protected SchemaOverride construct() { + return new SchemaOverride(this); + } + + @Nonnull + public Builder withProperties(@Nonnull Function>> fn) { + this.properties = Objects.requireNonNull(fn, "fn must not be null").apply(PropertyOverride.mapBuilder()).build(); + return this; + } + } + + public static final class MapBuilder extends MapBuilderBase { + private MapBuilder() {} + + @Nonnull + @Override + protected Builder valueBuilder() { + return builder(); + } + } +} diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/openapi/OpenApiSchema.java b/java-codegen/src/main/java/org/opensearch/client/codegen/openapi/OpenApiSchema.java index b301160c53..88f5d28779 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/openapi/OpenApiSchema.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/openapi/OpenApiSchema.java @@ -354,13 +354,12 @@ public static class Builder extends ObjectBuilderBase { @Nullable private Semver versionRemoved; - private Builder() { - super(OpenApiSchema::new); - } + private Builder() {} + @Nonnull @Override - protected @Nonnull Builder self() { - return this; + protected OpenApiSchema construct() { + return new OpenApiSchema(this); } @Nonnull diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/JavaCodeFormatter.java b/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/JavaCodeFormatter.java index f4589c7ec7..8f2717a120 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/JavaCodeFormatter.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/JavaCodeFormatter.java @@ -80,13 +80,12 @@ public static final class Builder extends ObjectBuilderBase values = new HashMap<>(); private TemplateRenderer renderer; - private Builder() { - super(TemplateGlobalContext::new); - } + private Builder() {} + @Nonnull @Override - protected @Nonnull Builder self() { - return this; + protected TemplateGlobalContext construct() { + return new TemplateGlobalContext(this); } @Nonnull diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateLoader.java b/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateLoader.java index 7988c6219a..6a6583125b 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateLoader.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateLoader.java @@ -61,13 +61,12 @@ public static Builder builder() { public static final class Builder extends ObjectBuilderBase { private String templatesResourceSubPath; - private Builder() { - super(TemplateLoader::new); - } + private Builder() {} + @Nonnull @Override - protected @Nonnull Builder self() { - return this; + protected TemplateLoader construct() { + return new TemplateLoader(this); } @Nonnull diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateRenderer.java b/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateRenderer.java index 1b22b4ede1..32daa020c2 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateRenderer.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/renderer/TemplateRenderer.java @@ -88,13 +88,12 @@ public static final class Builder extends ObjectBuilderBase { private final Map, Formatter> formatters = new HashMap<>(); - private Builder() { - super(TemplateValueFormatter::new); - } + private Builder() {} + @Nonnull @Override - protected @Nonnull Builder self() { - return this; + protected TemplateValueFormatter construct() { + return new TemplateValueFormatter(this); } @Nonnull diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/utils/MapBuilderBase.java b/java-codegen/src/main/java/org/opensearch/client/codegen/utils/MapBuilderBase.java new file mode 100644 index 0000000000..221d434d16 --- /dev/null +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/utils/MapBuilderBase.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.codegen.utils; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public abstract class MapBuilderBase, Self extends MapBuilderBase> extends + ObjectBuilderBase, Self> { + @Nullable + private Map map; + + protected MapBuilderBase() {} + + protected abstract @Nonnull VBuilder valueBuilder(); + + @Override + protected @Nonnull Map construct() { + return map != null ? Collections.unmodifiableMap(map) : Collections.emptyMap(); + } + + public @Nonnull Self with(@Nonnull Map map) { + this.map = _mapPutAll(this.map, map); + return self(); + } + + public @Nonnull Self with(@Nonnull K key, @Nonnull V value) { + this.map = _mapPut(this.map, key, value); + return self(); + } + + public @Nonnull Self with(@Nonnull K key, @Nonnull Function> fn) { + return with(key, Objects.requireNonNull(fn, "fn must not be null").apply(valueBuilder()).build()); + } +} diff --git a/java-codegen/src/main/java/org/opensearch/client/codegen/utils/ObjectBuilderBase.java b/java-codegen/src/main/java/org/opensearch/client/codegen/utils/ObjectBuilderBase.java index 31955ba570..8e71c9da70 100644 --- a/java-codegen/src/main/java/org/opensearch/client/codegen/utils/ObjectBuilderBase.java +++ b/java-codegen/src/main/java/org/opensearch/client/codegen/utils/ObjectBuilderBase.java @@ -6,29 +6,99 @@ * compatible open source license. */ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + package org.opensearch.client.codegen.utils; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; -import java.util.function.Function; import javax.annotation.Nonnull; public abstract class ObjectBuilderBase> implements ObjectBuilder { - private final Function ctor; private boolean _used = false; - protected ObjectBuilderBase(Function ctor) { - this.ctor = Objects.requireNonNull(ctor, "ctor must not be null"); + protected ObjectBuilderBase() {} + + protected @Nonnull Self self() { + // noinspection unchecked + return (Self) this; } - @Nonnull - protected abstract Self self(); + protected abstract @Nonnull T construct(); - @Override - public @Nonnull T build() { + protected final void _ensureSingleUse() { if (this._used) { throw new IllegalStateException("Object builders can only be used once"); } this._used = true; - return ctor.apply(self()); + } + + @Override + public final @Nonnull T build() { + var self = self(); + self._ensureSingleUse(); + return self.construct(); + } + + // ----- Map utilities + + private static final class OwnedMap extends HashMap { + OwnedMap() {} + + OwnedMap(Map m) { + super(m); + } + } + + private static Map _mutableMap(Map map) { + if (map == null) { + return new OwnedMap<>(); + } else if (map instanceof ObjectBuilderBase.OwnedMap) { + return map; + } else { + // Adding to a map we don't own: make a defensive copy, also ensuring it is mutable. + return new OwnedMap<>(map); + } + } + + protected static Map _mapPut(Map map, K key, V value) { + map = _mutableMap(map); + map.put(key, value); + return map; + } + + protected static Map _mapPutAll(Map map, Map entries) { + if (map == null) { + // Keep the original map to avoid an unnecessary copy. + // It will be copied if we add more entries. + return Objects.requireNonNull(entries); + } else { + map = _mutableMap(map); + map.putAll(entries); + return map; + } } } diff --git a/java-codegen/src/main/resources/org/opensearch/client/codegen/templates/Type/directSerializer.mustache b/java-codegen/src/main/resources/org/opensearch/client/codegen/templates/Type/directSerializer.mustache index 66ba3e4dd6..f8c2fb10cd 100644 --- a/java-codegen/src/main/resources/org/opensearch/client/codegen/templates/Type/directSerializer.mustache +++ b/java-codegen/src/main/resources/org/opensearch/client/codegen/templates/Type/directSerializer.mustache @@ -11,7 +11,12 @@ {{/type.isList}} {{^type.isListOrMap}} {{#type.isPrimitive}} - generator.write({{value}}); + {{#type.isNumber}} + generator.write({{value}}.doubleValue()); + {{/type.isNumber}} + {{^type.isNumber}} + generator.write({{value}}); + {{/type.isNumber}} {{/type.isPrimitive}} {{^type.isPrimitive}} {{value}}.serialize(generator, mapper);