From a2450842a114730927fe74d4c981a7feb686e2b8 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Tue, 12 Nov 2024 16:46:38 +0100 Subject: [PATCH 1/8] Add x-ms-enum-values management in CDP connectors --- .../ConnectorFunction.cs | 51 ++--- .../Environment/PowerFxConfigExtensions.cs | 7 +- .../Execution/FormulaValueSerializer.cs | 4 + .../Internal/OptionSetList.cs | 115 ++++++++++++ .../OpenApiExtensions.cs | 115 +++++++----- .../OpenApiParser.cs | 59 ++++-- .../Public/ConnectorParameter.cs | 17 +- .../Public/ConnectorSchema.cs | 9 +- .../Public/ConnectorType.cs | 20 +- .../Tabular/CdpTableResolver.cs | 11 +- .../Tabular/Services/CdpTable.cs | 6 +- .../CompatibilityTests.cs | 6 +- .../InternalTesting.cs | 5 +- ....PowerFx.Connectors.Tests.Shared.projitems | 1 + .../OpenApiParserTests.cs | 10 +- .../OptionSetListTests.cs | 66 +++++++ .../PowerPlatformTabularTests.cs | 9 +- .../Responses/ZD Tickets GetSchema.json | 174 ++++++++++++------ 18 files changed, 509 insertions(+), 176 deletions(-) create mode 100644 src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs create mode 100644 src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs diff --git a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs index c01b9b2b6a..816dd76cc0 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs @@ -27,6 +27,8 @@ using Microsoft.PowerFx.Intellisense; using Microsoft.PowerFx.Types; using static Microsoft.PowerFx.Connectors.ConnectorHelperFunctions; + +#pragma warning disable SA1117 namespace Microsoft.PowerFx.Connectors { @@ -292,15 +294,19 @@ public ConnectorType ReturnParameterType // Those properties are only used by HttpFunctionInvoker internal ConnectorParameterInternals _internals = null; - private readonly ConnectorLogger _configurationLogger = null; + private readonly ConnectorLogger _configurationLogger = null; + + internal readonly OptionSetList OptionSets; - internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues) + internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, + ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, OptionSetList optionSetList) { Operation = openApiOperation; Name = name; OperationPath = operationPath; HttpMethod = httpMethod; - ConnectorSettings = connectorSettings; + ConnectorSettings = connectorSettings; + OptionSets = optionSetList; GlobalContext = new ConnectorGlobalContext(functionList ?? throw new ArgumentNullException(nameof(functionList)), globalValues); _configurationLogger = configurationLogger; @@ -985,7 +991,7 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return null; } - ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, OptionSets, ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -999,15 +1005,16 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return connectorType; } - internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, OptionSetList optionSets, ConnectorCompatibility compatibility) { JsonElement je = ExtractFromJson(sv, valuePath); - return GetConnectorTypeInternal(compatibility, je); + return GetConnectorTypeInternal(optionSets, compatibility, je); } // Only called by ConnectorTable.GetSchema // Returns a FormulaType with AssociatedDataSources set (done in AddTabularDataSource) - internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue stringValue, List sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo) + internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue stringValue, List sqlRelationships, ConnectorCompatibility compatibility, string datasetName, + out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable optionSets) { // There are some errors when parsing this Json payload but that's not a problem here as we only need x-ms-capabilities parsing to work OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet }; @@ -1019,9 +1026,11 @@ internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, s JsonElement jsonElement = ExtractFromJson(stringValue, valuePath, out name, out displayName); bool isTableReadOnly = tablePermission == ConnectorPermission.PermissionReadOnly; IList referencedEntities = GetReferenceEntities(connectorName, stringValue); - - ConnectorType connectorType = new ConnectorType(jsonElement, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly); + + OptionSetList optionSetList = new OptionSetList(); + ConnectorType connectorType = new ConnectorType(jsonElement, optionSetList, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly); delegationInfo = ((DataSourceInfo)connectorType.FormulaType._type.AssociatedDataSources.First()).DelegationInfo; + optionSets = optionSetList.OptionSets; return connectorType; } @@ -1075,17 +1084,17 @@ internal class SalesForceReferencedEntity public bool RestrictedDelete { get; set; } } - private static ConnectorType GetConnectorTypeInternal(ConnectorCompatibility compatibility, JsonElement je) + private static ConnectorType GetConnectorTypeInternal(OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je) { OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet }; OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag); - return new ConnectorType(SwaggerSchema.New(schema), compatibility); + return new ConnectorType(SwaggerSchema.New(schema), optionSets, compatibility); } - private static ConnectorType GetJsonConnectorTypeInternal(ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) + private static ConnectorType GetJsonConnectorTypeInternal(OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) { - return new ConnectorType(je, compatibility, sqlRelationships); + return new ConnectorType(je, optionSets, compatibility, sqlRelationships); } private async Task GetConnectorSuggestionsFromDynamicPropertyAsync(NamedValue[] knownParameters, BaseRuntimeConnectorContext runtimeContext, ConnectorDynamicProperty cdp, CancellationToken cancellationToken) @@ -1106,7 +1115,7 @@ private async Task GetConnectorSuggestionsFromDynamicPropertyAsyn return null; } - ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, OptionSets, ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1413,7 +1422,7 @@ private ConnectorParameterInternals Initialize() return null; } - ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, ConnectorSettings.Compatibility)); + ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, OptionSets, ConnectorSettings.Compatibility)); if (connectorParameter.HiddenRecordType != null) { @@ -1482,12 +1491,12 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter = new OpenApiParameter() { Name = bodyPropertyName, Schema = bodyPropertySchema, Description = requestBody.Description, Required = bodyPropertyRequired, Extensions = bodyPropertySchema.Extensions }; - ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter2, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter2.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter2.HiddenRecordType != null) - { - hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, ConnectorSettings.Compatibility))); + { + hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, OptionSets, ConnectorSettings.Compatibility))); } List parameterList = !bodyPropertyRequired ? optionalParameters : bodyPropertyHiddenRequired ? hiddenRequiredParameters : requiredParameters; @@ -1506,7 +1515,7 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter2 = new OpenApiParameter() { Name = bodyName, Schema = bodySchema, Description = requestBody.Description, Required = requestBody.Required, Extensions = bodySchema.Extensions }; - ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter3, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter3.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter3.HiddenRecordType != null) @@ -1526,7 +1535,7 @@ private ConnectorParameterInternals Initialize() OpenApiSchema bodyParameterSchema = new OpenApiSchema() { Type = "string" }; OpenApiParameter bodyParameter3 = new OpenApiParameter() { Name = bodyName, Schema = bodyParameterSchema, Description = "Body", Required = requestBody.Required }; - ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, ConnectorSettings.Compatibility)); + ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyParameter, OpenApiExtensions.TryGetOpenApiValue(bodyParameter.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); List parameterList = requestBody.Required ? requiredParameters : optionalParameters; @@ -1569,7 +1578,7 @@ private ConnectorParameterInternals Initialize() _arityMax = _arityMin + (_optionalParameters.Length == 0 ? 0 : 1); _warnings = new List(); - _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(ConnectorSettings.Compatibility)); + _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(OptionSets, ConnectorSettings.Compatibility)); if (IsDeprecated) { diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs index ddffcfe24a..fea9f0cb83 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs @@ -68,12 +68,17 @@ internal static IReadOnlyList AddActionConnectorInternal(this return null; } - (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); + (List connectorFunctions, List texlFunctions, OptionSetList optionSets) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); foreach (TexlFunction function in texlFunctions) { config.AddFunction(function); } + foreach (OptionSet optionSet in optionSets.OptionSets) + { + config.AddOptionSet(optionSet); + } + return connectorFunctions; } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Execution/FormulaValueSerializer.cs b/src/libraries/Microsoft.PowerFx.Connectors/Execution/FormulaValueSerializer.cs index 6e0ae26012..048d6ed780 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Execution/FormulaValueSerializer.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Execution/FormulaValueSerializer.cs @@ -283,6 +283,10 @@ private async Task WritePropertyAsync(string propertyName, ISwaggerSchema proper { await WriteBlobValueAsync(bv).ConfigureAwait(false); } + else if (fv is OptionSetValue optionSetValue) + { + WriteStringValue(optionSetValue.Option); + } else { throw new PowerFxConnectorException($"Expected StringValue and got {fv?.GetType()?.Name ?? ""} value, for property {propertyName}"); diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs new file mode 100644 index 0000000000..ca94705968 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.PowerFx.Core; + +namespace Microsoft.PowerFx.Connectors +{ + internal class OptionSetList + { + private readonly Dictionary _optionSets; + + public OptionSetList() + { + _optionSets = new Dictionary(); + } + + public IEnumerable OptionSets => _optionSets.Values; + + // if the option set doesn't exist, it will be added to the list and we return optionSet + // if the option set exists with same name and options, it is not added, we return the existing optionSet + // if the option set exists with same name but different options, we rename it and return that new optionSet + public OptionSet TryAdd(OptionSet optionSet) + { + if (optionSet == null) + { + throw new ArgumentNullException("optionSet"); + } + + OptionSetStatus oss = GetStatus(optionSet); + + // new option set + if (oss == OptionSetStatus.New) + { + Add(optionSet); + return optionSet; + } + + // same optionset, no need to add anything + if (oss == OptionSetStatus.Same) + { + return Get(optionSet.EntityName); + } + + int i = 1; + SingleSourceDisplayNameProvider dnp = new SingleSourceDisplayNameProvider(optionSet.Options); + + while (oss == OptionSetStatus.Conflict && i < 20) + { + string name = $"{optionSet.EntityName}_{i++}"; + + OptionSet newOptionSet = new OptionSet(name, dnp); + oss = GetStatus(newOptionSet); + + // no conflict now, let's add it + if (oss == OptionSetStatus.New) + { + Add(newOptionSet); + return newOptionSet; + } + + // we found an existing one that matches + if (oss == OptionSetStatus.Same) + { + return Get(name); + } + } + + throw new InvalidOperationException("Too many option set conflicts"); + } + + private void Add(OptionSet optionSet) + { + _optionSets.Add(optionSet.EntityName, optionSet); + } + + private OptionSet Get(string name) + { + return _optionSets[name]; + } + + private enum OptionSetStatus + { + New, + Same, + Conflict + } + + private OptionSetStatus GetStatus(OptionSet optionSet) + { + if (optionSet == null) + { + return OptionSetStatus.New; + } + + string name = optionSet.EntityName; + + if (!_optionSets.ContainsKey(name)) + { + return OptionSetStatus.New; + } + + OptionSet existingOptionSet = Get(name); + + if (existingOptionSet.Equals(optionSet)) + { + return OptionSetStatus.Same; + } + + return OptionSetStatus.Conflict; + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs index 7df311ffd8..d9eae12154 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs @@ -10,6 +10,7 @@ using Microsoft.OpenApi.Any; using Microsoft.OpenApi.Interfaces; using Microsoft.OpenApi.Models; +using Microsoft.PowerFx.Core; using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Types; @@ -82,45 +83,58 @@ public static string GetBodyName(this OpenApiRequestBody requestBody) } // Get suggested options values. Returns null if none. - internal static (string[] options, ConnectorErrors errors) GetOptions(this OpenApiParameter openApiParameter) - { - ConnectorErrors errors = new ConnectorErrors(); - - // x-ms-enum-values is: array of { value :string, displayName:string}. + internal static DisplayNameProvider GetEnumValues(this ISwaggerParameter openApiParameter) + { + // x-ms-enum-values is: array of { value: string, displayName: string}. if (openApiParameter.Extensions.TryGetValue(XMsEnumValues, out var enumValues)) { if (enumValues is IList array) - { - var list = new List(array.Count); - + { + SingleSourceDisplayNameProvider displayNameProvider = new SingleSourceDisplayNameProvider(); + foreach (var item in array) - { + { + string logical = null; + string display = null; + if (item is IDictionary obj) - { - // has keys, "value", and "displayName" - if (obj.TryGetValue("value", out IOpenApiAny value)) + { + if (obj.TryGetValue("value", out IOpenApiAny openApiLogical)) { - if (value is OpenApiString str) + if (openApiLogical is OpenApiString logicalStr) { - list.Add(str.Value); - continue; + logical = logicalStr.Value; } - else if (value is OpenApiInteger i) + else if (openApiLogical is OpenApiInteger logicalInt) { - list.Add(i.Value.ToString(CultureInfo.InvariantCulture)); - continue; + logical = logicalInt.Value.ToString(CultureInfo.InvariantCulture); } + } + + if (obj.TryGetValue("displayName", out IOpenApiAny openApiDisplay)) + { + if (openApiDisplay is OpenApiString displayStr) + { + display = displayStr.Value; + } + else if (openApiDisplay is OpenApiInteger displayInt) + { + display = displayInt.Value.ToString(CultureInfo.InvariantCulture); + } + } + + if (!string.IsNullOrEmpty(logical) && !string.IsNullOrEmpty(display)) + { + displayNameProvider = displayNameProvider.AddField(new DName(logical), new DName(display)); } } - - errors.AddError($"Unrecognized {XMsEnumValues} schema ({item.GetType().Name})"); - } - - return (list.ToArray(), errors); + } + + return displayNameProvider.LogicalToDisplayPairs.Any() ? displayNameProvider : null; } } - return (null, null); + return null; } public static bool IsTrigger(this OpenApiOperation op) @@ -342,11 +356,11 @@ internal static bool TryGetOpenApiValue(IOpenApiAny openApiAny, FormulaType form internal static string GetVisibility(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsVisibility, out IOpenApiExtension openApiExt) && openApiExt is OpenApiString openApiStr ? openApiStr.Value : null; - internal static string GetEnumName(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsEnum, out IOpenApiExtension openApiExt) && + internal static string GetEnumName(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsEnum, out IOpenApiExtension openApiExt) && openApiExt is SwaggerJsonObject jsonObject && jsonObject.TryGetValue("name", out IOpenApiAny enumName) && - enumName is OpenApiString enumNameStr - ? enumNameStr.Value + enumName is OpenApiString enumNameStr + ? enumNameStr.Value : null; internal static string GetMediaKind(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsMediaKind, out IOpenApiExtension openApiExt) && openApiExt is OpenApiString openApiStr ? openApiStr.Value : null; @@ -378,10 +392,12 @@ internal class ConnectorTypeGetterSettings internal readonly IList SqlRelationships; internal Stack Chain = new Stack(); internal int Level = 0; + internal readonly OptionSetList OptionSets; - internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, IList sqlRelationships = null) + internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, OptionSetList optionSets, IList sqlRelationships = null) { Compatibility = connectorCompatibility; + OptionSets = optionSets; SqlRelationships = sqlRelationships; } @@ -399,9 +415,12 @@ internal void UnStack() } } - internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, ConnectorCompatibility compatibility, IList sqlRelationships = null) + internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships = null) { - return openApiParameter.GetConnectorType(new ConnectorTypeGetterSettings(compatibility, sqlRelationships)); + ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, optionSets, sqlRelationships); + ConnectorType connectorType = openApiParameter.GetConnectorType(settings); + + return connectorType; } // See https://swagger.io/docs/specification/data-models/data-types/ @@ -445,12 +464,25 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar return new ConnectorType(schema, openApiParameter, FormulaType.Blob); } + // Try getting enum from 'x-ms-enum-values' + DisplayNameProvider optionSetDisplayNameProvider = openApiParameter.GetEnumValues(); + + if (optionSetDisplayNameProvider != null && (settings.Compatibility.IsCDP() || schema.Format == "enum")) + { + string enumName = schema.GetEnumName() ?? openApiParameter.Name; + OptionSet optionSet = new OptionSet(enumName, optionSetDisplayNameProvider); + optionSet = settings.OptionSets.TryAdd(optionSet); + return new ConnectorType(schema, openApiParameter, optionSet.FormulaType); + } + + // Try getting enum from 'enum' if (schema.Enum != null && (settings.Compatibility.IsCDP() || schema.Format == "enum")) { if (schema.Enum.All(e => e is OpenApiString)) { - string enumName = schema.GetEnumName() ?? "enum"; + string enumName = schema.GetEnumName() ?? openApiParameter.Name; OptionSet optionSet = new OptionSet(enumName, schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary()); + optionSet = settings.OptionSets.TryAdd(optionSet); return new ConnectorType(schema, openApiParameter, optionSet.FormulaType); } else @@ -478,7 +510,7 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar case null: case "decimal": case "currency": - return new ConnectorType(schema, openApiParameter, FormulaType.Decimal); + return new ConnectorType(schema, openApiParameter, FormulaType.Decimal); default: return new ConnectorType(error: $"Unsupported type of number: {schema.Format}"); @@ -489,7 +521,7 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar return new ConnectorType(schema, openApiParameter, FormulaType.Number); // Always a boolean (Format not used) - case "boolean": + case "boolean": return new ConnectorType(schema, openApiParameter, FormulaType.Boolean); // OpenAPI spec: Format could be , int32, int64 @@ -734,8 +766,9 @@ public static HttpMethod ToHttpMethod(this OperationType key) public static FormulaType GetReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility) { - ConnectorType connectorType = openApiOperation.GetConnectorReturnType(compatibility); - FormulaType ft = connectorType.HasErrors ? ConnectorType.DefaultType : connectorType?.FormulaType ?? new BlankType(); + OptionSetList optionSetList = new OptionSetList(); + ConnectorType connectorType = openApiOperation.GetConnectorReturnType(optionSetList, compatibility); + FormulaType ft = connectorType.HasErrors ? ConnectorType.DefaultType : connectorType?.FormulaType ?? new BlankType(); return ft; } @@ -744,7 +777,7 @@ public static bool GetRequiresUserConfirmation(this OpenApiOperation op) return op.Extensions.TryGetValue(XMsRequireUserConfirmation, out IOpenApiExtension openExt) && openExt is OpenApiBoolean b && b.Value; } - internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, OptionSetList optionSets, ConnectorCompatibility compatibility) { OpenApiResponses responses = openApiOperation.Responses; OpenApiResponse response = responses.Where(kvp => kvp.Key?.Length == 3 && kvp.Key.StartsWith("2", StringComparison.Ordinal)).OrderBy(kvp => kvp.Key).FirstOrDefault().Value; @@ -760,7 +793,7 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA } if (response == null) - { + { // Returns UntypedObject by default, without error return new ConnectorType(null); } @@ -768,7 +801,7 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA if (response.Content.Count == 0) { OpenApiSchema schema = new OpenApiSchema() { Type = "string", Format = "no_format" }; - return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(compatibility); + return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(optionSets, compatibility); } // Responses is a list by content-type. Find "application/json" @@ -787,13 +820,13 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA if (openApiMediaType.Schema == null) { // Treat as void. - return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(compatibility); + return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(optionSets, compatibility); } - return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(compatibility); + return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(optionSets, compatibility); } } - + return new ConnectorType(error: $"Unsupported return type - found {string.Join(", ", response.Content.Select(kv4 => kv4.Key))}"); } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs index 08296f46c5..b447d59768 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs @@ -23,11 +23,18 @@ public static IEnumerable GetFunctions(string @namespace, Ope } public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) + { + return GetFunctions(@namespace, openApiDocument, globalValues, out var _, configurationLogger); + } + + public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, configurationLogger, globalValues); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); + OptionSetList optionSetList = new OptionSetList(); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, optionSetList, configurationLogger, globalValues); + optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); } @@ -36,7 +43,7 @@ public static IEnumerable GetFunctions(string @namespace, Ope configurationLogger?.LogException(ex, $"Exception in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, {nameof(ConnectorSettings)} Namespace {@namespace}, {LogException(ex)}"); throw; } - } + } public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null) { @@ -44,11 +51,18 @@ public static IEnumerable GetFunctions(ConnectorSettings conn } public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) + { + return GetFunctions(connectorSettings, openApiDocument, globalValues, out var _, configurationLogger); + } + + public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); + OptionSetList optionSetList = new OptionSetList(); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, optionSetList, configurationLogger, globalValues); + optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); } @@ -65,11 +79,18 @@ public static IEnumerable GetTriggers(string @namespace, Open } public static IEnumerable GetTriggers(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) + { + return GetTriggers(@namespace, openApiDocument, globalValues, out var _, configurationLogger); + } + + public static IEnumerable GetTriggers(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, configurationLogger, globalValues, FunctionType.Trigger); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); + OptionSetList optionSetList = new OptionSetList(); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); + optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); } @@ -86,11 +107,18 @@ public static IEnumerable GetTriggers(ConnectorSettings conne } public static IEnumerable GetTriggers(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) + { + return GetTriggers(connectorSettings, openApiDocument, globalValues, out var _, configurationLogger); + } + + public static IEnumerable GetTriggers(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, configurationLogger, globalValues, FunctionType.Trigger); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); + OptionSetList optionSetList = new OptionSetList(); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); + optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); } @@ -109,7 +137,7 @@ private static bool ShouldIncludeFunction(ConnectorFunction function, ConnectorS (function.IsSupported || settings?.AllowUnsupportedFunctions == true); } - internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) + internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, OptionSetList optionSetList, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) { bool connectorIsSupported = true; string connectorNotSupportedReason = string.Empty; @@ -215,7 +243,7 @@ internal static IEnumerable GetFunctionsInternal(ConnectorSet ? notSupportedReasonForPath : notSupportedReasonForOperation; - ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues) + ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues, optionSetList) { Servers = openApiDocument.Servers }; @@ -492,12 +520,13 @@ private static void ValidateSupportedOpenApiParameters(OpenApiOperation op, ref } // Parse an OpenApiDocument and return functions. - internal static (List connectorFunctions, List texlFunctions) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) - { - List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); + internal static (List connectorFunctions, List texlFunctions, OptionSetList optionsetList) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) + { + OptionSetList optionsetList = new OptionSetList(); + List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, optionsetList, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); List tFunctions = cFunctions.Select(f => new ConnectorTexlFunction(f)).ToList(); - return (cFunctions, tFunctions); + return (cFunctions, tFunctions, optionsetList); } internal static string GetServer(IEnumerable openApiServers, HttpMessageInvoker httpClient) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs index b69ac9779f..f3c9d3f8fc 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs @@ -22,30 +22,31 @@ public class ConnectorParameter : ConnectorSchema internal bool IsBodyParameter = false; - internal ConnectorParameter(OpenApiParameter openApiParameter, ConnectorCompatibility compatibility) - : this(openApiParameter, null, false, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, null, false, optionSets, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, ConnectorCompatibility compatibility) - : this(openApiParameter, null, useHiddenTypes, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, null, useHiddenTypes, optionSets, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, ConnectorCompatibility compatibility) - : this(openApiParameter, bodyExtensions, false, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, bodyExtensions, false, optionSets, compatibility) { IsBodyParameter = true; } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, ConnectorCompatibility compatibility) - : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, OptionSetList optionSets, ConnectorCompatibility compatibility) + : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, optionSets, compatibility) { Name = openApiParameter.Name; Description = openApiParameter.Description; Location = openApiParameter.In; } + // Intellisense only internal ConnectorParameter(ConnectorParameter connectorParameter, ConnectorType connectorType) : base(connectorParameter, connectorType) { diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs index 39bf54a073..74cab21832 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs @@ -29,17 +29,18 @@ public class ConnectorSchema : SupportsConnectorErrors public bool SupportsDynamicIntellisense => ConnectorType.SupportsDynamicIntellisense; - public bool? NotificationUrl => ConnectorType.NotificationUrl; + public bool? NotificationUrl => ConnectorType.NotificationUrl; - internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, ConnectorCompatibility compatibility) + internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, OptionSetList optionSets, ConnectorCompatibility compatibility) { Schema = openApiParameter.Schema; - UseHiddenTypes = useHiddenTypes; - ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(compatibility)); + UseHiddenTypes = useHiddenTypes; + ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(optionSets, compatibility)); DefaultValue = openApiParameter.Schema.TryGetDefaultValue(FormulaType, out FormulaValue defaultValue, this) && defaultValue is not BlankValue ? defaultValue : null; ConnectorExtensions = new ConnectorExtensions(openApiParameter, bodyExtensions); } + // Intellisense only internal ConnectorSchema(ConnectorSchema connectorSchema, ConnectorType connectorType) { Schema = connectorSchema.Schema; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs index 64b1bed879..ee196bb273 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs @@ -116,7 +116,7 @@ public class ConnectorType : SupportsConnectorErrors internal string RelationshipName { get; set; } - internal string ForeignKey { get; set; } + internal string ForeignKey { get; set; } internal ConnectorType(ISwaggerSchema schema, ISwaggerParameter openApiParameter, FormulaType formulaType, ErrorResourceKey warning = default) { @@ -197,20 +197,20 @@ internal ConnectorType(string error, ErrorResourceKey warning = default) FormulaType = DefaultType; } - internal ConnectorType(ISwaggerSchema schema, ConnectorCompatibility compatibility) - : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(compatibility)) - { + internal ConnectorType(ISwaggerSchema schema, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(optionSets, compatibility)) + { } - internal ConnectorType(JsonElement schema, ConnectorCompatibility compatibility, IList sqlRelationships) - : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(compatibility, sqlRelationships)) - { + internal ConnectorType(JsonElement schema, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships) + : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(optionSets, compatibility, sqlRelationships)) + { } - internal ConnectorType(JsonElement schema, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) - : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(compatibility, sqlRelationships)) + internal ConnectorType(JsonElement schema, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) + : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(optionSets, compatibility, sqlRelationships)) { - Name = name; + Name = name; foreach (ConnectorType field in Fields.Where(f => f.Capabilities != null)) { diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs index a28c470542..4ef9cbb800 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs @@ -11,12 +11,16 @@ using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Types; +#pragma warning disable SA1117 + namespace Microsoft.PowerFx.Connectors { internal class CdpTableResolver : ICdpTableResolver { public ConnectorLogger Logger { get; } + public IEnumerable OptionSets { get; private set; } + private readonly CdpTable _tabularTable; private readonly HttpClient _httpClient; @@ -87,7 +91,12 @@ public async Task ResolveTableAsync(string tableName, Cancellatio string connectorName = _uriPrefix.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1]; - return ConnectorFunction.GetCdpTableType(this, connectorName, "Schema/Items", FormulaValue.New(text), sqlRelationships, ConnectorCompatibility.CdpCompatibility, _tabularTable.DatasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo); + ConnectorType connectorType = ConnectorFunction.GetCdpTableType(this, connectorName, "Schema/Items", FormulaValue.New(text), sqlRelationships, ConnectorCompatibility.CdpCompatibility, _tabularTable.DatasetName, + out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable optionSets); + + OptionSets = optionSets; + + return connectorType; } private bool IsSql() => _uriPrefix.Contains("/sql/"); diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs index a2df070b77..73620a3d80 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs @@ -27,6 +27,8 @@ public class CdpTable : CdpService public override bool IsDelegable => (DelegationInfo?.SortRestriction != null) || (DelegationInfo?.FilterRestriction != null) || (DelegationInfo?.FilterSupportedFunctions != null); + public IEnumerable OptionSets { get; private set; } + internal TableDelegationInfo DelegationInfo => ((DataSourceInfo)TabularTableDescriptor.FormulaType._type.AssociatedDataSources.First()).DelegationInfo; internal override IReadOnlyDictionary Relationships => _relationships; @@ -80,6 +82,7 @@ public virtual async Task InitAsync(HttpClient httpClient, string uriPrefix, Can TabularTableDescriptor = await tableResolver.ResolveTableAsync(TableName, cancellationToken).ConfigureAwait(false); _relationships = TabularTableDescriptor.Relationships; + OptionSets = tableResolver.OptionSets; RecordType = (RecordType)TabularTableDescriptor.FormulaType; } @@ -87,8 +90,7 @@ public virtual async Task InitAsync(HttpClient httpClient, string uriPrefix, Can private async Task InitializeDatasetMetadata(HttpClient httpClient, string uriPrefix, ConnectorLogger logger, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - - CdpDataSource cds = new CdpDataSource(DatasetName); + DatasetMetadata dm = await CdpDataSource.GetDatasetsMetadataAsync(httpClient, uriPrefix, cancellationToken, logger).ConfigureAwait(false); DatasetMetadata = dm ?? throw new InvalidOperationException("Dataset metadata is not available"); diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs index 6a7a8c6e22..b66b29df2e 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs @@ -33,9 +33,9 @@ public void CompatibilityTest() string text = (string)LoggingTestServer.GetFileText(@"Responses\Compatibility GetSchema.json"); - ConnectorType ctCdp = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.CdpCompatibility, "dataset", out _, out _, out _); - ConnectorType ctPa = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.PowerAppsCompatibility, "dataset", out _, out _, out _); - ConnectorType ctSw = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.SwaggerCompatibility, "dataset", out _, out _, out _); + ConnectorType ctCdp = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.CdpCompatibility, "dataset", out _, out _, out _, out _); + ConnectorType ctPa = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.PowerAppsCompatibility, "dataset", out _, out _, out _, out _); + ConnectorType ctSw = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.SwaggerCompatibility, "dataset", out _, out _, out _, out _); string cdp = ctCdp.FormulaType.ToStringWithDisplayNames(); string pa = ctPa.FormulaType.ToStringWithDisplayNames(); diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs index 43d398e3b8..c319f56fcb 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs @@ -625,7 +625,7 @@ public void GenerateYamlFiles(string reference, int folderExclusionIndex = -1, s // Step 2: Get TexlFunctions to be exported // Notice that TexlFunction is internal and requires InternalVisibleTo - (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(connectorSettings, connector.Value.document, logger); + (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(connectorSettings, connector.Value.document, logger); // Step 3: Export TexlFunctions to Yaml ExportTexlFunctionsToYaml(reference, outFolderPath, connector.Key, texlFunctions.Cast().ToList(), false); @@ -1373,7 +1373,8 @@ public static class Exts public static string GetString(this OpenApiSchema schema) { StringBuilder sb = new StringBuilder(); - schema.GetStringInternal(new ConnectorTypeGetterSettings(0), sb); + OptionSetList optionSets = new OptionSetList(); + schema.GetStringInternal(new ConnectorTypeGetterSettings(0, optionSets), sb); return sb.ToString(); } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems index dc0b202d51..285e2f3f30 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems @@ -31,6 +31,7 @@ + diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs index 0317de8d4f..c3149e754c 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs @@ -64,7 +64,7 @@ public void ACSL_GetFunctionNames22() public void ACSL_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\Azure Cognitive Service for Language.json", _output); - (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL"), doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, OptionSetList optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL"), doc, new ConsoleLogger(_output)); Assert.Contains(connectorFunctions, func => func.Namespace == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); @@ -72,7 +72,7 @@ public void ACSL_Load() Assert.Equal("analysisInput:![documents:*[id:s, language:s, text:s]]|task:![parameters:![deploymentName:s, projectName:s, stringIndexType:s]]", string.Join("|", func1.RequiredParameters.Select(rp => $"{rp.Name}:{rp.FormulaType._type}"))); Assert.Equal("displayName:s", string.Join("|", func1.OptionalParameters.Select(rp => $"{rp.Name}:{rp.FormulaType._type}"))); - (connectorFunctions, texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); + (connectorFunctions, texlFunctions, optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); Assert.Contains(connectorFunctions, func => func.Namespace == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); @@ -85,7 +85,7 @@ public void ACSL_Load() public void SF_TextCsv() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\SalesForce.json", _output); - (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("SF") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(new ConnectorSettings("SF") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); // function returns text/csv ConnectorFunction func1 = connectorFunctions.First(f => f.Name == "GetJobRecordResults"); @@ -873,7 +873,7 @@ private void Visit(IUntypedObject untypedObject) public void LQA_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\Language - Question Answering.json", _output); - (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("LQA"), doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(new ConnectorSettings("LQA"), doc, new ConsoleLogger(_output)); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "LQA" && func.Name == "GetAnswersFromText"); } @@ -881,7 +881,7 @@ public void LQA_Load() public void SQL_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\SQL Server.json", _output); - (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("SQL") { IncludeInternalFunctions = true }, doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(new ConnectorSettings("SQL") { IncludeInternalFunctions = true }, doc, new ConsoleLogger(_output)); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "SQL" && func.Name == "GetProcedureV2"); } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs new file mode 100644 index 0000000000..355a10d10e --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Linq; +using Microsoft.PowerFx.Core; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Syntax; +using Xunit; + +namespace Microsoft.PowerFx.Connectors.Tests.Shared +{ + public class OptionSetListTests + { + [Fact] + public void AddOptionSetToList() + { + OptionSetList list = new OptionSetList(); + + SingleSourceDisplayNameProvider dnp = new SingleSourceDisplayNameProvider(); + dnp = dnp.AddField(new DName("logical1"), new DName("display1")); + dnp = dnp.AddField(new DName("logical2"), new DName("display2")); + + OptionSet os1 = new OptionSet("os1", dnp); + OptionSet os2 = new OptionSet("os1", dnp); + + OptionSet os = list.TryAdd(os1); + Assert.Same(os1, os); + + // twice the same, nothing added + os = list.TryAdd(os1); + Assert.Same(os1, os); + Assert.Single(list.OptionSets); + + // still the same, nothing added + os = list.TryAdd(os2); + Assert.Same(os1, os); + Assert.Single(list.OptionSets); + + dnp = dnp.AddField(new DName("logical3"), new DName("display3")); + OptionSet os3 = new OptionSet("os3", dnp); + + // new optionSet + os = list.TryAdd(os3); + Assert.NotSame(os, os1); + Assert.Equal(2, list.OptionSets.Count()); + + // try a name conflict now + OptionSet os4 = new OptionSet("os1", dnp); + OptionSet newOs = list.TryAdd(os4); + + Assert.Equal("os1_1", newOs.EntityName); + + OptionSet os5 = new OptionSet("os1", dnp); + os = list.TryAdd(os5); + + Assert.Same(newOs, os); + + // Once more + dnp = dnp.AddField(new DName("logical4"), new DName("display4")); + OptionSet os6 = new OptionSet("os1", dnp); + OptionSet newOs2 = list.TryAdd(os6); + + Assert.Equal("os1_2", newOs2.EntityName); + } + } +} diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs index 6145d83298..57b81d88d0 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs @@ -962,8 +962,8 @@ public async Task ZD_CdpTabular_GetTables2() Assert.Equal( "r![assignee_id:w, brand_id:w, collaborator_ids:s, created_at:d, custom_fields:s, description:s, due_at:d, external_id:s, followup_ids:s, forum_topic_id:w, group_id:w, has_incidents:b, " + - "id:w, organization_id:w, priority:l, problem_id:w, raw_subject:s, recipient:s, requester_id:w, satisfaction_rating:s, sharing_agreement_ids:s, status:s, subject:s, submitter_id:w, " + - "tags:s, ticket_form_id:w, type:s, updated_at:d, url:s, via:s]", ((CdpRecordType)zdTable.RecordType).ToStringWithDisplayNames()); + "id:w, organization_id:w, priority:l, problem_id:w, raw_subject:s, recipient:s, requester_id:w, satisfaction_rating:s, sharing_agreement_ids:s, status:l, subject:s, submitter_id:w, " + + "tags:s, ticket_form_id:w, type:l, updated_at:d, url:s, via:s]", ((CdpRecordType)zdTable.RecordType).ToStringWithDisplayNames()); SymbolValues symbolValues = new SymbolValues().Add("Tickets", zdTable); RuntimeConfig rc = new RuntimeConfig(symbolValues).AddService(logger); @@ -979,7 +979,10 @@ public async Task ZD_CdpTabular_GetTables2() OptionSetValue priority = Assert.IsType(result); Assert.Equal("normal", priority.Option); - Assert.Equal("normal", priority.DisplayName); + Assert.Equal("Normal", priority.DisplayName); + + Assert.NotNull(connectorTable.OptionSets); + Assert.Equal("priority, status, type", string.Join(", ", connectorTable.OptionSets.Select(os => os.EntityName.Value).OrderBy(x => x))); } } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json index b46cb8ba2d..7b6d06a2a4 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json @@ -18,254 +18,308 @@ "type": "array", "items": { "type": "object", - "required": [], "properties": { "id": { + "type": "integer", "title": "id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-keyOrder": 1, "x-ms-keyType": "primary", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "url": { + "type": "string", "title": "url", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "external_id": { + "type": "string", "title": "external_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "type": { + "type": "string", + "x-ms-enum-values": [ + { + "value": "problem", + "displayName": "Problem" + }, + { + "value": "incident", + "displayName": "Incident" + }, + { + "value": "question", + "displayName": "Question" + }, + { + "value": "task", + "displayName": "Task" + } + ], "title": "type", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "subject": { + "type": "string", "title": "subject", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "raw_subject": { + "type": "string", "title": "raw_subject", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "description": { + "type": "string", "title": "description", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "priority": { - "title": "priority", - "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, "type": "string", - "enum": [ - "low", - "normal", - "high", - "urgent" + "x-ms-enum-values": [ + { + "value": "low", + "displayName": "Low" + }, + { + "value": "normal", + "displayName": "Normal" + }, + { + "value": "high", + "displayName": "High" + }, + { + "value": "urgent", + "displayName": "Urgent" + } ], + "title": "priority", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "status": { + "type": "string", + "x-ms-enum-values": [ + { + "value": "new", + "displayName": "New" + }, + { + "value": "open", + "displayName": "Open" + }, + { + "value": "pending", + "displayName": "Pending" + }, + { + "value": "hold", + "displayName": "Hold" + }, + { + "value": "solved", + "displayName": "Solved" + }, + { + "value": "closed", + "displayName": "Closed" + } + ], "title": "status", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "recipient": { + "type": "string", "title": "recipient", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, - "type": "string", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "requester_id": { + "type": "integer", "title": "requester_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "submitter_id": { + "type": "integer", "title": "submitter_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "assignee_id": { + "type": "integer", "title": "assignee_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "organization_id": { + "type": "integer", "title": "organization_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "group_id": { + "type": "integer", "title": "group_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "collaborator_ids": { - "title": "collaborator_ids", "type": "string", + "title": "collaborator_ids", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "forum_topic_id": { + "type": "integer", "title": "forum_topic_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "problem_id": { + "type": "integer", "title": "problem_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "has_incidents": { + "type": "boolean", "title": "has_incidents", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "not", "and", "or" ] }, - "type": "boolean", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "due_at": { + "type": "string", "title": "due_at", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes" ] }, - "type": "string", "format": "date-time", "x-ms-permission": "read-write", "x-ms-sort": "none" }, "tags": { - "title": "tags", "type": "string", + "title": "tags", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "via": { - "title": "via", "type": "string", + "title": "via", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "custom_fields": { - "title": "custom_fields", "type": "string", + "title": "custom_fields", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "satisfaction_rating": { - "title": "satisfaction_rating", "type": "string", + "title": "satisfaction_rating", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "sharing_agreement_ids": { - "title": "sharing_agreement_ids", "type": "string", + "title": "sharing_agreement_ids", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "followup_ids": { - "title": "followup_ids", "type": "string", + "title": "followup_ids", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "ticket_form_id": { + "type": "integer", "title": "ticket_form_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "brand_id": { + "type": "integer", "title": "brand_id", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, - "type": "integer", "format": "int64", - "minimum": -9223372036854775808, - "maximum": 9223372036854775807, + "minimum": -9223372036854776000, + "maximum": 9223372036854776000, "x-ms-permission": "read-write", "x-ms-sort": "none" }, "created_at": { + "type": "string", "title": "created_at", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes" ] }, - "type": "string", "format": "date-time", "x-ms-permission": "read-only", "x-ms-sort": "none" }, "updated_at": { + "type": "string", "title": "updated_at", "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes" ] }, - "type": "string", "format": "date-time", "x-ms-permission": "read-only", "x-ms-sort": "none" } } - }, - "x-ms-permission": "read-write" + } } -} +} \ No newline at end of file From 7430132d4f9a20256039df331e0653865c041d23 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Wed, 13 Nov 2024 08:50:54 +0100 Subject: [PATCH 2/8] Update --- .../ConnectorFunction.cs | 42 ++++++++++--------- .../Internal/OptionSetList.cs | 30 +------------ .../OpenApiExtensions.cs | 41 ++++++++++++------ .../OpenApiParser.cs | 14 +++---- .../Public/ConnectorParameter.cs | 16 +++---- .../Public/ConnectorSchema.cs | 4 +- .../Public/ConnectorType.cs | 12 +++--- .../Tabular/CdpTableResolver.cs | 2 +- .../CompatibilityTests.cs | 6 +-- .../InternalTesting.cs | 2 +- .../OptionSetListTests.cs | 18 ++------ .../PowerPlatformTabularTests.cs | 2 +- 12 files changed, 86 insertions(+), 103 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs index 816dd76cc0..f9928c5b39 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs @@ -296,17 +296,21 @@ public ConnectorType ReturnParameterType private readonly ConnectorLogger _configurationLogger = null; - internal readonly OptionSetList OptionSets; + internal readonly OptionSetList OptionSets; + + // CDP only + internal readonly string TableName; internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, - ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, OptionSetList optionSetList) + ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, string tableName, OptionSetList optionSetList) { Operation = openApiOperation; Name = name; OperationPath = operationPath; HttpMethod = httpMethod; ConnectorSettings = connectorSettings; - OptionSets = optionSetList; + OptionSets = optionSetList; + TableName = tableName; GlobalContext = new ConnectorGlobalContext(functionList ?? throw new ArgumentNullException(nameof(functionList)), globalValues); _configurationLogger = configurationLogger; @@ -991,7 +995,7 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return null; } - ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, OptionSets, ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, null, new OptionSetList(), ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1005,15 +1009,15 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return connectorType; } - internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, OptionSetList optionSets, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) { JsonElement je = ExtractFromJson(sv, valuePath); - return GetConnectorTypeInternal(optionSets, compatibility, je); + return GetConnectorTypeInternal(tableName, optionSets, compatibility, je); } // Only called by ConnectorTable.GetSchema // Returns a FormulaType with AssociatedDataSources set (done in AddTabularDataSource) - internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue stringValue, List sqlRelationships, ConnectorCompatibility compatibility, string datasetName, + internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string tableName, string valuePath, StringValue stringValue, List sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable optionSets) { // There are some errors when parsing this Json payload but that's not a problem here as we only need x-ms-capabilities parsing to work @@ -1028,7 +1032,7 @@ internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, s IList referencedEntities = GetReferenceEntities(connectorName, stringValue); OptionSetList optionSetList = new OptionSetList(); - ConnectorType connectorType = new ConnectorType(jsonElement, optionSetList, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly); + ConnectorType connectorType = new ConnectorType(jsonElement, tableName, optionSetList, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly); delegationInfo = ((DataSourceInfo)connectorType.FormulaType._type.AssociatedDataSources.First()).DelegationInfo; optionSets = optionSetList.OptionSets; @@ -1084,17 +1088,17 @@ internal class SalesForceReferencedEntity public bool RestrictedDelete { get; set; } } - private static ConnectorType GetConnectorTypeInternal(OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je) + private static ConnectorType GetConnectorTypeInternal(string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je) { OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet }; OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag); - return new ConnectorType(SwaggerSchema.New(schema), optionSets, compatibility); + return new ConnectorType(SwaggerSchema.New(schema), tableName, optionSets, compatibility); } - private static ConnectorType GetJsonConnectorTypeInternal(OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) + private static ConnectorType GetJsonConnectorTypeInternal(string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) { - return new ConnectorType(je, optionSets, compatibility, sqlRelationships); + return new ConnectorType(je, tableName, optionSets, compatibility, sqlRelationships); } private async Task GetConnectorSuggestionsFromDynamicPropertyAsync(NamedValue[] knownParameters, BaseRuntimeConnectorContext runtimeContext, ConnectorDynamicProperty cdp, CancellationToken cancellationToken) @@ -1115,7 +1119,7 @@ private async Task GetConnectorSuggestionsFromDynamicPropertyAsyn return null; } - ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, OptionSets, ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, null, new OptionSetList(), ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1422,7 +1426,7 @@ private ConnectorParameterInternals Initialize() return null; } - ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, TableName, OptionSets, ConnectorSettings.Compatibility)); if (connectorParameter.HiddenRecordType != null) { @@ -1491,12 +1495,12 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter = new OpenApiParameter() { Name = bodyPropertyName, Schema = bodyPropertySchema, Description = requestBody.Description, Required = bodyPropertyRequired, Extensions = bodyPropertySchema.Extensions }; - ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, TableName, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter2, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter2.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter2.HiddenRecordType != null) { - hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, OptionSets, ConnectorSettings.Compatibility))); + hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, TableName, OptionSets, ConnectorSettings.Compatibility))); } List parameterList = !bodyPropertyRequired ? optionalParameters : bodyPropertyHiddenRequired ? hiddenRequiredParameters : requiredParameters; @@ -1515,7 +1519,7 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter2 = new OpenApiParameter() { Name = bodyName, Schema = bodySchema, Description = requestBody.Description, Required = requestBody.Required, Extensions = bodySchema.Extensions }; - ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, TableName, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter3, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter3.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter3.HiddenRecordType != null) @@ -1535,7 +1539,7 @@ private ConnectorParameterInternals Initialize() OpenApiSchema bodyParameterSchema = new OpenApiSchema() { Type = "string" }; OpenApiParameter bodyParameter3 = new OpenApiParameter() { Name = bodyName, Schema = bodyParameterSchema, Description = "Body", Required = requestBody.Required }; - ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, TableName, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyParameter, OpenApiExtensions.TryGetOpenApiValue(bodyParameter.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); List parameterList = requestBody.Required ? requiredParameters : optionalParameters; @@ -1578,7 +1582,7 @@ private ConnectorParameterInternals Initialize() _arityMax = _arityMin + (_optionalParameters.Length == 0 ? 0 : 1); _warnings = new List(); - _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(OptionSets, ConnectorSettings.Compatibility)); + _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(TableName, OptionSets, ConnectorSettings.Compatibility)); if (IsDeprecated) { diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs index ca94705968..1bfbe4a385 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs @@ -3,8 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; -using Microsoft.PowerFx.Core; namespace Microsoft.PowerFx.Connectors { @@ -43,32 +41,8 @@ public OptionSet TryAdd(OptionSet optionSet) { return Get(optionSet.EntityName); } - - int i = 1; - SingleSourceDisplayNameProvider dnp = new SingleSourceDisplayNameProvider(optionSet.Options); - - while (oss == OptionSetStatus.Conflict && i < 20) - { - string name = $"{optionSet.EntityName}_{i++}"; - - OptionSet newOptionSet = new OptionSet(name, dnp); - oss = GetStatus(newOptionSet); - - // no conflict now, let's add it - if (oss == OptionSetStatus.New) - { - Add(newOptionSet); - return newOptionSet; - } - - // we found an existing one that matches - if (oss == OptionSetStatus.Same) - { - return Get(name); - } - } - - throw new InvalidOperationException("Too many option set conflicts"); + + throw new InvalidOperationException($"Optionset name conflict ({optionSet.EntityName})"); } private void Add(OptionSet optionSet) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs index d9eae12154..aeddac5e61 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs @@ -7,6 +7,7 @@ using System.Globalization; using System.Linq; using System.Net.Http; +using System.Runtime; using Microsoft.OpenApi.Any; using Microsoft.OpenApi.Interfaces; using Microsoft.OpenApi.Models; @@ -394,11 +395,15 @@ internal class ConnectorTypeGetterSettings internal int Level = 0; internal readonly OptionSetList OptionSets; - internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, OptionSetList optionSets, IList sqlRelationships = null) + private readonly string _tableName; + + internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, string tableName, OptionSetList optionSets, IList sqlRelationships = null) { Compatibility = connectorCompatibility; OptionSets = optionSets; SqlRelationships = sqlRelationships; + + _tableName = tableName; } internal ConnectorTypeGetterSettings Stack(string identifier) @@ -413,11 +418,23 @@ internal void UnStack() Chain.Pop(); Level--; } + + internal string GetOptionSetName(string optionSetNameBase) + { + string optionSetName = optionSetNameBase; + + if (!string.IsNullOrEmpty(_tableName)) + { + optionSetName += $" ({_tableName})"; + } + + return optionSetName; + } } - internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships = null) + internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships = null) { - ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, optionSets, sqlRelationships); + ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, tableName, optionSets, sqlRelationships); ConnectorType connectorType = openApiParameter.GetConnectorType(settings); return connectorType; @@ -469,8 +486,8 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar if (optionSetDisplayNameProvider != null && (settings.Compatibility.IsCDP() || schema.Format == "enum")) { - string enumName = schema.GetEnumName() ?? openApiParameter.Name; - OptionSet optionSet = new OptionSet(enumName, optionSetDisplayNameProvider); + string optionSetName = settings.GetOptionSetName(schema.GetEnumName() ?? openApiParameter.Name); + OptionSet optionSet = new OptionSet(optionSetName, optionSetDisplayNameProvider); optionSet = settings.OptionSets.TryAdd(optionSet); return new ConnectorType(schema, openApiParameter, optionSet.FormulaType); } @@ -480,8 +497,8 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar { if (schema.Enum.All(e => e is OpenApiString)) { - string enumName = schema.GetEnumName() ?? openApiParameter.Name; - OptionSet optionSet = new OptionSet(enumName, schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary()); + string optionSetName = settings.GetOptionSetName(schema.GetEnumName() ?? openApiParameter.Name); + OptionSet optionSet = new OptionSet(optionSetName, schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary()); optionSet = settings.OptionSets.TryAdd(optionSet); return new ConnectorType(schema, openApiParameter, optionSet.FormulaType); } @@ -767,7 +784,7 @@ public static HttpMethod ToHttpMethod(this OperationType key) public static FormulaType GetReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility) { OptionSetList optionSetList = new OptionSetList(); - ConnectorType connectorType = openApiOperation.GetConnectorReturnType(optionSetList, compatibility); + ConnectorType connectorType = openApiOperation.GetConnectorReturnType(null, optionSetList, compatibility); FormulaType ft = connectorType.HasErrors ? ConnectorType.DefaultType : connectorType?.FormulaType ?? new BlankType(); return ft; } @@ -777,7 +794,7 @@ public static bool GetRequiresUserConfirmation(this OpenApiOperation op) return op.Extensions.TryGetValue(XMsRequireUserConfirmation, out IOpenApiExtension openExt) && openExt is OpenApiBoolean b && b.Value; } - internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, OptionSetList optionSets, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) { OpenApiResponses responses = openApiOperation.Responses; OpenApiResponse response = responses.Where(kvp => kvp.Key?.Length == 3 && kvp.Key.StartsWith("2", StringComparison.Ordinal)).OrderBy(kvp => kvp.Key).FirstOrDefault().Value; @@ -801,7 +818,7 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA if (response.Content.Count == 0) { OpenApiSchema schema = new OpenApiSchema() { Type = "string", Format = "no_format" }; - return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(optionSets, compatibility); + return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(tableName, optionSets, compatibility); } // Responses is a list by content-type. Find "application/json" @@ -820,10 +837,10 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA if (openApiMediaType.Schema == null) { // Treat as void. - return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(optionSets, compatibility); + return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(tableName, optionSets, compatibility); } - return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(optionSets, compatibility); + return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(tableName, optionSets, compatibility); } } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs index b447d59768..19432bd1a9 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs @@ -33,7 +33,7 @@ public static IEnumerable GetFunctions(string @namespace, Ope { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, optionSetList, configurationLogger, globalValues); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, optionSetList, configurationLogger, globalValues); optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); @@ -61,7 +61,7 @@ public static IEnumerable GetFunctions(ConnectorSettings conn { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, optionSetList, configurationLogger, globalValues); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionSetList, configurationLogger, globalValues); optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); @@ -89,7 +89,7 @@ public static IEnumerable GetTriggers(string @namespace, Open { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); @@ -117,7 +117,7 @@ public static IEnumerable GetTriggers(ConnectorSettings conne { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); optionSets = optionSetList.OptionSets; configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); @@ -137,7 +137,7 @@ private static bool ShouldIncludeFunction(ConnectorFunction function, ConnectorS (function.IsSupported || settings?.AllowUnsupportedFunctions == true); } - internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, OptionSetList optionSetList, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) + internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, string tableName, OptionSetList optionSetList, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) { bool connectorIsSupported = true; string connectorNotSupportedReason = string.Empty; @@ -243,7 +243,7 @@ internal static IEnumerable GetFunctionsInternal(ConnectorSet ? notSupportedReasonForPath : notSupportedReasonForOperation; - ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues, optionSetList) + ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues, tableName, optionSetList) { Servers = openApiDocument.Servers }; @@ -523,7 +523,7 @@ private static void ValidateSupportedOpenApiParameters(OpenApiOperation op, ref internal static (List connectorFunctions, List texlFunctions, OptionSetList optionsetList) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) { OptionSetList optionsetList = new OptionSetList(); - List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, optionsetList, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); + List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionsetList, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); List tFunctions = cFunctions.Select(f => new ConnectorTexlFunction(f)).ToList(); return (cFunctions, tFunctions, optionsetList); diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs index f3c9d3f8fc..4b6fbbd6e8 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs @@ -22,24 +22,24 @@ public class ConnectorParameter : ConnectorSchema internal bool IsBodyParameter = false; - internal ConnectorParameter(OpenApiParameter openApiParameter, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, null, false, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, null, false, tableName, optionSets, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, null, useHiddenTypes, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, null, useHiddenTypes, tableName, optionSets, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, bodyExtensions, false, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, bodyExtensions, false, tableName, optionSets, compatibility) { IsBodyParameter = true; } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, OptionSetList optionSets, ConnectorCompatibility compatibility) - : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, tableName, optionSets, compatibility) { Name = openApiParameter.Name; Description = openApiParameter.Description; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs index 74cab21832..2b678d1e3e 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs @@ -31,11 +31,11 @@ public class ConnectorSchema : SupportsConnectorErrors public bool? NotificationUrl => ConnectorType.NotificationUrl; - internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, OptionSetList optionSets, ConnectorCompatibility compatibility) + internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) { Schema = openApiParameter.Schema; UseHiddenTypes = useHiddenTypes; - ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(optionSets, compatibility)); + ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(tableName, optionSets, compatibility)); DefaultValue = openApiParameter.Schema.TryGetDefaultValue(FormulaType, out FormulaValue defaultValue, this) && defaultValue is not BlankValue ? defaultValue : null; ConnectorExtensions = new ConnectorExtensions(openApiParameter, bodyExtensions); } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs index ee196bb273..6ec2e207a9 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs @@ -197,18 +197,18 @@ internal ConnectorType(string error, ErrorResourceKey warning = default) FormulaType = DefaultType; } - internal ConnectorType(ISwaggerSchema schema, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(optionSets, compatibility)) + internal ConnectorType(ISwaggerSchema schema, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(tableName, optionSets, compatibility)) { } - internal ConnectorType(JsonElement schema, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships) - : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(optionSets, compatibility, sqlRelationships)) + internal ConnectorType(JsonElement schema, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships) + : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships)) { } - internal ConnectorType(JsonElement schema, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) - : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(optionSets, compatibility, sqlRelationships)) + internal ConnectorType(JsonElement schema, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) + : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships)) { Name = name; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs index 4ef9cbb800..7a64887298 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs @@ -91,7 +91,7 @@ public async Task ResolveTableAsync(string tableName, Cancellatio string connectorName = _uriPrefix.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1]; - ConnectorType connectorType = ConnectorFunction.GetCdpTableType(this, connectorName, "Schema/Items", FormulaValue.New(text), sqlRelationships, ConnectorCompatibility.CdpCompatibility, _tabularTable.DatasetName, + ConnectorType connectorType = ConnectorFunction.GetCdpTableType(this, connectorName, _tabularTable.TableName, "Schema/Items", FormulaValue.New(text), sqlRelationships, ConnectorCompatibility.CdpCompatibility, _tabularTable.DatasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable optionSets); OptionSets = optionSets; diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs index b66b29df2e..fa2ca3fb36 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs @@ -33,9 +33,9 @@ public void CompatibilityTest() string text = (string)LoggingTestServer.GetFileText(@"Responses\Compatibility GetSchema.json"); - ConnectorType ctCdp = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.CdpCompatibility, "dataset", out _, out _, out _, out _); - ConnectorType ctPa = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.PowerAppsCompatibility, "dataset", out _, out _, out _, out _); - ConnectorType ctSw = ConnectorFunction.GetCdpTableType(tableResolver, "name", "schema/items", StringValue.New(text), null, ConnectorCompatibility.SwaggerCompatibility, "dataset", out _, out _, out _, out _); + ConnectorType ctCdp = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), null, ConnectorCompatibility.CdpCompatibility, "dataset", out _, out _, out _, out _); + ConnectorType ctPa = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), null, ConnectorCompatibility.PowerAppsCompatibility, "dataset", out _, out _, out _, out _); + ConnectorType ctSw = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), null, ConnectorCompatibility.SwaggerCompatibility, "dataset", out _, out _, out _, out _); string cdp = ctCdp.FormulaType.ToStringWithDisplayNames(); string pa = ctPa.FormulaType.ToStringWithDisplayNames(); diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs index c319f56fcb..7b82b2b6ac 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs @@ -1374,7 +1374,7 @@ public static string GetString(this OpenApiSchema schema) { StringBuilder sb = new StringBuilder(); OptionSetList optionSets = new OptionSetList(); - schema.GetStringInternal(new ConnectorTypeGetterSettings(0, optionSets), sb); + schema.GetStringInternal(new ConnectorTypeGetterSettings(0, null, optionSets), sb); return sb.ToString(); } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs index 355a10d10e..499072ed9d 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Linq; using Microsoft.PowerFx.Core; using Microsoft.PowerFx.Core.Utils; -using Microsoft.PowerFx.Syntax; using Xunit; namespace Microsoft.PowerFx.Connectors.Tests.Shared @@ -46,21 +46,9 @@ public void AddOptionSetToList() // try a name conflict now OptionSet os4 = new OptionSet("os1", dnp); - OptionSet newOs = list.TryAdd(os4); - Assert.Equal("os1_1", newOs.EntityName); - - OptionSet os5 = new OptionSet("os1", dnp); - os = list.TryAdd(os5); - - Assert.Same(newOs, os); - - // Once more - dnp = dnp.AddField(new DName("logical4"), new DName("display4")); - OptionSet os6 = new OptionSet("os1", dnp); - OptionSet newOs2 = list.TryAdd(os6); - - Assert.Equal("os1_2", newOs2.EntityName); + InvalidOperationException ioe = Assert.Throws(() => list.TryAdd(os4)); + Assert.Equal("Optionset name conflict (os1)", ioe.Message); } } } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs index 57b81d88d0..f77dbd7a9d 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs @@ -982,7 +982,7 @@ public async Task ZD_CdpTabular_GetTables2() Assert.Equal("Normal", priority.DisplayName); Assert.NotNull(connectorTable.OptionSets); - Assert.Equal("priority, status, type", string.Join(", ", connectorTable.OptionSets.Select(os => os.EntityName.Value).OrderBy(x => x))); + Assert.Equal("priority (tickets), status (tickets), type (tickets)", string.Join(", ", connectorTable.OptionSets.Select(os => os.EntityName.Value).OrderBy(x => x))); } } From b30022522127c08af85c23ea18a9b7523a8a9a68 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Wed, 13 Nov 2024 08:54:20 +0100 Subject: [PATCH 3/8] Update --- .../Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs index 1bfbe4a385..c6a7f80e9a 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs @@ -19,7 +19,7 @@ public OptionSetList() // if the option set doesn't exist, it will be added to the list and we return optionSet // if the option set exists with same name and options, it is not added, we return the existing optionSet - // if the option set exists with same name but different options, we rename it and return that new optionSet + // if the option set exists with same name but different options, we throw an exception (conflicts not allowed) public OptionSet TryAdd(OptionSet optionSet) { if (optionSet == null) From 01a725dc49bc30559d123cba2f9a7028cc5c285c Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Wed, 13 Nov 2024 08:58:09 +0100 Subject: [PATCH 4/8] Update comment --- .../Microsoft.PowerFx.Connectors/OpenApiExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs index aeddac5e61..5b7b2b7c7e 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs @@ -7,7 +7,6 @@ using System.Globalization; using System.Linq; using System.Net.Http; -using System.Runtime; using Microsoft.OpenApi.Any; using Microsoft.OpenApi.Interfaces; using Microsoft.OpenApi.Models; @@ -419,6 +418,8 @@ internal void UnStack() Level--; } + // by default, optionset names will be 'propertyName (tableName)' in CDP case, where propertyName is replaced by x-ms-enum content, when provided + // in non-CDP case, tableName is null and will only be 'propertyName' (or x-ms-enum content) internal string GetOptionSetName(string optionSetNameBase) { string optionSetName = optionSetNameBase; From 3ea8194ea5a3e102161a1c00e480f1184093c40c Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Thu, 14 Nov 2024 14:34:01 +0100 Subject: [PATCH 5/8] Update --- .../ConnectorFunction.cs | 54 ++++++----- .../Environment/PowerFxConfigExtensions.cs | 6 +- .../Internal/OptionSetList.cs | 89 ------------------- .../OpenApiExtensions.cs | 58 +++++++++--- .../OpenApiParser.cs | 40 ++++----- .../Public/ConnectorParameter.cs | 16 ++-- .../Public/ConnectorSchema.cs | 10 +-- .../Public/ConnectorType.cs | 20 ++--- .../InternalTesting.cs | 4 +- ....PowerFx.Connectors.Tests.Shared.projitems | 2 +- .../OpenApiParserTests.cs | 8 +- ...ListTests.cs => SymbolTableTryAddTests.cs} | 22 ++--- 12 files changed, 133 insertions(+), 196 deletions(-) delete mode 100644 src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs rename src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/{OptionSetListTests.cs => SymbolTableTryAddTests.cs} (68%) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs index f9928c5b39..de4cc7e7c3 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs @@ -296,21 +296,17 @@ public ConnectorType ReturnParameterType private readonly ConnectorLogger _configurationLogger = null; - internal readonly OptionSetList OptionSets; - - // CDP only - internal readonly string TableName; + internal readonly SymbolTable OptionSets; - internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, - ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, string tableName, OptionSetList optionSetList) + internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, + ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, SymbolTable optionSets) { Operation = openApiOperation; Name = name; OperationPath = operationPath; HttpMethod = httpMethod; ConnectorSettings = connectorSettings; - OptionSets = optionSetList; - TableName = tableName; + OptionSets = optionSets; GlobalContext = new ConnectorGlobalContext(functionList ?? throw new ArgumentNullException(nameof(functionList)), globalValues); _configurationLogger = configurationLogger; @@ -995,7 +991,7 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return null; } - ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, null, new OptionSetList(), ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, null, new SymbolTable(), ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1009,7 +1005,7 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return connectorType; } - internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility) { JsonElement je = ExtractFromJson(sv, valuePath); return GetConnectorTypeInternal(tableName, optionSets, compatibility, je); @@ -1017,7 +1013,7 @@ internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, // Only called by ConnectorTable.GetSchema // Returns a FormulaType with AssociatedDataSources set (done in AddTabularDataSource) - internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string tableName, string valuePath, StringValue stringValue, List sqlRelationships, ConnectorCompatibility compatibility, string datasetName, + internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string tableName, string valuePath, StringValue stringValue, List sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable optionSets) { // There are some errors when parsing this Json payload but that's not a problem here as we only need x-ms-capabilities parsing to work @@ -1025,16 +1021,16 @@ internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, s ISwaggerSchema tableSchema = SwaggerSchema.New(new OpenApiStringReader(oars).ReadFragment(stringValue.Value, OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic _)); ServiceCapabilities serviceCapabilities = tableSchema.GetTableCapabilities(); - ConnectorPermission tablePermission = tableSchema.GetPermission(); - + ConnectorPermission tablePermission = tableSchema.GetPermission(); + JsonElement jsonElement = ExtractFromJson(stringValue, valuePath, out name, out displayName); bool isTableReadOnly = tablePermission == ConnectorPermission.PermissionReadOnly; IList referencedEntities = GetReferenceEntities(connectorName, stringValue); - OptionSetList optionSetList = new OptionSetList(); - ConnectorType connectorType = new ConnectorType(jsonElement, tableName, optionSetList, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly); + SymbolTable symbolTable = new SymbolTable(); + ConnectorType connectorType = new ConnectorType(jsonElement, tableName, symbolTable, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly); delegationInfo = ((DataSourceInfo)connectorType.FormulaType._type.AssociatedDataSources.First()).DelegationInfo; - optionSets = optionSetList.OptionSets; + optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); return connectorType; } @@ -1088,7 +1084,7 @@ internal class SalesForceReferencedEntity public bool RestrictedDelete { get; set; } } - private static ConnectorType GetConnectorTypeInternal(string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je) + private static ConnectorType GetConnectorTypeInternal(string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, JsonElement je) { OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet }; OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag); @@ -1096,7 +1092,7 @@ private static ConnectorType GetConnectorTypeInternal(string tableName, OptionSe return new ConnectorType(SwaggerSchema.New(schema), tableName, optionSets, compatibility); } - private static ConnectorType GetJsonConnectorTypeInternal(string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) + private static ConnectorType GetJsonConnectorTypeInternal(string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) { return new ConnectorType(je, tableName, optionSets, compatibility, sqlRelationships); } @@ -1119,7 +1115,7 @@ private async Task GetConnectorSuggestionsFromDynamicPropertyAsyn return null; } - ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, null, new OptionSetList(), ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, null, new SymbolTable(), ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1413,7 +1409,7 @@ private ConnectorParameterInternals Initialize() { // Ex: Api-Version hiddenRequired = true; - } + } } else if (ConnectorSettings.Compatibility.ExcludeInternals()) { @@ -1426,8 +1422,8 @@ private ConnectorParameterInternals Initialize() return null; } - ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, TableName, OptionSets, ConnectorSettings.Compatibility)); - + ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, OptionSets, ConnectorSettings.Compatibility)); + if (connectorParameter.HiddenRecordType != null) { errorsAndWarnings.AddError("[Internal error] Unexpected HiddenRecordType non-null value"); @@ -1495,12 +1491,12 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter = new OpenApiParameter() { Name = bodyPropertyName, Schema = bodyPropertySchema, Description = requestBody.Description, Required = bodyPropertyRequired, Extensions = bodyPropertySchema.Extensions }; - ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, TableName, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter2, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter2.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter2.HiddenRecordType != null) - { - hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, TableName, OptionSets, ConnectorSettings.Compatibility))); + { + hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, OptionSets, ConnectorSettings.Compatibility))); } List parameterList = !bodyPropertyRequired ? optionalParameters : bodyPropertyHiddenRequired ? hiddenRequiredParameters : requiredParameters; @@ -1519,7 +1515,7 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter2 = new OpenApiParameter() { Name = bodyName, Schema = bodySchema, Description = requestBody.Description, Required = requestBody.Required, Extensions = bodySchema.Extensions }; - ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, TableName, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter3, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter3.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter3.HiddenRecordType != null) @@ -1539,7 +1535,7 @@ private ConnectorParameterInternals Initialize() OpenApiSchema bodyParameterSchema = new OpenApiSchema() { Type = "string" }; OpenApiParameter bodyParameter3 = new OpenApiParameter() { Name = bodyName, Schema = bodyParameterSchema, Description = "Body", Required = requestBody.Required }; - ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, TableName, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, OptionSets, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyParameter, OpenApiExtensions.TryGetOpenApiValue(bodyParameter.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); List parameterList = requestBody.Required ? requiredParameters : optionalParameters; @@ -1582,8 +1578,8 @@ private ConnectorParameterInternals Initialize() _arityMax = _arityMin + (_optionalParameters.Length == 0 ? 0 : 1); _warnings = new List(); - _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(TableName, OptionSets, ConnectorSettings.Compatibility)); - + _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(null, OptionSets, ConnectorSettings.Compatibility)); + if (IsDeprecated) { _warnings.Add(ConnectorStringResources.WarnDeprecatedFunction); diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs index fea9f0cb83..8facd71684 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs @@ -68,15 +68,15 @@ internal static IReadOnlyList AddActionConnectorInternal(this return null; } - (List connectorFunctions, List texlFunctions, OptionSetList optionSets) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); + (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); foreach (TexlFunction function in texlFunctions) { config.AddFunction(function); } - foreach (OptionSet optionSet in optionSets.OptionSets) + foreach (KeyValuePair kvp in optionSets.OptionSets) { - config.AddOptionSet(optionSet); + config.AddOptionSet(kvp.Value); } return connectorFunctions; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs b/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs deleted file mode 100644 index c6a7f80e9a..0000000000 --- a/src/libraries/Microsoft.PowerFx.Connectors/Internal/OptionSetList.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; - -namespace Microsoft.PowerFx.Connectors -{ - internal class OptionSetList - { - private readonly Dictionary _optionSets; - - public OptionSetList() - { - _optionSets = new Dictionary(); - } - - public IEnumerable OptionSets => _optionSets.Values; - - // if the option set doesn't exist, it will be added to the list and we return optionSet - // if the option set exists with same name and options, it is not added, we return the existing optionSet - // if the option set exists with same name but different options, we throw an exception (conflicts not allowed) - public OptionSet TryAdd(OptionSet optionSet) - { - if (optionSet == null) - { - throw new ArgumentNullException("optionSet"); - } - - OptionSetStatus oss = GetStatus(optionSet); - - // new option set - if (oss == OptionSetStatus.New) - { - Add(optionSet); - return optionSet; - } - - // same optionset, no need to add anything - if (oss == OptionSetStatus.Same) - { - return Get(optionSet.EntityName); - } - - throw new InvalidOperationException($"Optionset name conflict ({optionSet.EntityName})"); - } - - private void Add(OptionSet optionSet) - { - _optionSets.Add(optionSet.EntityName, optionSet); - } - - private OptionSet Get(string name) - { - return _optionSets[name]; - } - - private enum OptionSetStatus - { - New, - Same, - Conflict - } - - private OptionSetStatus GetStatus(OptionSet optionSet) - { - if (optionSet == null) - { - return OptionSetStatus.New; - } - - string name = optionSet.EntityName; - - if (!_optionSets.ContainsKey(name)) - { - return OptionSetStatus.New; - } - - OptionSet existingOptionSet = Get(name); - - if (existingOptionSet.Equals(optionSet)) - { - return OptionSetStatus.Same; - } - - return OptionSetStatus.Conflict; - } - } -} diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs index 5b7b2b7c7e..e388580f5c 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs @@ -11,6 +11,8 @@ using Microsoft.OpenApi.Interfaces; using Microsoft.OpenApi.Models; using Microsoft.PowerFx.Core; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Binding.BindInfo; using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Types; @@ -392,11 +394,11 @@ internal class ConnectorTypeGetterSettings internal readonly IList SqlRelationships; internal Stack Chain = new Stack(); internal int Level = 0; - internal readonly OptionSetList OptionSets; + internal readonly SymbolTable OptionSets; private readonly string _tableName; - internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, string tableName, OptionSetList optionSets, IList sqlRelationships = null) + internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, string tableName, SymbolTable optionSets, IList sqlRelationships = null) { Compatibility = connectorCompatibility; OptionSets = optionSets; @@ -423,7 +425,7 @@ internal void UnStack() internal string GetOptionSetName(string optionSetNameBase) { string optionSetName = optionSetNameBase; - + if (!string.IsNullOrEmpty(_tableName)) { optionSetName += $" ({_tableName})"; @@ -433,11 +435,11 @@ internal string GetOptionSetName(string optionSetNameBase) } } - internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships = null) + internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList sqlRelationships = null) { ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, tableName, optionSets, sqlRelationships); ConnectorType connectorType = openApiParameter.GetConnectorType(settings); - + return connectorType; } @@ -489,7 +491,7 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar { string optionSetName = settings.GetOptionSetName(schema.GetEnumName() ?? openApiParameter.Name); OptionSet optionSet = new OptionSet(optionSetName, optionSetDisplayNameProvider); - optionSet = settings.OptionSets.TryAdd(optionSet); + optionSet = settings.OptionSets.TryAddOptionSet(optionSet); return new ConnectorType(schema, openApiParameter, optionSet.FormulaType); } @@ -500,7 +502,7 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar { string optionSetName = settings.GetOptionSetName(schema.GetEnumName() ?? openApiParameter.Name); OptionSet optionSet = new OptionSet(optionSetName, schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary()); - optionSet = settings.OptionSets.TryAdd(optionSet); + optionSet = settings.OptionSets.TryAddOptionSet(optionSet); return new ConnectorType(schema, openApiParameter, optionSet.FormulaType); } else @@ -715,6 +717,34 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar } } + // If an OptionSet doesn't exist, we add it (and return it) + // If an identical OptionSet exists (same name & list of options), we return it + // Otherwise we throw in case of conflict + internal static OptionSet TryAddOptionSet(this SymbolTable symbolTable, OptionSet optionSet) + { + if (optionSet == null) + { + throw new ArgumentNullException("optionSet"); + } + + string name = optionSet.EntityName; + + // No existing symbols with that name + if (!((INameResolver)symbolTable).Lookup(new DName(name), out NameLookupInfo info, NameLookupPreferences.None)) + { + symbolTable.AddOptionSet(optionSet); + return optionSet; + } + + // Same optionset already present in table + if (info.Kind == BindKind.OptionSet && info.Data is OptionSet existingOptionSet && existingOptionSet.Equals(optionSet)) + { + return existingOptionSet; + } + + throw new InvalidOperationException($"Optionset name conflict ({name})"); + } + internal static RecordType ToRecordType(this List<(string logicalName, string displayName, FormulaType type)> fields) { if (fields == null) @@ -784,9 +814,9 @@ public static HttpMethod ToHttpMethod(this OperationType key) public static FormulaType GetReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility) { - OptionSetList optionSetList = new OptionSetList(); - ConnectorType connectorType = openApiOperation.GetConnectorReturnType(null, optionSetList, compatibility); - FormulaType ft = connectorType.HasErrors ? ConnectorType.DefaultType : connectorType?.FormulaType ?? new BlankType(); + SymbolTable optionSets = new SymbolTable(); + ConnectorType connectorType = openApiOperation.GetConnectorReturnType(null, optionSets, compatibility); + FormulaType ft = connectorType.HasErrors ? ConnectorType.DefaultType : connectorType?.FormulaType ?? new BlankType(); return ft; } @@ -795,7 +825,7 @@ public static bool GetRequiresUserConfirmation(this OpenApiOperation op) return op.Extensions.TryGetValue(XMsRequireUserConfirmation, out IOpenApiExtension openExt) && openExt is OpenApiBoolean b && b.Value; } - internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility) { OpenApiResponses responses = openApiOperation.Responses; OpenApiResponse response = responses.Where(kvp => kvp.Key?.Length == 3 && kvp.Key.StartsWith("2", StringComparison.Ordinal)).OrderBy(kvp => kvp.Key).FirstOrDefault().Value; @@ -811,8 +841,8 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA } if (response == null) - { - // Returns UntypedObject by default, without error + { + // Returns UntypedObject by default, without error return new ConnectorType(null); } @@ -844,7 +874,7 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(tableName, optionSets, compatibility); } } - + return new ConnectorType(error: $"Unsupported return type - found {string.Join(", ", response.Content.Select(kv4 => kv4.Key))}"); } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs index 19432bd1a9..bd686483c4 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs @@ -24,7 +24,7 @@ public static IEnumerable GetFunctions(string @namespace, Ope public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) { - return GetFunctions(@namespace, openApiDocument, globalValues, out var _, configurationLogger); + return GetFunctions(@namespace, openApiDocument, globalValues, out var _, configurationLogger); } public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) @@ -32,9 +32,9 @@ public static IEnumerable GetFunctions(string @namespace, Ope try { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); - OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, optionSetList, configurationLogger, globalValues); - optionSets = optionSetList.OptionSets; + SymbolTable symbolTable = new SymbolTable(); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, symbolTable, configurationLogger, globalValues); + optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); } @@ -60,9 +60,9 @@ public static IEnumerable GetFunctions(ConnectorSettings conn try { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); - OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionSetList, configurationLogger, globalValues); - optionSets = optionSetList.OptionSets; + SymbolTable symbolTable = new SymbolTable(); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, symbolTable, configurationLogger, globalValues); + optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); } @@ -88,9 +88,9 @@ public static IEnumerable GetTriggers(string @namespace, Open try { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); - OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); - optionSets = optionSetList.OptionSets; + SymbolTable symbolTable = new SymbolTable(); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, symbolTable, configurationLogger, globalValues, FunctionType.Trigger); + optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); } @@ -116,9 +116,9 @@ public static IEnumerable GetTriggers(ConnectorSettings conne try { configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); - OptionSetList optionSetList = new OptionSetList(); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionSetList, configurationLogger, globalValues, FunctionType.Trigger); - optionSets = optionSetList.OptionSets; + SymbolTable symbolTable = new SymbolTable(); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, symbolTable, configurationLogger, globalValues, FunctionType.Trigger); + optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); } @@ -137,11 +137,11 @@ private static bool ShouldIncludeFunction(ConnectorFunction function, ConnectorS (function.IsSupported || settings?.AllowUnsupportedFunctions == true); } - internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, string tableName, OptionSetList optionSetList, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) + internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, string tableName, SymbolTable optionSets, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) { bool connectorIsSupported = true; string connectorNotSupportedReason = string.Empty; - List functions = new (); + List functions = new(); if (connectorSettings == null) { @@ -243,7 +243,7 @@ internal static IEnumerable GetFunctionsInternal(ConnectorSet ? notSupportedReasonForPath : notSupportedReasonForOperation; - ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues, tableName, optionSetList) + ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues, optionSets) { Servers = openApiDocument.Servers }; @@ -520,13 +520,13 @@ private static void ValidateSupportedOpenApiParameters(OpenApiOperation op, ref } // Parse an OpenApiDocument and return functions. - internal static (List connectorFunctions, List texlFunctions, OptionSetList optionsetList) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) + internal static (List connectorFunctions, List texlFunctions, SymbolTable optionSets) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) { - OptionSetList optionsetList = new OptionSetList(); - List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionsetList, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); + SymbolTable optionSets = new SymbolTable(); + List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionSets, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); List tFunctions = cFunctions.Select(f => new ConnectorTexlFunction(f)).ToList(); - return (cFunctions, tFunctions, optionsetList); + return (cFunctions, tFunctions, optionSets); } internal static string GetServer(IEnumerable openApiServers, HttpMessageInvoker httpClient) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs index 4b6fbbd6e8..a3caaba1b5 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs @@ -22,24 +22,24 @@ public class ConnectorParameter : ConnectorSchema internal bool IsBodyParameter = false; - internal ConnectorParameter(OpenApiParameter openApiParameter, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, null, false, tableName, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, SymbolTable optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, null, false, optionSets, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, null, useHiddenTypes, tableName, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, SymbolTable optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, null, useHiddenTypes, optionSets, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, bodyExtensions, false, tableName, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, SymbolTable optionSets, ConnectorCompatibility compatibility) + : this(openApiParameter, bodyExtensions, false, optionSets, compatibility) { IsBodyParameter = true; } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) - : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, tableName, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, SymbolTable optionSets, ConnectorCompatibility compatibility) + : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, optionSets, compatibility) { Name = openApiParameter.Name; Description = openApiParameter.Description; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs index 2b678d1e3e..e52bdb2d5f 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs @@ -29,13 +29,13 @@ public class ConnectorSchema : SupportsConnectorErrors public bool SupportsDynamicIntellisense => ConnectorType.SupportsDynamicIntellisense; - public bool? NotificationUrl => ConnectorType.NotificationUrl; - - internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + public bool? NotificationUrl => ConnectorType.NotificationUrl; + + internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, SymbolTable optionSets, ConnectorCompatibility compatibility) { Schema = openApiParameter.Schema; - UseHiddenTypes = useHiddenTypes; - ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(tableName, optionSets, compatibility)); + UseHiddenTypes = useHiddenTypes; + ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(null, optionSets, compatibility)); DefaultValue = openApiParameter.Schema.TryGetDefaultValue(FormulaType, out FormulaValue defaultValue, this) && defaultValue is not BlankValue ? defaultValue : null; ConnectorExtensions = new ConnectorExtensions(openApiParameter, bodyExtensions); } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs index 6ec2e207a9..970e4ea88b 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs @@ -116,7 +116,7 @@ public class ConnectorType : SupportsConnectorErrors internal string RelationshipName { get; set; } - internal string ForeignKey { get; set; } + internal string ForeignKey { get; set; } internal ConnectorType(ISwaggerSchema schema, ISwaggerParameter openApiParameter, FormulaType formulaType, ErrorResourceKey warning = default) { @@ -197,25 +197,25 @@ internal ConnectorType(string error, ErrorResourceKey warning = default) FormulaType = DefaultType; } - internal ConnectorType(ISwaggerSchema schema, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility) + internal ConnectorType(ISwaggerSchema schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility) : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(tableName, optionSets, compatibility)) - { + { } - internal ConnectorType(JsonElement schema, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships) + internal ConnectorType(JsonElement schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList sqlRelationships) : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships)) - { + { } - internal ConnectorType(JsonElement schema, string tableName, OptionSetList optionSets, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) + internal ConnectorType(JsonElement schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships)) { - Name = name; + Name = name; foreach (ConnectorType field in Fields.Where(f => f.Capabilities != null)) - { + { serviceCapabilities.AddColumnCapability(field.Name, field.Capabilities); - } + } FormulaType = new CdpRecordType(this, resolver, ServiceCapabilities.ToDelegationInfo(serviceCapabilities, name, isTableReadOnly, this, datasetName)); } @@ -295,7 +295,7 @@ internal void SetRelationship(SqlRelationship relationship) ExternalTables.Add(relationship.ReferencedTable); RelationshipName = relationship.RelationshipName; ForeignKey = relationship.ReferencedColumnName; - } + } private void AggregateErrors(ConnectorType[] types) { diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs index 7b82b2b6ac..d7109b1609 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs @@ -625,7 +625,7 @@ public void GenerateYamlFiles(string reference, int folderExclusionIndex = -1, s // Step 2: Get TexlFunctions to be exported // Notice that TexlFunction is internal and requires InternalVisibleTo - (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(connectorSettings, connector.Value.document, logger); + (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(connectorSettings, connector.Value.document, logger); // Step 3: Export TexlFunctions to Yaml ExportTexlFunctionsToYaml(reference, outFolderPath, connector.Key, texlFunctions.Cast().ToList(), false); @@ -1373,7 +1373,7 @@ public static class Exts public static string GetString(this OpenApiSchema schema) { StringBuilder sb = new StringBuilder(); - OptionSetList optionSets = new OptionSetList(); + SymbolTable optionSets = new SymbolTable(); schema.GetStringInternal(new ConnectorTypeGetterSettings(0, null, optionSets), sb); return sb.ToString(); } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems index 285e2f3f30..15838da7b3 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems @@ -31,7 +31,6 @@ - @@ -43,6 +42,7 @@ + diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs index c3149e754c..3eef6eb3be 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs @@ -64,7 +64,7 @@ public void ACSL_GetFunctionNames22() public void ACSL_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\Azure Cognitive Service for Language.json", _output); - (List connectorFunctions, List texlFunctions, OptionSetList optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL"), doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL"), doc, new ConsoleLogger(_output)); Assert.Contains(connectorFunctions, func => func.Namespace == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); @@ -85,7 +85,7 @@ public void ACSL_Load() public void SF_TextCsv() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\SalesForce.json", _output); - (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(new ConnectorSettings("SF") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("SF") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); // function returns text/csv ConnectorFunction func1 = connectorFunctions.First(f => f.Name == "GetJobRecordResults"); @@ -873,7 +873,7 @@ private void Visit(IUntypedObject untypedObject) public void LQA_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\Language - Question Answering.json", _output); - (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(new ConnectorSettings("LQA"), doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("LQA"), doc, new ConsoleLogger(_output)); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "LQA" && func.Name == "GetAnswersFromText"); } @@ -881,7 +881,7 @@ public void LQA_Load() public void SQL_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\SQL Server.json", _output); - (List connectorFunctions, List texlFunctions, OptionSetList optionSetList) = OpenApiParser.ParseInternal(new ConnectorSettings("SQL") { IncludeInternalFunctions = true }, doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("SQL") { IncludeInternalFunctions = true }, doc, new ConsoleLogger(_output)); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "SQL" && func.Name == "GetProcedureV2"); } diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/SymbolTableTryAddTests.cs similarity index 68% rename from src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs rename to src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/SymbolTableTryAddTests.cs index 499072ed9d..76f406861e 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OptionSetListTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/SymbolTableTryAddTests.cs @@ -9,12 +9,12 @@ namespace Microsoft.PowerFx.Connectors.Tests.Shared { - public class OptionSetListTests + public class SymbolTableTryAddOptionSetTests { [Fact] - public void AddOptionSetToList() + public void TryAddOptionSet() { - OptionSetList list = new OptionSetList(); + SymbolTable symbolTable = new SymbolTable(); SingleSourceDisplayNameProvider dnp = new SingleSourceDisplayNameProvider(); dnp = dnp.AddField(new DName("logical1"), new DName("display1")); @@ -23,31 +23,31 @@ public void AddOptionSetToList() OptionSet os1 = new OptionSet("os1", dnp); OptionSet os2 = new OptionSet("os1", dnp); - OptionSet os = list.TryAdd(os1); + OptionSet os = symbolTable.TryAddOptionSet(os1); Assert.Same(os1, os); // twice the same, nothing added - os = list.TryAdd(os1); + os = symbolTable.TryAddOptionSet(os1); Assert.Same(os1, os); - Assert.Single(list.OptionSets); + Assert.Single(symbolTable.OptionSets); // still the same, nothing added - os = list.TryAdd(os2); + os = symbolTable.TryAddOptionSet(os2); Assert.Same(os1, os); - Assert.Single(list.OptionSets); + Assert.Single(symbolTable.OptionSets); dnp = dnp.AddField(new DName("logical3"), new DName("display3")); OptionSet os3 = new OptionSet("os3", dnp); // new optionSet - os = list.TryAdd(os3); + os = symbolTable.TryAddOptionSet(os3); Assert.NotSame(os, os1); - Assert.Equal(2, list.OptionSets.Count()); + Assert.Equal(2, symbolTable.OptionSets.Count()); // try a name conflict now OptionSet os4 = new OptionSet("os1", dnp); - InvalidOperationException ioe = Assert.Throws(() => list.TryAdd(os4)); + InvalidOperationException ioe = Assert.Throws(() => symbolTable.TryAddOptionSet(os4)); Assert.Equal("Optionset name conflict (os1)", ioe.Message); } } From 60fa1ff08b0ef1233a389946c56dc9702a99c357 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Thu, 14 Nov 2024 14:50:57 +0100 Subject: [PATCH 6/8] missing space --- src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs index bd686483c4..2f3f18d471 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs @@ -141,7 +141,7 @@ internal static IEnumerable GetFunctionsInternal(ConnectorSet { bool connectorIsSupported = true; string connectorNotSupportedReason = string.Empty; - List functions = new(); + List functions = new (); if (connectorSettings == null) { From 8fb57cfb9028e9f18ed70ba91c13728e692d2206 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Thu, 14 Nov 2024 18:39:45 +0100 Subject: [PATCH 7/8] Cleanup --- .../ConnectorFunction.cs | 37 +++++++++---------- .../OpenApiExtensions.cs | 18 +++++---- .../Public/ConnectorParameter.cs | 16 ++++---- .../Public/ConnectorSchema.cs | 4 +- .../Public/ConnectorType.cs | 9 +++-- 5 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs index de4cc7e7c3..d51bf50e0a 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs @@ -294,19 +294,16 @@ public ConnectorType ReturnParameterType // Those properties are only used by HttpFunctionInvoker internal ConnectorParameterInternals _internals = null; - private readonly ConnectorLogger _configurationLogger = null; - - internal readonly SymbolTable OptionSets; + private readonly ConnectorLogger _configurationLogger = null; internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, - ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, SymbolTable optionSets) + ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, SymbolTable optionSets /* to be removed */) { Operation = openApiOperation; Name = name; OperationPath = operationPath; HttpMethod = httpMethod; - ConnectorSettings = connectorSettings; - OptionSets = optionSets; + ConnectorSettings = connectorSettings; GlobalContext = new ConnectorGlobalContext(functionList ?? throw new ArgumentNullException(nameof(functionList)), globalValues); _configurationLogger = configurationLogger; @@ -991,7 +988,7 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return null; } - ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, null, new SymbolTable(), ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1005,10 +1002,10 @@ private async Task GetConnectorSuggestionsFromDynamicSchemaAsync( return connectorType; } - internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, ConnectorCompatibility compatibility) { JsonElement je = ExtractFromJson(sv, valuePath); - return GetConnectorTypeInternal(tableName, optionSets, compatibility, je); + return GetConnectorTypeInternal(compatibility, je); } // Only called by ConnectorTable.GetSchema @@ -1084,17 +1081,17 @@ internal class SalesForceReferencedEntity public bool RestrictedDelete { get; set; } } - private static ConnectorType GetConnectorTypeInternal(string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, JsonElement je) + private static ConnectorType GetConnectorTypeInternal(ConnectorCompatibility compatibility, JsonElement je) { OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet }; OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag); - return new ConnectorType(SwaggerSchema.New(schema), tableName, optionSets, compatibility); + return new ConnectorType(SwaggerSchema.New(schema), compatibility); } - private static ConnectorType GetJsonConnectorTypeInternal(string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) + private static ConnectorType GetJsonConnectorTypeInternal(ConnectorCompatibility compatibility, JsonElement je, IList sqlRelationships) { - return new ConnectorType(je, tableName, optionSets, compatibility, sqlRelationships); + return new ConnectorType(je, compatibility, sqlRelationships); } private async Task GetConnectorSuggestionsFromDynamicPropertyAsync(NamedValue[] knownParameters, BaseRuntimeConnectorContext runtimeContext, ConnectorDynamicProperty cdp, CancellationToken cancellationToken) @@ -1115,7 +1112,7 @@ private async Task GetConnectorSuggestionsFromDynamicPropertyAsyn return null; } - ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, null, new SymbolTable(), ConnectorSettings.Compatibility); + ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, ConnectorSettings.Compatibility); if (connectorType.HasErrors) { @@ -1422,7 +1419,7 @@ private ConnectorParameterInternals Initialize() return null; } - ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter connectorParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(parameter, ConnectorSettings.Compatibility)); if (connectorParameter.HiddenRecordType != null) { @@ -1491,12 +1488,12 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter = new OpenApiParameter() { Name = bodyPropertyName, Schema = bodyPropertySchema, Description = requestBody.Description, Required = bodyPropertyRequired, Extensions = bodyPropertySchema.Extensions }; - ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter2 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, requestBody, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter2, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter2.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter2.HiddenRecordType != null) { - hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, OptionSets, ConnectorSettings.Compatibility))); + hiddenRequiredParameters.Add(errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter, true, ConnectorSettings.Compatibility))); } List parameterList = !bodyPropertyRequired ? optionalParameters : bodyPropertyHiddenRequired ? hiddenRequiredParameters : requiredParameters; @@ -1515,7 +1512,7 @@ private ConnectorParameterInternals Initialize() } OpenApiParameter bodyParameter2 = new OpenApiParameter() { Name = bodyName, Schema = bodySchema, Description = requestBody.Description, Required = requestBody.Required, Extensions = bodySchema.Extensions }; - ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyConnectorParameter3 = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter2, requestBody, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyConnectorParameter3, OpenApiExtensions.TryGetOpenApiValue(bodyConnectorParameter3.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); if (bodyConnectorParameter3.HiddenRecordType != null) @@ -1535,7 +1532,7 @@ private ConnectorParameterInternals Initialize() OpenApiSchema bodyParameterSchema = new OpenApiSchema() { Type = "string" }; OpenApiParameter bodyParameter3 = new OpenApiParameter() { Name = bodyName, Schema = bodyParameterSchema, Description = "Body", Required = requestBody.Required }; - ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, OptionSets, ConnectorSettings.Compatibility)); + ConnectorParameter bodyParameter = errorsAndWarnings.AggregateErrorsAndWarnings(new ConnectorParameter(bodyParameter3, requestBody, ConnectorSettings.Compatibility)); openApiBodyParameters.Add(bodyParameter, OpenApiExtensions.TryGetOpenApiValue(bodyParameter.Schema.Default, null, out FormulaValue defaultValue, errorsAndWarnings) ? defaultValue : null); List parameterList = requestBody.Required ? requiredParameters : optionalParameters; @@ -1578,7 +1575,7 @@ private ConnectorParameterInternals Initialize() _arityMax = _arityMin + (_optionalParameters.Length == 0 ? 0 : 1); _warnings = new List(); - _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(null, OptionSets, ConnectorSettings.Compatibility)); + _returnType = errorsAndWarnings.AggregateErrorsAndWarnings(Operation.GetConnectorReturnType(ConnectorSettings.Compatibility)); if (IsDeprecated) { diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs index e388580f5c..0b240d1ebc 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs @@ -435,6 +435,11 @@ internal string GetOptionSetName(string optionSetNameBase) } } + internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, ConnectorCompatibility compatibility, IList sqlRelationships = null) + { + return openApiParameter.GetConnectorType(tableName: null, optionSets: null, compatibility, sqlRelationships); + } + internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList sqlRelationships = null) { ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, tableName, optionSets, sqlRelationships); @@ -813,9 +818,8 @@ public static HttpMethod ToHttpMethod(this OperationType key) } public static FormulaType GetReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility) - { - SymbolTable optionSets = new SymbolTable(); - ConnectorType connectorType = openApiOperation.GetConnectorReturnType(null, optionSets, compatibility); + { + ConnectorType connectorType = openApiOperation.GetConnectorReturnType(compatibility); FormulaType ft = connectorType.HasErrors ? ConnectorType.DefaultType : connectorType?.FormulaType ?? new BlankType(); return ft; } @@ -825,7 +829,7 @@ public static bool GetRequiresUserConfirmation(this OpenApiOperation op) return op.Extensions.TryGetValue(XMsRequireUserConfirmation, out IOpenApiExtension openExt) && openExt is OpenApiBoolean b && b.Value; } - internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility) + internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility) { OpenApiResponses responses = openApiOperation.Responses; OpenApiResponse response = responses.Where(kvp => kvp.Key?.Length == 3 && kvp.Key.StartsWith("2", StringComparison.Ordinal)).OrderBy(kvp => kvp.Key).FirstOrDefault().Value; @@ -849,7 +853,7 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA if (response.Content.Count == 0) { OpenApiSchema schema = new OpenApiSchema() { Type = "string", Format = "no_format" }; - return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(tableName, optionSets, compatibility); + return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(compatibility); } // Responses is a list by content-type. Find "application/json" @@ -868,10 +872,10 @@ internal static ConnectorType GetConnectorReturnType(this OpenApiOperation openA if (openApiMediaType.Schema == null) { // Treat as void. - return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(tableName, optionSets, compatibility); + return new SwaggerParameter("response", true, new SwaggerSchema("string", "no_format"), response.Extensions).GetConnectorType(compatibility); } - return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(tableName, optionSets, compatibility); + return new SwaggerParameter("response", true, SwaggerSchema.New(openApiMediaType.Schema), openApiMediaType.Schema.Extensions).GetConnectorType(compatibility); } } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs index a3caaba1b5..4fde16dcd3 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs @@ -22,24 +22,24 @@ public class ConnectorParameter : ConnectorSchema internal bool IsBodyParameter = false; - internal ConnectorParameter(OpenApiParameter openApiParameter, SymbolTable optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, null, false, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, ConnectorCompatibility compatibility) + : this(openApiParameter, null, false, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, SymbolTable optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, null, useHiddenTypes, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, ConnectorCompatibility compatibility) + : this(openApiParameter, null, useHiddenTypes, compatibility) { } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, SymbolTable optionSets, ConnectorCompatibility compatibility) - : this(openApiParameter, bodyExtensions, false, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, ConnectorCompatibility compatibility) + : this(openApiParameter, bodyExtensions, false, compatibility) { IsBodyParameter = true; } - internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, SymbolTable optionSets, ConnectorCompatibility compatibility) - : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, optionSets, compatibility) + internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, ConnectorCompatibility compatibility) + : base(SwaggerParameter.New(openApiParameter), SwaggerExtensions.New(bodyExtensions), useHiddenTypes, compatibility) { Name = openApiParameter.Name; Description = openApiParameter.Description; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs index e52bdb2d5f..2c117fe0ed 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs @@ -31,11 +31,11 @@ public class ConnectorSchema : SupportsConnectorErrors public bool? NotificationUrl => ConnectorType.NotificationUrl; - internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, SymbolTable optionSets, ConnectorCompatibility compatibility) + internal ConnectorSchema(ISwaggerParameter openApiParameter, ISwaggerExtensions bodyExtensions, bool useHiddenTypes, ConnectorCompatibility compatibility) { Schema = openApiParameter.Schema; UseHiddenTypes = useHiddenTypes; - ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(null, optionSets, compatibility)); + ConnectorType = AggregateErrorsAndWarnings(openApiParameter.GetConnectorType(compatibility)); DefaultValue = openApiParameter.Schema.TryGetDefaultValue(FormulaType, out FormulaValue defaultValue, this) && defaultValue is not BlankValue ? defaultValue : null; ConnectorExtensions = new ConnectorExtensions(openApiParameter, bodyExtensions); } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs index 970e4ea88b..b896b3d9da 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs @@ -197,16 +197,17 @@ internal ConnectorType(string error, ErrorResourceKey warning = default) FormulaType = DefaultType; } - internal ConnectorType(ISwaggerSchema schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility) - : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(tableName, optionSets, compatibility)) + internal ConnectorType(ISwaggerSchema schema, ConnectorCompatibility compatibility) + : this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(compatibility)) { } - internal ConnectorType(JsonElement schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList sqlRelationships) - : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships)) + internal ConnectorType(JsonElement schema, ConnectorCompatibility compatibility, IList sqlRelationships) + : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(compatibility, sqlRelationships)) { } + // Called by ConnectorFunction.GetCdpTableType internal ConnectorType(JsonElement schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList sqlRelationships, IList referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly) : this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships)) { From 4f09e46032117d17d128a88729e00c707958d866 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Thu, 14 Nov 2024 18:47:20 +0100 Subject: [PATCH 8/8] Cleanup 2 --- .../ConnectorFunction.cs | 2 +- .../Environment/PowerFxConfigExtensions.cs | 7 +- .../OpenApiParser.cs | 65 +++++-------------- .../InternalTesting.cs | 2 +- .../OpenApiParserTests.cs | 10 +-- 5 files changed, 26 insertions(+), 60 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs index d51bf50e0a..a5bdcafad3 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs @@ -297,7 +297,7 @@ public ConnectorType ReturnParameterType private readonly ConnectorLogger _configurationLogger = null; internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List functionList, - ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues, SymbolTable optionSets /* to be removed */) + ConnectorLogger configurationLogger, IReadOnlyDictionary globalValues) { Operation = openApiOperation; Name = name; diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs index 8facd71684..ddffcfe24a 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs @@ -68,17 +68,12 @@ internal static IReadOnlyList AddActionConnectorInternal(this return null; } - (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); + (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); foreach (TexlFunction function in texlFunctions) { config.AddFunction(function); } - foreach (KeyValuePair kvp in optionSets.OptionSets) - { - config.AddOptionSet(kvp.Value); - } - return connectorFunctions; } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs index 2f3f18d471..7ef8f65cc8 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/OpenApiParser.cs @@ -21,20 +21,13 @@ public static IEnumerable GetFunctions(string @namespace, Ope { return GetFunctions(@namespace, openApiDocument, null, configurationLogger); } - - public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) - { - return GetFunctions(@namespace, openApiDocument, globalValues, out var _, configurationLogger); - } - public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) + public static IEnumerable GetFunctions(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); - SymbolTable symbolTable = new SymbolTable(); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, symbolTable, configurationLogger, globalValues); - optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, configurationLogger, globalValues); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); } @@ -48,21 +41,14 @@ public static IEnumerable GetFunctions(string @namespace, Ope public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null) { return GetFunctions(connectorSettings, openApiDocument, null, configurationLogger); - } - - public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) - { - return GetFunctions(connectorSettings, openApiDocument, globalValues, out var _, configurationLogger); } - public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) + public static IEnumerable GetFunctions(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); - SymbolTable symbolTable = new SymbolTable(); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, symbolTable, configurationLogger, globalValues); - optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, configurationLogger, globalValues); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetFunctions)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); } @@ -76,21 +62,14 @@ public static IEnumerable GetFunctions(ConnectorSettings conn public static IEnumerable GetTriggers(string @namespace, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null) { return GetTriggers(@namespace, openApiDocument, null, configurationLogger); - } - - public static IEnumerable GetTriggers(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) - { - return GetTriggers(@namespace, openApiDocument, globalValues, out var _, configurationLogger); } - public static IEnumerable GetTriggers(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) + public static IEnumerable GetTriggers(string @namespace, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); - SymbolTable symbolTable = new SymbolTable(); - IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, null, symbolTable, configurationLogger, globalValues, FunctionType.Trigger); - optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}"); + IEnumerable functions = GetFunctionsInternal(new ConnectorSettings(@namespace), openApiDocument, configurationLogger, globalValues, FunctionType.Trigger); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} Namespace {@namespace}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f)); } @@ -104,21 +83,14 @@ public static IEnumerable GetTriggers(string @namespace, Open public static IEnumerable GetTriggers(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null) { return GetTriggers(connectorSettings, openApiDocument, null, configurationLogger); - } - - public static IEnumerable GetTriggers(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) - { - return GetTriggers(connectorSettings, openApiDocument, globalValues, out var _, configurationLogger); } - public static IEnumerable GetTriggers(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, out IEnumerable optionSets, ConnectorLogger configurationLogger = null) + public static IEnumerable GetTriggers(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, IReadOnlyDictionary globalValues, ConnectorLogger configurationLogger = null) { try { - configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); - SymbolTable symbolTable = new SymbolTable(); - IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, null, symbolTable, configurationLogger, globalValues, FunctionType.Trigger); - optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value); + configurationLogger?.LogInformation($"Entering in {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}"); + IEnumerable functions = GetFunctionsInternal(connectorSettings, openApiDocument, configurationLogger, globalValues, FunctionType.Trigger); configurationLogger?.LogInformation($"Exiting {nameof(OpenApiParser)}.{nameof(GetTriggers)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}, returning {functions.Count()} functions"); return functions.Where(f => ShouldIncludeFunction(f, connectorSettings)); } @@ -137,7 +109,7 @@ private static bool ShouldIncludeFunction(ConnectorFunction function, ConnectorS (function.IsSupported || settings?.AllowUnsupportedFunctions == true); } - internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, string tableName, SymbolTable optionSets, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) + internal static IEnumerable GetFunctionsInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null, FunctionType functionType = FunctionType.Function) { bool connectorIsSupported = true; string connectorNotSupportedReason = string.Empty; @@ -243,7 +215,7 @@ internal static IEnumerable GetFunctionsInternal(ConnectorSet ? notSupportedReasonForPath : notSupportedReasonForOperation; - ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues, optionSets) + ConnectorFunction connectorFunction = new ConnectorFunction(op, isSupported, notSupportedReason, operationName, opPath, verb, connectorSettings, functions, configurationLogger, globalValues) { Servers = openApiDocument.Servers }; @@ -520,13 +492,12 @@ private static void ValidateSupportedOpenApiParameters(OpenApiOperation op, ref } // Parse an OpenApiDocument and return functions. - internal static (List connectorFunctions, List texlFunctions, SymbolTable optionSets) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) - { - SymbolTable optionSets = new SymbolTable(); - List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, null, optionSets, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); + internal static (List connectorFunctions, List texlFunctions) ParseInternal(ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary globalValues = null) + { + List cFunctions = GetFunctionsInternal(connectorSettings, openApiDocument, configurationLogger, globalValues).Where(f => ShouldIncludeFunction(f, connectorSettings)).ToList(); List tFunctions = cFunctions.Select(f => new ConnectorTexlFunction(f)).ToList(); - return (cFunctions, tFunctions, optionSets); + return (cFunctions, tFunctions); } internal static string GetServer(IEnumerable openApiServers, HttpMessageInvoker httpClient) diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs index d7109b1609..91c3f81afa 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/InternalTesting.cs @@ -625,7 +625,7 @@ public void GenerateYamlFiles(string reference, int folderExclusionIndex = -1, s // Step 2: Get TexlFunctions to be exported // Notice that TexlFunction is internal and requires InternalVisibleTo - (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(connectorSettings, connector.Value.document, logger); + (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(connectorSettings, connector.Value.document, logger); // Step 3: Export TexlFunctions to Yaml ExportTexlFunctionsToYaml(reference, outFolderPath, connector.Key, texlFunctions.Cast().ToList(), false); diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs index 3eef6eb3be..0317de8d4f 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/OpenApiParserTests.cs @@ -64,7 +64,7 @@ public void ACSL_GetFunctionNames22() public void ACSL_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\Azure Cognitive Service for Language.json", _output); - (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL"), doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL"), doc, new ConsoleLogger(_output)); Assert.Contains(connectorFunctions, func => func.Namespace == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); @@ -72,7 +72,7 @@ public void ACSL_Load() Assert.Equal("analysisInput:![documents:*[id:s, language:s, text:s]]|task:![parameters:![deploymentName:s, projectName:s, stringIndexType:s]]", string.Join("|", func1.RequiredParameters.Select(rp => $"{rp.Name}:{rp.FormulaType._type}"))); Assert.Equal("displayName:s", string.Join("|", func1.OptionalParameters.Select(rp => $"{rp.Name}:{rp.FormulaType._type}"))); - (connectorFunctions, texlFunctions, optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); + (connectorFunctions, texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("ACSL") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); Assert.Contains(connectorFunctions, func => func.Namespace == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "ACSL" && func.Name == "ConversationAnalysisAnalyzeConversationConversation"); @@ -85,7 +85,7 @@ public void ACSL_Load() public void SF_TextCsv() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\SalesForce.json", _output); - (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("SF") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("SF") { Compatibility = ConnectorCompatibility.SwaggerCompatibility }, doc, new ConsoleLogger(_output)); // function returns text/csv ConnectorFunction func1 = connectorFunctions.First(f => f.Name == "GetJobRecordResults"); @@ -873,7 +873,7 @@ private void Visit(IUntypedObject untypedObject) public void LQA_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\Language - Question Answering.json", _output); - (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("LQA"), doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("LQA"), doc, new ConsoleLogger(_output)); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "LQA" && func.Name == "GetAnswersFromText"); } @@ -881,7 +881,7 @@ public void LQA_Load() public void SQL_Load() { OpenApiDocument doc = Helpers.ReadSwagger(@"Swagger\SQL Server.json", _output); - (List connectorFunctions, List texlFunctions, SymbolTable optionSets) = OpenApiParser.ParseInternal(new ConnectorSettings("SQL") { IncludeInternalFunctions = true }, doc, new ConsoleLogger(_output)); + (List connectorFunctions, List texlFunctions) = OpenApiParser.ParseInternal(new ConnectorSettings("SQL") { IncludeInternalFunctions = true }, doc, new ConsoleLogger(_output)); Assert.Contains(texlFunctions, func => func.Namespace.Name.Value == "SQL" && func.Name == "GetProcedureV2"); }