Unify IAsyncTexlFunction (#2812)

Address #2751.  
This unifies most of them. To scope this PR, there are subissues on 2751
for remaining issues.



We have a plethora of IAsyncTexlFunction* interfaces. Want to
consolidate on 1, and make it public.

This 
Ttake the parameters on the various IAsyncTexlFunction* overloads and
move them as properties on a new public `FunctionInvokerInfo` class. And
then have a new IAsyncTexlFunction* that takes the invoker. And remove
all the others, migrating them onto this - and fixing the various
bugs/hacks along the way that would impede migration.

Most fundamentally, this FunctionInvokerInfo must have access to
interpreter state and so must live in the interpreter. (Whereas some of
the IAsyncFunction* interfaces live in core).

Successfully removes several here so we get a net reduction, and shows a
path to remove the rest.

This problem quickly touches other problems:
- **How to map from the TexlFunction to the invoker**. This is made more
complicated because we register on PowerFxConfig, but that lives in
Fx.Core and can't refer to interpreter implementations.
- **dll layering around Fx.Json**: Some function implementations live in
Fx.Json, but that specifically _does not_ depend on interpreter (because
it is included by PowerApps).
- **Standard Pipeline**: how to handle default args, common error
handling scenarios, etc. Today, that's still in interpreter. Whereas we
really want those checks to be in the IR and shared across front-ends
(and available to designer).
- **Split up Library.cs and class** - we have one giant class, partial
over many files, containing 100s of function impls. Just give each impl
its own file, similar to how we did for TexlFunction and the static
declarations.
- **lack of unit testing**: Today, most of our interpreter coverage
comes from .txt files. But we should have C# unit tests on interpreter
classes that provide coverage on core classes, like EvalVisitor.

Some related prereq fixes that will directly help here:
- Can we remove runner/context from Join? and leverage LambdaValue -
which already closes over these.
- fix Json layering problem. 
- IsType/AsType _UO shouldn't live in Json. They should work on any UO.
- Remove _additionalFunctions from the serviceProvider.

---------

Co-authored-by: Mike <jmstall@microsoft.com>

src/libraries/Microsoft.PowerFx.Connectors/Texl/ConnectorTexlFunction.cs

@@ -12,13 +12,14 @@
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Types;
 using Microsoft.PowerFx.Core.Utils;
+using Microsoft.PowerFx.Functions;
 using Microsoft.PowerFx.Intellisense;
 using Microsoft.PowerFx.Types;
 using static Microsoft.PowerFx.Connectors.ConnectorHelperFunctions;
 
 namespace Microsoft.PowerFx.Connectors
 {
-    internal class ConnectorTexlFunction : TexlFunction, IAsyncConnectorTexlFunction, IHasUnsupportedFunctions
+    internal class ConnectorTexlFunction : TexlFunction, IFunctionInvoker, IHasUnsupportedFunctions
     {
         public ConnectorFunction ConnectorFunction { get; }
 
@@ -85,8 +86,11 @@ public override bool TryGetParamDescription(string paramName, out string paramDe
 
         public override bool HasSuggestionsForParam(int argumentIndex) => argumentIndex <= MaxArity;
 
-        public async Task<FormulaValue> InvokeAsync(FormulaValue[] args, IServiceProvider serviceProvider, CancellationToken cancellationToken)
+        public async Task<FormulaValue> InvokeAsync(FunctionInvokeInfo invokeInfo, CancellationToken cancellationToken)
         {
+            FormulaValue[] args = invokeInfo.Args.ToArray(); // https://github.com/microsoft/Power-Fx/issues/2817
+            var serviceProvider = invokeInfo.FunctionServices;
+
             cancellationToken.ThrowIfCancellationRequested();
             BaseRuntimeConnectorContext runtimeContext = serviceProvider.GetService(typeof(BaseRuntimeConnectorContext)) as BaseRuntimeConnectorContext ?? throw new InvalidOperationException("RuntimeConnectorContext is missing from service provider");
 

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction.cs

@@ -8,6 +8,7 @@
 namespace Microsoft.PowerFx.Core.Functions
 {
     // A Texl function capable of async invokes. 
+    // This only lives in Core to enable Fx.Json funcs impl (which doesn't depend on interpreter).    
     internal interface IAsyncTexlFunction
     {
         Task<FormulaValue> InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken);

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction4.cs

@@ -9,6 +9,7 @@
 namespace Microsoft.PowerFx.Core.Functions
 {
     // A Texl function capable of async invokes, using TimeZoneInfo and IRContext. 
+    // Remove this: https://github.com/microsoft/Power-Fx/issues/2818
     internal interface IAsyncTexlFunction4
     {
         Task<FormulaValue> InvokeAsync(TimeZoneInfo timezoneInfo, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken);

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction5.cs

@@ -7,8 +7,10 @@
 using Microsoft.PowerFx.Types;
 
 namespace Microsoft.PowerFx.Core.Functions
-{    
+{
     // Texl function interface with IServiceProvider
+    // Only product impl is JsonFunctionImpl.
+    // Remove this: https://github.com/microsoft/Power-Fx/issues/2818
     internal interface IAsyncTexlFunction5
     {
         Task<FormulaValue> InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken);        

src/libraries/Microsoft.PowerFx.Core/Public/Config/PowerFxConfig.cs

@@ -52,6 +52,7 @@ public SymbolTable SymbolTable
             set => _symbolTable = value;
         }
 
+        // Remove this: https://github.com/microsoft/Power-Fx/issues/2821
         internal readonly Dictionary<TexlFunction, IAsyncTexlFunction> AdditionalFunctions = new ();
 
         [Obsolete("Use Config.EnumStore or symboltable directly")]

src/libraries/Microsoft.PowerFx.Interpreter/CustomFunction/CustomTexlFunction.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Numerics;
 using System.Threading;
 using System.Threading.Tasks;
@@ -22,7 +23,7 @@ namespace Microsoft.PowerFx
     /// <summary>
     /// Internal adapter for adding custom functions. 
     /// </summary>
-    internal class CustomTexlFunction : TexlFunction
+    internal class CustomTexlFunction : TexlFunction, IFunctionInvoker
     {
         public Func<IServiceProvider, FormulaValue[], CancellationToken, Task<FormulaValue>> _impl;
 
@@ -53,8 +54,10 @@ public CustomTexlFunction(DPath ns, string name, FunctionCategories functionCate
             yield return CustomFunctionUtility.GenerateArgSignature(_argNames, ParamTypes);
         }
 
-        public virtual Task<FormulaValue> InvokeAsync(IServiceProvider serviceProvider, FormulaValue[] args, CancellationToken cancellationToken)
+        public virtual Task<FormulaValue> InvokeAsync(FunctionInvokeInfo invokeInfo, CancellationToken cancellationToken)
         {
+            var serviceProvider = invokeInfo.FunctionServices;
+            var args = invokeInfo.Args.ToArray(); // remove ToArray: https://github.com/microsoft/Power-Fx/issues/2817
             return _impl(serviceProvider, args, cancellationToken);
         }
 

src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs

@@ -265,6 +265,100 @@ private async Task<FormulaValue> TryHandleSetProperty(CallNode node, EvalVisitor
 
             return result;
         }
+
+        // Given a TexlFunction, get the implementation to invoke. 
+        private IFunctionInvoker GetInvoker(TexlFunction func)
+        {
+            if (func is IFunctionInvoker invoker)
+            {
+                return invoker;
+            }
+
+            if (func is UserDefinedFunction userDefinedFunc)
+            {
+                return new UserDefinedFunctionAdapter(userDefinedFunc);
+            }
+
+            if (FunctionImplementations.TryGetValue(func, out AsyncFunctionPtr ptr))
+            {
+                return new AsyncFunctionPtrAdapter(ptr);
+            }
+
+            return null;
+        }
+
+        // Adapter for AsyncFunctionPtr to common invoker interface.
+        private class AsyncFunctionPtrAdapter : IFunctionInvoker
+        {
+            private readonly AsyncFunctionPtr _ptr;
+
+            public AsyncFunctionPtrAdapter(AsyncFunctionPtr ptr)
+            {
+                _ptr = ptr;
+            }
+
+            public async Task<FormulaValue> InvokeAsync(FunctionInvokeInfo invokeInfo, CancellationToken cancellationToken)
+            {
+                var args = invokeInfo.Args.ToArray();
+                var context = invokeInfo.Context;
+                var evalVisitor = invokeInfo.Runner;
+                var irContext = invokeInfo.IRContext;
+
+                var result = await _ptr(evalVisitor, context, irContext, args).ConfigureAwait(false);
+
+                return result;
+            }
+        }
+
+        // Adapter for UDF to common invoker. 
+        // This still ensures that *invoking* a UDF has the same semantics as invoking other function calls. 
+        private class UserDefinedFunctionAdapter : IFunctionInvoker
+        {
+            private readonly UserDefinedFunction _udf;
+
+            public UserDefinedFunctionAdapter(UserDefinedFunction udf)
+            {
+                _udf = udf;
+            }
+
+            public async Task<FormulaValue> InvokeAsync(FunctionInvokeInfo invokeInfo, CancellationToken cancellationToken)
+            {
+                var args = invokeInfo.Args.ToArray();
+                var context = invokeInfo.Context;
+                var evalVisitor = invokeInfo.Runner;
+
+                var udfStack = evalVisitor._udfStack;
+
+                UDFStackFrame frame = new UDFStackFrame(_udf, args);
+                UDFStackFrame framePop = null;
+                FormulaValue result = null;
+
+                try
+                {
+                    // Push this so that we have access to args. 
+                    udfStack.Push(frame);
+
+                    // https://github.com/microsoft/Power-Fx/issues/2822
+                    // This repeats IRTranslator each time. Do once and save. 
+                    (var irnode, _) = _udf.GetIRTranslator();
+
+                    evalVisitor.CheckCancel();
+
+                    result = await irnode.Accept(evalVisitor, context).ConfigureAwait(false);
+                }
+                finally
+                {
+                    framePop = udfStack.Pop();
+                }
+
+                if (frame != framePop)
+                {
+                    throw new Exception("Something went wrong. UDF stack values didn't match.");
+                }
+
+                return result;
+            }
+        }
 
         public override async ValueTask<FormulaValue> Visit(CallNode node, EvalVisitorContext context)
         {
@@ -300,40 +394,59 @@ public override async ValueTask<FormulaValue> Visit(CallNode node, EvalVisitorCo
                     args[i] = await child.Accept(this, context.IncrementStackDepthCounter()).ConfigureAwait(false);
                 }
                 else
-                {
+                {
+                    // This is where Lambdas are created. They close over key values to invoke.
                     args[i] = new LambdaFormulaValue(node.IRContext, child, this, context);
                 }
             }
 
             var childContext = context.SymbolContext.WithScope(node.Scope);
 
             FormulaValue result;
+
+            // Remove this: https://github.com/microsoft/Power-Fx/issues/2821
             IReadOnlyDictionary<TexlFunction, IAsyncTexlFunction> extraFunctions = _services.GetService<IReadOnlyDictionary<TexlFunction, IAsyncTexlFunction>>();
 
             try
             {
-                if (func is IAsyncTexlFunction asyncFunc || extraFunctions?.TryGetValue(func, out asyncFunc) == true)
+                IFunctionInvoker invoker = GetInvoker(func);
+
+                // Standard invoke path. Make everything go through here. 
+                // Eventually collapse all cases to this. 
+                if (invoker != null)
                 {
-                    result = await asyncFunc.InvokeAsync(args, _cancellationToken).ConfigureAwait(false);
+                    var invokeInfo = new FunctionInvokeInfo
+                    {
+                        Args = args,
+                        FunctionServices = _services,
+                        Runner = this,
+                        Context = context.IncrementStackDepthCounter(childContext),
+                        IRContext = node.IRContext,
+                    };
+
+                    result = await invoker.InvokeAsync(invokeInfo, _cancellationToken).ConfigureAwait(false);
                 }
-#pragma warning disable CS0618 // Type or member is obsolete
-                else if (func is IAsyncTexlFunction2 asyncFunc2)
-#pragma warning restore CS0618 // Type or member is obsolete
+                else if (func is IAsyncTexlFunction asyncFunc)
                 {
-                    result = await asyncFunc2.InvokeAsync(this.GetFormattingInfo(), args, _cancellationToken).ConfigureAwait(false);
+                    result = await asyncFunc.InvokeAsync(args, _cancellationToken).ConfigureAwait(false);
                 }
-                else if (func is IAsyncTexlFunction3 asyncFunc3)
+                else if (extraFunctions?.TryGetValue(func, out asyncFunc) == true)
                 {
-                    result = await asyncFunc3.InvokeAsync(node.IRContext.ResultType, args, _cancellationToken).ConfigureAwait(false);
+                    result = await asyncFunc.InvokeAsync(args, _cancellationToken).ConfigureAwait(false);
                 }
                 else if (func is IAsyncTexlFunction4 asyncFunc4)
                 {
+                    // https://github.com/microsoft/Power-Fx/issues/2818
+                    // This is used for Json() functions.  IsType, AsType
                     result = await asyncFunc4.InvokeAsync(TimeZoneInfo, node.IRContext.ResultType, args, _cancellationToken).ConfigureAwait(false);
                 }
                 else if (func is IAsyncTexlFunction5 asyncFunc5)
                 {
+                    // https://github.com/microsoft/Power-Fx/issues/2818
+                    // This is used for Json() functions.
                     BasicServiceProvider services2 = new BasicServiceProvider(_services);
 
+                    // Invocation should not get its own provider.  
                     if (services2.GetService(typeof(TimeZoneInfo)) == null)
                     {
                         services2.AddService(TimeZoneInfo);
@@ -346,63 +459,18 @@ public override async ValueTask<FormulaValue> Visit(CallNode node, EvalVisitorCo
 
                     result = await asyncFunc5.InvokeAsync(services2, node.IRContext.ResultType, args, _cancellationToken).ConfigureAwait(false);
                 }
-                else if (func is IAsyncConnectorTexlFunction asyncConnectorTexlFunction)
-                {
-                    return await asyncConnectorTexlFunction.InvokeAsync(args, _services, _cancellationToken).ConfigureAwait(false);
-                }
-                else if (func is CustomTexlFunction customTexlFunc)
-                {
-                    // If custom function throws an exception, don't catch it - let it propagate up to the host.
-                    result = await customTexlFunc.InvokeAsync(FunctionServices, args, _cancellationToken).ConfigureAwait(false);
-                }
-                else if (func is UserDefinedFunction userDefinedFunc)
+                else
                 {
-                    UDFStackFrame frame = new UDFStackFrame(userDefinedFunc, args);
-                    UDFStackFrame framePop = null;
-
-                    try
-                    {
-                        _udfStack.Push(frame);
-
-                        (var irnode, _) = userDefinedFunc.GetIRTranslator();
-
-                        this.CheckCancel();
-
-                        result = await irnode.Accept(this, context).ConfigureAwait(false);
-                    }
-                    finally
-                    {
-                        framePop = _udfStack.Pop();
-                    }
-
-                    if (frame != framePop)
-                    {
-                        throw new Exception("Something went wrong. UDF stack values didn't match.");
-                    }
+                    result = CommonErrors.NotYetImplementedFunctionError(node.IRContext, func.Name);
                 }
 
-#pragma warning disable CS0618 // Type or member is obsolete
-
-                // This is a temporary solution to release the Join function for host that want to use it.
-                else if (func is IAsyncTexlFunctionJoin asyncJoin)
+                // https://github.com/microsoft/Power-Fx/issues/2820
+                // We should remove this check that limits to just Adapter1, so we apply this check to all impls. 
+                if (invoker is AsyncFunctionPtrAdapter) 
                 {
-                    result = await asyncJoin.InvokeAsync(this, context.IncrementStackDepthCounter(childContext), node.IRContext, args).ConfigureAwait(false);
-                }
-#pragma warning restore CS0618 // Type or member is obsolete
-                else
-                {
-                    if (FunctionImplementations.TryGetValue(func, out AsyncFunctionPtr ptr))
-                    {
-                        result = await ptr(this, context.IncrementStackDepthCounter(childContext), node.IRContext, args).ConfigureAwait(false);
-
-                        if (!(result.IRContext.ResultType._type == node.IRContext.ResultType._type || result is ErrorValue || result.IRContext.ResultType is BlankType))
-                        {
-                            throw CommonExceptions.RuntimeMisMatch;
-                        }
-                    }
-                    else
+                    if (!(result.IRContext.ResultType._type == node.IRContext.ResultType._type || result is ErrorValue || result.IRContext.ResultType is BlankType))
                     {
-                        result = CommonErrors.NotYetImplementedFunctionError(node.IRContext, func.Name);
+                        throw CommonExceptions.RuntimeMisMatch;
                     }
                 }
             }

src/libraries/Microsoft.PowerFx.Interpreter/Functions/FileInfoImpl.cs

@@ -6,15 +6,18 @@
 using System.Threading.Tasks;
 using Microsoft.PowerFx.Core.Functions;
 using Microsoft.PowerFx.Core.Texl.Builtins;
+using Microsoft.PowerFx.Functions;
 using Microsoft.PowerFx.Interpreter;
 using Microsoft.PowerFx.Types;
 
 namespace Microsoft.PowerFx
 {
-    internal class FileInfoFunctionImpl : FileInfoFunction, IAsyncTexlFunction3
+    internal class FileInfoFunctionImpl : FileInfoFunction, IFunctionInvoker
     {
-        public async Task<FormulaValue> InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken)
+        public async Task<FormulaValue> InvokeAsync(FunctionInvokeInfo invokeInfo, CancellationToken cancellationToken)
         {
+            var args = invokeInfo.Args;
+
             var arg0 = args[0];
             if (arg0 is BlankValue || arg0 is ErrorValue)
             {