diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs index 5778346a41..5e2a59bdad 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs @@ -21,6 +21,7 @@ using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; using Microsoft.PowerFx.Core.Functions.Publish; using Microsoft.PowerFx.Core.Functions.TransportSchemas; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.IR.Symbols; using Microsoft.PowerFx.Core.Localization; @@ -31,6 +32,7 @@ using Microsoft.PowerFx.Intellisense; using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; +using static Microsoft.PowerFx.Core.IR.DependencyVisitor; using static Microsoft.PowerFx.Core.IR.IRTranslator; using CallNode = Microsoft.PowerFx.Syntax.CallNode; using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; @@ -1738,5 +1740,39 @@ internal ArgPreprocessor GetGenericArgPreprocessor(int index) return ArgPreprocessor.None; } + + /// + /// Visit all function nodes to compose dependency info. + /// + /// IR CallNode. + /// Dependency visitor. + /// Dependency context. + /// Static boolean value. + public virtual bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyContext context) + { + for (int i = 0; i < node.Args.Count; i++) + { + if (node.Scope != null && i < node.Function.ScopeArgs) + { + if (node.Args[i] is IRCallNode callNode) + { + callNode.Accept(visitor, context); + } + else + { + continue; + } + } + else + { + node.Args[i].Accept(visitor, context); + } + } + + // The return value is used by DepedencyScanFunctionTests test case. + // Returning false to indicate that the function runs a basic dependency scan. + // Other functions can override this method to return true if they have a custom dependency scan. + return false; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/Visitors/DependencyVisitor.cs b/src/libraries/Microsoft.PowerFx.Core/IR/Visitors/DependencyVisitor.cs new file mode 100644 index 0000000000..87f3dbe69e --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/IR/Visitors/DependencyVisitor.cs @@ -0,0 +1,350 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml; +using Microsoft.PowerFx.Core.IR.Nodes; +using Microsoft.PowerFx.Core.IR.Symbols; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Types; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; + +namespace Microsoft.PowerFx.Core.IR +{ + // IR has already: + // - resolved everything to logical names. + // - resolved implicit ThisRecord + internal class DependencyVisitor : IRNodeVisitor + { + // Track reults. + public DependencyInfo Info { get; private set; } = new DependencyInfo(); + + public DependencyVisitor() + { + } + + public override RetVal Visit(TextLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(NumberLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(BooleanLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(DecimalLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(ColorLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(RecordNode node, DependencyContext context) + { + // Read all the fields. The context will determine if the record is referencing a data source + foreach (var kv in node.Fields) + { + AddField(context, context.TableType?.TableSymbolName, kv.Key.Value); + } + + return null; + } + + public override RetVal Visit(ErrorNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(LazyEvalNode node, DependencyContext context) + { + return node.Child.Accept(this, context); + } + + private readonly Dictionary _scopeTypes = new Dictionary(); + + public override RetVal Visit(CallNode node, DependencyContext context) + { + if (node.Scope != null) + { + // Functions with more complex scoping will be handled by the function itself. + var arg0 = node.Args[0]; + _scopeTypes[node.Scope.Id] = arg0.IRContext.ResultType; + } + + node.Function.ComposeDependencyInfo(node, this, context); + + return null; + } + + public override RetVal Visit(BinaryOpNode node, DependencyContext context) + { + node.Left.Accept(this, context); + node.Right.Accept(this, context); + return null; + } + + public override RetVal Visit(UnaryOpNode node, DependencyContext context) + { + return node.Child.Accept(this, context); + } + + public override RetVal Visit(ScopeAccessNode node, DependencyContext context) + { + // Could be a symbol from RowScope. + // Price in "LookUp(t1,Price=255)" + if (node.Value is ScopeAccessSymbol sym) + { + if (_scopeTypes.TryGetValue(sym.Parent.Id, out var type)) + { + // Ignore ThisRecord scopeaccess node. e.g. Summarize(table, f1, Sum(ThisGroup, f2)) where ThisGroup should be ignored. + if (type is TableType tableType && node.IRContext.ResultType is not AggregateType) + { + var tableLogicalName = tableType.TableSymbolName; + var fieldLogicalName = sym.Name.Value; + + AddField(context, tableLogicalName, fieldLogicalName); + + return null; + } + } + } + + // Any symbol access here is some temporary local, and not a field. + return null; + } + + // field // IR will implicity recognize as ThisRecod.field + // ThisRecord.field // IR will get type of ThisRecord + // First(Remote).Data // IR will get type on left of dot. + public override RetVal Visit(RecordFieldAccessNode node, DependencyContext context) + { + node.From.Accept(this, context); + + var ltype = node.From.IRContext.ResultType; + if (ltype is RecordType ltypeRecord) + { + // Logical name of the table on left side. + // This will be null for non-dataverse records + var tableLogicalName = ltypeRecord.TableSymbolName; + if (tableLogicalName != null) + { + var fieldLogicalName = node.Field.Value; + AddField(context, tableLogicalName, fieldLogicalName); + } + } + + return null; + } + + public override RetVal Visit(ResolvedObjectNode node, DependencyContext context) + { + if (node.IRContext.ResultType is AggregateType aggregateType) + { + var tableLogicalName = aggregateType.TableSymbolName; + if (context.WriteState) + { + tableLogicalName = context.TableType?.TableSymbolName; + } + + if (tableLogicalName != null) + { + AddField(context, tableLogicalName, null); + } + } + + // Check if identifer is a field access on a table in row scope + var obj = node.Value; + if (obj is NameSymbol sym) + { + if (sym.Owner is SymbolTableOverRecordType symTable) + { + RecordType type = symTable.Type; + var tableLogicalName = type.TableSymbolName; + + if (symTable.IsThisRecord(sym)) + { + // "ThisRecord". Whole entity + AddField(context, tableLogicalName, null); + return null; + } + + // on current table + var fieldLogicalName = sym.Name; + + AddField(context, tableLogicalName, fieldLogicalName); + } + } + + return null; + } + + public override RetVal Visit(SingleColumnTableAccessNode node, DependencyContext context) + { + throw new NotImplementedException(); + } + + public override RetVal Visit(ChainingNode node, DependencyContext context) + { + foreach (var child in node.Nodes) + { + child.Accept(this, context); + } + + return null; + } + + public override RetVal Visit(AggregateCoercionNode node, DependencyContext context) + { + foreach (var kv in node.FieldCoercions) + { + kv.Value.Accept(this, context); + } + + return null; + } + + public class RetVal + { + } + + public class DependencyContext + { + public bool WriteState { get; set; } + + public TableType TableType { get; set; } + + public DependencyContext() + { + } + } + + // Translate relationship names to actual field references. + public string Translate(string tableLogicalName, string fieldLogicalName) + { + return fieldLogicalName; + } + + // if fieldLogicalName, then we're taking a dependency on entire record. + private void AddField(Dictionary> list, string tableLogicalName, string fieldLogicalName) + { + if (tableLogicalName == null) + { + return; + } + + if (!list.TryGetValue(tableLogicalName, out var fieldReads)) + { + fieldReads = new HashSet(); + list[tableLogicalName] = fieldReads; + } + + if (fieldLogicalName != null) + { + var name = Translate(tableLogicalName, fieldLogicalName); + fieldReads.Add(fieldLogicalName); + } + } + + public void AddFieldRead(string tableLogicalName, string fieldLogicalName) + { + if (Info.FieldReads == null) + { + Info.FieldReads = new Dictionary>(); + } + + AddField(Info.FieldReads, tableLogicalName, fieldLogicalName); + } + + public void AddFieldWrite(string tableLogicalName, string fieldLogicalName) + { + if (Info.FieldWrites == null) + { + Info.FieldWrites = new Dictionary>(); + } + + AddField(Info.FieldWrites, tableLogicalName, fieldLogicalName); + } + + public void AddField(DependencyContext context, string tableLogicalName, string fieldLogicalName) + { + if (context.WriteState) + { + AddFieldWrite(tableLogicalName, fieldLogicalName); + } + else + { + AddFieldRead(tableLogicalName, fieldLogicalName); + } + } + } + + /// + /// Capture Dataverse field-level reads and writes within a formula. + /// + public class DependencyInfo + { + /// + /// A dictionary of field logical names on related records, indexed by the related entity logical name. + /// + /// + /// On account, the formula "Name & 'Primary Contact'.'Full Name'" would return + /// "contact" => { "fullname" } + /// The formula "Name & 'Primary Contact'.'Full Name' & Sum(Contacts, 'Number Of Childeren')" would return + /// "contact" => { "fullname", "numberofchildren" }. + /// + public Dictionary> FieldReads { get; set; } + + public Dictionary> FieldWrites { get; set; } + + public bool HasWrites => FieldWrites != null && FieldWrites.Count > 0; + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + DumpHelper(sb, "Read", FieldReads); + DumpHelper(sb, "Write", FieldWrites); + + return sb.ToString(); + } + + private static void DumpHelper(StringBuilder sb, string kind, Dictionary> dict) + { + if (dict != null) + { + foreach (var kv in dict) + { + sb.Append(kind); + sb.Append(" "); + sb.Append(kv.Key); + sb.Append(": "); + + bool first = true; + foreach (var x in kv.Value) + { + if (!first) + { + sb.Append(", "); + } + + first = false; + sb.Append(x); + } + + sb.AppendLine("; "); + } + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs b/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs index fe999fd86b..5fea792bfe 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs @@ -246,7 +246,7 @@ public HashSet TopLevelIdentifiers { if (_topLevelIdentifiers == null) { - throw new InvalidOperationException($"Call {nameof(ApplyDependencyAnalysis)} first."); + throw new InvalidOperationException($"Call {nameof(ApplyTopLevelIdentifiersAnalysis)} first."); } return _topLevelIdentifiers; @@ -483,7 +483,27 @@ private void VerifyReturnTypeMatch() /// /// Compute the dependencies. Called after binding. /// - public void ApplyDependencyAnalysis() + public DependencyInfo ApplyDependencyAnalysis() + { + if (!IsSuccess) + { + return null; + } + + var ir = ApplyIR(); //throws on errors + + var ctx = new DependencyVisitor.DependencyContext(); + var visitor = new DependencyVisitor(); + + ir.TopNode.Accept(visitor, ctx); + + return visitor.Info; + } + + /// + /// Compute the dependencies. Called after binding. + /// + public void ApplyTopLevelIdentifiersAnalysis() { var binding = this.Binding; // will throw if binding wasn't run this._topLevelIdentifiers = DependencyFinder.FindDependencies(binding.Top, binding); diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs index 575e98dca4..e204eb311e 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs @@ -7,7 +7,6 @@ using System.Linq; using System.Reflection; using Microsoft.PowerFx.Core; -using Microsoft.PowerFx.Core.Annotations; using Microsoft.PowerFx.Core.App.Controls; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Entities.QueryOptions; @@ -292,7 +291,7 @@ private void CheckWorker(CheckResult check) { check.ApplyBindingInternal(); check.ApplyErrors(); - check.ApplyDependencyAnalysis(); + check.ApplyTopLevelIdentifiersAnalysis(); } // Called after check result, can inject additional errors or constraints. diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 297315def5..cc15abbac8 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -19,7 +19,8 @@ using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; using CallNode = Microsoft.PowerFx.Syntax.CallNode; -using RecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; +using IRRecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -450,7 +451,7 @@ protected static List CreateIRCallNodeCollect(CallNode node, I if (arg.IRContext.ResultType._type.IsPrimitive) { newArgs.Add( - new RecordNode( + new IRRecordNode( new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), new Dictionary { @@ -464,6 +465,22 @@ protected static List CreateIRCallNodeCollect(CallNode node, I } return newArgs; + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var newContext = new DependencyVisitor.DependencyContext() + { + WriteState = true, + TableType = node.Args[0].IRContext.ResultType as TableType + }; + + foreach (var arg in node.Args.Skip(1)) + { + arg.Accept(visitor, newContext); + } + + return true; } } @@ -489,6 +506,14 @@ public override DType GetCollectedType(Features features, DType argType) internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) { return base.CreateIRCallNode(node, context, CreateIRCallNodeCollect(node, context, args, scope), scope); + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + visitor.AddFieldWrite(tableType.TableSymbolName, "Value"); + + return true; } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs index 4ed7a644d3..b2efb9747b 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs @@ -238,5 +238,37 @@ internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node return new IRCallNode(context.GetIRContext(node), this, scope, newArgs); } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + // Skipping args 0 and 1. + for (int i = 2; i < node.Args.Count; i++) + { + var arg = node.Args[i]; + RecordNode recordNode; + + switch (i) + { + case 2: // Predicate arg. + arg.Accept(visitor, context); + break; + case 5: + case 6: // Left and right record args. + var sourceArg = node.Args[i - 5]; + recordNode = arg as RecordNode; + foreach (var field in recordNode.Fields) + { + var tableType = (TableType)sourceArg.IRContext.ResultType; + visitor.AddField(context, tableType.TableSymbolName, field.Key.Value); + } + + break; + default: + break; + } + } + + return true; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs index 810f781300..b3ec6bcced 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs @@ -10,14 +10,15 @@ using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.Functions.DLP; -using Microsoft.PowerFx.Core.IR.Nodes; -using Microsoft.PowerFx.Core.IR.Symbols; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; -using static Microsoft.PowerFx.Core.IR.IRTranslator; +using Microsoft.PowerFx.Types; using CallNode = Microsoft.PowerFx.Syntax.CallNode; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; +using IRRecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -447,6 +448,25 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(2).ToArray(), argTypes.Skip(2).ToArray(), errors); } } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + // Arg1 is the record to be found. All fields are readonly, so we don't need to add any writes here. + node.Args[1].Accept(visitor, new DependencyVisitor.DependencyContext() { TableType = node.Args[0].IRContext.ResultType as TableType }); + + var newContext = new DependencyVisitor.DependencyContext() + { + WriteState = true, + TableType = node.Args[0].IRContext.ResultType as TableType + }; + + foreach (var arg in node.Args.Skip(2).Select(arg => (IRRecordNode)arg)) + { + arg.Accept(visitor, newContext); + } + + return true; + } } // Patch(DS, record_with_keys_and_updates) @@ -474,6 +494,28 @@ public PatchSingleRecordFunction() { yield return new[] { TexlStrings.PatchArg_Source, TexlStrings.PatchArg_Record }; } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + var recordType = (RecordType)node.Args[1].IRContext.ResultType; + + var datasource = tableType._type.AssociatedDataSources.First(); + + foreach (var fieldName in recordType.FieldNames) + { + if (datasource != null && datasource.GetKeyColumns().Contains(fieldName)) + { + visitor.AddFieldRead(tableType.TableSymbolName, fieldName); + } + else + { + visitor.AddFieldWrite(tableType.TableSymbolName, fieldName); + } + } + + return true; + } } // Patch(DS, table_of_rows, table_of_updates) @@ -500,6 +542,27 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(2).ToArray(), argTypes.Skip(2).ToArray(), errors); } } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType0 = (TableType)node.Args[0].IRContext.ResultType; + var tableType1 = (TableType)node.Args[1].IRContext.ResultType; + var tableType2 = (TableType)node.Args[2].IRContext.ResultType; + + var datasource = tableType0._type.AssociatedDataSources.First(); + + foreach (var fieldName in tableType1.FieldNames) + { + visitor.AddFieldRead(tableType0.TableSymbolName, fieldName); + } + + foreach (var fieldName in tableType2.FieldNames) + { + visitor.AddFieldWrite(tableType0.TableSymbolName, fieldName); + } + + return true; + } } // Patch(DS, table_of_rows_with_updates) @@ -516,6 +579,28 @@ public PatchAggregateSingleTableFunction() { yield return new[] { TexlStrings.PatchArg_Source, TexlStrings.PatchArg_Rows }; } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType0 = (TableType)node.Args[0].IRContext.ResultType; + var tableType1 = (TableType)node.Args[1].IRContext.ResultType; + + var datasource = tableType0._type.AssociatedDataSources.First(); + + foreach (var fieldName in tableType1.FieldNames) + { + if (datasource != null && datasource.GetKeyColumns().Contains(fieldName)) + { + visitor.AddFieldRead(tableType0.TableSymbolName, fieldName); + } + else + { + visitor.AddFieldWrite(tableType0.TableSymbolName, fieldName); + } + } + + return true; + } } // Patch(Record, Updates1, Updates2,…) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs index f285151580..e21d5052f9 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; @@ -259,6 +260,27 @@ internal override IntermediateNode CreateIRCallNode(CallNode node, IRTranslatorC } return new IRCallNode(context.GetIRContext(node), this, scope, newArgs); + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableTypeName = ((AggregateType)node.Args[0].IRContext.ResultType).TableSymbolName; + + foreach (var arg in node.Args.Skip(1)) + { + if (arg is TextLiteralNode textLiteralNode) + { + visitor.AddFieldRead(tableTypeName, textLiteralNode.LiteralValue); + } + else if (arg is LazyEvalNode lazyEvalNode) + { + var recordNode = (RecordNode)lazyEvalNode.Child; + + recordNode.Fields.Values.First().Accept(visitor, context); + } + } + + return true; } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs index 1dfa1f1cc1..724e67444a 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs @@ -2,16 +2,17 @@ // Licensed under the MIT license. using System.Collections.Generic; -using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; -using static Microsoft.PowerFx.Core.Localization.TexlStrings; +using static Microsoft.PowerFx.Core.Localization.TexlStrings; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; namespace Microsoft.PowerFx.Interpreter { @@ -145,6 +146,19 @@ private bool CheckMutability(TexlBinding binding, TexlNode[] args, DType[] argTy } return false; + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var newContext = new DependencyVisitor.DependencyContext() + { + WriteState = true + }; + + node.Args[0].Accept(visitor, newContext); + node.Args[1].Accept(visitor, context); + + return true; } } -} +} diff --git a/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Public/EditorContextScope.cs b/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Public/EditorContextScope.cs index 47821b2608..869d8ffe4b 100644 --- a/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Public/EditorContextScope.cs +++ b/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Public/EditorContextScope.cs @@ -87,7 +87,7 @@ public CheckResult Check(string expression) // By default ... check.ApplyBindingInternal(); check.ApplyErrors(); - check.ApplyDependencyAnalysis(); + check.ApplyTopLevelIdentifiersAnalysis(); return check; } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs index 177c71f5a3..8a6fc34153 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs @@ -76,6 +76,7 @@ public static DType GetDType(bool hasCachedCountRows = false) displayNameMapping.Add("address1_line1", "Address 1: Street 1"); displayNameMapping.Add("nonsearchablestringcol", "Non-searchable string column"); displayNameMapping.Add("nonsortablestringcolumn", "Non-sortable string column"); + displayNameMapping.Add("numberofemployees", "Number of employees"); return DType.AttachDataSourceInfo(accountsType, dataSource); } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/CheckResultTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/CheckResultTests.cs index f0a438b8df..01783d0d03 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/CheckResultTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/CheckResultTests.cs @@ -270,7 +270,7 @@ public void Binding() // Binding doesn't compute dependency analysis. // These are other Apply* calls. Assert.Throws(() => check.TopLevelIdentifiers); - check.ApplyDependencyAnalysis(); + check.ApplyTopLevelIdentifiersAnalysis(); Assert.NotNull(check.TopLevelIdentifiers); } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs index 352871be4b..8bf0216089 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs @@ -214,7 +214,8 @@ public void PublicSurface_Tests() "Microsoft.PowerFx.Logging.ITracer", "Microsoft.PowerFx.Logging.TraceSeverity", "Microsoft.PowerFx.PowerFxFileInfo", - "Microsoft.PowerFx.UserInfo" + "Microsoft.PowerFx.UserInfo", + "Microsoft.PowerFx.Core.IR.DependencyInfo", }; var sb = new StringBuilder(); diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/DependencyTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/DependencyTests.cs new file mode 100644 index 0000000000..3aecae29ed --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/DependencyTests.cs @@ -0,0 +1,223 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; +using Microsoft.PowerFx.Core.Tests.AssociatedDataSourcesTests; +using Microsoft.PowerFx.Core.Tests.Helpers; +using Microsoft.PowerFx.Core.Texl; +using Microsoft.PowerFx.Core.Texl.Builtins; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Types.Enums; +using Microsoft.PowerFx.Interpreter; +using Microsoft.PowerFx.Types; +using Xunit; + +namespace Microsoft.PowerFx.Core.Tests +{ + public class DependencyTests : PowerFxTest + { + [Theory] + [InlineData("1+2", "")] // none + [InlineData("ThisRecord.'Address 1: City' & 'Account Name'", "Read Accounts: address1_city, name;")] // basic read + + [InlineData("numberofemployees%", "Read Accounts: numberofemployees;")] // unary op + [InlineData("ThisRecord", "Read Accounts: ;")] // whole scope + [InlineData("{x:5}.x", "")] + [InlineData("With({x : ThisRecord}, x.'Address 1: City')", "Read Accounts: address1_city;")] // alias + [InlineData("With({'Address 1: City' : \"Seattle\"}, 'Address 1: City' & 'Account Name')", "Read Accounts: name;")] // 'Address 1: City' is shadowed + [InlineData("With({'Address 1: City' : 5}, ThisRecord.'Address 1: City')", "")] // shadowed + [InlineData("LookUp(local,'Address 1: City'=\"something\")", "Read Accounts: address1_city;")] // Lookup and RowScope + [InlineData("Filter(local,numberofemployees > 200)", "Read Accounts: numberofemployees;")] + [InlineData("First(local)", "Read Accounts: ;")] + [InlineData("First(local).'Address 1: City'", "Read Accounts: address1_city;")] + [InlineData("Last(local)", "Read Accounts: ;")] + [InlineData("local", "Read Accounts: ;")] // whole table + [InlineData("12 & true & \"abc\" ", "")] // walker ignores literals + [InlineData("12;'Address 1: City';12", "Read Accounts: address1_city;")] // chaining + [InlineData("ParamLocal1.'Address 1: City'", "Read Accounts: address1_city;")] // basic read + + // Basic scoping + [InlineData("Min(local,numberofemployees)", "Read Accounts: numberofemployees;")] + [InlineData("Average(local,numberofemployees)", "Read Accounts: numberofemployees;")] + + // Patch + [InlineData("Patch(local, First(local), { 'Account Name' : \"some name\"})", "Read Accounts: ; Write Accounts: name;")] + [InlineData("Patch(local, {'Address 1: City':\"test\"}, { 'Account Name' : \"some name\"})", "Read Accounts: address1_city; Write Accounts: name;")] + [InlineData("Patch(local, {accountid:GUID(), 'Address 1: City':\"test\"})", "Read Accounts: accountid; Write Accounts: address1_city;")] + [InlineData("Patch(local, Table({accountid:GUID(), 'Address 1: City':\"test\"},{accountid:GUID(), 'Account Name':\"test\"}))", "Read Accounts: accountid; Write Accounts: address1_city, name;")] + [InlineData("Patch(local, Table({accountid:GUID(), 'Address 1: City':\"test\"},{accountid:GUID(), 'Account Name':\"test\"}),Table({'Address 1: City':\"test\"},{'Address 1: City':\"test\",'Account Name':\"test\"}))", "Read Accounts: accountid, address1_city, name; Write Accounts: address1_city, name;")] + + // Collect and ClearCollect. + [InlineData("Collect(local, Table({ 'Account Name' : \"some name\"}))", "Write Accounts: name;")] + [InlineData("Collect(local, local)", "Write Accounts: ;")] + [InlineData("ClearCollect(local, local)", "Write Accounts: ;")] + [InlineData("ClearCollect(local, Table({ 'Account Name' : \"some name\"}))", "Write Accounts: name;")] + + // Inside with. + [InlineData("With({r: local}, Filter(r, 'Number of employees' > 0))", "Read Accounts: numberofemployees;")] + [InlineData("With({r: local}, LookUp(r, 'Number of employees' > 0))", "Read Accounts: numberofemployees;")] + + // Option set. + [InlineData("Filter(local, dayofweek = StartOfWeek.Monday)", "Read Accounts: dayofweek;")] + + [InlineData("Filter(ForAll(local, ThisRecord.numberofemployees), Value < 20)", "Read Accounts: numberofemployees;")] + + // Summarize is special, becuase of ThisGroup. + [InlineData("Summarize(local, 'Account Name', Sum(ThisGroup, numberofemployees) As Employees)", "Read Accounts: name, numberofemployees;")] + [InlineData("Summarize(local, 'Account Name', Sum(ThisGroup, numberofemployees * 2) As TPrice)", "Read Accounts: name, numberofemployees;")] + + // Join + [InlineData("Join(remote As l, local As r, l.contactid = r.contactid, JoinType.Inner, r.name As AccountName)", "Read Contacts: contactid; Read Accounts: contactid, name;")] + [InlineData("Join(remote As l, local As r, l.contactid = r.contactid, JoinType.Inner, r.name As AccountName, l.contactnumber As NewContactNumber)", "Read Contacts: contactid, contactnumber; Read Accounts: contactid, name;")] + + // Set + [InlineData("Set(numberofemployees, 200)", "Write Accounts: numberofemployees;")] + [InlineData("Set('Address 1: City', 'Account Name')", "Read Accounts: name; Write Accounts: address1_city;")] + [InlineData("Set('Address 1: City', 'Address 1: City' & \"test\")", "Read Accounts: address1_city; Write Accounts: address1_city;")] + public void GetDependencies(string expr, string expected) + { + var opt = new ParserOptions() { AllowsSideEffects = true }; + var engine = new Engine(); + + var check = new CheckResult(engine) + .SetText(expr, opt) + .SetBindingInfo(GetSymbols()); + + check.ApplyBinding(); + + var info = check.ApplyDependencyAnalysis(); + var actual = info.ToString().Replace("\r", string.Empty).Replace("\n", string.Empty).Trim(); + Assert.Equal(expected, actual); + } + + private ReadOnlySymbolTable GetSymbols() + { + var localType = Accounts(); + var remoteType = Contacts(); + var customSymbols = new SymbolTable { DebugName = "Custom symbols " }; + var opt = new ParserOptions() { AllowsSideEffects = true }; + + var thisRecordScope = ReadOnlySymbolTable.NewFromRecord(localType.ToRecord(), allowThisRecord: true, allowMutable: true); + + customSymbols.AddFunction(new JoinFunction()); + customSymbols.AddFunction(new CollectFunction()); + customSymbols.AddFunction(new CollectScalarFunction()); + customSymbols.AddFunction(new ClearCollectFunction()); + customSymbols.AddFunction(new ClearCollectScalarFunction()); + customSymbols.AddFunction(new PatchFunction()); + customSymbols.AddFunction(new PatchAggregateFunction()); + customSymbols.AddFunction(new PatchAggregateSingleTableFunction()); + customSymbols.AddFunction(new PatchSingleRecordFunction()); + customSymbols.AddFunction(new SummarizeFunction()); + customSymbols.AddFunction(new RecalcEngineSetFunction()); + customSymbols.AddVariable("local", localType, mutable: true); + customSymbols.AddVariable("remote", remoteType, mutable: true); + + // Simulate a parameter + var parameterSymbols = new SymbolTable { DebugName = "Parameters " }; + parameterSymbols.AddVariable("ParamLocal1", localType.ToRecord(), mutable: true); + parameterSymbols.AddVariable("NewRecord", localType.ToRecord(), new SymbolProperties() { CanMutate = false, CanSet = false, CanSetMutate = true }); + + return ReadOnlySymbolTable.Compose(customSymbols, thisRecordScope, parameterSymbols); + } + + private TableType Accounts() + { + var tableType = (TableType)FormulaType.Build(AccountsTypeHelper.GetDType()); + tableType = tableType.Add("dayofweek", BuiltInEnums.StartOfWeekEnum.FormulaType); + tableType = tableType.Add("contactid", FormulaType.Guid); + + return tableType; + } + + private TableType Contacts() + { + var simplifiedAccountsSchema = "*[contactid:g, contactnumber:s, name`Contact Name`:s, address1_addresstypecode:l, address1_city`Address 1: City`:s, address1_composite:s, address1_country:s, address1_county:s, address1_line1`Address 1: Street 1`:s, numberofemployees:n]"; + + DType contactType = TestUtils.DT2(simplifiedAccountsSchema); + var dataSource = new TestDataSource( + "Contacts", + contactType, + keyColumns: new[] { "contactid" }, + selectableColumns: new[] { "name", "address1_city", "contactid", "address1_country", "address1_line1" }, + hasCachedCountRows: false); + var displayNameMapping = dataSource.DisplayNameMapping; + displayNameMapping.Add("name", "Contact Name"); + displayNameMapping.Add("address1_city", "Address 1: City"); + displayNameMapping.Add("address1_line1", "Address 1: Street 1"); + displayNameMapping.Add("numberofemployees", "Number of employees"); + + contactType = DType.AttachDataSourceInfo(contactType, dataSource); + + return (TableType)FormulaType.Build(contactType); + } + + // Some functions might require an different dependecy scan. This test case is to ensure that any new functions that + // is not self-contained or has a scope info has been assessed and either added to the exception list or has a dependency scan. + [Fact] + public void DepedencyScanFunctionTests() + { + var names = new List(); + var functions = new List(); + functions.AddRange(BuiltinFunctionsCore.BuiltinFunctionsLibrary); + + var exceptionList = new HashSet() + { + "AddColumns", + "Average", + "Concat", + "CountIf", + "DropColumns", + "Filter", + "ForAll", + "IfError", + "LookUp", + "Max", + "Min", + "Refresh", + "RenameColumns", + "Search", + "ShowColumns", + "Sort", + "SortByColumns", + "StdevP", + "Sum", + "Trace", + "VarP", + "With", + }; + + foreach (var func in functions) + { + if (!func.IsSelfContained || func.ScopeInfo != null) + { + var visitor = new DependencyVisitor(); + var context = new DependencyVisitor.DependencyContext(); + var node = new CallNode(IRContext.NotInSource(FormulaType.String), func); + var overwritten = func.ComposeDependencyInfo(node, visitor, context); + + if (!overwritten && !exceptionList.Contains(func.Name)) + { + names.Add(func.Name); + } + } + } + + if (names.Count > 0) + { + var sb = new StringBuilder(); + sb.AppendLine("The following functions do not have a dependency scan:"); + foreach (var name in names) + { + sb.AppendLine(name); + } + + Assert.Fail(sb.ToString()); + } + } + } +}