diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs index 5778346a41..e739d8dabf 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs @@ -21,6 +21,7 @@ using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; using Microsoft.PowerFx.Core.Functions.Publish; using Microsoft.PowerFx.Core.Functions.TransportSchemas; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.IR.Symbols; using Microsoft.PowerFx.Core.Localization; @@ -31,6 +32,7 @@ using Microsoft.PowerFx.Intellisense; using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; +using static Microsoft.PowerFx.Core.IR.DependencyVisitor; using static Microsoft.PowerFx.Core.IR.IRTranslator; using CallNode = Microsoft.PowerFx.Syntax.CallNode; using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; @@ -1738,5 +1740,25 @@ internal ArgPreprocessor GetGenericArgPreprocessor(int index) return ArgPreprocessor.None; } + + /// + /// Visit all function nodes to compose dependency info. + /// + /// IR CallNode. + /// Dependency visitor. + /// Dependency context. + /// + public virtual bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyContext context) + { + foreach (var arg in node.Args) + { + arg.Accept(visitor, context); + } + + // The return value is used by DepedencyScanFunctionTests test case. + // Returning false to indicate that the function runs a basic dependency scan. + // Other functions can override this method to return true if they have a custom dependency scan. + return false; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/Utils.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/Utils.cs index 0e3189f57b..f4d8ae1273 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Functions/Utils.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/Utils.cs @@ -2,10 +2,14 @@ // Licensed under the MIT license. using System.Globalization; +using System.Linq; using System.Numerics; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx.Core.Functions { @@ -23,5 +27,18 @@ public static string GetLocalizedName(this FunctionCategories category, CultureI { return StringResources.Get(category.ToString(), culture.Name); } + + public static void FunctionSupportColumnNamesAsIdentifiersDependencyUtil(this CallNode node, DependencyVisitor visitor) + { + var aggregateType0 = node.Args[0].IRContext.ResultType as AggregateType; + + foreach (TextLiteralNode arg in node.Args.Skip(1).Where(a => a is TextLiteralNode)) + { + if (aggregateType0.TryGetFieldType(arg.LiteralValue, out _)) + { + visitor.AddDependency(aggregateType0.TableSymbolName, arg.LiteralValue); + } + } + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs index f6619a7639..13d7117790 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs @@ -29,8 +29,13 @@ namespace Microsoft.PowerFx.Core.IR { internal class IRResult - { - public IntermediateNode TopNode; + { + // IR top node after transformations. + public IntermediateNode TopNode; + + // Original IR node, without transformations. + public IntermediateNode TopOriginalNode; + public ScopeSymbol RuleScopeSymbol; } diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/Visitors/DependencyVisitor.cs b/src/libraries/Microsoft.PowerFx.Core/IR/Visitors/DependencyVisitor.cs new file mode 100644 index 0000000000..9a754f7bc6 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/IR/Visitors/DependencyVisitor.cs @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.PowerFx.Core.IR.Nodes; +using Microsoft.PowerFx.Core.IR.Symbols; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Types; +using static Microsoft.PowerFx.Syntax.PrettyPrintVisitor; + +namespace Microsoft.PowerFx.Core.IR +{ + // IR has already: + // - resolved everything to logical names. + // - resolved implicit ThisRecord + internal sealed class DependencyVisitor : IRNodeVisitor + { + // Track reults. + public DependencyInfo Info { get; private set; } = new DependencyInfo(); + + public DependencyVisitor() + { + } + + public override RetVal Visit(TextLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(NumberLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(BooleanLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(DecimalLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(ColorLiteralNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(RecordNode node, DependencyContext context) + { + // Visit all field values in case there are CallNodes. Field keys should be handled by the function caller. + foreach (var kv in node.Fields) + { + kv.Value.Accept(this, context); + } + + return null; + } + + public override RetVal Visit(ErrorNode node, DependencyContext context) + { + return null; + } + + public override RetVal Visit(LazyEvalNode node, DependencyContext context) + { + return node.Child.Accept(this, context); + } + + private readonly Dictionary _scopeTypes = new Dictionary(); + + public override RetVal Visit(CallNode node, DependencyContext context) + { + if (node.Scope != null) + { + // Functions with more complex scoping will be handled by the function itself. + var arg0 = node.Args[0]; + _scopeTypes[node.Scope.Id] = arg0.IRContext.ResultType; + } + + node.Function.ComposeDependencyInfo(node, this, context); + + return null; + } + + public override RetVal Visit(BinaryOpNode node, DependencyContext context) + { + node.Left.Accept(this, context); + node.Right.Accept(this, context); + return null; + } + + public override RetVal Visit(UnaryOpNode node, DependencyContext context) + { + return node.Child.Accept(this, context); + } + + public override RetVal Visit(ScopeAccessNode node, DependencyContext context) + { + // Could be a symbol from RowScope. + // Price in "LookUp(t1,Price=255)" + if (node.Value is ScopeAccessSymbol sym) + { + if (_scopeTypes.TryGetValue(sym.Parent.Id, out var type)) + { + // Ignore ThisRecord scopeaccess node. e.g. Summarize(table, f1, Sum(ThisGroup, f2)) where ThisGroup should be ignored. + if (type is TableType tableType && tableType.TryGetFieldType(sym.Name.Value, out _)) + { + AddDependency(tableType.TableSymbolName, sym.Name.Value); + + return null; + } + } + } + + return null; + } + + // field // IR will implicity recognize as ThisRecod.field + // ThisRecord.field // IR will get type of ThisRecord + // First(Remote).Data // IR will get type on left of dot. + public override RetVal Visit(RecordFieldAccessNode node, DependencyContext context) + { + node.From.Accept(this, context); + + var ltype = node.From.IRContext.ResultType; + if (ltype is RecordType ltypeRecord) + { + // Logical name of the table on left side. + // This will be null for non-dataverse records + var tableLogicalName = ltypeRecord.TableSymbolName; + if (tableLogicalName != null) + { + var fieldLogicalName = node.Field.Value; + AddDependency(tableLogicalName, fieldLogicalName); + } + } + + return null; + } + + public override RetVal Visit(ResolvedObjectNode node, DependencyContext context) + { + if (node.IRContext.ResultType is AggregateType aggregateType) + { + AddDependency(aggregateType.TableSymbolName, null); + } + + CheckResolvedObjectNodeValue(node, context); + + return null; + } + + public void CheckResolvedObjectNodeValue(ResolvedObjectNode node, DependencyContext context) + { + if (node.Value is NameSymbol sym) + { + if (sym.Owner is SymbolTableOverRecordType symTable) + { + RecordType type = symTable.Type; + var tableLogicalName = type.TableSymbolName; + + if (symTable.IsThisRecord(sym)) + { + // "ThisRecord". Whole entity + AddDependency(type.TableSymbolName, null); + return; + } + + // on current table + var fieldLogicalName = sym.Name; + + AddDependency(type.TableSymbolName, fieldLogicalName); + } + } + } + + public override RetVal Visit(SingleColumnTableAccessNode node, DependencyContext context) + { + throw new NotImplementedException(); + } + + public override RetVal Visit(ChainingNode node, DependencyContext context) + { + foreach (var child in node.Nodes) + { + child.Accept(this, context); + } + + return null; + } + + public override RetVal Visit(AggregateCoercionNode node, DependencyContext context) + { + foreach (var kv in node.FieldCoercions) + { + kv.Value.Accept(this, context); + } + + return null; + } + + public class RetVal + { + } + + public class DependencyContext + { + public DependencyContext() + { + } + } + + // if fieldLogicalName, then we're taking a dependency on entire record. + public void AddDependency(string tableLogicalName, string fieldLogicalName) + { + if (tableLogicalName == null) + { + return; + } + + if (!Info.Dependencies.ContainsKey(tableLogicalName)) + { + Info.Dependencies[tableLogicalName] = new HashSet(); + } + + if (fieldLogicalName != null) + { + Info.Dependencies[tableLogicalName].Add(fieldLogicalName); + } + } + } + + /// + /// Capture Dataverse field-level reads and writes within a formula. + /// + public class DependencyInfo + { +#pragma warning disable CS1570 // XML comment has badly formed XML + /// + /// A dictionary of field logical names on related records, indexed by the related entity logical name. + /// + /// + /// On account, the formula "Name & 'Primary Contact'.'Full Name'" would return + /// "contact" => { "fullname" } + /// The formula "Name & 'Primary Contact'.'Full Name' & Sum(Contacts, 'Number Of Childeren')" would return + /// "contact" => { "fullname", "numberofchildren" }. + /// + public Dictionary> Dependencies { get; set; } + + public DependencyInfo() + { + Dependencies = new Dictionary>(); + } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + DumpHelper(sb, Dependencies); + + return sb.ToString(); + } + + private static void DumpHelper(StringBuilder sb, Dictionary> dict) + { + if (dict != null) + { + foreach (var kv in dict) + { + sb.Append("Entity"); + sb.Append(" "); + sb.Append(kv.Key); + sb.Append(": "); + + bool first = true; + foreach (var x in kv.Value) + { + if (!first) + { + sb.Append(", "); + } + + first = false; + sb.Append(x); + } + + sb.AppendLine("; "); + } + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs b/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs index fe999fd86b..862778772f 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/CheckResult.cs @@ -9,6 +9,7 @@ using Microsoft.CodeAnalysis; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Logging; using Microsoft.PowerFx.Core.Public; @@ -480,6 +481,23 @@ private void VerifyReturnTypeMatch() _errors.AddRange(ftErrors); } + /// + /// Compute the dependencies. Called after binding. + /// + [Obsolete("Preview")] + public DependencyInfo ApplyDependencyInfoScan() + { + var ir = ApplyIR(); //throws on errors + + var ctx = new DependencyVisitor.DependencyContext(); + var visitor = new DependencyVisitor(); + + // Using the original node without transformations. This simplifies the dependency analysis for PFx.DV side. + ir.TopOriginalNode.Accept(visitor, ctx); + + return visitor.Info; + } + /// /// Compute the dependencies. Called after binding. /// @@ -539,6 +557,8 @@ internal IRResult ApplyIR() this.ThrowOnErrors(); (var irnode, var ruleScopeSymbol) = IRTranslator.Translate(binding); + var originalIRNode = irnode; + var list = _engine.IRTransformList; if (list != null) { @@ -555,6 +575,7 @@ internal IRResult ApplyIR() _irresult = new IRResult { TopNode = irnode, + TopOriginalNode = originalIRNode, RuleScopeSymbol = ruleScopeSymbol }; } @@ -591,7 +612,7 @@ public FormulaType GetNodeType(TexlNode node) /// /// /// Null if the node is not bound. - public FunctionInfo GetFunctionInfo(CallNode node) + public FunctionInfo GetFunctionInfo(Syntax.CallNode node) { if (node == null) { diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs index 575e98dca4..23a1261ad2 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs @@ -7,7 +7,6 @@ using System.Linq; using System.Reflection; using Microsoft.PowerFx.Core; -using Microsoft.PowerFx.Core.Annotations; using Microsoft.PowerFx.Core.App.Controls; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Entities.QueryOptions; diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 297315def5..05fe602e7d 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -19,7 +19,8 @@ using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; using CallNode = Microsoft.PowerFx.Syntax.CallNode; -using RecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; +using IRRecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -450,7 +451,7 @@ protected static List CreateIRCallNodeCollect(CallNode node, I if (arg.IRContext.ResultType._type.IsPrimitive) { newArgs.Add( - new RecordNode( + new IRRecordNode( new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), new Dictionary { @@ -464,6 +465,34 @@ protected static List CreateIRCallNodeCollect(CallNode node, I } return newArgs; + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + + foreach (var arg in node.Args.Skip(1)) + { + var argType = arg.IRContext.ResultType; + string argTableSymbolName = null; + + if (argType is AggregateType aggregateType) + { + // If argN is a record/table and has a table symbol name, it means we are referencing another entity. + // We then need to add a dependency to the table symbol name as well. + argTableSymbolName = aggregateType.TableSymbolName; + } + + foreach (var name in argType._type.GetAllNames(DPath.Root)) + { + visitor.AddDependency(tableType.TableSymbolName, name.Name.Value); + visitor.AddDependency(argTableSymbolName, name.Name.Value); + } + + arg.Accept(visitor, context); + } + + return true; } } @@ -489,6 +518,14 @@ public override DType GetCollectedType(Features features, DType argType) internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) { return base.CreateIRCallNode(node, context, CreateIRCallNodeCollect(node, context, args, scope), scope); + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + visitor.AddDependency(tableType.TableSymbolName, "Value"); + + return true; } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs index 4ed7a644d3..892cde5428 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Join.cs @@ -238,5 +238,37 @@ internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node return new IRCallNode(context.GetIRContext(node), this, scope, newArgs); } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + // Skipping args 0 and 1. + for (int i = 2; i < node.Args.Count; i++) + { + var arg = node.Args[i]; + RecordNode recordNode; + + switch (i) + { + case 2: // Predicate arg. + arg.Accept(visitor, context); + break; + case 5: + case 6: // Left and right record args. + var sourceArg = node.Args[i - 5]; + recordNode = arg as RecordNode; + foreach (var field in recordNode.Fields) + { + var tableType = (TableType)sourceArg.IRContext.ResultType; + visitor.AddDependency(tableType.TableSymbolName, field.Key.Value); + } + + break; + default: + break; + } + } + + return true; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs index 810f781300..738f310bd1 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs @@ -10,14 +10,15 @@ using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.Functions.DLP; -using Microsoft.PowerFx.Core.IR.Nodes; -using Microsoft.PowerFx.Core.IR.Symbols; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; -using static Microsoft.PowerFx.Core.IR.IRTranslator; +using Microsoft.PowerFx.Types; using CallNode = Microsoft.PowerFx.Syntax.CallNode; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; +using IRRecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -447,6 +448,35 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(2).ToArray(), argTypes.Skip(2).ToArray(), errors); } } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + var recordType = (RecordType)node.Args[1].IRContext.ResultType; + + // arg1 might refere both to arg0 and arg1. + // Examples: + // Patch(t1, {...}, {...}) => arg1 is inmemory record and the fields refers to t1. + // Patch(t1, First(t2), {...}) => arg1 fields refers to t2 and t1. + foreach (var fieldName in recordType.FieldNames) + { + visitor.AddDependency(recordType.TableSymbolName, fieldName); + } + + foreach (var arg in node.Args.Skip(1)) + { + arg.Accept(visitor, context); + if (arg.IRContext.ResultType is AggregateType aggregateType) + { + foreach (var fieldName in aggregateType.FieldNames) + { + visitor.AddDependency(tableType.TableSymbolName, fieldName); + } + } + } + + return true; + } } // Patch(DS, record_with_keys_and_updates) @@ -474,6 +504,21 @@ public PatchSingleRecordFunction() { yield return new[] { TexlStrings.PatchArg_Source, TexlStrings.PatchArg_Record }; } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + var recordType = (RecordType)node.Args[1].IRContext.ResultType; + + var datasource = tableType._type.AssociatedDataSources.First(); + + foreach (var fieldName in recordType.FieldNames) + { + visitor.AddDependency(tableType.TableSymbolName, fieldName); + } + + return true; + } } // Patch(DS, table_of_rows, table_of_updates) @@ -500,6 +545,25 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(2).ToArray(), argTypes.Skip(2).ToArray(), errors); } } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType0 = (TableType)node.Args[0].IRContext.ResultType; + var tableType1 = (TableType)node.Args[1].IRContext.ResultType; + var tableType2 = (TableType)node.Args[2].IRContext.ResultType; + + foreach (var fieldName in tableType1.FieldNames) + { + visitor.AddDependency(tableType0.TableSymbolName, fieldName); + } + + foreach (var fieldName in tableType2.FieldNames) + { + visitor.AddDependency(tableType0.TableSymbolName, fieldName); + } + + return true; + } } // Patch(DS, table_of_rows_with_updates) @@ -516,6 +580,28 @@ public PatchAggregateSingleTableFunction() { yield return new[] { TexlStrings.PatchArg_Source, TexlStrings.PatchArg_Rows }; } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType0 = (TableType)node.Args[0].IRContext.ResultType; + var tableType1 = (TableType)node.Args[1].IRContext.ResultType; + + var datasource = tableType0._type.AssociatedDataSources.First(); + + foreach (var fieldName in tableType1.FieldNames) + { + if (datasource != null && datasource.GetKeyColumns().Contains(fieldName)) + { + visitor.AddDependency(tableType0.TableSymbolName, fieldName); + } + else + { + visitor.AddDependency(tableType0.TableSymbolName, fieldName); + } + } + + return true; + } } // Patch(Record, Updates1, Updates2,…) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/RenameColumns.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/RenameColumns.cs index 7044d0d9a5..3ce283dc20 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/RenameColumns.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/RenameColumns.cs @@ -3,15 +3,21 @@ using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Entities.QueryOptions; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Types; +using CallNode = Microsoft.PowerFx.Syntax.CallNode; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -265,5 +271,12 @@ public static ColumnReplacement Create(DName oldName, DName newName, DType type) }; } } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + node.FunctionSupportColumnNamesAsIdentifiersDependencyUtil(visitor); + + return true; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/ShowDropColumnsBase.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/ShowDropColumnsBase.cs index c9b315149a..b080db4f2c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/ShowDropColumnsBase.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/ShowDropColumnsBase.cs @@ -2,9 +2,12 @@ // Licensed under the MIT license. using System.Collections.Generic; +using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; @@ -193,5 +196,12 @@ public override ParamIdentifierStatus GetIdentifierParamStatus(TexlNode node, Fe return index > 0 ? ParamIdentifierStatus.AlwaysIdentifier : ParamIdentifierStatus.NeverIdentifier; } + + public override bool ComposeDependencyInfo(IR.Nodes.CallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + node.FunctionSupportColumnNamesAsIdentifiersDependencyUtil(visitor); + + return true; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/SortByColumns.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/SortByColumns.cs index d2b9bff807..ec4463cd84 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/SortByColumns.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/SortByColumns.cs @@ -7,18 +7,22 @@ using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; -using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Entities.QueryOptions; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.Functions.Delegation; using Microsoft.PowerFx.Core.Functions.Delegation.DelegationMetadata; using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Types.Enums; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Types; +using CallNode = Microsoft.PowerFx.Syntax.CallNode; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -543,5 +547,12 @@ public override ParamIdentifierStatus GetIdentifierParamStatus(TexlNode node, Fe return (index % 2) == 1 ? ParamIdentifierStatus.PossiblyIdentifier : ParamIdentifierStatus.NeverIdentifier; } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + node.FunctionSupportColumnNamesAsIdentifiersDependencyUtil(visitor); + + return true; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs index f285151580..4cb08064dc 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Summarize.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; @@ -259,6 +260,27 @@ internal override IntermediateNode CreateIRCallNode(CallNode node, IRTranslatorC } return new IRCallNode(context.GetIRContext(node), this, scope, newArgs); + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableTypeName = ((AggregateType)node.Args[0].IRContext.ResultType).TableSymbolName; + + foreach (var arg in node.Args.Skip(1)) + { + if (arg is TextLiteralNode textLiteralNode) + { + visitor.AddDependency(tableTypeName, textLiteralNode.LiteralValue); + } + else if (arg is LazyEvalNode lazyEvalNode) + { + var recordNode = (RecordNode)lazyEvalNode.Child; + + recordNode.Fields.Values.First().Accept(visitor, context); + } + } + + return true; } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs index 7ce73eeba3..cda01cf15f 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs @@ -10,13 +10,13 @@ using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; -using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; using static Microsoft.PowerFx.Core.Localization.TexlStrings; -using static Microsoft.PowerFx.Syntax.PrettyPrintVisitor; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; namespace Microsoft.PowerFx.Functions { @@ -235,8 +235,27 @@ public async Task InvokeAsync(FunctionInvokeInfo invokeInfo, Cance { result = returnType == FormulaType.Void ? FormulaValue.NewVoid() : FormulaValue.NewBlank(); } - + return result; + } + + public override bool ComposeDependencyInfo(IRCallNode node, DependencyVisitor visitor, DependencyVisitor.DependencyContext context) + { + var tableType = (TableType)node.Args[0].IRContext.ResultType; + + foreach (var arg in node.Args.Skip(1).Where(a => a.IRContext.ResultType is AggregateType)) + { + var argType = arg.IRContext.ResultType; + + foreach (var name in argType._type.GetAllNames(DPath.Root)) + { + visitor.AddDependency(tableType.TableSymbolName, name.Name.Value); + } + + arg.Accept(visitor, context); + } + + return true; } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs index 1dfa1f1cc1..18323d4797 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs @@ -2,16 +2,17 @@ // Licensed under the MIT license. using System.Collections.Generic; -using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; -using static Microsoft.PowerFx.Core.Localization.TexlStrings; +using static Microsoft.PowerFx.Core.Localization.TexlStrings; +using IRCallNode = Microsoft.PowerFx.Core.IR.Nodes.CallNode; namespace Microsoft.PowerFx.Interpreter { @@ -147,4 +148,4 @@ private bool CheckMutability(TexlBinding binding, TexlNode[] args, DType[] argTy return false; } } -} +} diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs index 177c71f5a3..8a6fc34153 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs @@ -76,6 +76,7 @@ public static DType GetDType(bool hasCachedCountRows = false) displayNameMapping.Add("address1_line1", "Address 1: Street 1"); displayNameMapping.Add("nonsearchablestringcol", "Non-searchable string column"); displayNameMapping.Add("nonsortablestringcolumn", "Non-sortable string column"); + displayNameMapping.Add("numberofemployees", "Number of employees"); return DType.AttachDataSourceInfo(accountsType, dataSource); } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs index 484d1bd13c..ee70f4b2db 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs @@ -215,7 +215,8 @@ public void PublicSurface_Tests() "Microsoft.PowerFx.Logging.ITracer", "Microsoft.PowerFx.Logging.TraceSeverity", "Microsoft.PowerFx.PowerFxFileInfo", - "Microsoft.PowerFx.UserInfo" + "Microsoft.PowerFx.UserInfo", + "Microsoft.PowerFx.Core.IR.DependencyInfo", }; var sb = new StringBuilder(); diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/DependencyTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/DependencyTests.cs new file mode 100644 index 0000000000..06d39006e5 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/DependencyTests.cs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; +using Microsoft.PowerFx.Core.Tests.AssociatedDataSourcesTests; +using Microsoft.PowerFx.Core.Tests.Helpers; +using Microsoft.PowerFx.Core.Texl; +using Microsoft.PowerFx.Core.Texl.Builtins; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Types.Enums; +using Microsoft.PowerFx.Functions; +using Microsoft.PowerFx.Interpreter; +using Microsoft.PowerFx.Types; +using Xunit; + +namespace Microsoft.PowerFx.Core.Tests +{ + public class DependencyTests : PowerFxTest + { + [Theory] + [InlineData("1+2", "")] // none + [InlineData("ThisRecord.'Address 1: City' & 'Account Name'", "Entity Accounts: address1_city, name;")] // basic read + + [InlineData("numberofemployees%", "Entity Accounts: numberofemployees;")] // unary op + [InlineData("ThisRecord", "Entity Accounts: ;")] // whole scope + [InlineData("{x:5}.x", "")] + [InlineData("With({x : ThisRecord}, x.'Address 1: City')", "Entity Accounts: address1_city;")] // alias + [InlineData("With({'Address 1: City' : \"Seattle\"}, 'Address 1: City' & 'Account Name')", "Entity Accounts: name;")] // 'Address 1: City' is shadowed + [InlineData("With({'Address 1: City' : 5}, ThisRecord.'Address 1: City')", "")] // shadowed + [InlineData("LookUp(local,'Address 1: City'=\"something\")", "Entity Accounts: address1_city;")] // Lookup and RowScope + [InlineData("Filter(local,numberofemployees > 200)", "Entity Accounts: numberofemployees;")] + [InlineData("First(local)", "Entity Accounts: ;")] + [InlineData("First(local).'Address 1: City'", "Entity Accounts: address1_city;")] + [InlineData("Last(local)", "Entity Accounts: ;")] + [InlineData("local", "Entity Accounts: ;")] // whole table + [InlineData("12 & true & \"abc\" ", "")] // walker ignores literals + [InlineData("12;'Address 1: City';12", "Entity Accounts: address1_city;")] // chaining + [InlineData("ParamLocal1.'Address 1: City'", "Entity Accounts: address1_city;")] // basic read + [InlineData("{test:First(local).name}", "Entity Accounts: name;")] + [InlineData("AddColumns(simple1, z, 1)", "Entity Simple1: ;")] + [InlineData("RenameColumns(simple1, b, c)", "Entity Simple1: b;")] + [InlineData("SortByColumns(simple1, a, SortOrder.Descending)", "Entity Simple1: a;")] + [InlineData("RenameColumns(If(false, simple1, Error({Kind:ErrorKind.Custom})), b, c)", "Entity Simple1: b;")] + [InlineData("SortByColumns(If(false, simple1, Error({Kind:ErrorKind.Custom})), a, SortOrder.Descending)", "Entity Simple1: a;")] + + // Basic scoping + [InlineData("Min(local,numberofemployees)", "Entity Accounts: numberofemployees;")] + [InlineData("Average(local,numberofemployees)", "Entity Accounts: numberofemployees;")] + + // Patch + [InlineData("Patch(simple2, First(simple1), { a : 1 })", "Entity Simple1: a, b; Entity Simple2: a, b;")] + [InlineData("Patch(local, {'Address 1: City':\"test\"}, { 'Account Name' : \"some name\"})", "Entity Accounts: address1_city, name;")] + [InlineData("Patch(local, {accountid:GUID(), 'Address 1: City':\"test\"})", "Entity Accounts: accountid, address1_city;")] + [InlineData("Patch(local, Table({accountid:GUID(), 'Address 1: City':\"test\"},{accountid:GUID(), 'Account Name':\"test\"}))", "Entity Accounts: accountid, address1_city, name;")] + [InlineData("Patch(local, Table({accountid:GUID(), 'Address 1: City':\"test\"},{accountid:GUID(), 'Account Name':\"test\"}),Table({'Address 1: City':\"test\"},{'Address 1: City':\"test\",'Account Name':\"test\"}))", "Entity Accounts: accountid, address1_city, name;")] + [InlineData("Patch(simple2, First(simple1), { a : First(simple1).b } )", "Entity Simple1: a, b; Entity Simple2: a, b;")] + [InlineData("Patch(simple1, First(simple1), { a : First(simple1).b } )", "Entity Simple1: a, b;")] + + // Remove + [InlineData("Remove(local, {name: First(remote).name})", "Entity Accounts: name; Entity Contacts: name;")] + + // Collect and ClearCollect. + [InlineData("Collect(local, Table({ 'Account Name' : \"some name\"}))", "Entity Accounts: name;")] + [InlineData("Collect(simple2, simple1)", "Entity Simple2: a, b; Entity Simple1: a, b;")] + [InlineData("Collect(simple2, { a : First(simple1).b })", "Entity Simple2: a; Entity Simple1: b;")] + [InlineData("Collect(local, { 'Address 1: City' : First(remote).'Contact Name' })", "Entity Accounts: address1_city; Entity Contacts: name;")] + [InlineData("ClearCollect(simple2, simple1)", "Entity Simple2: a, b; Entity Simple1: a, b;")] + [InlineData("ClearCollect(local, Table({ 'Account Name' : \"some name\"}))", "Entity Accounts: name;")] + + // Inside with. + [InlineData("With({r: local}, Filter(r, 'Number of employees' > 0))", "Entity Accounts: numberofemployees;")] + [InlineData("With({r: local}, LookUp(r, 'Number of employees' > 0))", "Entity Accounts: numberofemployees;")] + + // Option set. + [InlineData("Filter(local, dayofweek = StartOfWeek.Monday)", "Entity Accounts: dayofweek;")] + + [InlineData("Filter(ForAll(local, ThisRecord.numberofemployees), Value < 20)", "Entity Accounts: numberofemployees;")] + + // Summarize is special, becuase of ThisGroup. + [InlineData("Summarize(local, 'Account Name', Sum(ThisGroup, numberofemployees) As Employees)", "Entity Accounts: name, numberofemployees;")] + [InlineData("Summarize(local, 'Account Name', Sum(ThisGroup, numberofemployees * 2) As TPrice)", "Entity Accounts: name, numberofemployees;")] + + // Join + [InlineData("Join(remote As l, local As r, l.contactid = r.contactid, JoinType.Inner, r.name As AccountName)", "Entity Contacts: contactid; Entity Accounts: contactid, name;")] + [InlineData("Join(remote As l, local As r, l.contactid = r.contactid, JoinType.Inner, r.name As AccountName, l.contactnumber As NewContactNumber)", "Entity Contacts: contactid, contactnumber; Entity Accounts: contactid, name;")] + [InlineData("Join(remote, local, LeftRecord.contactid = RightRecord.contactid, JoinType.Inner, RightRecord.name As AccountName, LeftRecord.contactnumber As NewContactNumber)", "Entity Contacts: contactid, contactnumber; Entity Accounts: contactid, name;")] + + // Set + [InlineData("Set(numberofemployees, 200)", "Entity Accounts: numberofemployees;")] + [InlineData("Set('Address 1: City', 'Account Name')", "Entity Accounts: address1_city, name;")] + [InlineData("Set('Address 1: City', 'Address 1: City' & \"test\")", "Entity Accounts: address1_city;")] + [InlineData("Set(NewRecord.'Address 1: City', \"test\")", "Entity Accounts: address1_city;")] + + [InlineData("Filter(Distinct(ShowColumns(simple2, a, b), a), Value < 20)", "Entity Simple2: a, b;")] + [InlineData("Filter(Distinct(DropColumns(simple2, c), a), Value < 20)", "Entity Simple2: c, a;")] + + [InlineData("AddColumns(simple1, z, a+1)", "Entity Simple1: a;")] + public void GetDependencies(string expr, string expected) + { + var opt = new ParserOptions() { AllowsSideEffects = true }; + var engine = new Engine(); + + var check = new CheckResult(engine) + .SetText(expr, opt) + .SetBindingInfo(GetSymbols()); + + check.ApplyBinding(); + +#pragma warning disable CS0618 // Type or member is obsolete + var info = check.ApplyDependencyInfoScan(); +#pragma warning restore CS0618 // Type or member is obsolete + var actual = info.ToString().Replace("\r", string.Empty).Replace("\n", string.Empty).Trim(); + Assert.Equal(expected, actual); + } + + private ReadOnlySymbolTable GetSymbols() + { + var localType = Accounts(); + var remoteType = Contacts(); + var simple1Type = Simple1(); + var simple2Type = Simple2(); + var customSymbols = new SymbolTable { DebugName = "Custom symbols " }; + var opt = new ParserOptions() { AllowsSideEffects = true }; + + var thisRecordScope = ReadOnlySymbolTable.NewFromRecord(localType.ToRecord(), allowThisRecord: true, allowMutable: true); + + customSymbols.AddFunction(new JoinFunction()); + customSymbols.AddFunction(new CollectFunction()); + customSymbols.AddFunction(new CollectScalarFunction()); + customSymbols.AddFunction(new ClearCollectFunction()); + customSymbols.AddFunction(new ClearCollectScalarFunction()); + customSymbols.AddFunction(new PatchFunction()); + customSymbols.AddFunction(new PatchAggregateFunction()); + customSymbols.AddFunction(new PatchAggregateSingleTableFunction()); + customSymbols.AddFunction(new PatchSingleRecordFunction()); + customSymbols.AddFunction(new SummarizeFunction()); + customSymbols.AddFunction(new RecalcEngineSetFunction()); + customSymbols.AddFunction(new RemoveFunction()); + customSymbols.AddFunction(Library.DistinctInterpreterFunction); + customSymbols.AddVariable("local", localType, mutable: true); + customSymbols.AddVariable("remote", remoteType, mutable: true); + customSymbols.AddVariable("simple1", simple1Type, mutable: true); + customSymbols.AddVariable("simple2", simple2Type, mutable: true); + + // Simulate a parameter + var parameterSymbols = new SymbolTable { DebugName = "Parameters " }; + parameterSymbols.AddVariable("ParamLocal1", localType.ToRecord(), mutable: true); + parameterSymbols.AddVariable("NewRecord", localType.ToRecord(), new SymbolProperties() { CanMutate = false, CanSet = false, CanSetMutate = true }); + + return ReadOnlySymbolTable.Compose(customSymbols, thisRecordScope, parameterSymbols); + } + + private TableType Accounts() + { + var tableType = (TableType)FormulaType.Build(AccountsTypeHelper.GetDType()); + tableType = tableType.Add("dayofweek", BuiltInEnums.StartOfWeekEnum.FormulaType); + tableType = tableType.Add("contactid", FormulaType.Guid); + + return tableType; + } + + private TableType Contacts() + { + var simplifiedAccountsSchema = "*[contactid:g, contactnumber:s, name`Contact Name`:s, address1_addresstypecode:l, address1_city`Address 1: City`:s, address1_composite:s, address1_country:s, address1_county:s, address1_line1`Address 1: Street 1`:s, numberofemployees:n]"; + + DType contactType = TestUtils.DT2(simplifiedAccountsSchema); + var dataSource = new TestDataSource( + "Contacts", + contactType, + keyColumns: new[] { "contactid" }, + selectableColumns: new[] { "name", "address1_city", "contactid", "address1_country", "address1_line1" }, + hasCachedCountRows: false); + var displayNameMapping = dataSource.DisplayNameMapping; + displayNameMapping.Add("name", "Contact Name"); + displayNameMapping.Add("address1_city", "Address 1: City"); + displayNameMapping.Add("address1_line1", "Address 1: Street 1"); + displayNameMapping.Add("numberofemployees", "Number of employees"); + + contactType = DType.AttachDataSourceInfo(contactType, dataSource); + + return (TableType)FormulaType.Build(contactType); + } + + // Some test cases can produce a long list of dependencies. + // This method is used to simplify the schema for those cases. + private TableType Simple1() + { + var simplifiedchema = "*[a:w,b:w]"; + + DType type = TestUtils.DT2(simplifiedchema); + var dataSource = new TestDataSource( + "Simple1", + type, + keyColumns: new[] { "a" }); + + type = DType.AttachDataSourceInfo(type, dataSource); + + return (TableType)FormulaType.Build(type); + } + + private TableType Simple2() + { + var simplifiedchema = "*[a:w,b:w,c:s]"; + + DType type = TestUtils.DT2(simplifiedchema); + var dataSource = new TestDataSource( + "Simple2", + type, + keyColumns: new[] { "a" }); + + type = DType.AttachDataSourceInfo(type, dataSource); + + return (TableType)FormulaType.Build(type); + } + + /// + /// This test case is to ensure that all functions that are not self-contained or + /// have a scope info have been assessed and either added to the exception list or overrides . + /// + [Fact] + public void DepedencyScanFunctionTests() + { + var names = new List(); + var functions = new List(); + functions.AddRange(BuiltinFunctionsCore.BuiltinFunctionsLibrary); + + // These methods use default implementation of ComposeDependencyInfo and do not neeed to override it. + var exceptionList = new HashSet() + { + "AddColumns", + "Average", + "Concat", + "CountIf", + "Filter", + "ForAll", + "IfError", + "LookUp", + "Max", + "Min", + "Refresh", + "Search", + "Sort", + "StdevP", + "Set", + "Sum", + "Trace", + "VarP", + "With", + }; + + foreach (var func in functions) + { + if (!func.IsSelfContained || func.ScopeInfo != null) + { + var irContext = IRContext.NotInSource(FormulaType.String); + var node = new CallNode(irContext, func, new ErrorNode(irContext, "test")); + var visitor = new DependencyVisitor(); + var context = new DependencyVisitor.DependencyContext(); + var overwritten = func.ComposeDependencyInfo(node, visitor, context); + if (!overwritten && !exceptionList.Contains(func.Name)) + { + names.Add(func.Name); + } + } + } + + if (names.Count > 0) + { + var sb = new StringBuilder(); + sb.AppendLine("The following functions do not have a dependency scan:"); + foreach (var name in names) + { + sb.AppendLine(name); + } + + Assert.Fail(sb.ToString()); + } + } + } +}