Merge branch 'microsoft:main' into main

src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs

@@ -36,6 +36,8 @@ public ExternalCdpDataSource(DName name, string datasetName, ServiceCapabilities
 
         public TabularDataQueryOptions QueryOptions => new TabularDataQueryOptions(this);
 
+        public bool HasCachedCountRows => false;
+
         public string Name => EntityName.Value;
 
         public bool IsSelectable => ServiceCapabilities.IsSelectable;

src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs

@@ -11,6 +11,13 @@ internal interface IExternalTabularDataSource : IExternalDataSource, IDisplayMap
     {
         TabularDataQueryOptions QueryOptions { get; }
 
+        /// <summary>
+        /// Some data sources (like Dataverse) may return a cached value for
+        /// the number of rows (calls to CountRows) instead of always retrieving
+        /// the latest count.
+        /// </summary>
+        bool HasCachedCountRows { get; }
+
         IReadOnlyList<string> GetKeyColumns();
 
         IEnumerable<string> GetKeyColumns(IExpandInfo expandInfo);
@@ -23,4 +30,4 @@ internal interface IExternalTabularDataSource : IExternalDataSource, IDisplayMap
 
         bool CanIncludeExpand(IExpandInfo parentExpandInfo, IExpandInfo expandToAdd);
     }
-}
+}

src/libraries/Microsoft.PowerFx.Core/Lexer/TexlLexer.cs

@@ -81,7 +81,8 @@ public enum Flags
         public const string PunctuatorColon = ":";
         public const string PunctuatorAt = "@";
         public const char IdentifierDelimiter = '\'';
-        public const string PunctuatorDoubleBarrelArrow = "=>";
+        public const string PunctuatorDoubleBarrelArrow = "=>";
+        public const string PunctuatorColonEqual = ":=";
 
         // These puntuators are related to commenting in the formula bar
         public const string PunctuatorBlockComment = "/*";
@@ -305,7 +306,8 @@ private TexlLexer(string preferredDecimalSeparator)
             AddPunctuator(punctuators, PunctuatorAmpersand, TokKind.Ampersand);
             AddPunctuator(punctuators, PunctuatorPercent, TokKind.PercentSign);
             AddPunctuator(punctuators, PunctuatorAt, TokKind.At);
-            AddPunctuator(punctuators, PunctuatorDoubleBarrelArrow, TokKind.DoubleBarrelArrow);
+            AddPunctuator(punctuators, PunctuatorDoubleBarrelArrow, TokKind.DoubleBarrelArrow);
+            AddPunctuator(punctuators, PunctuatorColonEqual, TokKind.ColonEqual);
 
             // Commenting punctuators
             AddPunctuator(punctuators, PunctuatorBlockComment, TokKind.Comment);

src/libraries/Microsoft.PowerFx.Core/Lexer/TokKind.cs

@@ -303,6 +303,12 @@ public enum TokKind
         /// Start of body for user defined functions.
         /// <code>=></code>
         /// </summary>
-        DoubleBarrelArrow,
+        DoubleBarrelArrow,
+
+        /// <summary>
+        /// Colon immediately followed by equal.
+        /// <code>:=</code>
+        /// </summary>
+        ColonEqual,
     }
 }

src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs

@@ -750,6 +750,8 @@ internal static class TexlStrings
         public static ErrorResourceKey WarnDeferredType = new ErrorResourceKey("WarnDeferredType");
         public static ErrorResourceKey ErrColRenamedTwice_Name = new ErrorResourceKey("ErrColRenamedTwice_Name");
 
+        public static ErrorResourceKey WrnCountRowsMayReturnCachedValue = new ErrorResourceKey("WrnCountRowsMayReturnCachedValue");
+
         public static StringGetter InfoMessage = (b) => StringResources.Get("InfoMessage", b);
         public static StringGetter InfoNode_Node = (b) => StringResources.Get("InfoNode_Node", b);
         public static StringGetter InfoTok_Tok = (b) => StringResources.Get("InfoTok_Tok", b);
@@ -766,6 +768,7 @@ internal static class TexlStrings
 
         public static ErrorResourceKey ErrNamedFormula_MissingSemicolon = new ErrorResourceKey("ErrNamedFormula_MissingSemicolon");
         public static ErrorResourceKey ErrNamedFormula_MissingValue = new ErrorResourceKey("ErrNamedFormula_MissingValue");
+        public static ErrorResourceKey ErrNamedType_MissingTypeLiteral = new ErrorResourceKey("ErrNamedType_MissingTypeLiteral");
         public static ErrorResourceKey ErrUDF_MissingFunctionBody = new ErrorResourceKey("ErrUDF_MissingFunctionBody");
         public static ErrorResourceKey ErrNamedFormula_AlreadyDefined = new ErrorResourceKey("ErrNamedFormula_AlreadyDefined");
         public static ErrorResourceKey ErrorResource_NameConflict = new ErrorResourceKey("ErrorResource_NameConflict");
@@ -848,5 +851,7 @@ internal static class TexlStrings
         public static ErrorResourceKey ErrInvalidDataSourceForFunction = new ErrorResourceKey("ErrInvalidDataSourceForFunction");
         public static ErrorResourceKey ErrInvalidArgumentExpectedType = new ErrorResourceKey("ErrInvalidArgumentExpectedType");
         public static ErrorResourceKey ErrUnsupportedTypeInTypeArgument = new ErrorResourceKey("ErrUnsupportedTypeInTypeArgument");
+        public static ErrorResourceKey ErrReachedMaxJsonDepth = new ErrorResourceKey("ErrReachedMaxJsonDepth");
+        public static ErrorResourceKey ErrReachedMaxJsonLength = new ErrorResourceKey("ErrReachedMaxJsonLength");
     }
 }

src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs

@@ -300,15 +300,15 @@ private ParseUserDefinitionResult ParseUDFsAndNamedFormulas(string script, Parse
                     continue;
                 }
 
-                if (_curs.TidCur == TokKind.Equ)
+                if (_curs.TidCur == TokKind.ColonEqual && _flagsMode.Peek().HasFlag(Flags.AllowTypeLiteral))
                 {
                     var declaration = script.Substring(declarationStart, _curs.TokCur.Span.Min - declarationStart);
                     _curs.TokMove();
                     definitionBeforeTrivia.Add(ParseTrivia());
 
                     if (_curs.TidCur == TokKind.Semicolon)
                     {
-                        CreateError(thisIdentifier, TexlStrings.ErrNamedFormula_MissingValue);
+                        CreateError(thisIdentifier, TexlStrings.ErrNamedType_MissingTypeLiteral);
                     }
 
                     // Extract expression
@@ -334,6 +334,49 @@ private ParseUserDefinitionResult ParseUDFsAndNamedFormulas(string script, Parse
                             definitionBeforeTrivia = new List<ITexlSource>();
                             continue;
                         }
+                        else
+                        {
+                            CreateError(_curs.TokCur, TexlStrings.ErrNamedType_MissingTypeLiteral);
+                        }
+
+                        // If the result was an error, keep moving cursor until end of named type expression
+                        if (result.Kind == NodeKind.Error)
+                        {
+                            while (_curs.TidCur != TokKind.Semicolon && _curs.TidCur != TokKind.Eof)
+                            {
+                                _curs.TokMove();
+                            }
+                        }
+                    }
+
+                    declarationStart = _curs.TokCur.Span.Lim;
+                    _curs.TokMove();
+                    ParseTrivia();
+                }
+                else if (_curs.TidCur == TokKind.Equ)
+                {
+                    var declaration = script.Substring(declarationStart, _curs.TokCur.Span.Min - declarationStart);
+                    _curs.TokMove();
+                    definitionBeforeTrivia.Add(ParseTrivia());
+
+                    if (_curs.TidCur == TokKind.Semicolon)
+                    {
+                        CreateError(thisIdentifier, TexlStrings.ErrNamedFormula_MissingValue);
+                    }
+
+                    // Extract expression
+                    while (_curs.TidCur != TokKind.Semicolon)
+                    {
+                        // Check if we're at EOF before a semicolon is found
+                        if (_curs.TidCur == TokKind.Eof)
+                        {
+                            CreateError(_curs.TokCur, TexlStrings.ErrNamedFormula_MissingSemicolon);
+                            break;
+                        }
+
+                        // Parse expression
+                        definitionBeforeTrivia.Add(ParseTrivia());
+                        var result = ParseExpr(Precedence.None);
 
                         namedFormulas.Add(new NamedFormula(thisIdentifier.As<IdentToken>(), new Formula(result.GetCompleteSpan().GetFragment(script), result), _startingIndex, attribute));
                         userDefinitionSourceInfos.Add(new UserDefinitionSourceInfo(index++, UserDefinitionType.NamedFormula, thisIdentifier.As<IdentToken>(), declaration, new SourceList(definitionBeforeTrivia), GetExtraTriviaSourceList()));

src/libraries/Microsoft.PowerFx.Core/Public/Canceller.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.PowerFx
+{
+    internal class Canceller
+    {
+        private readonly List<Action> _cancellationAction;
+
+        public Canceller()
+        {
+            _cancellationAction = new List<Action>();
+        }
+
+        public Canceller(params Action[] cancellationActions)
+            : this()
+        {
+            if (cancellationActions != null)
+            {
+                foreach (Action cancellationAction in cancellationActions)
+                {
+                    AddAction(cancellationAction);
+                }
+            }
+        }
+
+        public void AddAction(Action cancellationAction)
+        {
+            if (cancellationAction != null)
+            {
+                _cancellationAction.Add(cancellationAction);
+            }
+        }
+
+        public void ThrowIfCancellationRequested()
+        {
+            foreach (Action cancellationAction in _cancellationAction)
+            {
+                cancellationAction();
+            }
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs

@@ -208,63 +208,39 @@ public void AddConstant(string name, FormulaValue data)
         }
 
         /// <summary>
-        /// Adds an user defined function.
+        /// Adds user defined functions in the script.
         /// </summary>
         /// <param name="script">String representation of the user defined function.</param>
         /// <param name="parseCulture">CultureInfo to parse the script againts. Default is invariant.</param>
         /// <param name="symbolTable">Extra symbols to bind UDF. Commonly coming from Engine.</param>
         /// <param name="extraSymbolTable">Additional symbols to bind UDF.</param>
         /// <param name="allowSideEffects">Allow for curly brace parsing.</param>
-        internal void AddUserDefinedFunction(string script, CultureInfo parseCulture = null, ReadOnlySymbolTable symbolTable = null, ReadOnlySymbolTable extraSymbolTable = null, bool allowSideEffects = false)
+        internal DefinitionsCheckResult AddUserDefinedFunction(string script, CultureInfo parseCulture = null, ReadOnlySymbolTable symbolTable = null, ReadOnlySymbolTable extraSymbolTable = null, bool allowSideEffects = false)
         {
             // Phase 1: Side affects are not allowed.
             // Phase 2: Introduces side effects and parsing of function bodies.
             var options = new ParserOptions() 
             { 
                 AllowsSideEffects = allowSideEffects, 
-                Culture = parseCulture ?? CultureInfo.InvariantCulture 
+                Culture = parseCulture ?? CultureInfo.InvariantCulture,
             };
-            var sb = new StringBuilder();
 
-            var parseResult = UserDefinitions.Parse(script, options);
-
-            // Compose will handle null symbols
             var composedSymbols = Compose(this, symbolTable, extraSymbolTable);
 
-            var udfs = UserDefinedFunction.CreateFunctions(parseResult.UDFs.Where(udf => udf.IsParseValid), composedSymbols, out var errors);
-
-            errors.AddRange(parseResult.Errors ?? Enumerable.Empty<TexlError>());
+            var checkResult = new DefinitionsCheckResult();
 
-            if (errors.Any(error => error.Severity > DocumentErrorSeverity.Warning))
-            {
-                sb.AppendLine("Something went wrong when parsing user defined functions.");
+            var udfs = checkResult.SetText(script, options)
+                .SetBindingInfo(composedSymbols)
+                .ApplyCreateUserDefinedFunctions();
 
-                foreach (var error in errors)
-                {
-                    error.FormatCore(sb);
-                }
-
-                throw new InvalidOperationException(sb.ToString());
-            }
+            Contracts.AssertValue(udfs);
 
-            foreach (var udf in udfs)
+            if (checkResult.IsSuccess)
             {
-                AddFunction(udf);
-                var config = new BindingConfig(allowsSideEffects: allowSideEffects, useThisRecordForRuleScope: false, numberIsFloat: false);
-                var binding = udf.BindBody(composedSymbols, new Glue2DocumentBinderGlue(), config);
-
-                List<TexlError> bindErrors = new List<TexlError>();
-
-                if (binding.ErrorContainer.GetErrors(ref bindErrors))
-                {
-                    sb.AppendLine(string.Join(", ", errors.Select(err => err.ToString())));
-                }
+                AddFunctions(udfs);
             }
 
-            if (sb.Length > 0)
-            {
-                throw new InvalidOperationException(sb.ToString());
-            }
+            return checkResult;
         }
 
         /// <summary>