From 166ceb2bfcddc1f26294204e438402fdc5e12146 Mon Sep 17 00:00:00 2001 From: Carlos Figueira Date: Wed, 18 Sep 2024 10:56:55 -0700 Subject: [PATCH 01/13] Add a warning to the CountRows function if a data source that caches its count is passed to it (#2640) Some data sources, like Dataverse, will cache its count and not return the accurate result if a CountRows call is made to it (see note at https://learn.microsoft.com/power-platform/power-fx/reference/function-table-counts#description). This adds a warning to the node if this happens, to make it more explicit to the makers that this is the case. --- .../Tabular/ExternalCdpDataSource.cs | 2 ++ .../External/IExternalTabularDataSource.cs | 9 ++++- .../Localization/Strings.cs | 2 ++ .../Texl/Builtins/CountRows.cs | 27 +++++++++++++++ src/strings/PowerFxResources.en-US.resx | 4 +++ .../TestDVEntity.cs | 14 ++++++-- .../TestDelegationValidation.cs | 34 +++++++++++++++++++ .../Helpers/TestTabularDataSource.cs | 6 +++- 8 files changed, 93 insertions(+), 5 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs index b213734b68..5badccba7c 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs @@ -36,6 +36,8 @@ public ExternalCdpDataSource(DName name, string datasetName, ServiceCapabilities public TabularDataQueryOptions QueryOptions => new TabularDataQueryOptions(this); + public bool HasCachedCountRows => false; + public string Name => EntityName.Value; public bool IsSelectable => ServiceCapabilities.IsSelectable; diff --git a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs index 1c30c12158..7d69e78e0c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs @@ -11,6 +11,13 @@ internal interface IExternalTabularDataSource : IExternalDataSource, IDisplayMap { TabularDataQueryOptions QueryOptions { get; } + /// + /// Some data sources (like Dataverse) may return a cached value for + /// the number of rows (calls to CountRows) instead of always retrieving + /// the latest count. + /// + bool HasCachedCountRows { get; } + IReadOnlyList GetKeyColumns(); IEnumerable GetKeyColumns(IExpandInfo expandInfo); @@ -23,4 +30,4 @@ internal interface IExternalTabularDataSource : IExternalDataSource, IDisplayMap bool CanIncludeExpand(IExpandInfo parentExpandInfo, IExpandInfo expandToAdd); } -} \ No newline at end of file +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 68008bb151..288586637b 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -745,6 +745,8 @@ internal static class TexlStrings public static ErrorResourceKey WarnDeferredType = new ErrorResourceKey("WarnDeferredType"); public static ErrorResourceKey ErrColRenamedTwice_Name = new ErrorResourceKey("ErrColRenamedTwice_Name"); + public static ErrorResourceKey WrnCountRowsMayReturnCachedValue = new ErrorResourceKey("WrnCountRowsMayReturnCachedValue"); + public static StringGetter InfoMessage = (b) => StringResources.Get("InfoMessage", b); public static StringGetter InfoNode_Node = (b) => StringResources.Get("InfoNode_Node", b); public static StringGetter InfoTok_Tok = (b) => StringResources.Get("InfoTok_Tok", b); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs index e04184c204..60caada7f3 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs @@ -5,6 +5,7 @@ using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Binding.BindInfo; using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; @@ -68,6 +69,32 @@ public override bool IsServerDelegatable(CallNode callNode, TexlBinding binding) return TryGetValidDataSourceForDelegation(callNode, binding, out var dataSource, out var preferredFunctionDelegationCapability); } + public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + base.CheckSemantics(binding, args, argTypes, errors); + if (args[0] is not FirstNameNode node) + { + // No additional check + return; + } + + var info = binding.GetInfo(args[0] as FirstNameNode); + if (info.Kind != BindKind.Data) + { + // No additional check + return; + } + + if (argTypes[0].AssociatedDataSources?.Count == 1) + { + var associatedDataSource = argTypes[0].AssociatedDataSources.Single(); + if (associatedDataSource.HasCachedCountRows) + { + errors.EnsureError(DocumentErrorSeverity.Warning, node, TexlStrings.WrnCountRowsMayReturnCachedValue); + } + } + } + // See if CountDistinct delegation is available. If true, we can make use of it on primary key as a workaround for CountRows delegation internal bool TryGetValidDataSourceForDelegation(CallNode callNode, TexlBinding binding, out IExternalDataSource dataSource, out DelegationCapability preferredFunctionDelegationCapability) { diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 3f818dcd1a..e063733cf3 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4332,6 +4332,10 @@ Can't delegate {0}: contains a behavior function '{1}'. Warning message. + + CountRows may return a cached value. Use CountIf(DataSource, true) to get the latest count. + {Locked=CountRows}. Warning message when an expression with the CountRows function is used with a data source that caches its size. + Determines if the supplied text has a match of the supplied text format. Description of 'IsMatch' function. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs index 7905edcdd9..177c71f5a3 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs @@ -15,6 +15,13 @@ namespace Microsoft.PowerFx.Core.Tests.AssociatedDataSourcesTests { public class AccountsEntity : IExternalEntity, IExternalDataSource { + private readonly bool _hasCachedCountRows; + + public AccountsEntity(bool hasCachedCountRows = false) + { + this._hasCachedCountRows = hasCachedCountRows; + } + public DName EntityName => new DName("Accounts"); public string Name => "Accounts"; @@ -33,7 +40,7 @@ public class AccountsEntity : IExternalEntity, IExternalDataSource public bool IsClearable => true; - DType IExternalEntity.Type => AccountsTypeHelper.GetDType(); + DType IExternalEntity.Type => AccountsTypeHelper.GetDType(this._hasCachedCountRows); IExternalDataEntityMetadataProvider IExternalDataSource.DataEntityMetadataProvider => throw new NotImplementedException(); @@ -54,14 +61,15 @@ internal static class AccountsTypeHelper "name`Account Name`:s, numberofemployees:n, primarytwitterid:s, stockexchange:s, telephone1:s, telephone2:s, telephone3:s, tickersymbol:s, versionnumber:n, " + "websiteurl:h, nonsearchablestringcol`Non-searchable string column`:s, nonsortablestringcolumn`Non-sortable string column`:s]"; - public static DType GetDType() + public static DType GetDType(bool hasCachedCountRows = false) { DType accountsType = TestUtils.DT2(SimplifiedAccountsSchema); var dataSource = new TestDataSource( "Accounts", accountsType, keyColumns: new[] { "accountid" }, - selectableColumns: new[] { "name", "address1_city", "accountid", "address1_country", "address1_line1" }); + selectableColumns: new[] { "name", "address1_city", "accountid", "address1_country", "address1_line1" }, + hasCachedCountRows: hasCachedCountRows); var displayNameMapping = dataSource.DisplayNameMapping; displayNameMapping.Add("name", "Account Name"); displayNameMapping.Add("address1_city", "Address 1: City"); diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs index b8c5761a88..50ff69317a 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using Microsoft.PowerFx.Core.Entities.QueryOptions; +using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Tests.Helpers; using Microsoft.PowerFx.Core.Texl; using Microsoft.PowerFx.Types; @@ -100,5 +101,38 @@ private void TestDelegableExpressions(Features features, string expression, bool // validate we can generate the display expression string displayExpr = engine.GetDisplayExpression(expression, symbolTable); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void TestCountRowsWarningForCachedData(bool isCachedData) + { + var symbolTable = new DelegatableSymbolTable(); + symbolTable.AddEntity(new AccountsEntity(isCachedData)); + var config = new PowerFxConfig(Features.PowerFxV1) + { + SymbolTable = symbolTable + }; + + var engine = new Engine(config); + var result = engine.Check("CountRows(Accounts)"); + Assert.True(result.IsSuccess); + + if (!isCachedData) + { + Assert.Empty(result.Errors); + } + else + { + Assert.Single(result.Errors); + var error = result.Errors.Single(); + Assert.Equal(ErrorSeverity.Warning, error.Severity); + } + + // Only shows warning if data source is passed directly to CountRows + result = engine.Check("CountRows(Filter(Accounts, IsBlank('Address 1: City')))"); + Assert.True(result.IsSuccess); + Assert.Empty(result.Errors); + } } } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs index 5d661e394a..cfc2945768 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs @@ -175,8 +175,9 @@ internal class TestDataSource : IExternalDataSource, IExternalTabularDataSource private readonly string[] _keyColumns; private readonly HashSet _selectableColumns; private readonly TabularDataQueryOptions _tabularDataQueryOptions; + private readonly bool _hasCachedCountRows; - internal TestDataSource(string name, DType schema, string[] keyColumns = null, IEnumerable selectableColumns = null) + internal TestDataSource(string name, DType schema, string[] keyColumns = null, IEnumerable selectableColumns = null, bool hasCachedCountRows = false) { ExternalDataEntityMetadataProvider = new ExternalDataEntityMetadataProvider(); Type = DType.AttachDataSourceInfo(schema, this); @@ -185,10 +186,13 @@ internal TestDataSource(string name, DType schema, string[] keyColumns = null, I _keyColumns = keyColumns ?? Array.Empty(); _selectableColumns = new HashSet(selectableColumns ?? Enumerable.Empty()); _tabularDataQueryOptions = new TabularDataQueryOptions(this); + _hasCachedCountRows = hasCachedCountRows; } public string Name { get; } + public bool HasCachedCountRows => this._hasCachedCountRows; + public virtual bool IsSelectable => true; public virtual bool IsDelegatable => throw new NotImplementedException(); From f4232bf481f54958f60d6269f2f2d7607c018e90 Mon Sep 17 00:00:00 2001 From: Jas Valgotar <32079188+jas-valgotar@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:46:18 -0400 Subject: [PATCH 02/13] Adds async way to get value from NamedValue (#2642) --- .../Public/Values/NamedValue.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Values/NamedValue.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Values/NamedValue.cs index f3a9d14ae5..9f62898a91 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Values/NamedValue.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Values/NamedValue.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Threading; using System.Threading.Tasks; using Microsoft.PowerFx.Core.Types; @@ -17,7 +18,17 @@ public class NamedValue { public string Name { get; } - public FormulaValue Value => _value ?? _getFormulaValue().ConfigureAwait(false).GetAwaiter().GetResult(); + public FormulaValue Value => ValueAsync().GetAwaiter().GetResult(); + + public async Task ValueAsync() + { + if (_value != null) + { + return _value; + } + + return await _getFormulaValue().ConfigureAwait(false); + } /// /// Useful for determining if the value is an entity or not. From b87018f486002dfc27b756680546c2c617158e98 Mon Sep 17 00:00:00 2001 From: Carlos Figueira Date: Thu, 19 Sep 2024 09:12:39 -0700 Subject: [PATCH 03/13] Prevent NullReferenceException found in telemetry (#2644) We found a NRE in Power Apps telemetry that was happening in the Intellisense code on Power Fx. We don't know exactly the cause, but this change adds a defense-in-depth that will help preventing that. --- .../Texl/Intellisense/Intellisense.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Intellisense/Intellisense.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Intellisense/Intellisense.cs index 58d9a85b0c..69c7f3e2dc 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Intellisense/Intellisense.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Intellisense/Intellisense.cs @@ -187,13 +187,14 @@ protected static void TypeMatchPriority(DType type, IList Date: Mon, 23 Sep 2024 09:25:43 -0700 Subject: [PATCH 04/13] Enable UDF on REPL (#2559) Adds ability to define User-defined functions in REPL. Parsing UDFs requires a different parser from the Texl Expression parser and this is addressed by using the DefinitionsParser as a fallback to check when regular parsing fails. If definitions parsing is successful and we find UDF, we add it to the Engine. ![image](https://github.com/user-attachments/assets/ee86d499-2ee6-4232-a7cd-3b5ca9e09d1e) Fixes #2546 --- .../Public/Config/SymbolTable.cs | 46 +++-------- .../Public/DefinitionsCheckResult.cs | 79 ++++++++++++++++++- .../Microsoft.PowerFx.Core/Public/Engine.cs | 4 +- .../RecalcEngine.cs | 7 +- src/libraries/Microsoft.PowerFx.Repl/Repl.cs | 27 +++++++ .../UserDefinedFunctionTests.cs | 2 +- .../RecalcEngineTests.cs | 44 +++++------ .../ReplTests.cs | 25 ++++++ src/tools/Repl/Program.cs | 23 +++++- 9 files changed, 191 insertions(+), 66 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs index 86b1d4e6ea..a8aa0d3ec9 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs @@ -208,63 +208,39 @@ public void AddConstant(string name, FormulaValue data) } /// - /// Adds an user defined function. + /// Adds user defined functions in the script. /// /// String representation of the user defined function. /// CultureInfo to parse the script againts. Default is invariant. /// Extra symbols to bind UDF. Commonly coming from Engine. /// Additional symbols to bind UDF. /// Allow for curly brace parsing. - internal void AddUserDefinedFunction(string script, CultureInfo parseCulture = null, ReadOnlySymbolTable symbolTable = null, ReadOnlySymbolTable extraSymbolTable = null, bool allowSideEffects = false) + internal DefinitionsCheckResult AddUserDefinedFunction(string script, CultureInfo parseCulture = null, ReadOnlySymbolTable symbolTable = null, ReadOnlySymbolTable extraSymbolTable = null, bool allowSideEffects = false) { // Phase 1: Side affects are not allowed. // Phase 2: Introduces side effects and parsing of function bodies. var options = new ParserOptions() { AllowsSideEffects = allowSideEffects, - Culture = parseCulture ?? CultureInfo.InvariantCulture + Culture = parseCulture ?? CultureInfo.InvariantCulture, }; - var sb = new StringBuilder(); - var parseResult = UserDefinitions.Parse(script, options); - - // Compose will handle null symbols var composedSymbols = Compose(this, symbolTable, extraSymbolTable); - var udfs = UserDefinedFunction.CreateFunctions(parseResult.UDFs.Where(udf => udf.IsParseValid), composedSymbols, out var errors); - - errors.AddRange(parseResult.Errors ?? Enumerable.Empty()); + var checkResult = new DefinitionsCheckResult(); - if (errors.Any(error => error.Severity > DocumentErrorSeverity.Warning)) - { - sb.AppendLine("Something went wrong when parsing user defined functions."); + var udfs = checkResult.SetText(script, options) + .SetBindingInfo(composedSymbols) + .ApplyCreateUserDefinedFunctions(); - foreach (var error in errors) - { - error.FormatCore(sb); - } - - throw new InvalidOperationException(sb.ToString()); - } + Contracts.AssertValue(udfs); - foreach (var udf in udfs) + if (checkResult.IsSuccess) { - AddFunction(udf); - var config = new BindingConfig(allowsSideEffects: allowSideEffects, useThisRecordForRuleScope: false, numberIsFloat: false); - var binding = udf.BindBody(composedSymbols, new Glue2DocumentBinderGlue(), config); - - List bindErrors = new List(); - - if (binding.ErrorContainer.GetErrors(ref bindErrors)) - { - sb.AppendLine(string.Join(", ", errors.Select(err => err.ToString()))); - } + AddFunctions(udfs); } - if (sb.Length > 0) - { - throw new InvalidOperationException(sb.ToString()); - } + return checkResult; } /// diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/DefinitionsCheckResult.cs b/src/libraries/Microsoft.PowerFx.Core/Public/DefinitionsCheckResult.cs index f4f6bfa9ac..4dad15d5c5 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/DefinitionsCheckResult.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/DefinitionsCheckResult.cs @@ -12,6 +12,9 @@ using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.PowerFx.Core.Binding; using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Glue; +using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Parser; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; @@ -30,11 +33,16 @@ public class DefinitionsCheckResult : IOperationStatus private IReadOnlyDictionary _resolvedTypes; + private TexlFunctionSet _userDefinedFunctions; + private CultureInfo _defaultErrorCulture; private ParserOptions _parserOptions; private ParseUserDefinitionResult _parse; + // Local symboltable to store new symbols in a given script and use in binding. + private readonly SymbolTable _localSymbolTable; + // Power Fx expression containing definitions private string _definitions; @@ -43,6 +51,7 @@ public class DefinitionsCheckResult : IOperationStatus public DefinitionsCheckResult() { + _localSymbolTable = new SymbolTable { DebugName = "LocalUserDefinitions" }; } internal DefinitionsCheckResult SetBindingInfo(ReadOnlySymbolTable symbols) @@ -59,7 +68,7 @@ internal DefinitionsCheckResult SetBindingInfo(ReadOnlySymbolTable symbols) return this; } - internal DefinitionsCheckResult SetText(string definitions, ParserOptions parserOptions = null) + public DefinitionsCheckResult SetText(string definitions, ParserOptions parserOptions = null) { Contracts.AssertValue(definitions); @@ -97,6 +106,8 @@ internal ParseUserDefinitionResult ApplyParse() public IReadOnlyDictionary ResolvedTypes => _resolvedTypes; + public bool ContainsUDF => _parse.UDFs.Any(); + internal IReadOnlyDictionary ApplyResolveTypes() { if (_parse == null) @@ -114,6 +125,7 @@ internal IReadOnlyDictionary ApplyResolveTypes() if (_parse.DefinedTypes.Any()) { this._resolvedTypes = DefinedTypeResolver.ResolveTypes(_parse.DefinedTypes.Where(dt => dt.IsParseValid), _symbols, out var errors); + this._localSymbolTable.AddTypes(this._resolvedTypes); _errors.AddRange(ExpressionError.New(errors, _defaultErrorCulture)); } else @@ -125,16 +137,79 @@ internal IReadOnlyDictionary ApplyResolveTypes() return this._resolvedTypes; } + internal TexlFunctionSet ApplyCreateUserDefinedFunctions() + { + if (_parse == null) + { + this.ApplyParse(); + } + + if (_symbols == null) + { + throw new InvalidOperationException($"Must call {nameof(SetBindingInfo)} before calling ApplyCreateUserDefinedFunctions()."); + } + + if (_resolvedTypes == null) + { + this.ApplyResolveTypes(); + } + + if (_userDefinedFunctions == null) + { + _userDefinedFunctions = new TexlFunctionSet(); + + var partialUDFs = UserDefinedFunction.CreateFunctions(_parse.UDFs.Where(udf => udf.IsParseValid), _symbols, out var errors); + + if (errors.Any()) + { + _errors.AddRange(ExpressionError.New(errors, _defaultErrorCulture)); + } + + var composedSymbols = ReadOnlySymbolTable.Compose(_localSymbolTable, _symbols); + foreach (var udf in partialUDFs) + { + var config = new BindingConfig(allowsSideEffects: _parserOptions.AllowsSideEffects, useThisRecordForRuleScope: false, numberIsFloat: false); + var binding = udf.BindBody(composedSymbols, new Glue2DocumentBinderGlue(), config); + + List bindErrors = new List(); + + if (binding.ErrorContainer.HasErrors()) + { + _errors.AddRange(ExpressionError.New(binding.ErrorContainer.GetErrors(), _defaultErrorCulture)); + } + else + { + _localSymbolTable.AddFunction(udf); + _userDefinedFunctions.Add(udf); + } + } + + return this._userDefinedFunctions; + } + + return this._userDefinedFunctions; + } + internal IEnumerable ApplyErrors() { if (_resolvedTypes == null) { - ApplyResolveTypes(); + this.ApplyCreateUserDefinedFunctions(); } return this.Errors; } + public IEnumerable ApplyParseErrors() + { + if (_parse == null) + { + this.ApplyParse(); + } + + return ExpressionError.New(_parse.Errors, _defaultErrorCulture); + } + /// /// List of all errors and warnings. Check . /// This can include Parse, ResolveType errors />, diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs index 869129ebc8..06152a8426 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Engine.cs @@ -546,10 +546,10 @@ public string GetDisplayExpression(string expressionText, ReadOnlySymbolTable sy return ExpressionLocalizationHelper.ConvertExpression(expressionText, ruleScope, GetDefaultBindingConfig(), CreateResolverInternal(symbolTable), CreateBinderGlue(), culture, Config.Features, toDisplay: true); } - internal void AddUserDefinedFunction(string script, CultureInfo parseCulture = null, ReadOnlySymbolTable symbolTable = null, bool allowSideEffects = false) + public DefinitionsCheckResult AddUserDefinedFunction(string script, CultureInfo parseCulture = null, ReadOnlySymbolTable symbolTable = null, bool allowSideEffects = false) { var engineTypesAndFunctions = ReadOnlySymbolTable.Compose(PrimitiveTypes, SupportedFunctions); - Config.SymbolTable.AddUserDefinedFunction(script, parseCulture, engineTypesAndFunctions, symbolTable, allowSideEffects); + return Config.SymbolTable.AddUserDefinedFunction(script, parseCulture, engineTypesAndFunctions, symbolTable, allowSideEffects); } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs b/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs index 8dd37baae8..c384cfe3d9 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs @@ -460,15 +460,18 @@ private void AddUserDefinedFunctions(IEnumerable parsedUdfs, ReadOnlySymbol foreach (var udf in udfs) { - Config.SymbolTable.AddFunction(udf); var binding = udf.BindBody(nameResolver, new Glue2DocumentBinderGlue(), BindingConfig.Default, Config.Features); List bindErrors = new List(); - if (binding.ErrorContainer.GetErrors(ref errors)) + if (binding.ErrorContainer.GetErrors(ref bindErrors)) { sb.AppendLine(string.Join(", ", bindErrors.Select(err => err.ToString()))); } + else + { + Config.SymbolTable.AddFunction(udf); + } } if (sb.Length > 0) diff --git a/src/libraries/Microsoft.PowerFx.Repl/Repl.cs b/src/libraries/Microsoft.PowerFx.Repl/Repl.cs index 6b1bf6c1b0..1d0657ff1b 100644 --- a/src/libraries/Microsoft.PowerFx.Repl/Repl.cs +++ b/src/libraries/Microsoft.PowerFx.Repl/Repl.cs @@ -34,6 +34,9 @@ public class PowerFxREPL // Allow repl to create new definitions, such as Set(). public bool AllowSetDefinitions { get; set; } + // Allow repl to create new UserDefinedFunctions. + public bool AllowUserDefinedFunctions { get; set; } + // Do we print each command before evaluation? // Useful if we're running a file and are debugging, or if input UI is separated from output UI. public bool Echo { get; set; } = false; @@ -405,6 +408,30 @@ await this.Output.WriteLineAsync($"Error: Can't set '{name}' to a Void value.", var errors = check.ApplyErrors(); if (!check.IsSuccess) { + var definitionsCheckResult = new DefinitionsCheckResult(); + + definitionsCheckResult.SetText(expression, this.ParserOptions) + .ApplyParseErrors(); + + if (this.AllowUserDefinedFunctions && definitionsCheckResult.IsSuccess && definitionsCheckResult.ContainsUDF) + { + var defCheckResult = this.Engine.AddUserDefinedFunction(expression, this.ParserOptions.Culture, extraSymbolTable); + + if (!defCheckResult.IsSuccess) + { + foreach (var error in defCheckResult.Errors) + { + var kind = error.IsWarning ? OutputKind.Warning : OutputKind.Error; + var msg = error.ToString(); + + await this.Output.WriteLineAsync(lineError + msg, kind, cancel) + .ConfigureAwait(false); + } + } + + return new ReplResult(); + } + foreach (var error in check.Errors) { var kind = error.IsWarning ? OutputKind.Warning : OutputKind.Error; diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedFunctionTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedFunctionTests.cs index 5716cd49c3..03dbc7e298 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedFunctionTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedFunctionTests.cs @@ -425,7 +425,7 @@ public void DefineEmpty() // Empty symbol table doesn't get builtins. var st = SymbolTable.WithPrimitiveTypes(); st.AddUserDefinedFunction("Foo1(x: Number): Number = x;"); // ok - Assert.Throws(() => st.AddUserDefinedFunction("Foo2(x: Number): Number = Abs(x);")); + Assert.False(st.AddUserDefinedFunction("Foo2(x: Number): Number = Abs(x);").IsSuccess); } // Show definitions on public symbol tables diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs index 8d476fa1a2..b681966a19 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs @@ -447,11 +447,6 @@ public void FormulaCantRedefine() "func1(x:Number/*comment*/): Number = x * 10;\nfunc2(x:Number): Number = y1 * 10;", null, true)] - [InlineData( - "foo(x:Number):Number = If(x=0,foo(1),If(x=1,foo(2),If(x=2,Float(2))));", - "foo(Float(0))", - false, - 2.0)] [InlineData( "foo():Blank = foo();", "foo()", @@ -472,7 +467,11 @@ public void FormulaCantRedefine() false, 14.0)] - // Recursive calls are not allowed + // Recursive calls are not allowed + [InlineData( + "foo(x:Number):Number = If(x=0,foo(1),If(x=1,foo(2),If(x=2,Float(2))));", + "foo(Float(0))", + true)] [InlineData( "hailstone(x:Number):Number = If(Not(x = 1), If(Mod(x, 2)=0, hailstone(x/2), hailstone(3*x+1)), x);", "hailstone(Float(192))", @@ -574,7 +573,7 @@ public void DefinedFunctionsErrorsTest(string script) { var engine = new RecalcEngine(); - Assert.Throws(() => engine.AddUserDefinedFunction(script, CultureInfo.InvariantCulture)); + Assert.False(engine.AddUserDefinedFunction(script, CultureInfo.InvariantCulture).IsSuccess); } // Overloads and conflict @@ -651,30 +650,32 @@ public void ShadowingFunctionPrecedenceTest() "F1(x:Number) : Boolean = { Set(a, x); Today(); };", null, true, - "AddUserDefinedFunction", + "ErrUDF_ReturnTypeDoesNotMatch", 0)] - public void ImperativeUserDefinedFunctionTest(string udfExpression, string expression, bool expectedError, string expectedMethodFailure, double expected) + public void ImperativeUserDefinedFunctionTest(string udfExpression, string expression, bool expectedError, string errorKey, double expected) { var config = new PowerFxConfig(); config.EnableSetFunction(); var recalcEngine = new RecalcEngine(config); recalcEngine.UpdateVariable("a", 1m); + + var definitionsCheckResult = recalcEngine.AddUserDefinedFunction(udfExpression, CultureInfo.InvariantCulture, symbolTable: recalcEngine.EngineSymbols, allowSideEffects: true); - try - { - recalcEngine.AddUserDefinedFunction(udfExpression, CultureInfo.InvariantCulture, symbolTable: recalcEngine.EngineSymbols, allowSideEffects: true); + if (!expectedError) + { + Assert.True(definitionsCheckResult.IsSuccess); var result = recalcEngine.Eval(expression, options: _opts); var fvExpected = FormulaValue.New(expected); Assert.Equal(fvExpected.AsDecimal(), result.AsDecimal()); - Assert.False(expectedError); } - catch (Exception ex) + else { - Assert.True(expectedError, ex.Message); - Assert.Contains(expectedMethodFailure, ex.StackTrace); + Assert.False(definitionsCheckResult.IsSuccess); + Assert.Single(definitionsCheckResult.Errors); + Assert.Contains(definitionsCheckResult.Errors, err => err.MessageKey == errorKey); } } @@ -708,7 +709,7 @@ public void DelegableUDFTest() var recalcEngine = new RecalcEngine(config); - recalcEngine.AddUserDefinedFunction("A():MyDataSourceTableType = Filter(MyDataSource, Value > 10);C():MyDataSourceTableType = A(); B():MyDataSourceTableType = Filter(C(), Value > 11); D():MyDataSourceTableType = { Filter(B(), Value > 12); }; E():Void = { E(); };", CultureInfo.InvariantCulture, symbolTable: recalcEngine.EngineSymbols, allowSideEffects: true); + recalcEngine.AddUserDefinedFunction("A():MyDataSourceTableType = Filter(MyDataSource, Value > 10);C():MyDataSourceTableType = A(); B():MyDataSourceTableType = Filter(C(), Value > 11); D():MyDataSourceTableType = { Filter(B(), Value > 12); };", CultureInfo.InvariantCulture, symbolTable: recalcEngine.EngineSymbols, allowSideEffects: true); var func = recalcEngine.Functions.WithName("A").First() as UserDefinedFunction; Assert.True(func.IsAsync); @@ -730,11 +731,8 @@ public void DelegableUDFTest() Assert.True(func.IsAsync); Assert.True(!func.IsDelegatable); - func = recalcEngine.Functions.WithName("E").First() as UserDefinedFunction; - - // Imperative function is not delegable - // E():Void = { E() }; ---> binding will be null so no attempt to get datasource should happen - Assert.True(!func.IsDelegatable); + // Binding fails for recursive definitions and hence function is not added. + Assert.False(recalcEngine.AddUserDefinedFunction("E():Void = { E(); };", CultureInfo.InvariantCulture, symbolTable: recalcEngine.EngineSymbols, allowSideEffects: true).IsSuccess); } // Binding to inner functions does not impact outer functions. @@ -1836,7 +1834,7 @@ public void UDFImperativeVsRecordAmbiguityTest(string udf, string evalExpression } else { - Assert.Throws(() => recalcEngine.AddUserDefinedFunction(udf, CultureInfo.InvariantCulture, extraSymbols, true)); + Assert.False(recalcEngine.AddUserDefinedFunction(udf, CultureInfo.InvariantCulture, extraSymbols, true).IsSuccess); } } diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ReplTests.cs b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ReplTests.cs index 8ae0ecdfcf..bdfaa4d9e8 100644 --- a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ReplTests.cs +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ReplTests.cs @@ -34,6 +34,8 @@ public ReplTests() Engine = engine, Output = _output, AllowSetDefinitions = true, + AllowUserDefinedFunctions = true, + ParserOptions = new ParserOptions() { AllowsSideEffects = true } }; } @@ -268,6 +270,29 @@ public void BadRedefinedNamedFormula() Assert.True(log.Length > 0); } + [Fact] + public void UserDefinedFunctions() + { + _repl.HandleLine("F(x: Number): Number = x;"); + _repl.HandleLine("F(42)"); + var log = _output.Get(OutputKind.Repl); + Assert.Equal("42", log); + + // we do not have a clear semantics defined yet for the below test + // should be addressed in future + /* + _repl.HandleLine("F(x: Text): Text = x;"); + var error1 = _output.Get(OutputKind.Error); + Assert.Equal("Error 0-1: Function F is already defined.", error1); + */ + + _repl.HandleLine("G(x: Currency): Currency = x;"); + var error2 = _output.Get(OutputKind.Error); + Assert.Equal( + @"Error 5-13: Unknown type Currency. +Error 16-24: Unknown type Currency.", error2); + } + // test that Exit() informs the host that an exit has been requested [Fact] public void Exit() diff --git a/src/tools/Repl/Program.cs b/src/tools/Repl/Program.cs index 37cabab2b6..f23481dc62 100644 --- a/src/tools/Repl/Program.cs +++ b/src/tools/Repl/Program.cs @@ -40,6 +40,9 @@ public static class ConsoleRepl private const string OptionTextFirst = "TextFirst"; private static bool _textFirst = false; + private const string OptionUDF = "UserDefinedFunctions"; + private static bool _enableUDFs = true; + private static readonly Features _features = Features.PowerFxV1; private static StandardFormatter _standardFormatter; @@ -64,7 +67,8 @@ private static RecalcEngine ReplRecalcEngine() { OptionPowerFxV1, OptionPowerFxV1 }, { OptionHashCodes, OptionHashCodes }, { OptionStackTrace, OptionStackTrace }, - { OptionTextFirst, OptionTextFirst } + { OptionTextFirst, OptionTextFirst }, + { OptionUDF, OptionUDF }, }; foreach (var featureProperty in typeof(Features).GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) @@ -133,6 +137,7 @@ public MyRepl() this.HelpProvider = new MyHelpProvider(); this.AllowSetDefinitions = true; + this.AllowUserDefinedFunctions = _enableUDFs; this.EnableSampleUserObject(); this.AddPseudoFunction(new IRPseudoFunction()); this.AddPseudoFunction(new SuggestionsPseudoFunction()); @@ -255,6 +260,7 @@ public FormulaValue Execute() sb.Append(CultureInfo.InvariantCulture, $"{"LargeCallDepth:",-42}{_largeCallDepth}\n"); sb.Append(CultureInfo.InvariantCulture, $"{"StackTrace:",-42}{_stackTrace}\n"); sb.Append(CultureInfo.InvariantCulture, $"{"TextFirst:",-42}{_textFirst}\n"); + sb.Append(CultureInfo.InvariantCulture, $"{"UserDefinedFunctions:",-42}{_enableUDFs}\n"); foreach (var prop in typeof(Features).GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) { @@ -303,6 +309,11 @@ public FormulaValue Execute(StringValue option) return BooleanValue.New(_stackTrace); } + if (string.Equals(option.Value, OptionUDF, StringComparison.OrdinalIgnoreCase)) + { + return BooleanValue.New(_enableUDFs); + } + return FormulaValue.NewError(new ExpressionError() { Kind = ErrorKind.InvalidArgument, @@ -343,6 +354,13 @@ public FormulaValue Execute(StringValue option, BooleanValue value) return value; } + if (string.Equals(option.Value, OptionUDF, StringComparison.OrdinalIgnoreCase)) + { + _enableUDFs = value.Value; + _reset = true; + return value; + } + if (string.Equals(option.Value, OptionLargeCallDepth, StringComparison.OrdinalIgnoreCase)) { _largeCallDepth = value.Value; @@ -441,6 +459,9 @@ Displays the full stack trace when an exception is encountered. Options.None Removed all the feature flags, which is even less than Canvas uses. +Options.EnableUDFs + Enables UserDefinedFunctions to be added. + "; await WriteAsync(repl, msg, cancel) From ff7aeb87731dc51ac7c3c66837daa688154f6068 Mon Sep 17 00:00:00 2001 From: Adithya Selvaprithiviraj Date: Mon, 23 Sep 2024 12:51:13 -0700 Subject: [PATCH 05/13] Fix CompletionItemKind for Types to display correct Intellisense Icon (#2647) Fixes CompletionItemKind for SuggestioonKind.Types to display correct Intellisense Icon . ![image](https://github.com/user-attachments/assets/59f5806c-0046-42ca-85e8-46c3995eb255) --- .../Completions/CompletionsLanguageServerOperationHandler.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Completions/CompletionsLanguageServerOperationHandler.cs b/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Completions/CompletionsLanguageServerOperationHandler.cs index c46feeda13..1a25bcc941 100644 --- a/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Completions/CompletionsLanguageServerOperationHandler.cs +++ b/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Completions/CompletionsLanguageServerOperationHandler.cs @@ -145,6 +145,8 @@ private static CompletionItemKind GetCompletionItemKind(SuggestionKind kind) return CompletionItemKind.Module; case SuggestionKind.ScopeVariable: return CompletionItemKind.Variable; + case SuggestionKind.Type: + return CompletionItemKind.TypeParameter; default: return CompletionItemKind.Text; } From 71818556f4724ef499cd51324c72c94ad0cb41a1 Mon Sep 17 00:00:00 2001 From: Anderson Silva Date: Mon, 23 Sep 2024 17:48:48 -0500 Subject: [PATCH 06/13] UO support for functions (#2649) 'Column' and 'CountRows' functions now support PAD UO values. --- .../Functions/LibraryUntypedObject.cs | 4 ++-- .../PadUntypedObjectTests.cs | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUntypedObject.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUntypedObject.cs index f5e4bdb5c6..3c83962512 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUntypedObject.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUntypedObject.cs @@ -423,7 +423,7 @@ public static FormulaValue CountRows_UO(IRContext irContext, UntypedObjectValue[ { var impl = args[0].Impl; - if (impl.Type is ExternalType externalType && externalType.Kind == ExternalTypeKind.Array) + if (impl.Type is ExternalType externalType && (externalType.Kind == ExternalTypeKind.Array || externalType.Kind == ExternalTypeKind.ArrayAndObject)) { return new NumberValue(irContext, impl.GetArrayLength()); } @@ -452,7 +452,7 @@ public static FormulaValue Column_UO(IRContext irContext, FormulaValue[] args) var impl = (args[0] as UntypedObjectValue).Impl; var propertyName = (args[1] as StringValue).Value; - if (impl.Type is ExternalType externalType && externalType.Kind == ExternalTypeKind.Object) + if (impl.Type is ExternalType externalType && (externalType.Kind == ExternalTypeKind.Object || externalType.Kind == ExternalTypeKind.ArrayAndObject)) { if (impl.TryGetProperty(propertyName, out var propertyValue)) { diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PadUntypedObjectTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PadUntypedObjectTests.cs index 353873c5a1..b36c437348 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PadUntypedObjectTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PadUntypedObjectTests.cs @@ -178,6 +178,27 @@ public void PadUntypedObject2ColumnNamesTest() var result = engine.Eval(@"ColumnNames(Index(padTable, 1))"); Assert.IsAssignableFrom(result); + } + + [Theory] + [InlineData("Column(First(padTable),\"Id\")")] + [InlineData("CountRows(padTable)")] + public void PadUntypedObjectFunctionsSupportTest(string expression) + { + var dt = GetDataTable(); + var uoTable = new PadUntypedObject(dt); + var uoRow = new PadUntypedObject(dt.Rows[0]); // First row + + var uovTable = new UntypedObjectValue(IRContext.NotInSource(FormulaType.UntypedObject), uoTable); + + PowerFxConfig config = new PowerFxConfig(Features.PowerFxV1); + RecalcEngine engine = new RecalcEngine(config); + + engine.Config.SymbolTable.EnableMutationFunctions(); + engine.UpdateVariable("padTable", uovTable); + + var result = engine.Eval(expression); + Assert.IsNotType(result); } private DataTable GetDataTable() From c0e1ad7c42e723fc5e9384e25979ba6a1356fa5a Mon Sep 17 00:00:00 2001 From: Luc Genetier <69138830+LucGenetier@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:51:56 +0200 Subject: [PATCH 07/13] Update SAP tabular test (#2652) --- ....PowerFx.Connectors.Tests.Shared.projitems | 1 + .../PowerPlatformTabularTests.cs | 18 +++++- .../Responses/SAP GetData.json | 57 +++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SAP GetData.json diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems index 58658cea68..86400c1585 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems @@ -225,6 +225,7 @@ + diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs index ed57736f80..4dd691f33b 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs @@ -274,9 +274,9 @@ public async Task SAP_CDP() ConsoleLogger logger = new ConsoleLogger(_output); using var httpClient = new HttpClient(testConnector); - string connectionId = "66108f1684944d4994ea38b13d1ee70f"; + string connectionId = "1e702ce4f10c482684cee1465e686764"; string jwt = "eyJ0eXAi..."; - using var client = new PowerPlatformConnectorClient("f5de196a-41e6-ee09-92cf-664b4f31a6b2.06.common.tip1002.azure-apihub.net", "f5de196a-41e6-ee09-92cf-664b4f31a6b2", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" }; + using var client = new PowerPlatformConnectorClient("066d5714-1ffc-e316-90bd-affc61d8e6fd.18.common.tip2.azure-apihub.net", "066d5714-1ffc-e316-90bd-affc61d8e6fd", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" }; testConnector.SetResponseFromFile(@"Responses\SAP GetDataSetMetadata.json"); DatasetMetadata dm = await CdpDataSource.GetDatasetsMetadataAsync(client, $"/apim/sapodata/{connectionId}", CancellationToken.None, logger); @@ -292,6 +292,20 @@ public async Task SAP_CDP() CdpTableValue sapTableValue = sapTable.GetTableValue(); Assert.Equal("*[ALL_EMPLOYEES:s, APP_MODE:s, BEGIN_DATE:s, BEGIN_DATE_CHAR:s, COMMAND:s, DESCRIPTION:s, EMP_PERNR:s, END_DATE:s, END_DATE_CHAR:s, EVENT_NAME:s, FLAG:s, GetMessages:*[MESSAGE:s, PERNR:s], HIDE_PEERS:s, LEGEND:s, LEGENDID:s, LEGEND_TEXT:s, PERNR:s, PERNR_MEM_ID:s, TYPE:s]", sapTableValue.Type._type.ToString()); + + string expr = "First(TeamCalendarCollection).LEGEND_TEXT"; + + SymbolValues symbolValues = new SymbolValues().Add("TeamCalendarCollection", sapTableValue); + RuntimeConfig rc = new RuntimeConfig(symbolValues).AddService(logger); + + CheckResult check = engine.Check(expr, options: new ParserOptions() { AllowsSideEffects = true }, symbolTable: symbolValues.SymbolTable); + Assert.True(check.IsSuccess); + + testConnector.SetResponseFromFile(@"Responses\SAP GetData.json"); + FormulaValue result = await check.GetEvaluator().EvalAsync(CancellationToken.None, rc); + + StringValue sv = Assert.IsType(result); + Assert.Equal("Holiday", sv.Value); } [Fact] diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SAP GetData.json b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SAP GetData.json new file mode 100644 index 0000000000..eb0b640ec1 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SAP GetData.json @@ -0,0 +1,57 @@ +{ + "@odata.context": "https://066d5714-1ffc-e316-90bd-affc61d8e6fd.18.common.tip2.azure-apihub.net/apim/sapodata/1e702ce4f10c482684cee1465e686764/datasets/http%253A%252F%252Fsapecckerb.roomsofthehouse.com%253A8080%252Fsap%252Fopu%252Fodata%252Fsap%252FHRESS_TEAM_CALENDAR_SERVICE%252F/tables/TeamCalendarCollection/items?%24select=END_DATE&%24top=1", + "value": [ + { + "GetMessages": { + "IsCollection": true, + "Name": "GetMessages", + "Url": "http://sapecckerb.roomsofthehouse.com:8080/sap/opu/odata/sap/HRESS_TEAM_CALENDAR_SERVICE/TeamCalendarCollection('00000000')/GetMessages", + "AssociationLinkUrl": null + }, + "PERNR": "00000000", + "EVENT_NAME": "", + "BEGIN_DATE": null, + "END_DATE": null, + "DESCRIPTION": "", + "LEGENDID": "", + "LEGEND": "RGB(246,243,135)", + "LEGEND_TEXT": "Holiday", + "FLAG": "04", + "TYPE": "", + "APP_MODE": "", + "ALL_EMPLOYEES": "", + "COMMAND": "", + "EMP_PERNR": "00000000", + "HIDE_PEERS": "", + "PERNR_MEM_ID": "", + "BEGIN_DATE_CHAR": "00000000", + "END_DATE_CHAR": "00000000" + }, + { + "GetMessages": { + "IsCollection": true, + "Name": "GetMessages", + "Url": "http://sapecckerb.roomsofthehouse.com:8080/sap/opu/odata/sap/HRESS_TEAM_CALENDAR_SERVICE/TeamCalendarCollection('00000000')/GetMessages", + "AssociationLinkUrl": null + }, + "PERNR": "00000000", + "EVENT_NAME": "", + "BEGIN_DATE": null, + "END_DATE": null, + "DESCRIPTION": "", + "LEGENDID": "", + "LEGEND": "RGB(220,220,220)", + "LEGEND_TEXT": "Non-Working Day", + "FLAG": "04", + "TYPE": "", + "APP_MODE": "", + "ALL_EMPLOYEES": "", + "COMMAND": "", + "EMP_PERNR": "00000000", + "HIDE_PEERS": "", + "PERNR_MEM_ID": "", + "BEGIN_DATE_CHAR": "00000000", + "END_DATE_CHAR": "00000000" + } + ] +} \ No newline at end of file From 93daa65514e68a8d9d7a4abee642ce2d5480c2d6 Mon Sep 17 00:00:00 2001 From: Carlos Figueira Date: Wed, 25 Sep 2024 09:29:51 -0700 Subject: [PATCH 08/13] Update the Trace function to return a Void type (#2650) The Trace function doesn't return anything. For legacy reasons, it currently returns DType.Boolean (when it was created, we didn't have DType.Void). This PR updates it to return Void instead. --- .../Texl/Builtins/Trace.cs | 7 +++++++ .../Functions/Library.cs | 2 +- .../TexlTests.cs | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Trace.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Trace.cs index 39d2d1904d..116bf89839 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Trace.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Trace.cs @@ -44,6 +44,13 @@ public override IEnumerable GetRequiredEnumNames() yield return new[] { TexlStrings.TraceArg1, TexlStrings.TraceArg2, TexlStrings.TraceArg3, TexlStrings.TraceArg4 }; } + public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType returnType, out Dictionary nodeToCoercedTypeMap) + { + var result = base.CheckTypes(context, args, argTypes, errors, out returnType, out nodeToCoercedTypeMap); + returnType = context.Features.PowerFxV1CompatibilityRules ? DType.Void : DType.Boolean; + return result; + } + public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) { Contracts.AssertValue(args); diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Library.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Library.cs index 8cc07d81b6..367e632f10 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Library.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Library.cs @@ -2797,7 +2797,7 @@ public static async ValueTask TraceFunction(EvalVisitor runner, Ev throw new CustomFunctionErrorException(ex.Message); } - return FormulaValue.New(true); + return irContext.ResultType._type.Kind == DKind.Boolean ? FormulaValue.New(true) : FormulaValue.NewVoid(); } } } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs index dedb99e263..c5e682c4fb 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs @@ -2693,6 +2693,25 @@ public void TexlFunctionTypeSemanticsTable_Negative(string script, string expect features: Features.PowerFxV1); } + [Theory] + [InlineData("Trace(\"hello\")")] + [InlineData("Trace(\"hello\", TraceSeverity.Warning)")] + [InlineData("Trace(\"hello\", TraceSeverity.Warning, { a: 1 })")] + public void TexlFunctionTypeSemanticsTrace(string expression) + { + foreach (var powerFxV1 in new[] { false, true }) + { + var features = powerFxV1 ? Features.PowerFxV1 : Features.None; + var engine = new Engine(new PowerFxConfig(features)); + var options = new ParserOptions() { AllowsSideEffects = true }; + var result = engine.Check(expression, options); + + var expectedType = powerFxV1 ? DType.Void : DType.Boolean; + Assert.Equal(expectedType, result.Binding.ResultType); + Assert.True(result.IsSuccess); + } + } + [Theory] [InlineData("Concat([], \"\")")] [InlineData("Concat([1, 2, 3], Text(Value))")] From ee0ca2c67efdfa2ccf551c068cbf0020b7808582 Mon Sep 17 00:00:00 2001 From: Luc Genetier <69138830+LucGenetier@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:09:23 +0200 Subject: [PATCH 09/13] JSON function: serialize UO (#2639) >> JSON(ParseJSON("{""a"": 1}"), JSONFormat.IgnoreUnsupportedTypes) "{""a"":1}" --- .../Localization/Strings.cs | 2 + .../Public/Canceller.cs | 46 +++++ .../Texl/Builtins/Json.cs | 25 ++- .../EvalVisitor.cs | 14 +- .../Functions/AsTypeFunction_UOImpl.cs | 3 +- .../Functions/IsTypeFunction_UOImpl.cs | 3 +- .../Functions/JsonFunctionImpl.cs | 162 ++++++++++++++-- .../Functions/TypedParseJSONImpl.cs | 3 +- src/strings/PowerFxResources.en-US.resx | 8 + .../ExpressionTestCases/JSON.txt | 98 +++++++++- .../JsonSerializeUOTests.cs | 181 ++++++++++++++++++ 11 files changed, 510 insertions(+), 35 deletions(-) create mode 100644 src/libraries/Microsoft.PowerFx.Core/Public/Canceller.cs create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/JsonSerializeUOTests.cs diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 288586637b..f1dd1963b3 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -845,5 +845,7 @@ internal static class TexlStrings public static ErrorResourceKey ErrInvalidDataSourceForFunction = new ErrorResourceKey("ErrInvalidDataSourceForFunction"); public static ErrorResourceKey ErrInvalidArgumentExpectedType = new ErrorResourceKey("ErrInvalidArgumentExpectedType"); public static ErrorResourceKey ErrUnsupportedTypeInTypeArgument = new ErrorResourceKey("ErrUnsupportedTypeInTypeArgument"); + public static ErrorResourceKey ErrReachedMaxJsonDepth = new ErrorResourceKey("ErrReachedMaxJsonDepth"); + public static ErrorResourceKey ErrReachedMaxJsonLength = new ErrorResourceKey("ErrReachedMaxJsonLength"); } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Canceller.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Canceller.cs new file mode 100644 index 0000000000..ce55553618 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Canceller.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; + +namespace Microsoft.PowerFx +{ + internal class Canceller + { + private readonly List _cancellationAction; + + public Canceller() + { + _cancellationAction = new List(); + } + + public Canceller(params Action[] cancellationActions) + : this() + { + if (cancellationActions != null) + { + foreach (Action cancellationAction in cancellationActions) + { + AddAction(cancellationAction); + } + } + } + + public void AddAction(Action cancellationAction) + { + if (cancellationAction != null) + { + _cancellationAction.Add(cancellationAction); + } + } + + public void ThrowIfCancellationRequested() + { + foreach (Action cancellationAction in _cancellationAction) + { + cancellationAction(); + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs index 75006f6005..b80f141819 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs @@ -17,7 +17,7 @@ namespace Microsoft.PowerFx.Core.Texl.Builtins { // JSON(data:any, [format:s]) internal class JsonFunction : BuiltinFunction - { + { private const char _includeBinaryDataEnumValue = 'B'; private const char _ignoreBinaryDataEnumValue = 'G'; private const char _ignoreUnsupportedTypesEnumValue = 'I'; @@ -31,26 +31,25 @@ internal class JsonFunction : BuiltinFunction DKind.DataEntity, DKind.LazyRecord, DKind.LazyTable, - DKind.View, + DKind.View, DKind.ViewValue }; private static readonly DKind[] _unsupportedTypes = new[] { - DKind.Control, + DKind.Control, DKind.LazyRecord, DKind.LazyTable, DKind.Metadata, - DKind.OptionSet, - DKind.PenImage, + DKind.OptionSet, + DKind.PenImage, DKind.Polymorphic, - DKind.UntypedObject, DKind.Void }; public override bool IsSelfContained => true; - public override bool IsAsync => true; + public override bool IsAsync => true; public override bool SupportsParamCoercion => false; @@ -78,13 +77,13 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp // Do not call base.CheckTypes for arg0 if (args.Length > 1) { - if (context.Features.StronglyTypedBuiltinEnums && + if (context.Features.StronglyTypedBuiltinEnums && !base.CheckType(context, args[1], argTypes[1], BuiltInEnums.JSONFormatEnum.FormulaType._type, errors, ref nodeToCoercedTypeMap)) { return false; } - TexlNode optionsNode = args[1]; + TexlNode optionsNode = args[1]; if (!IsConstant(context, argTypes, optionsNode, out string nodeValue)) { errors.EnsureError(optionsNode, TexlStrings.ErrFunctionArg2ParamMustBeConstant, "JSON", TexlStrings.JSONArg2.Invoke()); @@ -117,11 +116,11 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ bool includeBinaryData = false; bool ignoreUnsupportedTypes = false; - bool ignoreBinaryData = false; + bool ignoreBinaryData = false; if (args.Length > 1) { - TexlNode optionsNode = args[1]; + TexlNode optionsNode = args[1]; if (!IsConstant(binding.CheckTypesContext, argTypes, optionsNode, out string nodeValue)) { return; @@ -180,12 +179,12 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ } if (!ignoreUnsupportedTypes) - { + { if (HasUnsupportedType(dataArgType, supportsLazyTypes, out DType unsupportedNestedType, out var unsupportedColumnName)) { errors.EnsureError(dataNode, TexlStrings.ErrJSONArg1UnsupportedNestedType, unsupportedColumnName, unsupportedNestedType.GetKindString()); } - } + } } private static bool IsConstant(CheckTypesContext context, DType[] argTypes, TexlNode optionsNode, out string nodeValue) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs b/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs index 0a24c16b5a..fe1334bc1e 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs @@ -332,7 +332,19 @@ public override async ValueTask Visit(CallNode node, EvalVisitorCo } else if (func is IAsyncTexlFunction5 asyncFunc5) { - result = await asyncFunc5.InvokeAsync(_services, node.IRContext.ResultType, args, _cancellationToken).ConfigureAwait(false); + BasicServiceProvider services2 = new BasicServiceProvider(_services); + + if (services2.GetService(typeof(TimeZoneInfo)) == null) + { + services2.AddService(TimeZoneInfo); + } + + if (services2.GetService(typeof(Canceller)) == null) + { + services2.AddService(new Canceller(CheckCancel)); + } + + result = await asyncFunc5.InvokeAsync(services2, node.IRContext.ResultType, args, _cancellationToken).ConfigureAwait(false); } else if (func is IAsyncConnectorTexlFunction asyncConnectorTexlFunction) { diff --git a/src/libraries/Microsoft.PowerFx.Json/Functions/AsTypeFunction_UOImpl.cs b/src/libraries/Microsoft.PowerFx.Json/Functions/AsTypeFunction_UOImpl.cs index c09190ea34..d072648229 100644 --- a/src/libraries/Microsoft.PowerFx.Json/Functions/AsTypeFunction_UOImpl.cs +++ b/src/libraries/Microsoft.PowerFx.Json/Functions/AsTypeFunction_UOImpl.cs @@ -6,9 +6,7 @@ using System.Threading.Tasks; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.IR; -using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; -using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx.Core.Texl.Builtins @@ -18,6 +16,7 @@ internal class AsTypeFunction_UOImpl : AsTypeFunction_UO, IAsyncTexlFunction4 public async Task InvokeAsync(TimeZoneInfo timezoneInfo, FormulaType ft, FormulaValue[] args, CancellationToken cancellationToken) { Contracts.Assert(args.Length == 2); + cancellationToken.ThrowIfCancellationRequested(); var irContext = IRContext.NotInSource(ft); var typeString = (StringValue)args[1]; diff --git a/src/libraries/Microsoft.PowerFx.Json/Functions/IsTypeFunction_UOImpl.cs b/src/libraries/Microsoft.PowerFx.Json/Functions/IsTypeFunction_UOImpl.cs index 588f97d3ff..16b1b7fa08 100644 --- a/src/libraries/Microsoft.PowerFx.Json/Functions/IsTypeFunction_UOImpl.cs +++ b/src/libraries/Microsoft.PowerFx.Json/Functions/IsTypeFunction_UOImpl.cs @@ -6,9 +6,7 @@ using System.Threading.Tasks; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.IR; -using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; -using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx.Core.Texl.Builtins @@ -18,6 +16,7 @@ internal class IsTypeFunction_UOImpl : IsTypeFunction_UO, IAsyncTexlFunction4 public async Task InvokeAsync(TimeZoneInfo timezoneInfo, FormulaType ft, FormulaValue[] args, CancellationToken cancellationToken) { Contracts.Assert(args.Length == 2); + cancellationToken.ThrowIfCancellationRequested(); var irContext = IRContext.NotInSource(FormulaType.UntypedObject); var typeString = (StringValue)args[1]; diff --git a/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs b/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs index bc584c180c..0163316dce 100644 --- a/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs +++ b/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs @@ -16,24 +16,30 @@ using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types.Enums; using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx.Core.Texl.Builtins { - internal class JsonFunctionImpl : JsonFunction, IAsyncTexlFunction4 + internal class JsonFunctionImpl : JsonFunction, IAsyncTexlFunction5 { - public Task InvokeAsync(TimeZoneInfo timezoneInfo, FormulaType type, FormulaValue[] args, CancellationToken cancellationToken) + public Task InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType type, FormulaValue[] args, CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); - return Task.FromResult(new JsonProcessing(timezoneInfo, type, args, supportsLazyTypes).Process()); + TimeZoneInfo timeZoneInfo = runtimeServiceProvider.GetService(typeof(TimeZoneInfo)) as TimeZoneInfo ?? throw new InvalidOperationException("TimeZoneInfo is required"); + Canceller canceller = runtimeServiceProvider.GetService(typeof(Canceller)) as Canceller ?? new Canceller(() => cancellationToken.ThrowIfCancellationRequested()); + + return Task.FromResult(new JsonProcessing(timeZoneInfo, type, args, supportsLazyTypes).Process(canceller)); } internal class JsonProcessing { private readonly FormulaValue[] _arguments; + private readonly FormulaType _type; + private readonly TimeZoneInfo _timeZoneInfo; + private readonly bool _supportsLazyTypes; internal JsonProcessing(TimeZoneInfo timezoneInfo, FormulaType type, FormulaValue[] args, bool supportsLazyTypes) @@ -44,8 +50,10 @@ internal JsonProcessing(TimeZoneInfo timezoneInfo, FormulaType type, FormulaValu _supportsLazyTypes = supportsLazyTypes; } - internal FormulaValue Process() + internal FormulaValue Process(Canceller canceller) { + canceller.ThrowIfCancellationRequested(); + JsonFlags flags = GetFlags(); if (flags == null || JsonFunction.HasUnsupportedType(_arguments[0].Type._type, _supportsLazyTypes, out _, out _)) @@ -61,10 +69,21 @@ internal FormulaValue Process() using MemoryStream memoryStream = new MemoryStream(); using Utf8JsonWriter writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions() { Indented = flags.IndentFour, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); - Utf8JsonWriterVisitor jsonWriterVisitor = new Utf8JsonWriterVisitor(writer, _timeZoneInfo, flattenValueTables: flags.FlattenValueTables); + Utf8JsonWriterVisitor jsonWriterVisitor = new Utf8JsonWriterVisitor(writer, _timeZoneInfo, flattenValueTables: flags.FlattenValueTables, canceller); - _arguments[0].Visit(jsonWriterVisitor); - writer.Flush(); + try + { + _arguments[0].Visit(jsonWriterVisitor); + writer.Flush(); + } + catch (InvalidOperationException) + { + if (!jsonWriterVisitor.ErrorValues.Any()) + { + // Unexpected error, rethrow + throw; + } + } if (jsonWriterVisitor.ErrorValues.Any()) { @@ -104,7 +123,7 @@ private JsonFlags GetFlags() optionString = sv.Value; break; - // if not one of these, will check optionString != null below + // if not one of these, will check optionString != null below } if (optionString != null) @@ -129,17 +148,58 @@ private JsonFlags GetFlags() private class Utf8JsonWriterVisitor : IValueVisitor { + private const int _maxDepth = 20; // maximum depth of UO + + private const int _maxLength = 1024 * 1024; // 1 MB, maximum number of bytes allowed to be sent to Utf8JsonWriter + private readonly Utf8JsonWriter _writer; + private readonly TimeZoneInfo _timeZoneInfo; + private readonly bool _flattenValueTables; + private readonly Canceller _canceller; + internal readonly List ErrorValues = new List(); - internal Utf8JsonWriterVisitor(Utf8JsonWriter writer, TimeZoneInfo timeZoneInfo, bool flattenValueTables) + internal Utf8JsonWriterVisitor(Utf8JsonWriter writer, TimeZoneInfo timeZoneInfo, bool flattenValueTables, Canceller canceller) { _writer = writer; _timeZoneInfo = timeZoneInfo; _flattenValueTables = flattenValueTables; + + _canceller = canceller; + } + + private void CheckLimitsAndCancellation(int index) + { + _canceller.ThrowIfCancellationRequested(); + + if (index > _maxDepth) + { + IRContext irContext = IRContext.NotInSource(FormulaType.UntypedObject); + ErrorValues.Add(new ErrorValue(irContext, new ExpressionError() + { + ResourceKey = TexlStrings.ErrReachedMaxJsonDepth, + Span = irContext.SourceContext, + Kind = ErrorKind.InvalidArgument + })); + + throw new InvalidOperationException($"Maximum depth {_maxDepth} reached while traversing JSON payload."); + } + + if (_writer.BytesCommitted + _writer.BytesPending > _maxLength) + { + IRContext irContext = IRContext.NotInSource(FormulaType.UntypedObject); + ErrorValues.Add(new ErrorValue(irContext, new ExpressionError() + { + ResourceKey = TexlStrings.ErrReachedMaxJsonLength, + Span = irContext.SourceContext, + Kind = ErrorKind.InvalidArgument + })); + + throw new InvalidOperationException($"Maximum length {_maxLength} reached in JSON function."); + } } public void Visit(BlankValue blankValue) @@ -173,7 +233,7 @@ public void Visit(DecimalValue decimalValue) } public void Visit(ErrorValue errorValue) - { + { ErrorValues.Add(errorValue); _writer.WriteStringValue("ErrorValue"); } @@ -256,8 +316,10 @@ public void Visit(RecordValue recordValue) { _writer.WriteStartObject(); - foreach (NamedValue namedValue in recordValue.Fields) + foreach (NamedValue namedValue in recordValue.Fields.OrderBy(f => f.Name, StringComparer.Ordinal)) { + CheckLimitsAndCancellation(0); + _writer.WritePropertyName(namedValue.Name); namedValue.Value.Visit(this); } @@ -287,6 +349,8 @@ public void Visit(TableValue tableValue) foreach (DValue row in tableValue.Rows) { + CheckLimitsAndCancellation(0); + if (row.IsBlank) { row.Blank.Visit(this); @@ -319,7 +383,77 @@ public void Visit(TimeValue timeValue) public void Visit(UntypedObjectValue untypedObjectValue) { - throw new ArgumentException($"Unable to serialize type {untypedObjectValue.GetType().FullName} to Json format."); + Visit(untypedObjectValue.Impl); + } + + private void Visit(IUntypedObject untypedObject, int depth = 0) + { + FormulaType type = untypedObject.Type; + + CheckLimitsAndCancellation(depth); + + if (type is StringType) + { + _writer.WriteStringValue(untypedObject.GetString()); + } + else if (type is DecimalType) + { + _writer.WriteNumberValue(untypedObject.GetDecimal()); + } + else if (type is NumberType) + { + _writer.WriteNumberValue(untypedObject.GetDouble()); + } + else if (type is BooleanType) + { + _writer.WriteBooleanValue(untypedObject.GetBoolean()); + } + else if (type is ExternalType externalType) + { + if (externalType.Kind == ExternalTypeKind.Array || externalType.Kind == ExternalTypeKind.ArrayAndObject) + { + _writer.WriteStartArray(); + + for (var i = 0; i < untypedObject.GetArrayLength(); i++) + { + CheckLimitsAndCancellation(depth); + + IUntypedObject row = untypedObject[i]; + Visit(row, depth + 1); + } + + _writer.WriteEndArray(); + } + else if ((externalType.Kind == ExternalTypeKind.Object || externalType.Kind == ExternalTypeKind.ArrayAndObject) && untypedObject.TryGetPropertyNames(out IEnumerable propertyNames)) + { + _writer.WriteStartObject(); + + foreach (var propertyName in propertyNames.OrderBy(prop => prop, StringComparer.Ordinal)) + { + CheckLimitsAndCancellation(depth); + + if (untypedObject.TryGetProperty(propertyName, out IUntypedObject res)) + { + _writer.WritePropertyName(propertyName); + Visit(res, depth + 1); + } + } + + _writer.WriteEndObject(); + } + else if (externalType.Kind == ExternalTypeKind.UntypedNumber) + { + _writer.WriteRawValue(untypedObject.GetUntypedNumber()); + } + else + { + throw new NotSupportedException("Unknown ExternalType"); + } + } + else + { + throw new NotSupportedException("Unknown IUntypedObject"); + } } public void Visit(BlobValue value) @@ -332,7 +466,7 @@ public void Visit(BlobValue value) { _writer.WriteBase64StringValue(value.GetAsByteArrayAsync(CancellationToken.None).Result); } - } + } } internal static string GetColorString(Color color) => $"#{color.R:x2}{color.G:x2}{color.B:x2}{color.A:x2}"; diff --git a/src/libraries/Microsoft.PowerFx.Json/Functions/TypedParseJSONImpl.cs b/src/libraries/Microsoft.PowerFx.Json/Functions/TypedParseJSONImpl.cs index 9fc5e6d122..fcc3e5667a 100644 --- a/src/libraries/Microsoft.PowerFx.Json/Functions/TypedParseJSONImpl.cs +++ b/src/libraries/Microsoft.PowerFx.Json/Functions/TypedParseJSONImpl.cs @@ -6,9 +6,7 @@ using System.Threading.Tasks; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.IR; -using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; -using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx.Core.Texl.Builtins @@ -18,6 +16,7 @@ internal class TypedParseJSONFunctionImpl : TypedParseJSONFunction, IAsyncTexlFu public async Task InvokeAsync(TimeZoneInfo timezoneInfo, FormulaType ft, FormulaValue[] args, CancellationToken cancellationToken) { Contracts.Assert(args.Length == 2); + cancellationToken.ThrowIfCancellationRequested(); var irContext = IRContext.NotInSource(ft); var typeString = (StringValue)args[1]; diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index e063733cf3..84509cbaf5 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4760,4 +4760,12 @@ Unsupported untyped/JSON conversion type '{0}' in argument. Error Message shown to user when a unsupported type is passed in type argument of AsType, IsType and ParseJSON functions. + + Maximum depth reached while traversing JSON payload. + Error message returned by the {Locked=JSON} function when a document that is too deeply nested is passed to it. The term JSON refers to the data format described in www.json.org. + + + Maximum length reached in JSON function. + Error message returned by the {Locked=JSON} function when the result generated by this function would be too long. The term JSON refers to the data format described in www.json.org. + \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt index 6753c48dac..eb9c58e861 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt @@ -1,4 +1,4 @@ -#SETUP: EnableJsonFunctions +#SETUP: EnableJsonFunctions, PowerFxV1CompatibilityRules >> JSON() Errors: Error 0-6: Invalid number of arguments: received 0, expected 1-2. @@ -101,6 +101,10 @@ Error({Kind:ErrorKind.Div0}) >> JSON({a:1,b:Sqrt(-1),c:true}) Error({Kind:ErrorKind.Numeric}) +// Reordering of properties in culture-invariant order +>> JSON({b:2,a:1}) +"{""a"":1,""b"":2}" + >> JSON([{a:1,b:[2]},{a:3,b:[4,5]},{a:6,b:[7,1/0,9]}]) Error({Kind:ErrorKind.Div0}) @@ -133,3 +137,95 @@ Error({Kind:ErrorKind.Div0}) // Flattening nested tables >> JSON([[1,2,3],[4,5],[6]], JSONFormat.FlattenValueTables) "[[1,2,3],[4,5],[6]]" + +>> JSON(ParseJSON("{}")) +"{}" + +>> JSON(ParseJSON("[]")) +"[]" + +>> JSON(ParseJSON("1")) +"1" + +>> JSON(ParseJSON("1.77")) +"1.77" + +>> JSON(ParseJSON("-871")) +"-871" + +>> JSON(ParseJSON("""John""")) +"""John""" + +>> JSON(ParseJSON("true")) +"true" + +>> JSON(ParseJSON("false")) +"false" + +>> JSON(ParseJSON("{""a"": 1}")) +"{""a"":1}" + +// Reordering of properties in culture-invariant order +>> JSON(ParseJSON("{""b"": 2, ""a"": 1}")) +"{""a"":1,""b"":2}" + +>> JSON(ParseJSON("{""a"": ""x""}")) +"{""a"":""x""}" + +>> JSON(ParseJSON("[1]")) +"[1]" + +>> JSON(ParseJSON("{""a"": 1.5}")) +"{""a"":1.5}" + +>> JSON(ParseJSON("[1.5]")) +"[1.5]" + +>> JSON(ParseJSON("{""a"":[1]}")) +"{""a"":[1]}" + +>> JSON(ParseJSON("[{""a"": -17}]")) +"[{""a"":-17}]" + +>> JSON(ParseJSON("[true, false]")) +"[true,false]" + +>> JSON(ParseJSON("[""True"", ""False""]")) +"[""True"",""False""]" + +// Round-trip is not guaranteed +>> JSON(ParseJSON(" { ""a"" : 1 } ")) +"{""a"":1}" + +>> JSON(ParseJSON("{""a"": {""a"": 1}}")) +"{""a"":{""a"":1}}" + +// Depth 21 +>> JSON(ParseJSON("{""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": {""a"": 1}}}}}}}}}}}}}}}}}}}}}")) +Error({Kind:ErrorKind.InvalidArgument}) + +>> JSON(ParseJSON("[[1]]")) +"[[1]]" + +// Depth 21 +>> JSON(ParseJSON("[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]")) +Error({Kind:ErrorKind.InvalidArgument}) + +>> JSON(Decimal(ParseJSON("123456789012345.6789012345678"))) +"123456789012345.6789012345678" + +>> JSON(ParseJSON("123456789012345.6789012345678")) +"123456789012345.6789012345678" + +// Round-trip is not guaranteed - escaped characters that don't need escaping will not be re-escaped when JSON-ified +>> JSON(ParseJSON("""\u0048\u0065\u006c\u006c\u006f""")) +"""Hello""" + +>> JSON(ParseJSON("1e300")) +"1e300" + +>> JSON(ParseJSON("1111111111111111111111111111111.2222222222222222222222222222222222")) +"1111111111111111111111111111111.2222222222222222222222222222222222" + +>> JSON(ParseJSON("1e700")) +"1e700" diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/JsonSerializeUOTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/JsonSerializeUOTests.cs new file mode 100644 index 0000000000..98df7fc6ba --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/JsonSerializeUOTests.cs @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Core.Tests; +using Microsoft.PowerFx.Types; +using Xunit; + +#pragma warning disable CA1065 + +namespace Microsoft.PowerFx.Tests +{ + public class JsonSerializeUOTests : PowerFxTest + { + [Fact] + public async Task JsonSerializeUOTest() + { + PowerFxConfig config = new PowerFxConfig(); + config.EnableJsonFunctions(); + + SymbolTable symbolTable = new SymbolTable(); + ISymbolSlot objSlot = symbolTable.AddVariable("obj", FormulaType.UntypedObject); + + foreach ((int id, TestUO uo, string expectedResult) in GetUOTests()) + { + SymbolValues symbolValues = new SymbolValues(symbolTable); + symbolValues.Set(objSlot, FormulaValue.New(uo)); + + RuntimeConfig runtimeConfig = new RuntimeConfig(symbolValues); + RecalcEngine engine = new RecalcEngine(config); + + FormulaValue fv = await engine.EvalAsync("JSON(obj)", CancellationToken.None, runtimeConfig: runtimeConfig); + Assert.IsNotType(fv); + + string str = fv.ToExpression().ToString(); + Assert.True(expectedResult == str, $"[{id}: Expected={expectedResult}, Result={str}]"); + } + } + + private IEnumerable<(int id, TestUO uo, string expectedResult)> GetUOTests() + { + yield return (1, new TestUO(true), @"""true"""); + yield return (2, new TestUO(false), @"""false"""); + yield return (3, new TestUO(string.Empty), @""""""""""""""); + yield return (4, new TestUO("abc"), @"""""""abc"""""""); + yield return (5, new TestUO(null), @"""null"""); + yield return (6, new TestUO(0), @"""0"""); + yield return (7, new TestUO(1.3f), @"""1.3"""); + yield return (8, new TestUO(-1.7m), @"""-1.7"""); + yield return (9, new TestUO(new[] { true, false }), @"""[true,false]"""); + yield return (10, new TestUO(new bool[0]), @"""[]"""); + yield return (11, new TestUO(new[] { "abc", "def" }), @"""[""""abc"""",""""def""""]"""); + yield return (12, new TestUO(new string[0]), @"""[]"""); + yield return (13, new TestUO(new[] { 11.5m, -7.5m }), @"""[11.5,-7.5]"""); + yield return (14, new TestUO(new string[0]), @"""[]"""); + yield return (15, new TestUO(new[] { new[] { 1, 2 }, new[] { 3, 4 } }), @"""[[1,2],[3,4]]"""); + yield return (16, new TestUO(new[] { new object[] { 1, 2 }, new object[] { true, "a", 7 } }), @"""[[1,2],[true,""""a"""",7]]"""); + yield return (17, new TestUO(new { a = 10, b = -20m, c = "abc" }), @"""{""""a"""":10,""""b"""":-20,""""c"""":""""abc""""}"""); + yield return (18, new TestUO(new { x = new { y = true } }), @"""{""""x"""":{""""y"""":true}}"""); + yield return (19, new TestUO(new { x = new { y = new[] { 1 }, z = "a", t = new { } }, a = false }), @"""{""""a"""":false,""""x"""":{""""t"""":{},""""y"""":[1],""""z"""":""""a""""}}"""); + yield return (20, new TestUO(123456789012345.6789012345678m), @"""123456789012345.6789012345678"""); + } + + public class TestUO : IUntypedObject + { + private enum UOType + { + Unknown = -1, + Array, + Object, + Bool, + Decimal, + String + } + + private readonly dynamic _o; + + public TestUO(object o) + { + _o = o; + } + + public IUntypedObject this[int index] => GetUOType(_o) == UOType.Array ? new TestUO(_o[index]) : throw new Exception("Not an array"); + + public FormulaType Type => _o == null ? FormulaType.String : _o switch + { + string => FormulaType.String, + int or double or float => new ExternalType(ExternalTypeKind.UntypedNumber), + decimal => FormulaType.Decimal, + bool => FormulaType.Boolean, + Array => new ExternalType(ExternalTypeKind.Array), + object o => new ExternalType(ExternalTypeKind.Object), + _ => throw new Exception("Not a valid type") + }; + + private static UOType GetUOType(object o) => o switch + { + bool => UOType.Bool, + int or decimal or double => UOType.Decimal, + string => UOType.String, + Array => UOType.Array, + _ => UOType.Object + }; + + public int GetArrayLength() + { + return _o is Array a ? a.Length : throw new Exception("Not an array"); + } + + public bool GetBoolean() + { + return _o is bool b ? b + : throw new Exception("Not a boolean"); + } + + public decimal GetDecimal() + { + return _o is int i ? (decimal)i + : _o is float f ? (decimal)f + : _o is decimal dec ? dec + : throw new Exception("Not a decimal"); + } + + public double GetDouble() + { + return _o is int i ? (double)i + : _o is float f ? (double)f + : _o is double dbl ? dbl + : throw new Exception("Not a double"); + } + + public string GetString() + { + return _o == null ? null + : _o is string str ? str + : throw new Exception("Not a string"); + } + + public string GetUntypedNumber() + { + return _o is int i ? i.ToString() + : _o is float f ? f.ToString() + : _o is double dbl ? dbl.ToString() + : _o is decimal dec ? dec.ToString() + : throw new Exception("Not valid untyped number"); + } + + public bool TryGetProperty(string value, out IUntypedObject result) + { + if (_o is object o && o.GetType().GetProperties().Any(pi => pi.Name == value)) + { + PropertyInfo pi = o.GetType().GetProperty(value); + object prop = pi.GetValue(_o); + + result = new TestUO(prop); + return true; + } + + result = null; + return false; + } + + public bool TryGetPropertyNames(out IEnumerable propertyNames) + { + if (_o is object o) + { + propertyNames = o.GetType().GetProperties().Select(pi => pi.Name); + return true; + } + + propertyNames = null; + return false; + } + } + } +} From dd32f3ce9cdb451072a6b972ae3245daf1a85f2f Mon Sep 17 00:00:00 2001 From: Luc Genetier <69138830+LucGenetier@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:06:50 +0200 Subject: [PATCH 10/13] JSON UO update (#2655) --- src/strings/PowerFxResources.en-US.resx | 4 ++-- .../ExpressionTestCases/JSON.txt | 5 +---- .../ExpressionTestCases/JSON_V1Compat.txt | 3 +++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 84509cbaf5..6ac8be115e 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4762,10 +4762,10 @@ Maximum depth reached while traversing JSON payload. - Error message returned by the {Locked=JSON} function when a document that is too deeply nested is passed to it. The term JSON refers to the data format described in www.json.org. + Error message returned by the JSON function when a document that is too deeply nested is passed to it. The term JSON in the error refers to the data format described in www.json.org. Maximum length reached in JSON function. - Error message returned by the {Locked=JSON} function when the result generated by this function would be too long. The term JSON refers to the data format described in www.json.org. + Error message returned by the {Locked=JSON} function when the result generated by this function would be too long. \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt index eb9c58e861..00375c062a 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON.txt @@ -1,4 +1,4 @@ -#SETUP: EnableJsonFunctions, PowerFxV1CompatibilityRules +#SETUP: EnableJsonFunctions >> JSON() Errors: Error 0-6: Invalid number of arguments: received 0, expected 1-2. @@ -211,9 +211,6 @@ Error({Kind:ErrorKind.InvalidArgument}) >> JSON(ParseJSON("[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]")) Error({Kind:ErrorKind.InvalidArgument}) ->> JSON(Decimal(ParseJSON("123456789012345.6789012345678"))) -"123456789012345.6789012345678" - >> JSON(ParseJSON("123456789012345.6789012345678")) "123456789012345.6789012345678" diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON_V1Compat.txt index c64f936248..1b57b3c355 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/JSON_V1Compat.txt @@ -3,3 +3,6 @@ // Error for unknown options in the second argument >> JSON({a:1,b:[1,2,3]}, "_U") Errors: Error 22-26: Invalid argument type (Text). Expecting a Enum (JSONFormat) value instead. + +>> JSON(Decimal(ParseJSON("123456789012345.6789012345678"))) +"123456789012345.6789012345678" From a5c092036622a4f7c323a0185bc12147b3782d6d Mon Sep 17 00:00:00 2001 From: Carlos Figueira Date: Thu, 26 Sep 2024 07:13:51 -0700 Subject: [PATCH 11/13] Add new test to validate the DType always orders its fields (#2657) Currently the tree that stores fields of a complex DType (records or arrays) stores those in an ordered way (ordinal comparison). There is some code that relies on that, so codifying it as a test to make it a "specification" --- .../TypeSystemTests/DTypeTests.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TypeSystemTests/DTypeTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TypeSystemTests/DTypeTests.cs index c432b6dfb1..290660d1ce 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TypeSystemTests/DTypeTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TypeSystemTests/DTypeTests.cs @@ -1102,6 +1102,34 @@ public void RecordAndTableDTypeTests() Assert.True(type11.GetType(DPath.Root.Append(new DName("A"))) == DType.Error); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RecordAndTableFieldOrder(bool useRecord) + { + var type = useRecord ? DType.EmptyRecord : DType.EmptyTable; + + var fieldNames = new[] { "C", "ß", "ç", "A", "b", "á", "é", "ss", "word", "wórd" }; + var expectedFieldOrder = new List(fieldNames); + expectedFieldOrder.Sort(StringComparer.Ordinal); + + foreach (var fieldName in fieldNames) + { + type = type.Add(new DName(fieldName), DType.String); + } + + var expectedTypeStr = + (useRecord ? "!" : "*") + + "[" + + string.Join(", ", expectedFieldOrder.Select(f => f + ":s")) + + "]"; + + Assert.Equal(expectedTypeStr, type.ToString()); + + var actualFieldNames = type.GetNames(DPath.Root).Select(tn => tn.Name.Value).ToArray(); + Assert.Equal(expectedFieldOrder, actualFieldNames); + } + [Fact] public void EnumDTypeTests() { From 6a096a18c226ec885ee6296649ea7f24deb42faa Mon Sep 17 00:00:00 2001 From: Luc Genetier <69138830+LucGenetier@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:21:44 +0200 Subject: [PATCH 12/13] Add test for ZD CDP connector, using enums (#2658) Validate that we can process option sets properly with CDP connectors --- ....PowerFx.Connectors.Tests.Shared.projitems | 2 + .../PowerPlatformTabularTests.cs | 74 ++++- .../Responses/ZD GetTables.json | 2 +- .../Responses/ZD Tickets GetRows.json | 55 ++++ .../Responses/ZD Tickets GetSchema.json | 271 ++++++++++++++++++ 5 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetRows.json create mode 100644 src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems index 86400c1585..dc0b202d51 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems @@ -270,6 +270,8 @@ + + diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs index 4dd691f33b..599f5898d1 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs @@ -297,7 +297,7 @@ public async Task SAP_CDP() SymbolValues symbolValues = new SymbolValues().Add("TeamCalendarCollection", sapTableValue); RuntimeConfig rc = new RuntimeConfig(symbolValues).AddService(logger); - + CheckResult check = engine.Check(expr, options: new ParserOptions() { AllowsSideEffects = true }, symbolTable: symbolValues.SymbolTable); Assert.True(check.IsSuccess); @@ -916,6 +916,78 @@ public async Task ZD_CdpTabular_GetTables() StringValue userName = Assert.IsType(result); Assert.Equal("Ram Sitwat", userName.Value); } + + [Fact] + public async Task ZD_CdpTabular_GetTables2() + { + using var testConnector = new LoggingTestServer(null /* no swagger */, _output); + var config = new PowerFxConfig(Features.PowerFxV1); + var engine = new RecalcEngine(config); + + ConsoleLogger logger = new ConsoleLogger(_output); + using var httpClient = new HttpClient(testConnector); + string connectionId = "ca06d34f4b684e38b7cf4c0f517a7e99"; + string uriPrefix = $"/apim/zendesk/{connectionId}"; + string jwt = "eyJ0eXA..."; + using var client = new PowerPlatformConnectorClient("4d4a8e81-17a4-4a92-9bfe-8d12e607fb7f.08.common.tip1.azure-apihub.net", "4d4a8e81-17a4-4a92-9bfe-8d12e607fb7f", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" }; + + testConnector.SetResponseFromFile(@"Responses\ZD GetDatasetsMetadata.json"); + DatasetMetadata dm = await CdpDataSource.GetDatasetsMetadataAsync(client, uriPrefix, CancellationToken.None, logger); + + Assert.NotNull(dm); + Assert.Null(dm.Blob); + Assert.Null(dm.DatasetFormat); + Assert.Null(dm.Parameters); + + Assert.NotNull(dm.Tabular); + Assert.Equal("dataset", dm.Tabular.DisplayName); + Assert.Equal("singleton", dm.Tabular.Source); + Assert.Equal("table", dm.Tabular.TableDisplayName); + Assert.Equal("tables", dm.Tabular.TablePluralName); + Assert.Equal("double", dm.Tabular.UrlEncoding); + + CdpDataSource cds = new CdpDataSource("default"); + + // only one network call as we already read metadata + testConnector.SetResponseFromFiles(@"Responses\ZD GetDatasetsMetadata.json", @"Responses\ZD GetTables.json"); + IEnumerable tables = await cds.GetTablesAsync(client, uriPrefix, CancellationToken.None, logger); + + Assert.NotNull(tables); + Assert.Equal(18, tables.Count()); + + CdpTable connectorTable = tables.First(t => t.DisplayName == "Tickets"); + Assert.Equal("tickets", connectorTable.TableName); + Assert.False(connectorTable.IsInitialized); + + testConnector.SetResponseFromFile(@"Responses\ZD Tickets GetSchema.json"); + await connectorTable.InitAsync(client, uriPrefix, CancellationToken.None, logger); + Assert.True(connectorTable.IsInitialized); + + CdpTableValue zdTable = connectorTable.GetTableValue(); + Assert.True(zdTable._tabularService.IsInitialized); + Assert.True(zdTable.IsDelegable); + + Assert.Equal( + "![assignee_id:w, brand_id:w, collaborator_ids:s, created_at:d, custom_fields:s, description:s, due_at:d, external_id:s, followup_ids:s, forum_topic_id:w, group_id:w, has_incidents:b, " + + "id:w, organization_id:w, priority:l, problem_id:w, raw_subject:s, recipient:s, requester_id:w, satisfaction_rating:s, sharing_agreement_ids:s, status:s, subject:s, submitter_id:w, " + + "tags:s, ticket_form_id:w, type:s, updated_at:d, url:s, via:s]", ((CdpRecordType)zdTable.TabularRecordType).ToStringWithDisplayNames()); + + SymbolValues symbolValues = new SymbolValues().Add("Tickets", zdTable); + RuntimeConfig rc = new RuntimeConfig(symbolValues).AddService(logger); + + // Expression with tabular connector + string expr = @"First(Tickets).priority"; + CheckResult check = engine.Check(expr, options: new ParserOptions() { AllowsSideEffects = true }, symbolTable: symbolValues.SymbolTable); + Assert.True(check.IsSuccess); + + // Use tabular connector. Internally we'll call CdpTableValue.GetRowsInternal to get the data + testConnector.SetResponseFromFile(@"Responses\ZD Tickets GetRows.json"); + FormulaValue result = await check.GetEvaluator().EvalAsync(CancellationToken.None, rc); + + OptionSetValue priority = Assert.IsType(result); + Assert.Equal("normal", priority.Option); + Assert.Equal("normal", priority.DisplayName); + } } public static class Exts2 diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD GetTables.json b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD GetTables.json index ca4d0fb329..dad12db94a 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD GetTables.json +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD GetTables.json @@ -1,5 +1,5 @@ { - "@odata.context": "https://zendesk-dfscus.azconn-dfscus-002.p.azurewebsites.net/$metadata#datasets('default')/tables", + "@odata.context": "https://zendesk-dfwus.azconn-dfwus-001.p.azurewebsites.net/$metadata#datasets('default')/tables", "value": [ { "Name": "activities", diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetRows.json b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetRows.json new file mode 100644 index 0000000000..eded034927 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetRows.json @@ -0,0 +1,55 @@ +{ + "@odata.context":"https://4d4a8e81-17a4-4a92-9bfe-8d12e607fb7f.08.common.tip1.azure-apihub.net/apim/zendesk/ca06d34f4b684e38b7cf4c0f517a7e99/$metadata#datasets('default')/tables('tickets')/items","value":[ + { + "@odata.etag":"","ItemInternalId":"f41edfa3-1246-430c-b165-c7fe223d04ae","id":24,"url":"https://startup8613.zendesk.com/api/v2/tickets/24.json","subject":"SAMPLE TICKET: Gift card","raw_subject":"SAMPLE TICKET: Gift card","description":"Hi there, I have a friend who recently moved overseas and I was thinking of sending her a housewarming gift. I saw that you offer international gift cards, but I\u2019m a little unsure about how the whole process works.\n\nCould you explain? Like what the denominations are, how we determine the recipient\u2019s currency, and how to personalize the gift card?\n\nCheers,\nBlake Jackson","priority":"normal","status":"open","requester_id":33720018216979,"submitter_id":33720018216979,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:33Z","updated_at":"2024-09-24T01:34:33Z","key":"24" + },{ + "@odata.etag":"","ItemInternalId":"207d5fd0-5d44-47cc-ac86-27488ce83a04","id":25,"url":"https://startup8613.zendesk.com/api/v2/tickets/25.json","subject":"SAMPLE TICKET: Gift card expiring","raw_subject":"SAMPLE TICKET: Gift card expiring","description":"Hey there, I was lucky enough to receive a gift card from a friend as a housewarming gift. Small problem, I\u2019ve been so swamped with the move I totally forgot about it until now and it expires in a week!\n\nCan you extend the expiration date?\n\nHelp,\nLuka Jensen","priority":"normal","status":"open","requester_id":33720000283795,"submitter_id":33720000283795,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:33Z","updated_at":"2024-09-24T01:34:33Z","key":"25" + },{ + "@odata.etag":"","ItemInternalId":"b6d12103-9595-4028-b45d-8b1cd02dcc1b","id":23,"url":"https://startup8613.zendesk.com/api/v2/tickets/23.json","subject":"SAMPLE TICKET: Does this impact my credit?","raw_subject":"SAMPLE TICKET: Does this impact my credit?","description":"Hey hey, I\u2019m totally loving your Minimalist Elegance collection! But before I fill up my cart, I\u2019ve got some questions about your payment plans.\n\nI\u2019m curious about how choosing them might be beneficial and will this impact my credit score? Could you break down the advantages of both the Homebuy Layaway and Installment Plan for me specifically in terms of their benefits related to price and instant ownership?\n\nLastly, I wanted to know whether the payment plan I choose might impact the delivery time slot of my chosen pieces.\n\nAppreciate your help!","priority":"normal","status":"open","requester_id":33719972942611,"submitter_id":33719972942611,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:32Z","updated_at":"2024-09-24T01:34:32Z","key":"23" + },{ + "@odata.etag":"","ItemInternalId":"9d91baf9-b20c-47f9-906d-d442c8d11f43","id":19,"url":"https://startup8613.zendesk.com/api/v2/tickets/19.json","subject":"SAMPLE TICKET: Do I put it together?","raw_subject":"SAMPLE TICKET: Do I put it together?","description":"Hey there, I\u2019ve been browsing your site and I keep seeing this term \"Flat Pack Delivery\".\n\nI\u2019m not sure what this really means. Does it shorten my waiting time for delivery or does it just make the package smaller?\n\nAnd once the flat packed item arrives, does this mean I have to put it together?\n\nYour help in clarifying this would be much appreciated!\n\nThanks,\nSoobin Do","priority":"normal","status":"open","requester_id":33720000167699,"submitter_id":33720000167699,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:30Z","updated_at":"2024-09-24T01:34:30Z","key":"19" + },{ + "@odata.etag":"","ItemInternalId":"84c715f4-f310-4b57-b81c-54b130e5f704","id":16,"url":"https://startup8613.zendesk.com/api/v2/tickets/16.json","subject":"SAMPLE TICKET: Where\u2019s my order?","raw_subject":"SAMPLE TICKET: Where\u2019s my order?","description":"Hiya, I\u2019ve recently splurged on some really cool furniture (first time, yay). I got an email confirming all the details with an order number, but here\u2019s the hiccup \u2026 I\u2019m completely lost about how to see my order status.\n\nIs there a way to check this? Or can you do it for me?\nThanks a bunch,\nElla Rivera","priority":"normal","status":"open","requester_id":33719955916179,"submitter_id":33719955916179,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:29Z","updated_at":"2024-09-24T01:34:29Z","key":"16" + },{ + "@odata.etag":"","ItemInternalId":"43abf833-e563-4132-bb79-718525f8ebec","id":11,"url":"https://startup8613.zendesk.com/api/v2/tickets/11.json","subject":"SAMPLE TICKET: Do you do gift wrapping?","raw_subject":"SAMPLE TICKET: Do you do gift wrapping?","description":"Hello! Does your store do gift wrapping? I want to order a dinner set for a friend and it would be great if I could just send it straight to her, already gift wrapped.","priority":"normal","status":"open","requester_id":33719972942611,"submitter_id":33719972942611,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:26Z","updated_at":"2024-09-24T01:35:45Z","key":"11" + },{ + "@odata.etag":"","ItemInternalId":"1a2ed272-c63f-4139-b928-818ccb3225c7","id":8,"url":"https://startup8613.zendesk.com/api/v2/tickets/8.json","subject":"SAMPLE TICKET: Ordered the wrong color","raw_subject":"SAMPLE TICKET: Ordered the wrong color","description":"Hi! I accidentally ordered my item in the wrong color, is it possible to change my order?","priority":"urgent","status":"open","requester_id":33719972845331,"submitter_id":33719972845331,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:24Z","updated_at":"2024-09-24T01:37:06Z","key":"8" + },{ + "@odata.etag":"","ItemInternalId":"dd0dd780-710e-434f-b476-7abf778ea5a2","id":21,"url":"https://startup8613.zendesk.com/api/v2/tickets/21.json","subject":"SAMPLE TICKET: Exchange my order","raw_subject":"SAMPLE TICKET: Exchange my order","description":"I want to swap my purchase for store credit. I recently ordered a piece from y\u2019all, but it doesn\u2019t quite gel with my living room aesthetics like I thought it would.\n\nI see that there\u2019s an option to exchange my item for store credit. Can you shed some light on how this works? Do I have to return the item in its original packaging? Will I get full credit?\n\nAlso, could you briefly guide me on initiating the exchange process through your website?\n\nBest\nMarcus Allen","priority":"normal","status":"open","requester_id":33720000222995,"submitter_id":33720000222995,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:31Z","updated_at":"2024-09-24T01:34:31Z","key":"21" + },{ + "@odata.etag":"","ItemInternalId":"c0018f2c-628f-4a4b-b8ae-9765eb3e0c8a","id":20,"url":"https://startup8613.zendesk.com/api/v2/tickets/20.json","subject":"SAMPLE TICKET: Return my order","raw_subject":"SAMPLE TICKET: Return my order","description":"Hey Homebuy team, I recently ordered a piece of furniture which, unfortunately, doesn\u2019t quite fit the way I hoped it would in my home.\n\nHow do I go about returning it? Is there a time frame within which I need to make the return?\n\nAlso, how will my refund be calculated? I remember reading something about it depending on whether or not the item has been assembled, but I\u2019m not certain.\n\nHoping I can still return this \u2026 thanks.\n\nRam Sitwat","priority":"normal","status":"open","requester_id":33719972845331,"submitter_id":33719972845331,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:30Z","updated_at":"2024-09-24T01:34:30Z","key":"20" + },{ + "@odata.etag":"","ItemInternalId":"bccbd2fe-edf8-4cd9-a72f-9afad7b66bb7","id":13,"url":"https://startup8613.zendesk.com/api/v2/tickets/13.json","subject":"SAMPLE TICKET: Need less items than ordered","raw_subject":"SAMPLE TICKET: Need less items than ordered","description":"Hi! Yesterday I placed an order on your site but accidentaly ordered 3 lamps instead of 2. Can I still change that?","priority":"urgent","status":"open","requester_id":33719984468371,"submitter_id":33719984468371,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:27Z","updated_at":"2024-09-24T01:36:59Z","key":"13" + },{ + "@odata.etag":"","ItemInternalId":"064dfb3e-da9a-46f1-83e2-d4e49956bcb5","id":12,"url":"https://startup8613.zendesk.com/api/v2/tickets/12.json","subject":"SAMPLE TICKET: Cancel order","raw_subject":"SAMPLE TICKET: Cancel order","description":"Can I still cancel my order?","priority":"urgent","status":"open","requester_id":33720000283795,"submitter_id":33720000283795,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:27Z","updated_at":"2024-09-24T01:37:20Z","key":"12" + },{ + "@odata.etag":"","ItemInternalId":"bbbbcec7-c189-473d-ace6-65e1b928ad37","id":9,"url":"https://startup8613.zendesk.com/api/v2/tickets/9.json","subject":"SAMPLE TICKET: Free repair","raw_subject":"SAMPLE TICKET: Free repair","description":"G\u2019day. My cat shredded my brand new armchair today. The leather now has holes in it. Do you offer a repair service or is there a warranty on my chair?","priority":"normal","status":"open","requester_id":33720000222995,"submitter_id":33720000222995,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:25Z","updated_at":"2024-09-24T01:37:09Z","key":"9" + },{ + "@odata.etag":"","ItemInternalId":"857f203b-3920-43ef-ab68-3348e94be439","id":18,"url":"https://startup8613.zendesk.com/api/v2/tickets/18.json","subject":"SAMPLE TICKET: Shipping cost","raw_subject":"SAMPLE TICKET: Shipping cost","description":"Hello, I\u2019ve got some cool items in my cart on your site, but before I take the plunge, I want to understand how much I\u2019ll be paying for shipping. The numbers can be a bit scary when you don\u2019t know what they\u2019re for.\n\nCan you help me understand what all influences the shipping costs? Is there a calculator or formula I can use first?","priority":"normal","status":"open","requester_id":33719937265939,"submitter_id":33719937265939,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:30Z","updated_at":"2024-09-24T01:34:30Z","key":"18" + },{ + "@odata.etag":"","ItemInternalId":"e172d475-e80b-4f15-abbd-345c8855afe3","id":15,"url":"https://startup8613.zendesk.com/api/v2/tickets/15.json","subject":"SAMPLE TICKET: Putting it together?","raw_subject":"SAMPLE TICKET: Putting it together?","description":"Hello, I just placed my order and I\u2019m eager to set up my new furniture when it arrives, but must admit, I\u2019m not very handy, so, uh, I\u2019m not confident about the assembly part.\n\nAre there online instructions? What should I do if I find the assembly steps too difficult or complicated?\n\nKind regards,\nCarlos Garcia","priority":"normal","status":"open","requester_id":33720000038803,"submitter_id":33720000038803,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:28Z","updated_at":"2024-09-24T01:34:28Z","key":"15" + },{ + "@odata.etag":"","ItemInternalId":"51a15f8c-ea29-4b31-b2d2-dcc85b142bd0","id":14,"url":"https://startup8613.zendesk.com/api/v2/tickets/14.json","subject":"SAMPLE TICKET: Where\u2019s it made?","raw_subject":"SAMPLE TICKET: Where\u2019s it made?","description":"Hi, I\u2019m relatively new to furniture shopping and was introduced to Homebuy by a friend. I visited your online store and I\u2019m really interested in your furniture collections, but I\u2019m concerned about where things are made.\n\nAlso, I see that you offer something called \"direct-to-consumer\" payment plans. Could you explain a bit more about what that involves?\n\nThanks in advance for your time,\nTaylor Moore","priority":"normal","status":"open","requester_id":33719937084435,"submitter_id":33719937084435,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:28Z","updated_at":"2024-09-24T01:34:28Z","key":"14" + },{ + "@odata.etag":"","ItemInternalId":"f9dea1b4-199a-47e1-aedb-25741b62b158","id":22,"url":"https://startup8613.zendesk.com/api/v2/tickets/22.json","subject":"SAMPLE TICKET: How does layaway work?","raw_subject":"SAMPLE TICKET: How does layaway work?","description":"Hello. I\u2019m about to purchase a few items from your Urban Chic collection, but the total is a bit much to pay up front. I\u2019ve noticed that you offer a couple of payment options that I\u2019m considering, but I would love some more details. Can you explain the difference between how the Homebuy Layaway and Homebuy Installment Plan work? Can I do both?\n\nAlso, are there any specific thresholds or conditions that I should be aware of while opting for these payment plans?\n\nLooking forward to your response,\nJakub W\u00f3jcik","priority":"normal","status":"open","requester_id":33719969757587,"submitter_id":33719969757587,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:31Z","updated_at":"2024-09-24T01:34:31Z","key":"22" + },{ + "@odata.etag":"","ItemInternalId":"1b7d1102-c808-4c31-a27a-e6ea8d482ad1","id":17,"url":"https://startup8613.zendesk.com/api/v2/tickets/17.json","subject":"SAMPLE TICKET: New delivery address","raw_subject":"SAMPLE TICKET: New delivery address","description":"Hi there! Looks like I jumped the gun and, uh-oh, put the wrong delivery address on my order. I need to fix this fast but I\u2019m not sure how to go about it.\nCan you help me change the delivery address for my order? Or can I do it on the website? Also, is there a deadline for when I can make this change?\n\nThanks for your help!\n\nIngrid Van Dijk","priority":"normal","status":"open","requester_id":33719955946643,"submitter_id":33719955946643,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:29Z","updated_at":"2024-09-24T01:34:29Z","key":"17" + },{ + "@odata.etag":"","ItemInternalId":"42bdcae9-bbd5-4102-a69b-1d3de6ffae47","id":10,"url":"https://startup8613.zendesk.com/api/v2/tickets/10.json","subject":"SAMPLE TICKET: Missing assembly instructions","raw_subject":"SAMPLE TICKET: Missing assembly instructions","description":"Hi, I recently ordered a wardrobe from your website. It arrived yesterday but it looks like the assembly instructions are missing.","priority":"urgent","status":"open","requester_id":33719969757587,"submitter_id":33719969757587,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:25Z","updated_at":"2024-09-24T01:36:52Z","key":"10" + },{ + "@odata.etag":"","ItemInternalId":"11123242-c3d2-4bc6-8655-a10214fe6231","id":4,"url":"https://startup8613.zendesk.com/api/v2/tickets/4.json","subject":"SAMPLE TICKET: Item restock","raw_subject":"SAMPLE TICKET: Item restock","description":"Hello! I noticed that your wattle cushion is out of stock. I want to purchase it for my friend\u2019s birthday next month, do you know when it will be restocked?","priority":"normal","status":"open","requester_id":33719955916179,"submitter_id":33719955916179,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:21Z","updated_at":"2024-09-24T01:35:21Z","key":"4" + },{ + "@odata.etag":"","ItemInternalId":"fff0231c-3f40-4e5d-8775-95b6516b9795","id":7,"url":"https://startup8613.zendesk.com/api/v2/tickets/7.json","subject":"SAMPLE TICKET: Wrong address","raw_subject":"SAMPLE TICKET: Wrong address","description":"Hi there, I just purchased an item and realized I used the wrong delivery address. Can you help me?","priority":"urgent","status":"open","requester_id":33720000167699,"submitter_id":33720000167699,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:23Z","updated_at":"2024-09-24T01:37:16Z","key":"7" + },{ + "@odata.etag":"","ItemInternalId":"4db1c873-30cc-4d21-a0b6-a2ae730797ef","id":3,"url":"https://startup8613.zendesk.com/api/v2/tickets/3.json","subject":"SAMPLE TICKET: Order status says it is still processing","raw_subject":"SAMPLE TICKET: Order status says it is still processing","description":"My order status has been stuck on processing for 3 days. Why hasn\u2019t it been shipped yet?","priority":"urgent","status":"open","requester_id":33720000038803,"submitter_id":33720000038803,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:21Z","updated_at":"2024-09-24T01:36:39Z","key":"3" + },{ + "@odata.etag":"","ItemInternalId":"184eb7ea-8cb6-4b1c-b0b4-4deb01500fba","id":2,"url":"https://startup8613.zendesk.com/api/v2/tickets/2.json","subject":"SAMPLE TICKET: Damaged product","raw_subject":"SAMPLE TICKET: Damaged product","description":"Hi there, I received my armchair today in the mail and I noticed that the material has a tear in it.\n\nI would like a replacement sent to me or to be issued a refund.","priority":"urgent","status":"open","requester_id":33719937084435,"submitter_id":33719937084435,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:20Z","updated_at":"2024-09-24T01:36:35Z","key":"2" + },{ + "@odata.etag":"","ItemInternalId":"21a25035-d6a3-4c0f-b196-ec8964c05ee7","id":5,"url":"https://startup8613.zendesk.com/api/v2/tickets/5.json","subject":"SAMPLE TICKET: International shipping","raw_subject":"SAMPLE TICKET: International shipping","description":"Hello, I would like to buy one of your products and send it to my friend in Portugal. Do you ship overseas?","priority":"normal","status":"open","requester_id":33719955946643,"submitter_id":33719955946643,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:22Z","updated_at":"2024-09-24T01:35:25Z","key":"5" + },{ + "@odata.etag":"","ItemInternalId":"9436acec-75a0-4678-aeac-b2ce5515c94b","id":6,"url":"https://startup8613.zendesk.com/api/v2/tickets/6.json","subject":"SAMPLE TICKET: Are products ethically sourced","raw_subject":"SAMPLE TICKET: Are products ethically sourced","description":"Hi, I\u2019m interested in some of your products, but want to know if your materials are sustainably and ethically sourced before purchasing.","priority":"low","status":"open","requester_id":33719937265939,"submitter_id":33719937265939,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:34:23Z","updated_at":"2024-09-24T01:35:28Z","key":"6" + },{ + "@odata.etag":"","ItemInternalId":"c0a50c4f-5c67-44bf-a0c0-aab0ef66b0ae","id":1,"url":"https://startup8613.zendesk.com/api/v2/tickets/1.json","type":"incident","subject":"SAMPLE TICKET: Meet the ticket","raw_subject":"SAMPLE TICKET: Meet the ticket","description":"Hi there,\n\nI\u2019m sending an email because I\u2019m having a problem setting up your new product. Can you help me troubleshoot?\n\nThanks,\n The Customer","priority":"normal","status":"open","requester_id":33719944623507,"submitter_id":33719960414739,"assignee_id":33719960414739,"group_id":33719944452883,"has_incidents":false,"ticket_form_id":33719944367635,"brand_id":33719928618899,"created_at":"2024-09-24T01:32:27Z","updated_at":"2024-09-24T01:32:27Z","key":"1" + } + ] +} diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json new file mode 100644 index 0000000000..b46cb8ba2d --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/ZD Tickets GetSchema.json @@ -0,0 +1,271 @@ +{ + "name": "tickets", + "title": "tickets", + "x-ms-permission": "read-write", + "x-ms-capabilities": { + "sortRestrictions": { + "sortable": true, + "unsortableProperties": [ "collaborator_ids", "tags", "via", "custom_fields", "satisfaction_rating", "sharing_agreement_ids", "followup_ids" ] + }, + "filterRestrictions": { + "filterable": true, + "nonFilterableProperties": [ "collaborator_ids", "tags", "via", "custom_fields", "satisfaction_rating", "sharing_agreement_ids", "followup_ids" ] + }, + "selectRestrictions": { "selectable": true }, + "filterFunctionSupport": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "now", "not", "and", "or", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes", "totalseconds", "round", "floor", "ceiling", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat", "sum", "min", "max", "average", "countdistinct", "null" ] + }, + "schema": { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "id": { + "title": "id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-keyOrder": 1, + "x-ms-keyType": "primary", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "url": { + "title": "url", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "external_id": { + "title": "external_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "type": { + "title": "type", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "subject": { + "title": "subject", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "raw_subject": { + "title": "raw_subject", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "description": { + "title": "description", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "priority": { + "title": "priority", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "enum": [ + "low", + "normal", + "high", + "urgent" + ], + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "status": { + "title": "status", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "recipient": { + "title": "recipient", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] }, + "type": "string", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "requester_id": { + "title": "requester_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "submitter_id": { + "title": "submitter_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "assignee_id": { + "title": "assignee_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "organization_id": { + "title": "organization_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "group_id": { + "title": "group_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "collaborator_ids": { + "title": "collaborator_ids", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "forum_topic_id": { + "title": "forum_topic_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "problem_id": { + "title": "problem_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "has_incidents": { + "title": "has_incidents", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "not", "and", "or" ] }, + "type": "boolean", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "due_at": { + "title": "due_at", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes" ] }, + "type": "string", + "format": "date-time", + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "tags": { + "title": "tags", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "via": { + "title": "via", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "custom_fields": { + "title": "custom_fields", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "satisfaction_rating": { + "title": "satisfaction_rating", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "sharing_agreement_ids": { + "title": "sharing_agreement_ids", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "followup_ids": { + "title": "followup_ids", + "type": "string", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "ticket_form_id": { + "title": "ticket_form_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "brand_id": { + "title": "brand_id", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] }, + "type": "integer", + "format": "int64", + "minimum": -9223372036854775808, + "maximum": 9223372036854775807, + "x-ms-permission": "read-write", + "x-ms-sort": "none" + }, + "created_at": { + "title": "created_at", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes" ] }, + "type": "string", + "format": "date-time", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + }, + "updated_at": { + "title": "updated_at", + "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes" ] }, + "type": "string", + "format": "date-time", + "x-ms-permission": "read-only", + "x-ms-sort": "none" + } + } + }, + "x-ms-permission": "read-write" + } +} From de16049af3981f14a2fd464641b6f76e74e5f50c Mon Sep 17 00:00:00 2001 From: Adithya Selvaprithiviraj Date: Fri, 27 Sep 2024 17:47:50 -0700 Subject: [PATCH 13/13] Change NamedType definition syntax (#2616) Changes NamedType definitions to use `:=` --- .../Microsoft.PowerFx.Core/Lexer/TexlLexer.cs | 6 +- .../Microsoft.PowerFx.Core/Lexer/TokKind.cs | 8 +- .../Localization/Strings.cs | 1 + .../Parser/TexlParser.cs | 47 +++++++++- src/strings/PowerFxResources.en-US.resx | 8 ++ .../NamedFormulasTests.cs | 6 +- .../UserDefinedTypeTests.cs | 94 ++++++++++--------- .../RecalcEngineTests.cs | 42 ++++----- .../AsTypeIsTypeParseJSONTests.cs | 6 +- 9 files changed, 140 insertions(+), 78 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Lexer/TexlLexer.cs b/src/libraries/Microsoft.PowerFx.Core/Lexer/TexlLexer.cs index fb6a05e1b5..e59d6713ef 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Lexer/TexlLexer.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Lexer/TexlLexer.cs @@ -81,7 +81,8 @@ public enum Flags public const string PunctuatorColon = ":"; public const string PunctuatorAt = "@"; public const char IdentifierDelimiter = '\''; - public const string PunctuatorDoubleBarrelArrow = "=>"; + public const string PunctuatorDoubleBarrelArrow = "=>"; + public const string PunctuatorColonEqual = ":="; // These puntuators are related to commenting in the formula bar public const string PunctuatorBlockComment = "/*"; @@ -305,7 +306,8 @@ private TexlLexer(string preferredDecimalSeparator) AddPunctuator(punctuators, PunctuatorAmpersand, TokKind.Ampersand); AddPunctuator(punctuators, PunctuatorPercent, TokKind.PercentSign); AddPunctuator(punctuators, PunctuatorAt, TokKind.At); - AddPunctuator(punctuators, PunctuatorDoubleBarrelArrow, TokKind.DoubleBarrelArrow); + AddPunctuator(punctuators, PunctuatorDoubleBarrelArrow, TokKind.DoubleBarrelArrow); + AddPunctuator(punctuators, PunctuatorColonEqual, TokKind.ColonEqual); // Commenting punctuators AddPunctuator(punctuators, PunctuatorBlockComment, TokKind.Comment); diff --git a/src/libraries/Microsoft.PowerFx.Core/Lexer/TokKind.cs b/src/libraries/Microsoft.PowerFx.Core/Lexer/TokKind.cs index a91473efe9..55ef86bd0b 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Lexer/TokKind.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Lexer/TokKind.cs @@ -303,6 +303,12 @@ public enum TokKind /// Start of body for user defined functions. /// => /// - DoubleBarrelArrow, + DoubleBarrelArrow, + + /// + /// Colon immediately followed by equal. + /// := + /// + ColonEqual, } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index f1dd1963b3..63ce198629 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -763,6 +763,7 @@ internal static class TexlStrings public static ErrorResourceKey ErrNamedFormula_MissingSemicolon = new ErrorResourceKey("ErrNamedFormula_MissingSemicolon"); public static ErrorResourceKey ErrNamedFormula_MissingValue = new ErrorResourceKey("ErrNamedFormula_MissingValue"); + public static ErrorResourceKey ErrNamedType_MissingTypeLiteral = new ErrorResourceKey("ErrNamedType_MissingTypeLiteral"); public static ErrorResourceKey ErrUDF_MissingFunctionBody = new ErrorResourceKey("ErrUDF_MissingFunctionBody"); public static ErrorResourceKey ErrNamedFormula_AlreadyDefined = new ErrorResourceKey("ErrNamedFormula_AlreadyDefined"); public static ErrorResourceKey ErrorResource_NameConflict = new ErrorResourceKey("ErrorResource_NameConflict"); diff --git a/src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs b/src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs index 8e642a5bd1..8f95c9a50f 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs @@ -300,7 +300,7 @@ private ParseUserDefinitionResult ParseUDFsAndNamedFormulas(string script, Parse continue; } - if (_curs.TidCur == TokKind.Equ) + if (_curs.TidCur == TokKind.ColonEqual && _flagsMode.Peek().HasFlag(Flags.AllowTypeLiteral)) { var declaration = script.Substring(declarationStart, _curs.TokCur.Span.Min - declarationStart); _curs.TokMove(); @@ -308,7 +308,7 @@ private ParseUserDefinitionResult ParseUDFsAndNamedFormulas(string script, Parse if (_curs.TidCur == TokKind.Semicolon) { - CreateError(thisIdentifier, TexlStrings.ErrNamedFormula_MissingValue); + CreateError(thisIdentifier, TexlStrings.ErrNamedType_MissingTypeLiteral); } // Extract expression @@ -334,6 +334,49 @@ private ParseUserDefinitionResult ParseUDFsAndNamedFormulas(string script, Parse definitionBeforeTrivia = new List(); continue; } + else + { + CreateError(_curs.TokCur, TexlStrings.ErrNamedType_MissingTypeLiteral); + } + + // If the result was an error, keep moving cursor until end of named type expression + if (result.Kind == NodeKind.Error) + { + while (_curs.TidCur != TokKind.Semicolon && _curs.TidCur != TokKind.Eof) + { + _curs.TokMove(); + } + } + } + + declarationStart = _curs.TokCur.Span.Lim; + _curs.TokMove(); + ParseTrivia(); + } + else if (_curs.TidCur == TokKind.Equ) + { + var declaration = script.Substring(declarationStart, _curs.TokCur.Span.Min - declarationStart); + _curs.TokMove(); + definitionBeforeTrivia.Add(ParseTrivia()); + + if (_curs.TidCur == TokKind.Semicolon) + { + CreateError(thisIdentifier, TexlStrings.ErrNamedFormula_MissingValue); + } + + // Extract expression + while (_curs.TidCur != TokKind.Semicolon) + { + // Check if we're at EOF before a semicolon is found + if (_curs.TidCur == TokKind.Eof) + { + CreateError(_curs.TokCur, TexlStrings.ErrNamedFormula_MissingSemicolon); + break; + } + + // Parse expression + definitionBeforeTrivia.Add(ParseTrivia()); + var result = ParseExpr(Precedence.None); namedFormulas.Add(new NamedFormula(thisIdentifier.As(), new Formula(result.GetCompleteSpan().GetFragment(script), result), _startingIndex, attribute)); userDefinitionSourceInfos.Add(new UserDefinitionSourceInfo(index++, UserDefinitionType.NamedFormula, thisIdentifier.As(), declaration, new SourceList(definitionBeforeTrivia), GetExtraTriviaSourceList())); diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 6ac8be115e..06c0828c9a 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4115,6 +4115,14 @@ Named formula must be an expression. This error message shows up when Named formula is not an expression. For example, a = ; + + Named type must be a type literal. + + This error message shows up when Named type is not a type literal. A valid type literal expression is of syntax "Type(Expression)". + Some examples for valid named type declarations - "Point := Type({x: Number, y: Number});" , "T1 := Type(Number);" , "T2 := Type([Boolean]);". + Some examples for invalid named type declarations - "T1 := 5;" , "T2 := ;" , "T3 := [1, 2, 3];". + + User-defined function must have a body. This error message shows up when user-defined function does not have a body diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/NamedFormulasTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/NamedFormulasTests.cs index 9a81a045ac..400377e58d 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/NamedFormulasTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/NamedFormulasTests.cs @@ -17,7 +17,7 @@ public class NamedFormulasTests : PowerFxTest private readonly ParserOptions _parseOptions = new ParserOptions() { AllowsSideEffects = true }; [Theory] - [InlineData("Foo = Type(Number);")] + [InlineData("Foo := Type(Number);")] public void DefSimpleTypeTest(string script) { var parserOptions = new ParserOptions() @@ -33,7 +33,7 @@ public void DefSimpleTypeTest(string script) } [Theory] - [InlineData("Foo = Type({ Age: Number });")] + [InlineData("Foo := Type({ Age: Number });")] public void DefRecordTypeTest(string script) { var parserOptions = new ParserOptions() @@ -63,7 +63,7 @@ public void AsTypeTest(string script) } [Theory] - [InlineData("Foo = Type({Age: Number}; Bar(x: Number): Number = Abs(x);")] + [InlineData("Foo := Type({Age: Number}; Bar(x: Number): Number = Abs(x);")] public void FailParsingTest(string script) { var parserOptions = new ParserOptions() diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedTypeTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedTypeTests.cs index a9d10f5337..163e732dc1 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedTypeTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/UserDefinedTypeTests.cs @@ -22,24 +22,24 @@ public class UserDefinedTypeTests : PowerFxTest [Theory] // Check record, table types with primitive types - [InlineData("Point = Type({ x: Number, y: Number })", "![x:n,y:n]", true)] - [InlineData("Points = Type([{ x: Number, y: Number }])", "*[x:n,y:n]", true)] - [InlineData("Person = Type({ name: Text, dob: Date })", "![name:s,dob:D]", true)] - [InlineData("People = Type([{ name: Text, isReady: Boolean }])", "*[name:s,isReady:b]", true)] - [InlineData("Heights = Type([Number])", "*[Value:n]", true)] - [InlineData("Palette = Type([Color])", "*[Value:c]", true)] + [InlineData("Point := Type({ x: Number, y: Number })", "![x:n,y:n]", true)] + [InlineData("Points := Type([{ x: Number, y: Number }])", "*[x:n,y:n]", true)] + [InlineData("Person := Type({ name: Text, dob: Date })", "![name:s,dob:D]", true)] + [InlineData("People := Type([{ name: Text, isReady: Boolean }])", "*[name:s,isReady:b]", true)] + [InlineData("Heights := Type([Number])", "*[Value:n]", true)] + [InlineData("Palette := Type([Color])", "*[Value:c]", true)] // Type alias - [InlineData("DTNZ = Type(DateTimeTZInd)", "Z", true)] + [InlineData("DTNZ := Type(DateTimeTZInd)", "Z", true)] // Nested record types - [InlineData("Nested = Type({a: {b: DateTime, c: {d: GUID, e: Hyperlink}}, x: Time})", "![a:![b:d, c:![d:g, e:h]], x:T]", true)] + [InlineData("Nested := Type({a: {b: DateTime, c: {d: GUID, e: Hyperlink}}, x: Time})", "![a:![b:d, c:![d:g, e:h]], x:T]", true)] // Invalid types - [InlineData("Pics = Type([Image])", "*[Value:i]", false)] - [InlineData("A = Type(B)", "", false)] - [InlineData("A = Type([])", "", false)] - [InlineData("A = Type({})", "", false)] + [InlineData("Pics := Type([Image])", "*[Value:i]", false)] + [InlineData("A := Type(B)", "", false)] + [InlineData("A := Type([])", "", false)] + [InlineData("A := Type({})", "", false)] public void TestUserDefinedType(string typeDefinition, string expectedDefinedTypeString, bool isValid) { var parseOptions = new ParserOptions @@ -66,38 +66,38 @@ public void TestUserDefinedType(string typeDefinition, string expectedDefinedTyp } [Theory] - [InlineData("X = 5; Point = Type({ x: Number, y: Number })", 1)] - [InlineData("Point = Type({ x: Number, y: Number }); Points = Type([Point])", 2)] + [InlineData("X = 5; Point := Type({ x: Number, y: Number })", 1)] + [InlineData("Point := Type({ x: Number, y: Number }); Points := Type([Point])", 2)] // Mix named formula with named type - [InlineData("X = 5; Point = Type({ x: X, y: Number }); Points = Type([Point])", 0)] + [InlineData("X = 5; Point := Type({ x: X, y: Number }); Points := Type([Point])", 0)] // Have invalid type expression - [InlineData("WrongType = Type(5+5); WrongTypes = Type([WrongType]); People = Type([{ name: Text, age: Number }])", 1)] + [InlineData("WrongType := Type(5+5); WrongTypes := Type([WrongType]); People := Type([{ name: Text, age: Number }])", 1)] // Have incomplete expressions and parse errors - [InlineData("Point = Type({a:); Points = Type([Point]); People = Type([{ name: Text, age })", 0)] - [InlineData("Point = Type({a:; Points = Type([Point]); People = Type([{ name: Text, age: Number })", 1)] + [InlineData("Point := Type({a:); Points = Type([Point]); People := Type([{ name: Text, age })", 0)] + [InlineData("Point := Type({a:; Points = Type([Point]); People := Type([{ name: Text, age: Number })", 1)] // Redeclare type - [InlineData("Point = Type({ x: Number, y: Number }); Point = Type(Number);", 1)] + [InlineData("Point := Type({ x: Number, y: Number }); Point := Type(Number);", 1)] // Redeclare typed name in record - [InlineData("X= Type({ f:Number, f:Number});", 0)] + [InlineData("X:= Type({ f:Number, f:Number});", 0)] // Cyclic definition - [InlineData("B = Type({ x: A }); A = Type(B);", 0)] - [InlineData("B = Type(B);", 0)] + [InlineData("B := Type({ x: A }); A := Type(B);", 0)] + [InlineData("B := Type(B);", 0)] // Complex resolutions - [InlineData("C = Type({x: Boolean, y: Date, f: B});B = Type({ x: A }); A = Type(Number);", 3)] - [InlineData("D = Type({nArray: [Number]}), C = Type({x: Boolean, y: Date, f: B});B = Type({ x: A }); A = Type([C]);", 1)] + [InlineData("C := Type({x: Boolean, y: Date, f: B});B := Type({ x: A }); A := Type(Number);", 3)] + [InlineData("D := Type({nArray: [Number]}), C := Type({x: Boolean, y: Date, f: B});B := Type({ x: A }); A := Type([C]);", 1)] // With Invalid types - [InlineData("A = Type(Blob); B = Type({x: Currency}); C = Type([DateTime]); D = Type(None)", 2)] + [InlineData("A := Type(Blob); B := Type({x: Currency}); C := Type([DateTime]); D := Type(None)", 2)] // Have named formulas and udf in the script - [InlineData("NAlias = Type(Number);X = 5; ADDX(n:Number): Number = n + X; SomeType = Type(UntypedObject)", 2)] + [InlineData("NAlias := Type(Number);X := 5; ADDX(n:Number): Number = n + X; SomeType := Type(UntypedObject)", 2)] public void TestValidUDTCounts(string typeDefinition, int expectedDefinedTypesCount) { var parseOptions = new ParserOptions @@ -118,11 +118,13 @@ public void TestValidUDTCounts(string typeDefinition, int expectedDefinedTypesCo [Theory] //To test DefinitionsCheckResult.ApplyErrors method and error messages - [InlineData("Point = Type({ x: Number, y: Number }); Point = Type(Number);", 1, "ErrNamedType_TypeAlreadyDefined")] - [InlineData("X= Type({ f:Number, f:Number});", 1, "ErrNamedType_InvalidTypeDefinition")] - [InlineData("B = Type({ x: A }); A = Type(B);", 2, "ErrNamedType_InvalidCycles")] - [InlineData("B = Type(B);", 1, "ErrNamedType_InvalidCycles")] - [InlineData("Currency = Type({x: Text}); Record = Type([DateTime]); D = Type(None);", 2, "ErrNamedType_InvalidTypeName")] + [InlineData("Point := Type({ x: Number, y: Number }); Point := Type(Number);", 1, "ErrNamedType_TypeAlreadyDefined")] + [InlineData("X:= Type({ f:Number, f:Number});", 1, "ErrNamedType_InvalidTypeDefinition")] + [InlineData("B := Type({ x: A }); A := Type(B);", 2, "ErrNamedType_InvalidCycles")] + [InlineData("B := Type(B);", 1, "ErrNamedType_InvalidCycles")] + [InlineData("Currency := Type({x: Text}); Record := Type([DateTime]); D := Type(None);", 2, "ErrNamedType_InvalidTypeName")] + [InlineData("A = 5;C :=; B := Type(Number);", 1, "ErrNamedType_MissingTypeLiteral")] + [InlineData("C := 5; D := [1,2,3];", 2, "ErrNamedType_MissingTypeLiteral")] public void TestUDTErrors(string typeDefinition, int expectedErrorCount, string expectedMessageKey) { var parseOptions = new ParserOptions @@ -140,21 +142,21 @@ public void TestUDTErrors(string typeDefinition, int expectedErrorCount, string } [Theory] - [InlineData("T = Type({ x: 5+5, y: -5 });", 2)] - [InlineData("T = Type(Type(Number));", 1)] - [InlineData("T = Type({+});", 1)] - [InlineData("T = Type({);", 1)] - [InlineData("T = Type({x: true, y: \"Number\"});", 2)] - [InlineData("T1 = Type({A: Number}); T2 = Type(T1.A);", 1)] - [InlineData("T = Type((1, 2));", 1)] - [InlineData("T1 = Type(UniChar(955)); T2 = Type([Table(Number)])", 2)] - [InlineData("T = Type((1; 2));", 1)] - [InlineData("T = Type(Self.T);", 1)] - [InlineData("T = Type(Parent.T);", 1)] - [InlineData("T = Type(Number As T1);", 1)] - [InlineData("T = Type(Text); T1 = Type(Not T);", 1)] - [InlineData("T1 = Type({V: Number}); T2 = Type(T1[@V]);", 1)] - [InlineData("T = Type([{a: {b: {c: [{d: 10e+4}]}}}]);", 1)] + [InlineData("T := Type({ x: 5+5, y: -5 });", 2)] + [InlineData("T := Type(Type(Number));", 1)] + [InlineData("T := Type({+});", 1)] + [InlineData("T := Type({);", 1)] + [InlineData("T := Type({x: true, y: \"Number\"});", 2)] + [InlineData("T1 := Type({A: Number}); T2 := Type(T1.A);", 1)] + [InlineData("T := Type((1, 2));", 1)] + [InlineData("T1 := Type(UniChar(955)); T2 := Type([Table(Number)])", 2)] + [InlineData("T := Type((1; 2));", 1)] + [InlineData("T := Type(Self.T);", 1)] + [InlineData("T := Type(Parent.T);", 1)] + [InlineData("T := Type(Number As T1);", 1)] + [InlineData("T := Type(Text); T1 := Type(Not T);", 1)] + [InlineData("T1 := Type({V: Number}); T2 := Type(T1[@V]);", 1)] + [InlineData("T := Type([{a: {b: {c: [{d: 10e+4}]}}}]);", 1)] public void TestUDTParseErrors(string typeDefinition, int expectedErrorCount) { var parseOptions = new ParserOptions diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs index b681966a19..0fee0cc1a1 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/RecalcEngineTests.cs @@ -617,7 +617,7 @@ public void ShadowingFunctionPrecedenceTest() result = check.GetEvaluator().Eval(); Assert.Equal(11111, result.AsDouble()); - engine.AddUserDefinitions("Test = Type({A: Number}); TestTable = Type([{A: Number}]);" + + engine.AddUserDefinitions("Test := Type({A: Number}); TestTable := Type([{A: Number}]);" + "Filter(X: TestTable):Test = First(X); ShowColumns(X: TestTable):TestTable = FirstN(X, 3);"); check = engine.Check("Filter([{A: 123}]).A"); @@ -1651,57 +1651,57 @@ public void LookupBuiltinOptionSets() [Theory] [InlineData( - "Point = Type({x : Number, y : Number}); distance(a: Point, b: Point): Number = Sqrt(Power(b.x-a.x, 2) + Power(b.y-a.y, 2));", + "Point := Type({x : Number, y : Number}); distance(a: Point, b: Point): Number = Sqrt(Power(b.x-a.x, 2) + Power(b.y-a.y, 2));", "distance({x: 0, y: 0}, {x: 0, y: 5})", true, 5.0)] // Table types are accepted [InlineData( - "People = Type([{Id:Number, Age: Number}]); countMinors(p: People): Number = CountRows(Filter(p, Age < 18));", + "People := Type([{Id:Number, Age: Number}]); countMinors(p: People): Number = CountRows(Filter(p, Age < 18));", "countMinors([{Id: 1, Age: 17}, {Id: 2, Age: 21}])", true, 1.0)] [InlineData( - "Numbers = Type([Number]); countEven(nums: Numbers): Number = CountRows(Filter(nums, Mod(Value, 2) = 0));", + "Numbers := Type([Number]); countEven(nums: Numbers): Number = CountRows(Filter(nums, Mod(Value, 2) = 0));", "countEven([1,2,3,4,5,6,7,8,9,10])", true, 5.0)] // Type Aliases are allowed [InlineData( - "CarYear = Type(Number); Car = Type({Model: Text, ModelYear: CarYear}); createCar(model:Number, year: Number): Car = {Model:model, ModelYear: year};", + "CarYear := Type(Number); Car := Type({Model: Text, ModelYear: CarYear}); createCar(model:Number, year: Number): Car = {Model:model, ModelYear: year};", "createCar(\"Model Y\", 2024).ModelYear", true, 2024.0)] // Type definitions order shouldn't matter [InlineData( - "Person = Type({Id: IdType, Age: Number}); IdType = Type(Number); createUser(id:Number, a: Number): Person = {Id:id, Age: a};", + "Person := Type({Id: IdType, Age: Number}); IdType := Type(Number); createUser(id:Number, a: Number): Person = {Id:id, Age: a};", "createUser(1, 42).Age", true, 42.0)] // Functions accept record with more/less fields [InlineData( - "People = Type([{Name: Text, Age: Number}]); countMinors(p: People): Number = CountRows(Filter(p, Age < 18));", + "People := Type([{Name: Text, Age: Number}]); countMinors(p: People): Number = CountRows(Filter(p, Age < 18));", "countMinors([{Name: \"Bob\", Age: 21, Title: \"Engineer\"}, {Name: \"Alice\", Age: 25, Title: \"Manager\"}])", true, 0.0)] [InlineData( - "Employee = Type({Name: Text, Age: Number, Title: Text}); getAge(e: Employee): Number = e.Age;", + "Employee := Type({Name: Text, Age: Number, Title: Text}); getAge(e: Employee): Number = e.Age;", "getAge({Name: \"Bob\", Age: 21})", true, 21.0)] [InlineData( - @"Employee = Type({Name: Text, Age: Number, Title: Text}); Employees = Type([Employee]); EmployeeNames = Type([{Name: Text}]); + @"Employee := Type({Name: Text, Age: Number, Title: Text}); Employees := Type([Employee]); EmployeeNames := Type([{Name: Text}]); getNames(e: Employees):EmployeeNames = ShowColumns(e, Name); getNamesCount(e: EmployeeNames):Number = CountRows(getNames(e));", "getNamesCount([{Name: \"Jim\", Age:25}, {Name: \"Tony\", Age:42}])", true, 2.0)] [InlineData( - @"Employee = Type({Name: Text, Age: Number, Title: Text}); + @"Employee := Type({Name: Text, Age: Number, Title: Text}); getAge(e: Employee): Number = e.Age; hasNoAge(e: Employee): Number = IsBlank(getAge(e));", "hasNoAge({Name: \"Bob\", Title: \"CEO\"})", @@ -1710,8 +1710,8 @@ public void LookupBuiltinOptionSets() // Types with UDF restricted primitive types resolve successfully [InlineData( - @"Patient = Type({DOB: DateTimeTZInd, Weight: Decimal, Dummy: None}); - Patients = Type([Patient]); + @"Patient := Type({DOB: DateTimeTZInd, Weight: Decimal, Dummy: None}); + Patients := Type([Patient]); Dummy():Number = CountRows([]);", "Dummy()", true, @@ -1719,45 +1719,45 @@ public void LookupBuiltinOptionSets() // Aggregate types with restricted types are not allowed in UDF [InlineData( - @"Patient = Type({DOB: DateTimeTZInd, Weight: Decimal, Dummy: None}); - Patients = Type([Patient]); + @"Patient := Type({DOB: DateTimeTZInd, Weight: Decimal, Dummy: None}); + Patients := Type([Patient]); getAnomaly(p: Patients): Patients = Filter(p, Weight < 0);", "", false)] [InlineData( - @"Patient = Type({Name: Text, Details: {h: Number, w:Decimal}}); + @"Patient := Type({Name: Text, Details: {h: Number, w:Decimal}}); getPatient(): Patient = {Name:""Alice"", Details: {h: 1, w: 2}};", "", false)] // Cycles not allowed [InlineData( - "Z = Type([{a: {b: Z}}]);", + "Z := Type([{a: {b: Z}}]);", "", false)] [InlineData( - "X = Type(Y); Y = Type(X);", + "X := Type(Y); Y := Type(X);", "", false)] [InlineData( - "C = Type({x: Boolean, y: Date, f: B});B = Type({ x: A }); A = Type([C]);", + "C := Type({x: Boolean, y: Date, f: B});B := Type({ x: A }); A := Type([C]);", "", false)] // Redeclaration not allowed [InlineData( - "Number = Type(Text);", + "Number := Type(Text);", "", false)] [InlineData( - "Point = Type({x : Number, y : Number}); Point = Type({x : Number, y : Number, z: Number})", + "Point := Type({x : Number, y : Number}); Point := Type({x : Number, y : Number, z: Number})", "", false)] // UDFs with body errors should fail [InlineData( - "S = Type({x:Text}); f():S = ({);", + "S := Type({x:Text}); f():S = ({);", "", false)] diff --git a/src/tests/Microsoft.PowerFx.Json.Tests.Shared/AsTypeIsTypeParseJSONTests.cs b/src/tests/Microsoft.PowerFx.Json.Tests.Shared/AsTypeIsTypeParseJSONTests.cs index 5b74f1c72e..c031431f43 100644 --- a/src/tests/Microsoft.PowerFx.Json.Tests.Shared/AsTypeIsTypeParseJSONTests.cs +++ b/src/tests/Microsoft.PowerFx.Json.Tests.Shared/AsTypeIsTypeParseJSONTests.cs @@ -39,7 +39,7 @@ public void PrimitivesTest() var engine = SetupEngine(); // custom-type type alias - engine.AddUserDefinitions("T = Type(Number);"); + engine.AddUserDefinitions("T := Type(Number);"); // Positive tests CheckIsTypeAsTypeParseJSON(engine, "\"42\"", "Number", 42D); @@ -74,7 +74,7 @@ public void RecordsTest() { var engine = SetupEngine(); - engine.AddUserDefinitions("T = Type({a: Number});"); + engine.AddUserDefinitions("T := Type({a: Number});"); dynamic obj1 = new ExpandoObject(); obj1.a = 5D; @@ -103,7 +103,7 @@ public void TablesTest() { var engine = SetupEngine(); - engine.AddUserDefinitions("T = Type([{a: Number}]);"); + engine.AddUserDefinitions("T := Type([{a: Number}]);"); var t1 = new object[] { 5D }; var t2 = new object[] { 1m, 2m, 3m, 4m };