Culture support for in operator and Sort function

src/libraries/Microsoft.PowerFx.Core/Public/Values/StringValue.cs

@@ -36,11 +36,6 @@ public override void Visit(IValueVisitor visitor)
             visitor.Visit(this);
         }
 
-        internal StringValue ToLower()
-        {
-            return new StringValue(IRContext.NotInSource(FormulaType.String), Value.ToLowerInvariant());
-        }
-
         public override void ToExpression(StringBuilder sb, FormulaValueSerializerSettings settings)
         {
             sb.Append($"\"{CharacterUtils.ExcelEscapeString(Value)}\"");

src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryOperators.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Globalization;
 using System.Linq;
 using Microsoft.PowerFx.Core.IR;
 using Microsoft.PowerFx.Core.Utils;
@@ -719,9 +720,9 @@ private static BooleanValue NotEqualPolymorphic(IRContext irContext, FormulaValu
         }
 
         // See in_SS in JScript membershipReplacementFunctions
-        public static Func<IRContext, FormulaValue[], FormulaValue> StringInOperator(bool exact)
+        public static Func<IServiceProvider, IRContext, FormulaValue[], FormulaValue> StringInOperator(bool exact)
         {
-            return (irContext, args) =>
+            return (services, irContext, args) =>
             {
                 var left = args[0];
                 var right = args[1];
@@ -737,23 +738,25 @@ public static Func<IRContext, FormulaValue[], FormulaValue> StringInOperator(boo
 
                 var leftStr = (StringValue)left;
                 var rightStr = (StringValue)right;
-
-                return new BooleanValue(irContext, rightStr.Value.IndexOf(leftStr.Value, exact ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase) >= 0);
+
+                return new BooleanValue(irContext, services.GetService<CultureInfo>().CompareInfo.IndexOf(rightStr.Value, leftStr.Value, exact ? CompareOptions.Ordinal : CompareOptions.IgnoreCase) >= 0);
             };
         }
 
         // Left is a scalar. Right is a single-column table.
         // See in_ST()
-        public static Func<IRContext, FormulaValue[], FormulaValue> InScalarTableOperator(bool exact)
+        public static Func<IServiceProvider, IRContext, FormulaValue[], FormulaValue> InScalarTableOperator(bool exact)
         {
-            return (irContext, args) =>
+            return (services, irContext, args) =>
             {
                 var left = args[0];
-                var right = args[1];
-
+                var right = args[1];
+
+                var cultureInfo = services.GetService<CultureInfo>();
+
                 if (!exact && left is StringValue strLhs)
                 {
-                    left = strLhs.ToLower();
+                    left = new StringValue(IRContext.NotInSource(FormulaType.String), cultureInfo.TextInfo.ToLower(strLhs.Value));
                 }
 
                 var source = (TableValue)right;
@@ -766,7 +769,7 @@ public static Func<IRContext, FormulaValue[], FormulaValue> InScalarTableOperato
 
                         if (!exact && rhs is StringValue strRhs)
                         {
-                            rhs = strRhs.ToLower();
+                            rhs = new StringValue(IRContext.NotInSource(FormulaType.String), cultureInfo.TextInfo.ToLower(strRhs.Value));
                         }
 
                         if (RuntimeHelpers.AreEqual(left, rhs))

src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs

@@ -830,31 +830,31 @@ public static async ValueTask<FormulaValue> SortTable(EvalVisitor runner, EvalVi
 
             if (allNumbers)
             {
-                return SortValueType<NumberValue, double>(pairs, irContext, compareToResultModifier);
+                return SortValueType<NumberValue, double>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allDecimals)
             {
-                return SortValueType<DecimalValue, decimal>(pairs, irContext, compareToResultModifier);
+                return SortValueType<DecimalValue, decimal>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allStrings)
             {
-                return SortValueType<StringValue, string>(pairs, irContext, compareToResultModifier);
+                return SortValueType<StringValue, string>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allBooleans)
             {
-                return SortValueType<BooleanValue, bool>(pairs, irContext, compareToResultModifier);
+                return SortValueType<BooleanValue, bool>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allDatetimes)
             {
-                return SortValueType<DateTimeValue, DateTime>(pairs, irContext, compareToResultModifier);
+                return SortValueType<DateTimeValue, DateTime>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allDates)
             {
-                return SortValueType<DateValue, DateTime>(pairs, irContext, compareToResultModifier);
+                return SortValueType<DateValue, DateTime>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allTimes)
             {
-                return SortValueType<TimeValue, TimeSpan>(pairs, irContext, compareToResultModifier);
+                return SortValueType<TimeValue, TimeSpan>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allOptionSets)
             {
@@ -1281,7 +1281,7 @@ private static FormulaValue DistinctValueType(List<(DValue<RecordValue> row, For
             return new InMemoryTableValue(irContext, result);
         }
 
-        private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<(DValue<RecordValue> row, FormulaValue sortValue)> pairs, IRContext irContext, int compareToResultModifier)
+        private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<(DValue<RecordValue> row, FormulaValue sortValue)> pairs, EvalVisitor runner, IRContext irContext, int compareToResultModifier)
             where TPFxPrimitive : PrimitiveValue<TDotNetPrimitive>
             where TDotNetPrimitive : IComparable<TDotNetPrimitive>
         {
@@ -1297,8 +1297,16 @@ private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<
                 }
 
                 var n1 = a.sortValue as TPFxPrimitive;
-                var n2 = b.sortValue as TPFxPrimitive;
-                return n1.Value.CompareTo(n2.Value) * compareToResultModifier;
+                var n2 = b.sortValue as TPFxPrimitive;
+                CultureInfo culture;
+                if (n1.Value is string n1s && n2.Value is string n2s && (culture = runner.GetService<CultureInfo>()) != null)
+                {
+                    return culture.CompareInfo.Compare(n1s, n2s) * compareToResultModifier;
+                }
+                else
+                {
+                    return n1.Value.CompareTo(n2.Value) * compareToResultModifier;
+                }
             });
 
             return new InMemoryTableValue(irContext, pairs.Select(pair => pair.row));

src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Culture_en-US.txt

@@ -0,0 +1,178 @@
+#SETUP: RegEx,CultureInfo("en-US"),PowerFxV1CompatibilityRules,ConsistentOneColumnTableResult
+
+// Four types of letter I
+//        Dotted     Dotless
+// Upper  İ U+0130   I U+0049
+// Lower  i U+0069   ı U+0131
+
+>> Language()
+"en-US"
+
+>> "İ" = UniChar( Hex2Dec( "0130") )
+true
+
+>> "ı" = UniChar( Hex2Dec( "0131" ) )
+true
+
+// UPPER, LOWER, PROPER
+
+>> Upper( "i" )
+"I"
+
+>> Lower( "I" )
+"i"
+
+>> Upper( "i" ) = "I"
+true
+
+>> Lower( "I" ) = "i"
+true
+
+>> Lower( "quit" ) = Lower( "QUIT" )
+true
+
+>> Lower( "quit" ) = Lower( "QUİT" )
+true
+
+>> Lower( "quıt" ) = Lower( "QUIT" )
+false
+
+>> Upper( "quit" ) = Upper( "QUIT" )
+true
+
+>> Proper( "Iabc" )
+"Iabc"
+
+>> Proper( "iabc" )
+"Iabc"
+
+// VALUE, DECIMAL, FLOAT
+
+>> Value( "123,456" )
+123456
+
+>> Value( "123,456", "tr-TR" )
+123.456
+
+>> Decimal( "123,456" )
+123456
+
+>> Decimal( "123,456", "tr-TR" )
+123.456
+
+>> Float( "123,456" )
+123456
+
+>> Float( "123,456", "tr-TR" )
+123.456
+
+// TEXT
+
+>> Text( DateTime(2010,1,1,14,0,0,0), "mmm ddd yyyy AM/PM" )
+"Jan Fri 2010 PM"
+
+>> Text( DateTime(2020,1,1,2,0,0,0), "mmmm dddd yyyy AM/PM" )
+"January Wednesday 2020 AM"
+
+>> Text( 123456789, "#,###.00" )
+"123,456,789.00"
+
+>> Text( 123456789, "#.###,00" )
+"123456789.00000"
+
+// IN AND EXACTIN
+
+>> "i" in "SIGH"
+true
+
+>> "I" in "sigh"
+true
+
+>> "i" exactin "SIGH"
+false
+
+>> "I" exactin "sigh"
+false
+
+>> "I" exactin "SIGH"
+true
+
+>> "i" exactin "sigh"
+true
+
+>> "sIGh" in ["sigh","bcde"]
+true
+
+>> "siGh" in ["SIGH","bcde"]
+true
+
+>> "sIGH" in ["sigh","bcde"]
+true
+
+>> "siGH" in ["bcde","sIgh"]
+true
+
+>> "SIgh" in ["bcde","sigh"]
+true
+
+// SORT
+// Relative order of i, I, ı, İ are different between en-US and tr-TR
+
+>> Sort( [ "Z", "İ", "z", "I", "J", "j", "ı", "a", "h", "i", "A", "H"], Value )
+Table({Value:"a"},{Value:"A"},{Value:"h"},{Value:"H"},{Value:"i"},{Value:"I"},{Value:"İ"},{Value:"ı"},{Value:"j"},{Value:"J"},{Value:"z"},{Value:"Z"})
+
+>> SortByColumns( [ "Z", "İ", "z", "I", "J", "j", "ı", "a", "h", "i", "A", "H"], "Value" )
+Table({Value:"a"},{Value:"A"},{Value:"h"},{Value:"H"},{Value:"i"},{Value:"I"},{Value:"İ"},{Value:"ı"},{Value:"j"},{Value:"J"},{Value:"z"},{Value:"Z"})
+
+>> Concat( Sort( Split( "j J k K l L m M n N o O p P r R s S t T u U v V y Y z Z Ç ç Ş ş Ü ü Ö ö İ ı Ğ ğ a A b B c C d D e E f F g G h H i I", " " ), Value ), Value, " " )
+"a A b B c C ç Ç d D e E f F g G ğ Ğ h H i I İ ı j J k K l L m M n N o O ö Ö p P r R s S ş Ş t T u U ü Ü v V y Y z Z"
+
+>> Concat( SortByColumns( Split( "d D e E f F g G h H i I j J k K l L m M n N o O p P r R s S t T u U v V y Y z Z Ç ç Ş ş Ü ü Ö ö İ ı Ğ ğ a A b B c C", " " ), "Value" ), Value, " " )
+"a A b B c C ç Ç d D e E f F g G ğ Ğ h H i I İ ı j J k K l L m M n N o O ö Ö p P r R s S ş Ş t T u U ü Ü v V y Y z Z"
+
+// REGULAR EXPRESSIONS
+// Always uses invariant even though tr-TR is set, subject of https://github.com/microsoft/Power-Fx/issues/2538
+
+// Results when using C#                                  // Invariant   tr-TR    en-US
+
+>> IsMatch( "İ", "i", MatchOptions.IgnoreCase )           // false       TRUE     TRUE
+false
+
+>> IsMatch( "i", "İ", MatchOptions.IgnoreCase )           // false       TRUE     TRUE
+false
+
+>> IsMatch( "ı", "I", MatchOptions.IgnoreCase )           // false       TRUE     false
+false
+
+>> IsMatch( "I", "ı", MatchOptions.IgnoreCase )           // false       TRUE     false
+false
+
+>> IsMatch( "İ", "I", MatchOptions.IgnoreCase )           // false       false    TRUE
+false
+
+>> IsMatch( "I", "İ", MatchOptions.IgnoreCase )           // false       false    TRUE
+false
+
+>> IsMatch( "ı", "i", MatchOptions.IgnoreCase )           // false       false    false
+false
+
+>> IsMatch( "i", "ı", MatchOptions.IgnoreCase )           // false       false    false
+false
+
+>> IsMatch( "i", "I", MatchOptions.IgnoreCase )           // TRUE        false    TRUE
+true
+
+>> IsMatch( "I", "i", MatchOptions.IgnoreCase )           // TRUE        false    TRUE
+true
+
+>> IsMatch( "ı", "İ", MatchOptions.IgnoreCase )           // false       false    false
+false
+
+>> IsMatch( "İ", "ı", MatchOptions.IgnoreCase )           // false       false    false
+false
+
+>> Match( "hiIıİİıIhi", "\u0130+" )
+{FullMatch:"İİ",StartMatch:5,SubMatches:Table()}
+
+>> IsMatch( "Sıgh", "\u0131", MatchOptions.Contains )
+true