Culture support for in operator and Sort function

src/libraries/Microsoft.PowerFx.Core/Public/Values/StringValue.cs

@@ -36,11 +36,6 @@ public override void Visit(IValueVisitor visitor)
             visitor.Visit(this);
         }
 
-        internal StringValue ToLower()
-        {
-            return new StringValue(IRContext.NotInSource(FormulaType.String), Value.ToLowerInvariant());
-        }
-
         public override void ToExpression(StringBuilder sb, FormulaValueSerializerSettings settings)
         {
             sb.Append($"\"{CharacterUtils.ExcelEscapeString(Value)}\"");

src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryOperators.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Globalization;
 using System.Linq;
 using Microsoft.PowerFx.Core.IR;
 using Microsoft.PowerFx.Core.Utils;
@@ -719,9 +720,9 @@ private static BooleanValue NotEqualPolymorphic(IRContext irContext, FormulaValu
         }
 
         // See in_SS in JScript membershipReplacementFunctions
-        public static Func<IRContext, FormulaValue[], FormulaValue> StringInOperator(bool exact)
+        public static Func<IServiceProvider, IRContext, FormulaValue[], FormulaValue> StringInOperator(bool exact)
         {
-            return (irContext, args) =>
+            return (services, irContext, args) =>
             {
                 var left = args[0];
                 var right = args[1];
@@ -737,23 +738,25 @@ public static Func<IRContext, FormulaValue[], FormulaValue> StringInOperator(boo
 
                 var leftStr = (StringValue)left;
                 var rightStr = (StringValue)right;
-
-                return new BooleanValue(irContext, rightStr.Value.IndexOf(leftStr.Value, exact ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase) >= 0);
+
+                return new BooleanValue(irContext, services.GetService<CultureInfo>().CompareInfo.IndexOf(rightStr.Value, leftStr.Value, exact ? CompareOptions.Ordinal : CompareOptions.IgnoreCase) >= 0);
             };
         }
 
         // Left is a scalar. Right is a single-column table.
         // See in_ST()
-        public static Func<IRContext, FormulaValue[], FormulaValue> InScalarTableOperator(bool exact)
+        public static Func<IServiceProvider, IRContext, FormulaValue[], FormulaValue> InScalarTableOperator(bool exact)
         {
-            return (irContext, args) =>
+            return (services, irContext, args) =>
             {
                 var left = args[0];
-                var right = args[1];
-
+                var right = args[1];
+
+                var cultureInfo = services.GetService<CultureInfo>();
+
                 if (!exact && left is StringValue strLhs)
                 {
-                    left = strLhs.ToLower();
+                    left = new StringValue(IRContext.NotInSource(FormulaType.String), cultureInfo.TextInfo.ToLower(strLhs.Value));
                 }
 
                 var source = (TableValue)right;
@@ -766,7 +769,7 @@ public static Func<IRContext, FormulaValue[], FormulaValue> InScalarTableOperato
 
                         if (!exact && rhs is StringValue strRhs)
                         {
-                            rhs = strRhs.ToLower();
+                            rhs = new StringValue(IRContext.NotInSource(FormulaType.String), cultureInfo.TextInfo.ToLower(strRhs.Value));
                         }
 
                         if (RuntimeHelpers.AreEqual(left, rhs))

src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs

@@ -830,31 +830,31 @@ public static async ValueTask<FormulaValue> SortTable(EvalVisitor runner, EvalVi
 
             if (allNumbers)
             {
-                return SortValueType<NumberValue, double>(pairs, irContext, compareToResultModifier);
+                return SortValueType<NumberValue, double>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allDecimals)
             {
-                return SortValueType<DecimalValue, decimal>(pairs, irContext, compareToResultModifier);
+                return SortValueType<DecimalValue, decimal>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allStrings)
             {
-                return SortValueType<StringValue, string>(pairs, irContext, compareToResultModifier);
+                return SortValueType<StringValue, string>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allBooleans)
             {
-                return SortValueType<BooleanValue, bool>(pairs, irContext, compareToResultModifier);
+                return SortValueType<BooleanValue, bool>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allDatetimes)
             {
-                return SortValueType<DateTimeValue, DateTime>(pairs, irContext, compareToResultModifier);
+                return SortValueType<DateTimeValue, DateTime>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allDates)
             {
-                return SortValueType<DateValue, DateTime>(pairs, irContext, compareToResultModifier);
+                return SortValueType<DateValue, DateTime>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allTimes)
             {
-                return SortValueType<TimeValue, TimeSpan>(pairs, irContext, compareToResultModifier);
+                return SortValueType<TimeValue, TimeSpan>(pairs, runner, irContext, compareToResultModifier);
             }
             else if (allOptionSets)
             {
@@ -1281,7 +1281,7 @@ private static FormulaValue DistinctValueType(List<(DValue<RecordValue> row, For
             return new InMemoryTableValue(irContext, result);
         }
 
-        private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<(DValue<RecordValue> row, FormulaValue sortValue)> pairs, IRContext irContext, int compareToResultModifier)
+        private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<(DValue<RecordValue> row, FormulaValue sortValue)> pairs, EvalVisitor runner, IRContext irContext, int compareToResultModifier)
             where TPFxPrimitive : PrimitiveValue<TDotNetPrimitive>
             where TDotNetPrimitive : IComparable<TDotNetPrimitive>
         {
@@ -1297,8 +1297,15 @@ private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<
                 }
 
                 var n1 = a.sortValue as TPFxPrimitive;
-                var n2 = b.sortValue as TPFxPrimitive;
-                return n1.Value.CompareTo(n2.Value) * compareToResultModifier;
+                var n2 = b.sortValue as TPFxPrimitive;
+                if (n1.Value is string n1s && n2.Value is string n2s && runner.CultureInfo != null)
+                {
+                    return runner.CultureInfo.CompareInfo.Compare(n1s, n2s) * compareToResultModifier;
+                }
+                else
+                {
+                    return n1.Value.CompareTo(n2.Value) * compareToResultModifier;
+                }
             });
 
             return new InMemoryTableValue(irContext, pairs.Select(pair => pair.row));

src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Culture_da-DK.txt

@@ -0,0 +1,144 @@
+#SETUP: RegEx,CultureInfo("da-DK"),PowerFxV1CompatibilityRules,ConsistentOneColumnTableResult,SupportColumnNamesAsIdentifiers
+
+>> Language()
+"da-DK"
+
+>> "Ø" = UniChar( Hex2Dec( "00d8") )
+true
+
+>> "ø" = UniChar( Hex2Dec( "00f8") )
+true
+
+// UPPER, LOWER, PROPER
+
+>> Upper( "ø" )
+"Ø"
+
+>> Lower( "Ø" )
+"ø"
+
+>> Upper( "ø" ) = "Ø"
+true
+
+>> Lower( "Ø" ) = "ø"
+true
+
+>> Lower( "ørkesløse" ) = Lower( "ØRKESLØSE" )
+true
+
+>> Upper( "ørkesløse" ) = Upper( "ØRKESLØSE" )
+true
+
+>> Proper( "ørkesløse" )
+"Ørkesløse"
+
+>> Proper( "ØRKESLØSE" )
+"Ørkesløse"
+
+// VALUE, DECIMAL, FLOAT
+// Comma decimal seperator
+
+>> Value( "123,456" )
+123.456
+
+>> Value( "123,456", "da-DK" )
+123.456
+
+>> Decimal( "123,456" )
+123.456
+
+>> Decimal( "123,456", "da-DK" )
+123.456
+
+>> Float( "123,456" )
+123.456
+
+>> Float( "123,456", "da-DK" )
+123.456
+
+// TEXT
+
+>> Text( DateTime(2010,1,1,14,0,0,0), "mmm ddd yyyy AM/PM" )
+"jan. fre. 2010 PM"
+
+>> Text( DateTime(2020,1,1,2,0,0,0), "mmmm dddd yyyy AM/PM" )
+"januar onsdag 2020 AM"
+
+>> Text( 123456789, "#,###.00" )
+"123456789,00000"
+
+>> Text( 123456789, "#.###,00" )
+"123.456.789,00"
+
+// IN AND EXACTIN
+
+>> "å" in "ømtåligt"
+true
+
+>> "å" in "ØMTÅLIGT"
+true
+
+>> "Å" in "ømtåligt"
+true
+
+>> "Å" in "ØMTÅLIGT"
+true
+
+>> "å" exactin "ømtåligt"
+true
+
+>> "å" exactin "ØMTÅLIGT"
+false
+
+>> "Å" exactin "ØMTÅLIGT"
+true
+
+>> "ØMtålIGT" in ["ømtåligt","bcde"]
+true
+
+>> "ømtålIgt" in ["bcde", "ØMTÅLiGT"]
+true
+
+>> "ømtålIgt" in ["bcde", "MTÅLiGT"]
+false
+
+// SORT
+
+>> Concat( Sort( Split( "n F X W o i j x B m I R G S h Ø L p K t A k l y J æ u v s T a ø N D z Æ e O U E H r Z å g b q Å P d f C M c Y w V Q", " " ), Value ), Value, " " )
+"A a B b C c D d E e F f G g H h I i J j K k L l M m N n O o P p Q q R r S s T t U u V v W w X x Y y Z z Æ æ Ø ø Å å"
+
+>> Concat( SortByColumns( Split( "U c q s X Å P L i I u d J å M E l k W v j Æ n a B K C D e ø æ f O y m Ø r Q R A x h T H N Z F V w o S g t p G Y b z", " " ), "Value" ), Value, " " )
+"A a B b C c D d E e F f G g H h I i J j K k L l M m N n O o P p Q q R r S s T t U u V v W w X x Y y Z z Æ æ Ø ø Å å"
+
+// REGULAR EXPRESSIONS
+// Always uses invariant in all locales, even in en-US and tr-TR (industry standard)
+
+>> IsMatch( "å", "Å", MatchOptions.IgnoreCase )           
+true
+
+>> IsMatch( "Å", "å", MatchOptions.IgnoreCase )           
+true
+
+>> IsMatch( "Å", "A", MatchOptions.IgnoreCase )           
+false
+
+>> IsMatch( "Å", "a", MatchOptions.IgnoreCase )           
+false
+
+>> IsMatch( "ø", "Ø", MatchOptions.IgnoreCase )
+true
+
+>> IsMatch( "Ø", "ø", MatchOptions.IgnoreCase )
+true
+
+>> IsMatch( "æ", "Æ", MatchOptions.IgnoreCase )
+true
+
+>> IsMatch( "Æ", "æ", MatchOptions.IgnoreCase )
+true
+
+>> ShowColumns( Match( "ØMTÅÅLIGT", "\u00c5+" ), FullMatch, StartMatch )
+{FullMatch:"ÅÅ",StartMatch:4}
+
+>> IsMatch( "ØMTÅÅLIGT", "Ø", MatchOptions.Contains )
+true