From 2cf0856dafca9f6e32671ac35aca7f15aeaf912c Mon Sep 17 00:00:00 2001 From: darrenge Date: Fri, 21 Feb 2025 15:52:16 -0800 Subject: [PATCH] Fixed a bug in the BDN script and updated expected values. --- test/BDNPerfTests/BDN_Benchmark_Config.json | 78 ++++++++++----------- test/BDNPerfTests/run_bdnperftest.ps1 | 3 +- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/test/BDNPerfTests/BDN_Benchmark_Config.json b/test/BDNPerfTests/BDN_Benchmark_Config.json index fc79c08598..feaf448261 100644 --- a/test/BDNPerfTests/BDN_Benchmark_Config.json +++ b/test/BDNPerfTests/BDN_Benchmark_Config.json @@ -31,7 +31,7 @@ "expected_HMGet_ACL": 0, "expected_HMSet_ACL": 0, "expected_HRandField_ACL": 0, - "expected_HScan_ACL": 776, + "expected_HScan_ACL": 5100, "expected_HSetNx_ACL": 0, "expected_HStrLen_ACL": 0, "expected_HVals_ACL": 0, @@ -43,26 +43,26 @@ "expected_HIncrbyFloat_AOF": 14400, "expected_HKeys_AOF": 3200, "expected_HLen_AOF": 3200, - "expected_HMGet_AOF": 64000, + "expected_HMGet_AOF": 76000, "expected_HMSet_AOF": 16000, "expected_HRandField_AOF": 8800, - "expected_HScan_AOF": 776, - "expected_HSetNx_AOF": 9600, - "expected_HStrLen_AOF": 6400, + "expected_HScan_AOF": 4900, + "expected_HSetNx_AOF": 69000, + "expected_HStrLen_AOF": 47000, "expected_HVals_AOF": 3200, - "expected_HSetDel_None": 100000, + "expected_HSetDel_None": 120000, "expected_HExists_None": 6400, "expected_HGet_None": 6400, "expected_HGetAll_None": 3200, - "expected_HIncrby_None": 14400, + "expected_HIncrby_None": 58400, "expected_HIncrbyFloat_None": 14400, "expected_HKeys_None": 3200, "expected_HLen_None": 3200, "expected_HMGet_None": 67000, "expected_HMSet_None": 114000, "expected_HRandField_None": 8800, - "expected_HScan_None": 776, - "expected_HSetNx_None": 9600, + "expected_HScan_None": 4500, + "expected_HSetNx_None": 74000, "expected_HStrLen_None": 6400, "expected_HVals_None": 3200 }, @@ -81,7 +81,7 @@ "expected_ZMScore_ACL": 0, "expected_ZPopMax_ACL": 6400, "expected_ZPopMin_ACL": 6400, - "expected_ZRandMember_ACL": 6005, + "expected_ZRandMember_ACL": 7400, "expected_ZRange_ACL": 0, "expected_ZRangeStore_ACL": 0, "expected_ZRank_ACL": 0, @@ -96,10 +96,10 @@ "expected_ZAddRem_AOF": 18400, "expected_ZCard_AOF": 3200, "expected_ZCount_AOF": 135000, - "expected_ZDiff_AOF": 147000, + "expected_ZDiff_AOF": 303000, "expected_ZDiffStore_AOF": 27200, "expected_ZIncrby_AOF": 12000, - "expected_ZInter_AOF": 34400, + "expected_ZInter_AOF": 221000, "expected_ZInterCard_AOF": 212000, "expected_ZInterStore_AOF": 123200, "expected_ZLexCount_AOF": 66400, @@ -107,25 +107,25 @@ "expected_ZMScore_AOF": 6400, "expected_ZPopMax_AOF": 48001, "expected_ZPopMin_AOF": 48001, - "expected_ZRandMember_AOF": 1184, + "expected_ZRandMember_AOF": 6900, "expected_ZRange_AOF": 209588, - "expected_ZRangeStore_AOF": 12800, - "expected_ZRank_AOF": 13600, + "expected_ZRangeStore_AOF": 580000, + "expected_ZRank_AOF": 100000, "expected_ZRemRangeByLex_AOF": 111201, "expected_ZRemRangeByRank_AOF": 84801, "expected_ZRemRangeByScore_AOF": 84001, - "expected_ZRevRank_AOF": 13600, - "expected_ZScan_AOF": 776, + "expected_ZRevRank_AOF": 87600, + "expected_ZScan_AOF": 4800, "expected_ZScore_AOF": 6400, - "expected_ZUnion_AOF": 34400, - "expected_ZUnionStore_AOF": 129600, - "expected_ZAddRem_None": 119000, + "expected_ZUnion_AOF": 260000, + "expected_ZUnionStore_AOF": 452600, + "expected_ZAddRem_None": 149000, "expected_ZCard_None": 3200, - "expected_ZCount_None": 130000, + "expected_ZCount_None": 150000, "expected_ZDiff_None": 179000, "expected_ZDiffStore_None": 27200, "expected_ZIncrby_None": 12000, - "expected_ZInter_None": 224000, + "expected_ZInter_None": 250000, "expected_ZInterCard_None": 222071, "expected_ZInterStore_None": 537000, "expected_ZLexCount_None": 66400, @@ -134,17 +134,17 @@ "expected_ZPopMax_None": 48001, "expected_ZPopMin_None": 48001, "expected_ZRandMember_None": 7150, - "expected_ZRange_None": 181000, + "expected_ZRange_None": 320000, "expected_ZRangeStore_None": 572000, - "expected_ZRank_None": 13600, + "expected_ZRank_None": 86000, "expected_ZRemRangeByLex_None": 111201, "expected_ZRemRangeByRank_None": 84801, "expected_ZRemRangeByScore_None": 84001, - "expected_ZRevRank_None": 13600, + "expected_ZRevRank_None": 80000, "expected_ZScan_None": 776, "expected_ZScore_None": 6400, "expected_ZUnion_None": 220000, - "expected_ZUnionStore_None": 84800 + "expected_ZUnionStore_None": 528000 }, "BDN.benchmark.Cluster.ClusterOperations.*": { "expected_Get_DSV": 0, @@ -193,7 +193,7 @@ "expected_ConstructLarge_Managed,Limit": 2100666, "expected_CompileForSessionSmall_Managed,Limit": 1984, "expected_CompileForSessionLarge_Managed,Limit": 3000, - "expected_ResetParametersSmall_Managed,None": 1984, + "expected_ResetParametersSmall_Managed,None": 3000, "expected_ResetParametersLarge_Managed,None": 1984, "expected_ConstructSmall_Managed,None": 2097674, "expected_ConstructLarge_Managed,None": 2100738, @@ -201,21 +201,21 @@ "expected_CompileForSessionLarge_Managed,None": 1984, "expected_ResetParametersSmall_Native,None": 1984, "expected_ResetParametersLarge_Native,None": 1984, - "expected_ConstructSmall_Native,None": 2400, + "expected_ConstructSmall_Native,None": 3024, "expected_ConstructLarge_Native,None": 5500, "expected_CompileForSessionSmall_Native,None": 2000, "expected_CompileForSessionLarge_Native,None": 1984, "expected_ResetParametersSmall_Tracked,Limit": 1984, "expected_ResetParametersLarge_Tracked,Limit": 1984, "expected_ConstructSmall_Tracked,Limit": 2400, - "expected_ConstructLarge_Tracked,Limit": 5500, + "expected_ConstructLarge_Tracked,Limit": 6500, "expected_CompileForSessionSmall_Tracked,Limit": 1984, "expected_CompileForSessionLarge_Tracked,Limit": 1984, "expected_ResetParametersSmall_Tracked,None": 1984, "expected_ResetParametersLarge_Tracked,None": 1984, "expected_ConstructSmall_Tracked,None": 2400, "expected_ConstructLarge_Tracked,None": 5500, - "expected_CompileForSessionSmall_Tracked,None": 1984, + "expected_CompileForSessionSmall_Tracked,None": 3000, "expected_CompileForSessionLarge_Tracked,None": 1984 }, "BDN.benchmark.Lua.LuaScriptCacheOperations.*": { @@ -236,12 +236,12 @@ "expected_LoadOuterHit_Native,None": 2700, "expected_LoadInnerHit_Native,None": 2350, "expected_LoadMiss_Native,None": 1984, - "expected_Digest_Native,None": 1984, + "expected_Digest_Native,None": 3000, "expected_LookupHit_Tracked,Limit": 2000, "expected_LookupMiss_Tracked,Limit": 2000, - "expected_LoadOuterHit_Tracked,Limit": 1984, + "expected_LoadOuterHit_Tracked,Limit": 3000, "expected_LoadInnerHit_Tracked,Limit": 2350, - "expected_LoadMiss_Tracked,Limit": 1984, + "expected_LoadMiss_Tracked,Limit": 2700, "expected_Digest_Tracked,Limit": 1984, "expected_LookupHit_Tracked,None": 1984, "expected_LookupMiss_Tracked,None": 1984, @@ -327,7 +327,7 @@ "expected_Eval_Tracked,Limit": 0, "expected_EvalSha_Tracked,Limit": 0, "expected_SmallScript_Tracked,Limit": 0, - "expected_LargeScript_Tracked,Limit": 1800, + "expected_LargeScript_Tracked,Limit": 2800, "expected_ArrayReturn_Tracked,Limit": 1, "expected_ScriptLoad_Tracked,None": 9600, "expected_ScriptExistsTrue_Tracked,None": 0, @@ -335,22 +335,22 @@ "expected_Eval_Tracked,None": 0, "expected_EvalSha_Tracked,None": 0, "expected_SmallScript_Tracked,None": 0, - "expected_LargeScript_Tracked,None": 52, + "expected_LargeScript_Tracked,None": 1000, "expected_ArrayReturn_Tracked,None": 1 }, "BDN.benchmark.Operations.ModuleOperations.*": { "expected_ModuleNoOpRawStringReadCommand_ACL": 0, "expected_ModuleNoOpRawStringRmwCommand_ACL": 0, "expected_ModuleNoOpObjRmwCommand_ACL": 3200, - "expected_ModuleNoOpObjReadCommand_ACL": 3200, + "expected_ModuleNoOpObjReadCommand_ACL": 21000, "expected_ModuleNoOpProc_ACL": 0, "expected_ModuleNoOpTxn_ACL": 0, "expected_ModuleJsonGetCommand_ACL": 400000, "expected_ModuleJsonSetCommand_ACL": 223200, "expected_ModuleNoOpRawStringReadCommand_AOF": 0, "expected_ModuleNoOpRawStringRmwCommand_AOF": 0, - "expected_ModuleNoOpObjRmwCommand_AOF": 3200, - "expected_ModuleNoOpObjReadCommand_AOF": 3200, + "expected_ModuleNoOpObjRmwCommand_AOF": 24000, + "expected_ModuleNoOpObjReadCommand_AOF": 26000, "expected_ModuleNoOpProc_AOF": 0, "expected_ModuleNoOpTxn_AOF": 0, "expected_ModuleJsonGetCommand_AOF": 405000, @@ -361,7 +361,7 @@ "expected_ModuleNoOpObjReadCommand_None": 3200, "expected_ModuleNoOpProc_None": 0, "expected_ModuleNoOpTxn_None": 0, - "expected_ModuleJsonGetCommand_None": 72800, + "expected_ModuleJsonGetCommand_None": 430000, "expected_ModuleJsonSetCommand_None": 223200 }, "BDN.benchmark.Network.RawStringOperations.*": { diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index f3f72a0efe..376f16420a 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -48,6 +48,7 @@ function AnalyzeResult { if ($warnonly) { Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark found Allocated value ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " + return $true # Since it is warning, don't want to cause a fail } else { Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark found Allocated value ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" @@ -235,7 +236,7 @@ Get-Content $resultsFile | ForEach-Object { # Check if found value is not equal to expected value Write-Host "** Config: "$expectedResultsArray[$currentExpectedProp, 0].Substring(2) $expectedResultsArray[$currentExpectedProp, 1] - $currentResults = AnalyzeResult $foundValue $expectedResultsArray[$currentExpectedProp, 2] $acceptableAllocatedRange $true + $currentResults = AnalyzeResult $foundValue $expectedResultsArray[$currentExpectedProp, 2] $acceptableAllocatedRange $false if ($currentResults -eq $false) { $testSuiteResult = $false }