diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Sysbench/SysbenchExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Sysbench/SysbenchExample1.txt new file mode 100644 index 0000000000..0ee7860991 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Sysbench/SysbenchExample1.txt @@ -0,0 +1,38 @@ +sysbench 1.1.0 (using bundled LuaJIT 2.1.0-beta3) + +Running the test with following options: +Number of threads: 8 +Initializing random number generator from current time + + +Initializing worker threads... + +Threads started! + +SQL statistics: + queries performed: + read: 9720106 + write: 0 + other: 0 + total: 9720106 + transactions: 9720106 (32400.24 per sec.) + queries: 9720106 (32400.24 per sec.) + ignored errors: 0 (0.00 per sec.) + reconnects: 0 (0.00 per sec.) + +Throughput: + events/s (eps): 32400.2399 + time elapsed: 300.0011s + total number of events: 9720106 + +Latency (ms): + min: 0.16 + avg: 0.25 + max: 13.68 + 95th percentile: 0.28 + sum: 2389444.97 + +Threads fairness: + events (avg/stddev): 1215013.2500/22596.17 + execution time (avg/stddev): 298.6806/0.05 + diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs index 98268dbf24..4bf12bbe70 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs @@ -77,7 +77,7 @@ public async Task SysbenchConfigurationSkipsSysbenchInitialization() string[] expectedCommands = { - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --tableCount 10 --recordCount 1000 --threadCount 8 --password [A-Za-z0-9+/=]+", + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --threadCount 8 --tableCount 10 --recordCount 1000 --password [A-Za-z0-9+/=]+", }; int commandNumber = 0; @@ -122,7 +122,7 @@ public async Task SysbenchConfigurationPreparesDatabase() string[] expectedCommands = { $"python3 {this.mockPackagePath}/configure-workload-generator.py --distro Ubuntu --databaseSystem MySQL --packagePath {this.mockPackagePath}", - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --tableCount 10 --recordCount 1000 --threadCount 8 --password [A-Za-z0-9+/=]+", + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --threadCount 8 --tableCount 10 --recordCount 1000 --password [A-Za-z0-9+/=]+", }; int commandNumber = 0; @@ -173,7 +173,7 @@ public async Task SysbenchConfigurationUsesDefinedParametersWhenRunningTheWorklo string[] expectedCommands = { $"python3 {this.mockPackagePath}/configure-workload-generator.py --distro Ubuntu --databaseSystem MySQL --packagePath {this.mockPackagePath}", - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --tableCount 40 --recordCount 1000 --threadCount 16 --password [A-Za-z0-9+/=]+", + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --threadCount 16 --tableCount 40 --recordCount 1000 --password [A-Za-z0-9+/=]+", }; int commandNumber = 0; @@ -386,7 +386,7 @@ public async Task SysbenchConfigurationProperlyExecutesPostgreSQLOLTPConfigurabl string[] expectedCommands = { - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark OLTP --tableCount 40 --recordCount 1000 --threadCount 16 --password [A-Za-z0-9+/=]+" + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark OLTP --threadCount 16 --tableCount 40 --recordCount 1000 --password [A-Za-z0-9+/=]+" }; int commandNumber = 0; diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchMetricsParserTests.cs index 170837e6aa..2dbd5d8d66 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchMetricsParserTests.cs @@ -46,5 +46,17 @@ public void SysbenchParserParsesCorrectly() MetricAssert.Exists(metrics, "latency p95", 68.05, "milliseconds"); MetricAssert.Exists(metrics, "latency sum", 7458385.25, "milliseconds"); } + + [Test] + public void SysbenchParserParsesMetricsMetadataCorrectly() + { + string rawText = File.ReadAllText(Path.Combine(examplesDirectory, "SysbenchExample1.txt")); + SysbenchMetricsParser parser = new SysbenchMetricsParser(rawText); + + IList metrics = parser.Parse(); + Assert.AreEqual(17, metrics.Count); + Assert.IsTrue(metrics[0].Metadata.ContainsKey("sysbench_version")); + Assert.IsTrue(metrics[0].Metadata["sysbench_version"].Equals("1.1.0")); + } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index 9c891a8df0..e153aee502 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -10,6 +10,7 @@ namespace VirtualClient.Actions using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; + using Microsoft.Identity.Client; using Polly; using VirtualClient.Common; using VirtualClient.Common.Extensions; @@ -140,6 +141,20 @@ private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext { SysbenchMetricsParser parser = new SysbenchMetricsParser(text); IList metrics = parser.Parse(); + string sysbenchVersion = null; + + var sysbenchVersionMetric = metrics.FirstOrDefault(); + if (sysbenchVersionMetric != null && sysbenchVersionMetric.Metadata.TryGetValue("sysbench_version", out var versionValue)) + { + sysbenchVersion = versionValue?.ToString(); + } + + if (!string.IsNullOrEmpty(sysbenchVersion)) + { + this.MetadataContract.Add("sysbench_version", sysbenchVersion, MetadataContractCategory.Dependencies); + } + + this.MetadataContract.Apply(telemetryContext); this.Logger.LogMetrics( toolName: "Sysbench", @@ -150,7 +165,8 @@ private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext null, scenarioArguments: this.sysbenchLoggingArguments, this.Tags, - telemetryContext); + telemetryContext, + toolVersion: sysbenchVersion); } catch (Exception exc) { @@ -182,6 +198,10 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok { await this.PrepareOLTPMySQLDatabase(telemetryContext, cancellationToken); } + else if (this.Action == ClientAction.Cleanup) + { + await this.CleanUpDatabase(telemetryContext, cancellationToken); + } else { await this.RunOLTPWorkloadAsync(telemetryContext, cancellationToken); @@ -229,6 +249,33 @@ private async Task RunOLTPWorkloadAsync(EventContext telemetryContext, Cancellat } } + private async Task CleanUpDatabase(EventContext telemetryContext, CancellationToken cancellationToken) + { + int tableCount = GetTableCount(this.DatabaseScenario, this.TableCount, this.Workload); + + string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty()) + .FirstOrDefault()?.IPAddress + ?? "localhost"; + + string sysbenchCleanupArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --hostIpAddress {serverIp}"; + + string script = $"{this.SysbenchPackagePath}/cleanup-database.py "; + + using (IProcessProxy process = await this.ExecuteCommandAsync( + SysbenchExecutor.PythonCommand, + script + sysbenchCleanupArguments, + this.SysbenchPackagePath, + telemetryContext, + cancellationToken)) + { + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); + process.ThrowIfErrored(process.StandardError.ToString(), ErrorReason.WorkloadFailed); + } + } + } + private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, CancellationToken cancellationToken) { int tableCount = GetTableCount(this.Scenario, this.TableCount, this.Workload); @@ -288,7 +335,7 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); - this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}"; + this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --threadCount {threadCount} --tableCount {tableCount} --recordCount {recordCount}"; this.sysbenchPrepareArguments = $"{this.sysbenchLoggingArguments} --password {this.SuperUserPassword}"; string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty()) diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index 87587facc3..caa31e733b 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -82,7 +82,7 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); - string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}"; + string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --threadCount {threadCount} --tableCount {tableCount} --recordCount {recordCount}"; this.sysbenchPrepareArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword}"; string serverIp = "localhost"; diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index b90d003818..0f8c8fd5d9 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -417,7 +417,7 @@ protected void AddMetric(string arguments, IProcessProxy process, EventContext t List metrics = new List(); double duration = (process.ExitTime - process.StartTime).TotalMinutes; - metrics.Add(new Metric("PopulateDatabaseTime_Minutes ", duration, "minutes", MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric("PopulateDatabaseDuration", duration, "minutes", MetricRelativity.LowerIsBetter)); this.Logger.LogMetrics( toolName: "Sysbench", @@ -532,9 +532,14 @@ internal class ClientAction public const string TruncateDatabase = nameof(TruncateDatabase); /// - /// Truncates all tables existing in database + /// Runs specified workload. /// public const string RunWorkload = nameof(RunWorkload); + + /// + /// Runs sysbench cleanup action. + /// + public const string Cleanup = nameof(Cleanup); } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchMetricsParser.cs index 26053cc35d..8d5e75b598 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchMetricsParser.cs @@ -19,6 +19,7 @@ public class SysbenchMetricsParser : MetricsParser private const string ReconnectsPerSecond = "reconnects/sec"; private const string Second = "seconds"; private const string MilliSecond = "milliseconds"; + private Dictionary metricMetadata = new Dictionary(); /// /// Constructor for @@ -32,29 +33,32 @@ public SysbenchMetricsParser(string rawText) /// public override IList Parse() { + var match = Regex.Match(this.RawText, @"sysbench\s+([\d]+\.[\d]+\.[\d]+(?:-\w+)?)"); + string sysbenchversion = match.Success ? match.Groups[1].Value : string.Empty; + this.metricMetadata["sysbench_version"] = sysbenchversion; this.Preprocess(); List metrics = new List(); // Create list of Metrics Info List metricInfoList = new List() { - new MetricInfo("# read queries", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("# write queries", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("# other queries", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("# transactions", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("transactions/sec", TransactionsPerSecond, MetricRelativity.HigherIsBetter), - new MetricInfo("# queries", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("queries/sec", QueriesPerSecond, MetricRelativity.HigherIsBetter), - new MetricInfo("# ignored errors", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("ignored errors/sec", IgnoredErrorsPerSecond, MetricRelativity.HigherIsBetter), - new MetricInfo("# reconnects", string.Empty, MetricRelativity.HigherIsBetter), - new MetricInfo("reconnects/sec", ReconnectsPerSecond, MetricRelativity.HigherIsBetter), - new MetricInfo("elapsed time", Second, MetricRelativity.LowerIsBetter), - new MetricInfo("latency min", MilliSecond, MetricRelativity.LowerIsBetter), - new MetricInfo("latency avg", MilliSecond, MetricRelativity.LowerIsBetter), - new MetricInfo("latency max", MilliSecond, MetricRelativity.LowerIsBetter), - new MetricInfo("latency p95", MilliSecond, MetricRelativity.LowerIsBetter), - new MetricInfo("latency sum", MilliSecond, MetricRelativity.LowerIsBetter), + new MetricInfo("# read queries", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("# write queries", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("# other queries", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("# transactions", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("transactions/sec", TransactionsPerSecond, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("# queries", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("queries/sec", QueriesPerSecond, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("# ignored errors", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("ignored errors/sec", IgnoredErrorsPerSecond, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("# reconnects", string.Empty, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("reconnects/sec", ReconnectsPerSecond, MetricRelativity.HigherIsBetter, metadata: this.metricMetadata), + new MetricInfo("elapsed time", Second, MetricRelativity.LowerIsBetter, metadata: this.metricMetadata), + new MetricInfo("latency min", MilliSecond, MetricRelativity.LowerIsBetter, metadata: this.metricMetadata), + new MetricInfo("latency avg", MilliSecond, MetricRelativity.LowerIsBetter, metadata: this.metricMetadata), + new MetricInfo("latency max", MilliSecond, MetricRelativity.LowerIsBetter, metadata: this.metricMetadata), + new MetricInfo("latency p95", MilliSecond, MetricRelativity.LowerIsBetter, metadata: this.metricMetadata), + new MetricInfo("latency sum", MilliSecond, MetricRelativity.LowerIsBetter, metadata: this.metricMetadata), }; if (!string.IsNullOrEmpty(this.PreprocessedText)) @@ -84,7 +88,7 @@ public override IList Parse() { MetricInfo metricInfo = metricInfoList[metricInfoIndex]; Match m = mc[mcIndex]; - metrics.Add(new Metric($"{metricInfo.Name}", Convert.ToDouble(m.Value), metricInfo.Unit, metricInfo.Relativity)); + metrics.Add(new Metric($"{metricInfo.Name}", Convert.ToDouble(m.Value), metricInfo.Unit, metricInfo.Relativity, metadata: metricInfo.Metadata)); metricInfoIndex++; } @@ -107,11 +111,12 @@ protected override void Preprocess() // helper class that contains Metric Name, Unit, and Relativity private class MetricInfo { - public MetricInfo(string name, string unit, MetricRelativity relativity) + public MetricInfo(string name, string unit, MetricRelativity relativity, Dictionary metadata) { this.Name = name; this.Unit = unit; this.Relativity = relativity; + this.Metadata = metadata; } public string Name { get; set; } @@ -119,6 +124,8 @@ public MetricInfo(string name, string unit, MetricRelativity relativity) public string Unit { get; set; } public MetricRelativity Relativity { get; set; } + + public Dictionary Metadata { get; set; } } } } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json index 8895ebf569..18a75f445c 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json @@ -182,7 +182,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev2.zip", + "BlobName": "sysbench-1.0.20.rev3.zip", "PackageName": "sysbench", "Extract": true }