Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MySQL adding fixing bugs and adding new features #403

Merged
merged 12 commits into from
Feb 5, 2025
Merged

MySQL adding fixing bugs and adding new features #403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f1c4e7d
adding truncate tables feature for MYSQL sysbench OTLP recipe
nmalkapuram Oct 18, 2024
4f1f079
removing unnecessary files
nmalkapuram Nov 15, 2024
4fe4cc1
removing unnecessary comments
nmalkapuram Nov 15, 2024
6f6de38
resolving comments
nmalkapuram Nov 18, 2024
c5d25ff
updating profile
nmalkapuram Nov 29, 2024
bae410d
updating documentation
nmalkapuram Nov 29, 2024
f9c27d7
optimizing code
nmalkapuram Dec 2, 2024
7a7b572
adding changes+ extra debugging
nmalkapuram Dec 19, 2024
1982cf4
allow striped disk to be used by mysql, updating mysql server package…
nmalkapuram Jan 5, 2025
b8b403f
adding new parameter while database population and some code clean up
nmalkapuram Feb 4, 2025
b691507
updating documentation
nmalkapuram Feb 4, 2025
a5d6866
Merge branch 'main' into users/nmalkapuram/MySQLCompeteRecipe
nmalkapuram Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 5 additions & 14 deletions src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ namespace VirtualClient.Actions
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using MathNet.Numerics;
using Microsoft.Extensions.DependencyInjection;
using VirtualClient;
using VirtualClient.Common;
Expand Down Expand Up @@ -98,7 +100,9 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>(
cancellationToken,
this.PollingInterval);

string targetIPAddress = this.GetServerIPAddress(cancellationToken);
string targetIPAddress = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>())
.FirstOrDefault()?.IPAddress
?? "localhost";

string ctsTrafficCommandArgs = $"-Target:{targetIPAddress} -Consoleverbosity:1 -StatusFilename:{this.StatusFileName} " +
$@"-ConnectionFilename:{this.ConnectionsFileName} -ErrorFileName:{this.ErrorFileName} -Port:{this.Port} " +
Expand Down Expand Up @@ -145,18 +149,5 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>(
}

}

private string GetServerIPAddress(CancellationToken cancellationToken)
{
string targetIPAddress = "localhost";

if (this.IsMultiRoleLayout())
{
ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First();
targetIPAddress = serverInstance.IPAddress;
}

return targetIPAddress;
}
}
}
91 changes: 82 additions & 9 deletions src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,13 @@ namespace VirtualClient.Actions
{
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Polly;
using VirtualClient.Common;
using VirtualClient.Common.Contracts;
using VirtualClient.Common.Extensions;
using VirtualClient.Common.Telemetry;
using VirtualClient.Contracts;
Expand All @@ -27,6 +24,8 @@ public class SysbenchClientExecutor : SysbenchExecutor
{
private string sysbenchExecutionArguments;
private string sysbenchLoggingArguments;
private string sysbenchPrepareArguments;
private string packageDirectory;

/// <summary>
/// Initializes a new instance of the <see cref="SysbenchClientExecutor"/> class.
Expand Down Expand Up @@ -169,7 +168,24 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
{
if (this.Benchmark == BenchmarkName.OLTP)
{
await this.RunOLTPWorkloadAsync(telemetryContext, cancellationToken);
if (this.Action == ClientAction.TruncateDatabase)
{
DependencyPath workloadPackage = await this.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false);
nmalkapuram marked this conversation as resolved.
Show resolved Hide resolved
workloadPackage.ThrowIfNull(this.PackageName);

DependencyPath package = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken);
this.packageDirectory = package.Path;

await this.TruncateMySQLDatabaseAsync(telemetryContext, cancellationToken).ConfigureAwait(false);
nmalkapuram marked this conversation as resolved.
Show resolved Hide resolved
}
else if (this.Action == ClientAction.PopulateDatabase)
{
await this.PrepareOLTPMySQLDatabase(telemetryContext, cancellationToken);
}
else
{
await this.RunOLTPWorkloadAsync(telemetryContext, cancellationToken);
}
}
else if (this.Benchmark == BenchmarkName.TPCC)
{
Expand All @@ -194,11 +210,10 @@ private async Task RunOLTPWorkloadAsync(EventContext telemetryContext, Cancellat
this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --workload {this.Workload} --threadCount {threadCount} --tableCount {tableCount} --recordCount {recordCount} ";
this.sysbenchExecutionArguments = this.sysbenchLoggingArguments + $"--hostIpAddress {this.ServerIpAddress} --durationSecs {this.Duration.TotalSeconds} --password {this.SuperUserPassword}";

string command = "python3";
string script = $"{this.SysbenchPackagePath}/run-workload.py ";

using (IProcessProxy process = await this.ExecuteCommandAsync(
command,
SysbenchExecutor.PythonCommand,
script + this.sysbenchExecutionArguments,
this.SysbenchPackagePath,
telemetryContext,
Expand All @@ -223,11 +238,10 @@ private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, Cancellat
this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --workload tpcc --threadCount {threadCount} --tableCount {tableCount} --warehouses {warehouseCount} ";
this.sysbenchExecutionArguments = this.sysbenchLoggingArguments + $"--hostIpAddress {this.ServerIpAddress} --durationSecs {this.Duration.TotalSeconds} --password {this.SuperUserPassword}";

string command = "python3";
string script = $"{this.SysbenchPackagePath}/run-workload.py ";

using (IProcessProxy process = await this.ExecuteCommandAsync(
command,
SysbenchExecutor.PythonCommand,
script + this.sysbenchExecutionArguments,
this.SysbenchPackagePath,
telemetryContext,
Expand All @@ -242,5 +256,64 @@ private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, Cancellat
}
}
}

private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken)
{
string arguments = $"{this.packageDirectory}/truncate-tables.py --dbName {this.DatabaseName}";
nmalkapuram marked this conversation as resolved.
Show resolved Hide resolved

string serverIps = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>())
.FirstOrDefault()?.IPAddress
?? "localhost";

arguments += $" --clientIps \"{serverIps}\"";

using (IProcessProxy process = await this.ExecuteCommandAsync(
SysbenchExecutor.PythonCommand,
arguments,
Environment.CurrentDirectory,
telemetryContext,
cancellationToken))
{
if (!cancellationToken.IsCancellationRequested)
{
await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true);
process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString());
}
}
}

private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, CancellationToken cancellationToken)
{
int tableCount = GetTableCount(this.DatabaseScenario, this.TableCount, this.Workload);
int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads);
int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount);

this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}";
this.sysbenchPrepareArguments = $"{this.sysbenchLoggingArguments} --password {this.SuperUserPassword}";

string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>())
.FirstOrDefault()?.IPAddress
?? "localhost";

this.sysbenchPrepareArguments += $" --host \"{serverIp}\"";

string arguments = $"{this.SysbenchPackagePath}/populate-database.py ";

using (IProcessProxy process = await this.ExecuteCommandAsync(
SysbenchExecutor.PythonCommand,
arguments + this.sysbenchPrepareArguments,
this.SysbenchPackagePath,
telemetryContext,
cancellationToken))
{
if (!cancellationToken.IsCancellationRequested)
{
await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true);
process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly);

this.AddMetric(this.sysbenchLoggingArguments, process, telemetryContext, cancellationToken);
}
}
}
}
}
23 changes: 17 additions & 6 deletions src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ namespace VirtualClient.Actions
{
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
Expand Down Expand Up @@ -79,13 +82,17 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance
int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads);
int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount);

this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount} --password {this.SuperUserPassword}";
string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}";
this.sysbenchPrepareArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword}";

string serverIp = "localhost";

this.sysbenchPrepareArguments += $" --host \"{serverIp}\"";

string command = $"python3";
string arguments = $"{this.SysbenchPackagePath}/populate-database.py ";

using (IProcessProxy process = await this.ExecuteCommandAsync(
command,
SysbenchExecutor.PythonCommand,
arguments + this.sysbenchPrepareArguments,
this.SysbenchPackagePath,
telemetryContext,
Expand All @@ -95,6 +102,8 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance
{
await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true);
process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly);

this.AddMetric(sysbenchLoggingArguments, process, telemetryContext, cancellationToken);
}
}
}
Expand All @@ -105,13 +114,13 @@ private async Task PrepareTPCCMySQLDatabase(EventContext telemetryContext, Cance
int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads);
int warehouseCount = GetWarehouseCount(this.DatabaseScenario, this.WarehouseCount);

this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --warehouses {warehouseCount} --threadCount {threadCount} --password {this.SuperUserPassword}";
string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --warehouses {warehouseCount} --threadCount {threadCount}";
this.sysbenchPrepareArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword}";

string command = $"python3";
string arguments = $"{this.SysbenchPackagePath}/populate-database.py ";

using (IProcessProxy process = await this.ExecuteCommandAsync(
command,
SysbenchExecutor.PythonCommand,
arguments + this.sysbenchPrepareArguments,
this.SysbenchPackagePath,
telemetryContext,
Expand All @@ -121,6 +130,8 @@ private async Task PrepareTPCCMySQLDatabase(EventContext telemetryContext, Cance
{
await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true);
process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly);

this.AddMetric(sysbenchLoggingArguments, process, telemetryContext, cancellationToken);
}
}
}
Expand Down
81 changes: 80 additions & 1 deletion src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ namespace VirtualClient.Actions
using VirtualClient.Common.Extensions;
using VirtualClient.Common.Telemetry;
using VirtualClient.Contracts;
using VirtualClient.Contracts.Metadata;

/// <summary>
/// The Sysbench workload executor.
Expand All @@ -35,6 +36,11 @@ public class SysbenchExecutor : VirtualClientComponent
/// </summary>
public const int SelectWorkloadDefaultTableCount = 1;

/// <summary>
/// const for python command.
/// </summary>
protected const string PythonCommand = "python3";

private readonly IStateManager stateManager;
private static readonly string[] SelectWorkloads =
{
Expand Down Expand Up @@ -175,6 +181,18 @@ public string Workload
}
}

/// <summary>
/// The specifed action that controls the execution of the dependency.
/// </summary>
public string Action
{
get
{
this.Parameters.TryGetValue(nameof(this.Action), out IConvertible action);
return action?.ToString();
}
}

/// <summary>
/// Client used to communicate with the hosted instance of the
/// Virtual Client API at server side.
Expand Down Expand Up @@ -299,7 +317,10 @@ await this.CheckDistroSupportAsync(telemetryContext, cancellationToken)
DependencyPath package = await this.GetPackageAsync(this.PackageName, cancellationToken);
this.SysbenchPackagePath = package.Path;

await this.InitializeExecutablesAsync(telemetryContext, cancellationToken);
if (this.Action != ClientAction.TruncateDatabase)
{
await this.InitializeExecutablesAsync(telemetryContext, cancellationToken);
}

this.InitializeApiClients(telemetryContext, cancellationToken);

Expand Down Expand Up @@ -377,6 +398,43 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C
await this.stateManager.SaveStateAsync<SysbenchState>(nameof(SysbenchState), state, cancellationToken);
}

/// <summary>
/// Add metrics to telemtry.
/// </summary>
/// <param name="arguments"></param>
/// <param name="process"></param>
/// <param name="telemetryContext"></param>
/// <param name="cancellationToken"></param>
protected void AddMetric(string arguments, IProcessProxy process, EventContext telemetryContext, CancellationToken cancellationToken)
{
if (!cancellationToken.IsCancellationRequested)
{
this.MetadataContract.AddForScenario(
"Sysbench",
process.FullCommand(),
toolVersion: null);

this.MetadataContract.Apply(telemetryContext);

string text = process.StandardOutput.ToString();

List<Metric> metrics = new List<Metric>();
double duration = (process.ExitTime - process.StartTime).TotalMinutes;
metrics.Add(new Metric("PopulateDatabaseTime_Minutes ", duration, "minutes", MetricRelativity.LowerIsBetter));

this.Logger.LogMetrics(
toolName: "Sysbench",
scenarioName: this.MetricScenario ?? this.Scenario,
process.StartTime,
process.ExitTime,
metrics,
null,
scenarioArguments: arguments,
this.Tags,
telemetryContext);
}
}

private async Task CheckDistroSupportAsync(EventContext telemetryContext, CancellationToken cancellationToken)
{
if (this.Platform == PlatformID.Unix)
Expand Down Expand Up @@ -460,5 +518,26 @@ internal class BenchmarkName
public const string OLTP = nameof(OLTP);
public const string TPCC = nameof(TPCC);
}

/// <summary>
/// Supported Sysbench Client actions.
/// </summary>
internal class ClientAction
{
/// <summary>
/// Creates Database on MySQL server and Users on Server and any Clients.
/// </summary>
public const string PopulateDatabase = nameof(PopulateDatabase);

/// <summary>
/// Truncates all tables existing in database
/// </summary>
public const string TruncateDatabase = nameof(TruncateDatabase);

/// <summary>
/// Truncates all tables existing in database
/// </summary>
public const string RunWorkload = nameof(RunWorkload);
}
}
}
5 changes: 2 additions & 3 deletions src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public class ProfileExpressionEvaluator : IExpressionEvaluator
// {fn(512 / 16)]}
// {fn(512 / {LogicalThreadCount})}
private static readonly Regex CalculateExpression = new Regex(
@"\{calculate\(([0-9\*\/\+\-\(\)\s]+)\)\}",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
@"\{calculate\(([0-9L\*\/\+\-\(\)\s]+)\)\}",
RegexOptions.Compiled | RegexOptions.IgnoreCase);

// e.g.
// {calculate({IsTLSEnabled} ? "Yes" : "No")}
Expand Down Expand Up @@ -441,7 +441,6 @@ public class ProfileExpressionEvaluator : IExpressionEvaluator
{
string function = match.Groups[1].Value;
long result = await Microsoft.CodeAnalysis.CSharp.Scripting.CSharpScript.EvaluateAsync<long>(function);

evaluatedExpression = evaluatedExpression.Replace(match.Value, result.ToString());
}
}
Expand Down
Loading
Loading