diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 40beacaca0..2f1bf6c486 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -339,14 +339,118 @@ dotnet test --collect:"XPlat Code Coverage" ## Run Performance Tests -### Running Performance test project directly +The performance tests live here: +`src\Microsoft.Data.SqlClient\tests\PerformanceTests\` -Project location from Root: `src\Microsoft.Data.SqlClient\tests\PerformanceTests\Microsoft.Data.SqlClient.PerformanceTests.csproj` -Configure `runnerconfig.json` file with connection string and preferred settings to run Benchmark Jobs. +They can be run from the command line by following the instructions below. + +Launch a shell and change into the project directory: + +PowerShell: + +```pwsh +> cd src\Microsoft.Data.SqlClient\tests\PerformanceTests +``` + +Bash: + +```bash +$ cd src/Microsoft.Data.SqlClient/tests/PerformanceTests +``` + +### Create Database + +Create an empty database for the benchmarks to use. This example assumes +a local SQL server instance using SQL authentication: ```bash -cd src\Microsoft.Data.SqlClient\tests\PerformanceTests -dotnet run -c Release -f net8.0 +$ sqlcmd -S localhost -U sa -P password +1> create database [sqlclient-perf-db] +2> go +1> quit +``` + +The default `runnerconfig.json` expects a database named `sqlclient-perf-db`, +but you may change the config to use any existing database. All tables in +the database will be dropped when running the benchmarks. + +### Configure Runner + +Configure the benchmarks by editing the `runnerconfig.json` file directly in the +`PerformanceTests` directory with an appropriate connection string and benchmark +settings: + +```json +{ + "ConnectionString": "Server=tcp:localhost; Integrated Security=true; Initial Catalog=sqlclient-perf-db;", + "UseManagedSniOnWindows": false, + "Benchmarks": + { + "SqlConnectionRunnerConfig": + { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 50, + "InvocationCount":30, + "WarmupCount": 5, + "RowCount": 0 + }, + ... + } +} ``` -_Only "**Release** Configuration" applies to Performance Tests_ +Individual benchmarks may be enabled or disabled, and each has several +benchmarking options for fine tuning. + +After making edits to `runnerconfig.json` you must perform a build which will +copy the file into the `artifacts` directory alongside the benchmark DLL. By +default, the benchmarks look for `runnerconfig.json` in the same directory as +the DLL. + +Optionally, to avoid polluting your git workspace and requring a build after +each config change, copy `runnerconfig.json` to a new file, make your edits +there, and then specify the new file with the RUNNER_CONFIG environment +variable. + +PowerShell: + +```pwsh +> copy runnerconfig.json $HOME\.configs\runnerconfig.json + +# Make edits to $HOME\.configs\runnerconfig.json + +# You must set the RUNNER_CONFIG environment variable for the current shell. +> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.json" +``` + +Bash: + +```bash +$ cp runnerconfig.json ~/.configs/runnerconfig.json + +# Make edits to ~/.configs/runnerconfig.json + +# Optionally export RUNNER_CONFIG. +$ export RUNNER_CONFIG=~/.configs/runnerconfig.json +``` + +### Run Benchmarks + +All benchmarks must be compiled and run in **Release** configuration. + +PowerShell: + +```pwsh +> dotnet run -c Release -f net9.0 +``` + +Bash: + +```bash +# Omit RUNNER_CONFIG if you exported it earlier, or if you're using the +# copy prepared by the build. +$ dotnet run -c Release -f net9.0 + +$ RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0 +``` diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs index 8ec9be9cf9..62cce36f94 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs @@ -2,17 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.IO; -using Newtonsoft.Json; - namespace Microsoft.Data.SqlClient.PerformanceTests { public abstract class BaseRunner { public BaseRunner() { - s_config = JsonConvert.DeserializeObject(File.ReadAllText("runnerconfig.json")); - s_datatypes = JsonConvert.DeserializeObject(File.ReadAllText("datatypes.json")); + s_config = Config.Load(); + s_datatypes = DataTypes.Load(); } internal static Config s_config; diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs index 3fe22e45b3..f52fb1b554 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs @@ -12,6 +12,28 @@ public class Config public string ConnectionString; public bool UseManagedSniOnWindows; public Benchmarks Benchmarks; + + /// + /// Load the benchmark configuration from a JSON file. + /// + /// If the environment variable "RUNNER_CONFIG" is set, it will be used + /// as the path to the config file. Otherwise, the file + /// "runnerconfig.json" in the current working directory will be used. + /// + /// + /// + /// The Config instance populated from the JSON config file. + /// + /// + /// + /// Thrown if the config file cannot be read or deserialized. + /// + /// + public static Config Load() + { + return Loader.FromJsonFile( + "runnerconfig.json", "RUNNER_CONFIG"); + } } public class Benchmarks diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs index 3140fd6b95..54b59a1015 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs @@ -18,6 +18,28 @@ public class DataTypes public MaxLengthBinaryType[] Binary; public MaxLengthValueLengthType[] MaxTypes; public DataType[] Others; + + /// + /// Load the data types configuration from a JSON file. + /// + /// If the environment variable "DATATYPES_CONFIG" is set, it will be + /// used as the path to the config file. Otherwise, the file + /// "datatypes.json" in the current working directory will be used. + /// + /// + /// + /// The DataTypes instance populated from the JSON file. + /// + /// + /// + /// Thrown if the config file cannot be read or deserialized. + /// + /// + public static DataTypes Load() + { + return Loader.FromJsonFile( + "datatypes.json", "DATATYPES_CONFIG"); + } } /// diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs new file mode 100644 index 0000000000..a452216b66 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text.Json; + +namespace Microsoft.Data.SqlClient.PerformanceTests +{ + public static class Loader + { + /// + /// Load a JSON config file into the given type. + /// + /// + /// + /// The type to deserialize the JSON into. + /// + /// + /// + /// The path to the JSON config file. + /// + /// + /// + /// An optional environment variable that, if set, will be used as + /// the config file path, ignoring path. + /// + /// + /// + /// The T instance populated from the JSON config file. + /// + /// + /// + /// Thrown if the config file cannot be read or deserialized. + /// + /// + public static T FromJsonFile( + string path, + string envOverride = null) + where T : class + { + string configFile = + envOverride is null + ? path + : Environment.GetEnvironmentVariable(envOverride) + ?? path; + + T config = null; + Exception error = null; + try + { + using var stream = File.OpenRead(configFile); + config = + JsonSerializer.Deserialize( + stream, + new JsonSerializerOptions + { + IncludeFields = true, + ReadCommentHandling = JsonCommentHandling.Skip + }); + } + catch (Exception ex) + { + error = ex; + } + + if (config is null || error is not null) + { + throw new InvalidOperationException( + $"Failed to load {typeof(T).Name} config from file=" + + $"{configFile}", error); + } + + return config; + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj index 8fff3a5cab..cc06945c54 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj @@ -3,16 +3,17 @@ Exe PerformanceTests net8.0;net9.0 - false Debug;Release; $(ObjFolder)$(Configuration).$(Platform).$(AssemblyName) $(BinFolder)$(Configuration).$(Platform).$(AssemblyName) Microsoft.Data.SqlClient.PerformanceTests.Program + + @@ -20,29 +21,13 @@ + - + - - - - - - - - - - - - - - - - - diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs index 2b98162de9..8df9d36a3e 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs @@ -2,22 +2,20 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using BenchmarkDotNet.Running; -using Newtonsoft.Json; using System; -using System.IO; +using BenchmarkDotNet.Running; namespace Microsoft.Data.SqlClient.PerformanceTests { public class Program { - private static Config s_config; + private readonly Config _config; - public static void Main() + public Program() { // Load config file - s_config = JsonConvert.DeserializeObject(File.ReadAllText("runnerconfig.json")); - if (s_config.UseManagedSniOnWindows) + _config = Config.Load(); + if (_config.UseManagedSniOnWindows) { AppContext.SetSwitch("Switch.Microsoft.Data.SqlClient.UseManagedNetworkingOnWindows", true); } @@ -37,44 +35,50 @@ public static void Main() // Always Encrypted } - private static void Run_SqlConnectionBenchmark() + private void Run_SqlConnectionBenchmark() { - if (s_config.Benchmarks.SqlConnectionRunnerConfig?.Enabled == true) + if (_config.Benchmarks.SqlConnectionRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.SqlConnectionRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.SqlConnectionRunnerConfig)); } } - private static void Run_SqlCommandBenchmark() + private void Run_SqlCommandBenchmark() { - if (s_config.Benchmarks.SqlCommandRunnerConfig?.Enabled == true) + if (_config.Benchmarks.SqlCommandRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.SqlCommandRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.SqlCommandRunnerConfig)); } } - private static void Run_DataTypeReaderBenchmark() + private void Run_DataTypeReaderBenchmark() { - if (s_config.Benchmarks.DataTypeReaderRunnerConfig?.Enabled == true) + if (_config.Benchmarks.DataTypeReaderRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.DataTypeReaderRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.DataTypeReaderRunnerConfig)); } } - private static void Run_DataTypeReaderAsyncBenchmark() + private void Run_DataTypeReaderAsyncBenchmark() { - if (s_config.Benchmarks.DataTypeReaderAsyncRunnerConfig?.Enabled == true) + if (_config.Benchmarks.DataTypeReaderAsyncRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.DataTypeReaderAsyncRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.DataTypeReaderAsyncRunnerConfig)); } } - private static void Run_SqlBulkCopyBenchmark() + private void Run_SqlBulkCopyBenchmark() { - if (s_config.Benchmarks.SqlBulkCopyRunnerConfig?.Enabled == true) + if (_config.Benchmarks.SqlBulkCopyRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.SqlBulkCopyRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.SqlBulkCopyRunnerConfig)); } } + + public static void Main() + { + // Run the benchmarks. + new Program(); + } } } diff --git a/tools/props/Versions.props b/tools/props/Versions.props index c0f6a3fdf2..407a1d68a2 100644 --- a/tools/props/Versions.props +++ b/tools/props/Versions.props @@ -53,7 +53,7 @@ - 0.13.2 + 0.14.0 3.1.6 10.0.0-beta.24564.1 8.0.0-beta.24123.1