From 73654670765f4ab9013ff106569d8debc28212c9 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:24:35 -0300 Subject: [PATCH 1/2] User Story 34145: Tidy up Bulk Copy unmatched column name work - Tidied up the existing Performance benchmarks. - Added environment variables to specify config files. - Removed Newtonsoft.Json as a dependency. - Simplified csproj a bit. --- BUILDGUIDE.md | 21 ++++- .../BenchmarkRunners/BaseRunner.cs | 7 +- .../tests/PerformanceTests/Config/Config.cs | 22 ++++++ .../PerformanceTests/Config/DataTypes.cs | 22 ++++++ .../tests/PerformanceTests/Config/Loader.cs | 78 +++++++++++++++++++ ...oft.Data.SqlClient.PerformanceTests.csproj | 23 +----- .../tests/PerformanceTests/Program.cs | 48 ++++++------ tools/props/Versions.props | 2 +- 8 files changed, 175 insertions(+), 48 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 40beacaca0..8506d7328a 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -342,11 +342,30 @@ dotnet test --collect:"XPlat Code Coverage" ### Running Performance test project directly Project location from Root: `src\Microsoft.Data.SqlClient\tests\PerformanceTests\Microsoft.Data.SqlClient.PerformanceTests.csproj` -Configure `runnerconfig.json` file with connection string and preferred settings to run Benchmark Jobs. + +Create an empty database for the benchmarks to use: +```bash +> create database [sqlclient-perf-db] +> go +``` + +Configure `runnerconfig.json` file with connection string and preferred settings +to run Benchmark Jobs. ```bash cd src\Microsoft.Data.SqlClient\tests\PerformanceTests dotnet run -c Release -f net8.0 ``` +Optionally, to avoid polluting your git workspace, copy `runnerconfig.json` to a +new file, make your edits there, and then specify the new file with the +RUNNER_CONFIG environment variable. + +```bash +cd src/Microsoft.Data.SqlClient/tests/PerformanceTests +cp runnerconfig.json ~/.configs/runnerconfig.json +# Make edits to ~/.configs/runnerconfig.json +RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0 +``` + _Only "**Release** Configuration" applies to Performance Tests_ diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs index 8ec9be9cf9..62cce36f94 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs @@ -2,17 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.IO; -using Newtonsoft.Json; - namespace Microsoft.Data.SqlClient.PerformanceTests { public abstract class BaseRunner { public BaseRunner() { - s_config = JsonConvert.DeserializeObject(File.ReadAllText("runnerconfig.json")); - s_datatypes = JsonConvert.DeserializeObject(File.ReadAllText("datatypes.json")); + s_config = Config.Load(); + s_datatypes = DataTypes.Load(); } internal static Config s_config; diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs index 3fe22e45b3..f52fb1b554 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs @@ -12,6 +12,28 @@ public class Config public string ConnectionString; public bool UseManagedSniOnWindows; public Benchmarks Benchmarks; + + /// + /// Load the benchmark configuration from a JSON file. + /// + /// If the environment variable "RUNNER_CONFIG" is set, it will be used + /// as the path to the config file. Otherwise, the file + /// "runnerconfig.json" in the current working directory will be used. + /// + /// + /// + /// The Config instance populated from the JSON config file. + /// + /// + /// + /// Thrown if the config file cannot be read or deserialized. + /// + /// + public static Config Load() + { + return Loader.FromJsonFile( + "runnerconfig.json", "RUNNER_CONFIG"); + } } public class Benchmarks diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs index 3140fd6b95..54b59a1015 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs @@ -18,6 +18,28 @@ public class DataTypes public MaxLengthBinaryType[] Binary; public MaxLengthValueLengthType[] MaxTypes; public DataType[] Others; + + /// + /// Load the data types configuration from a JSON file. + /// + /// If the environment variable "DATATYPES_CONFIG" is set, it will be + /// used as the path to the config file. Otherwise, the file + /// "datatypes.json" in the current working directory will be used. + /// + /// + /// + /// The DataTypes instance populated from the JSON file. + /// + /// + /// + /// Thrown if the config file cannot be read or deserialized. + /// + /// + public static DataTypes Load() + { + return Loader.FromJsonFile( + "datatypes.json", "DATATYPES_CONFIG"); + } } /// diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs new file mode 100644 index 0000000000..a452216b66 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text.Json; + +namespace Microsoft.Data.SqlClient.PerformanceTests +{ + public static class Loader + { + /// + /// Load a JSON config file into the given type. + /// + /// + /// + /// The type to deserialize the JSON into. + /// + /// + /// + /// The path to the JSON config file. + /// + /// + /// + /// An optional environment variable that, if set, will be used as + /// the config file path, ignoring path. + /// + /// + /// + /// The T instance populated from the JSON config file. + /// + /// + /// + /// Thrown if the config file cannot be read or deserialized. + /// + /// + public static T FromJsonFile( + string path, + string envOverride = null) + where T : class + { + string configFile = + envOverride is null + ? path + : Environment.GetEnvironmentVariable(envOverride) + ?? path; + + T config = null; + Exception error = null; + try + { + using var stream = File.OpenRead(configFile); + config = + JsonSerializer.Deserialize( + stream, + new JsonSerializerOptions + { + IncludeFields = true, + ReadCommentHandling = JsonCommentHandling.Skip + }); + } + catch (Exception ex) + { + error = ex; + } + + if (config is null || error is not null) + { + throw new InvalidOperationException( + $"Failed to load {typeof(T).Name} config from file=" + + $"{configFile}", error); + } + + return config; + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj index 8fff3a5cab..cc06945c54 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj @@ -3,16 +3,17 @@ Exe PerformanceTests net8.0;net9.0 - false Debug;Release; $(ObjFolder)$(Configuration).$(Platform).$(AssemblyName) $(BinFolder)$(Configuration).$(Platform).$(AssemblyName) Microsoft.Data.SqlClient.PerformanceTests.Program + + @@ -20,29 +21,13 @@ + - + - - - - - - - - - - - - - - - - - diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs index 2b98162de9..8df9d36a3e 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs @@ -2,22 +2,20 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using BenchmarkDotNet.Running; -using Newtonsoft.Json; using System; -using System.IO; +using BenchmarkDotNet.Running; namespace Microsoft.Data.SqlClient.PerformanceTests { public class Program { - private static Config s_config; + private readonly Config _config; - public static void Main() + public Program() { // Load config file - s_config = JsonConvert.DeserializeObject(File.ReadAllText("runnerconfig.json")); - if (s_config.UseManagedSniOnWindows) + _config = Config.Load(); + if (_config.UseManagedSniOnWindows) { AppContext.SetSwitch("Switch.Microsoft.Data.SqlClient.UseManagedNetworkingOnWindows", true); } @@ -37,44 +35,50 @@ public static void Main() // Always Encrypted } - private static void Run_SqlConnectionBenchmark() + private void Run_SqlConnectionBenchmark() { - if (s_config.Benchmarks.SqlConnectionRunnerConfig?.Enabled == true) + if (_config.Benchmarks.SqlConnectionRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.SqlConnectionRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.SqlConnectionRunnerConfig)); } } - private static void Run_SqlCommandBenchmark() + private void Run_SqlCommandBenchmark() { - if (s_config.Benchmarks.SqlCommandRunnerConfig?.Enabled == true) + if (_config.Benchmarks.SqlCommandRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.SqlCommandRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.SqlCommandRunnerConfig)); } } - private static void Run_DataTypeReaderBenchmark() + private void Run_DataTypeReaderBenchmark() { - if (s_config.Benchmarks.DataTypeReaderRunnerConfig?.Enabled == true) + if (_config.Benchmarks.DataTypeReaderRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.DataTypeReaderRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.DataTypeReaderRunnerConfig)); } } - private static void Run_DataTypeReaderAsyncBenchmark() + private void Run_DataTypeReaderAsyncBenchmark() { - if (s_config.Benchmarks.DataTypeReaderAsyncRunnerConfig?.Enabled == true) + if (_config.Benchmarks.DataTypeReaderAsyncRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.DataTypeReaderAsyncRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.DataTypeReaderAsyncRunnerConfig)); } } - private static void Run_SqlBulkCopyBenchmark() + private void Run_SqlBulkCopyBenchmark() { - if (s_config.Benchmarks.SqlBulkCopyRunnerConfig?.Enabled == true) + if (_config.Benchmarks.SqlBulkCopyRunnerConfig?.Enabled == true) { - BenchmarkRunner.Run(BenchmarkConfig.s_instance(s_config.Benchmarks.SqlBulkCopyRunnerConfig)); + BenchmarkRunner.Run(BenchmarkConfig.s_instance(_config.Benchmarks.SqlBulkCopyRunnerConfig)); } } + + public static void Main() + { + // Run the benchmarks. + new Program(); + } } } diff --git a/tools/props/Versions.props b/tools/props/Versions.props index c0f6a3fdf2..407a1d68a2 100644 --- a/tools/props/Versions.props +++ b/tools/props/Versions.props @@ -53,7 +53,7 @@ - 0.13.2 + 0.14.0 3.1.6 10.0.0-beta.24564.1 8.0.0-beta.24123.1 From 55938e5db8782b6aeebdcde30e78be6e1b61e4da Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 20 Mar 2025 08:47:21 -0300 Subject: [PATCH 2/2] User Story 34145: Tidy up Bulk Copy unmatched column name work - Improved benchmark running instructions. --- BUILDGUIDE.md | 117 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 101 insertions(+), 16 deletions(-) diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 8506d7328a..2f1bf6c486 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -339,33 +339,118 @@ dotnet test --collect:"XPlat Code Coverage" ## Run Performance Tests -### Running Performance test project directly +The performance tests live here: +`src\Microsoft.Data.SqlClient\tests\PerformanceTests\` -Project location from Root: `src\Microsoft.Data.SqlClient\tests\PerformanceTests\Microsoft.Data.SqlClient.PerformanceTests.csproj` +They can be run from the command line by following the instructions below. + +Launch a shell and change into the project directory: + +PowerShell: + +```pwsh +> cd src\Microsoft.Data.SqlClient\tests\PerformanceTests +``` + +Bash: -Create an empty database for the benchmarks to use: ```bash -> create database [sqlclient-perf-db] -> go +$ cd src/Microsoft.Data.SqlClient/tests/PerformanceTests ``` -Configure `runnerconfig.json` file with connection string and preferred settings -to run Benchmark Jobs. +### Create Database + +Create an empty database for the benchmarks to use. This example assumes +a local SQL server instance using SQL authentication: ```bash -cd src\Microsoft.Data.SqlClient\tests\PerformanceTests -dotnet run -c Release -f net8.0 +$ sqlcmd -S localhost -U sa -P password +1> create database [sqlclient-perf-db] +2> go +1> quit +``` + +The default `runnerconfig.json` expects a database named `sqlclient-perf-db`, +but you may change the config to use any existing database. All tables in +the database will be dropped when running the benchmarks. + +### Configure Runner + +Configure the benchmarks by editing the `runnerconfig.json` file directly in the +`PerformanceTests` directory with an appropriate connection string and benchmark +settings: + +```json +{ + "ConnectionString": "Server=tcp:localhost; Integrated Security=true; Initial Catalog=sqlclient-perf-db;", + "UseManagedSniOnWindows": false, + "Benchmarks": + { + "SqlConnectionRunnerConfig": + { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 50, + "InvocationCount":30, + "WarmupCount": 5, + "RowCount": 0 + }, + ... + } +} ``` -Optionally, to avoid polluting your git workspace, copy `runnerconfig.json` to a -new file, make your edits there, and then specify the new file with the -RUNNER_CONFIG environment variable. +Individual benchmarks may be enabled or disabled, and each has several +benchmarking options for fine tuning. + +After making edits to `runnerconfig.json` you must perform a build which will +copy the file into the `artifacts` directory alongside the benchmark DLL. By +default, the benchmarks look for `runnerconfig.json` in the same directory as +the DLL. + +Optionally, to avoid polluting your git workspace and requring a build after +each config change, copy `runnerconfig.json` to a new file, make your edits +there, and then specify the new file with the RUNNER_CONFIG environment +variable. + +PowerShell: + +```pwsh +> copy runnerconfig.json $HOME\.configs\runnerconfig.json + +# Make edits to $HOME\.configs\runnerconfig.json + +# You must set the RUNNER_CONFIG environment variable for the current shell. +> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.json" +``` + +Bash: ```bash -cd src/Microsoft.Data.SqlClient/tests/PerformanceTests -cp runnerconfig.json ~/.configs/runnerconfig.json +$ cp runnerconfig.json ~/.configs/runnerconfig.json + # Make edits to ~/.configs/runnerconfig.json -RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0 + +# Optionally export RUNNER_CONFIG. +$ export RUNNER_CONFIG=~/.configs/runnerconfig.json ``` -_Only "**Release** Configuration" applies to Performance Tests_ +### Run Benchmarks + +All benchmarks must be compiled and run in **Release** configuration. + +PowerShell: + +```pwsh +> dotnet run -c Release -f net9.0 +``` + +Bash: + +```bash +# Omit RUNNER_CONFIG if you exported it earlier, or if you're using the +# copy prepared by the build. +$ dotnet run -c Release -f net9.0 + +$ RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0 +```