From 7a03a4f189f1e327db805240fcc87dcf61a8bb3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 3 May 2026 14:26:57 +0200 Subject: [PATCH 1/3] feat: collect benchmark data into 'benchmarks' branch Adds two Nuke targets and a docs-site-compatible aggregator so the matrix Benchmarks-* artifacts produced by the build workflow can be turned into the data.js / limited-data.js files consumed by the documentation site (mirroring the aweXpect setup). - PublishBenchmarkReport: downloads all Benchmarks-* artifacts of the current run, parses *-report-full-compressed.json files, and commits updated data.js / limited-data.js (last 50 commits) to the 'benchmarks' branch via the GitHub Contents API. The branch is auto-created from main on first run. - BenchmarkSeedHistory: one-shot tool that walks past successful build.yml runs on main, downloads each run's Benchmarks-* artifacts, fetches commit metadata from the workflow run head_commit, and seeds the benchmark data files. Idempotent - already-recorded SHAs are skipped. - PageBenchmarkReportGenerator: chart key incorporates BenchmarkDotNet Parameters (e.g. 'Method (N=1)'), so each [Params] combination becomes its own chart. Includes Mockolate, Moq, NSubstitute, FakeItEasy, TUnitMocks, Imposter as datasets. - build.yml: new publish-benchmark-report job that runs after the matrix benchmarks job on push to main only, with contents: write. --- .github/workflows/build.yml | 24 ++ .nuke/build.schema.json | 7 + Pipeline/Build.Benchmarks.cs | 210 +++++++++++++++ Pipeline/BuildExtensions.cs | 178 +++++++++++++ Pipeline/PageBenchmarkReportGenerator.cs | 316 +++++++++++++++++++++++ 5 files changed, 735 insertions(+) create mode 100644 Pipeline/PageBenchmarkReportGenerator.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ffcba82..ba6d3b44 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,6 +103,30 @@ jobs: ./Artifacts/* if-no-files-found: ignore + publish-benchmark-report: + name: "Publish Benchmark Report" + needs: [ benchmarks ] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + env: + DOTNET_NOLOGO: true + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Setup .NET SDKs + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 10.0.x + - name: Publish benchmark report + run: ./build.sh PublishBenchmarkReport + env: + GithubToken: ${{ secrets.GITHUB_TOKEN }} + WorkflowRunId: ${{ github.run_id }} + mutation-tests: name: "Mutation tests" runs-on: ubuntu-latest diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index b86fdf90..4654f133 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -29,6 +29,7 @@ "BenchmarkDotNet", "BenchmarkResult", "Benchmarks", + "BenchmarkSeedHistory", "CalculateNugetVersion", "Clean", "CodeAnalysis", @@ -42,6 +43,7 @@ "MutationTestExecution", "MutationTests", "Pack", + "PublishBenchmarkReport", "Restore", "UnitTests", "UpdateReadme" @@ -122,6 +124,11 @@ "type": "string", "description": "Filter for BenchmarkDotNet - Default is '*'" }, + "BenchmarkSeedRunLimit": { + "type": "integer", + "description": "Maximum number of historic Build runs to seed via 'BenchmarkSeedHistory' - Default is 30", + "format": "int32" + }, "Configuration": { "type": "string", "description": "Configuration to build - Default is 'Debug' (local) or 'Release' (server)", diff --git a/Pipeline/Build.Benchmarks.cs b/Pipeline/Build.Benchmarks.cs index 852e5e19..3b93aefc 100644 --- a/Pipeline/Build.Benchmarks.cs +++ b/Pipeline/Build.Benchmarks.cs @@ -1,10 +1,13 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using Nuke.Common; using Nuke.Common.IO; +using Nuke.Common.Tooling; using Nuke.Common.Tools.DotNet; +using Nuke.Common.Tools.Git; using Octokit; using Serilog; using static Nuke.Common.Tools.DotNet.DotNetTasks; @@ -15,8 +18,16 @@ namespace Build; partial class Build { + private const string BenchmarkBranch = "benchmarks"; + private const string BenchmarkDataPath = "Docs/pages/static/js/data.js"; + private const string BenchmarkLimitedDataPath = "Docs/pages/static/js/limited-data.js"; + private const int BenchmarkLimit = 50; + [Parameter("Filter for BenchmarkDotNet - Default is '*'")] readonly string BenchmarkFilter = "*"; + [Parameter("Maximum number of historic Build runs to seed via 'BenchmarkSeedHistory' - Default is 30")] + readonly int BenchmarkSeedRunLimit = 30; + Target BenchmarkDotNet => _ => _ .Executes(() => { @@ -126,6 +137,205 @@ partial class Build .DependsOn(BenchmarkDotNet) .DependsOn(BenchmarkResult); + Target PublishBenchmarkReport => _ => _ + .Description("Aggregates BenchmarkDotNet JSON results from the matrix Benchmarks-* artifacts of the " + + "current workflow run, then commits an updated data.js / limited-data.js to the " + + $"'{BenchmarkBranch}' branch.") + .Requires(() => GithubToken) + .Executes(async () => + { + await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken); + + List benchmarkReports = LoadBenchmarkJsonReports(ArtifactsDirectory / "Benchmarks" / "results"); + if (benchmarkReports.Count == 0) + { + Log.Warning("Skip benchmark report: no benchmark JSON reports found."); + return; + } + + PageBenchmarkReportGenerator.CommitInfo commitInfo = ReadCurrentCommitInfo(); + Log.Information( + "Appending benchmark data for commit {Sha} ({Author}, {Date}): {Message}", + commitInfo.Sha, commitInfo.Author, commitInfo.Date, commitInfo.Message); + + await BuildExtensions.EnsureBranchExistsAsync(BenchmarkBranch, "main", GithubToken); + + BuildExtensions.GithubFile dataFile = + await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken); + BuildExtensions.GithubFile limitedFile = + await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken); + + (string updated, string limited) = PageBenchmarkReportGenerator.Append( + commitInfo, + dataFile?.Content ?? string.Empty, + benchmarkReports, + BenchmarkLimit); + + if (string.IsNullOrWhiteSpace(updated)) + { + Log.Information("No changes to publish (commit already recorded)."); + return; + } + + string commitMessage = + $"Update benchmark for {commitInfo.Sha.Substring(0, 8)}: {commitInfo.Message} by {commitInfo.Author}"; + await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, updated, + dataFile?.Sha, GithubToken); + await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage, + limited, limitedFile?.Sha, GithubToken); + }); + + Target BenchmarkSeedHistory => _ => _ + .Description("Seeds the benchmark data files on the 'benchmarks' branch with results from past " + + "successful 'Build' workflow runs on main, by downloading their Benchmarks-* artifacts. " + + "One-shot tool; safe to re-run (commits already recorded are skipped).") + .Requires(() => GithubToken) + .Executes(async () => + { + List runs = await BuildExtensions.ListSuccessfulRunsAsync( + "build.yml", "main", BenchmarkSeedRunLimit, GithubToken); + if (runs.Count == 0) + { + Log.Warning("No successful 'Build' runs found on main - nothing to seed."); + return; + } + + runs.Reverse(); + Log.Information("Seeding from {Count} historical runs (oldest first).", runs.Count); + + await BuildExtensions.EnsureBranchExistsAsync(BenchmarkBranch, "main", GithubToken); + + BuildExtensions.GithubFile dataFile = + await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken); + BuildExtensions.GithubFile limitedFile = + await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken); + + string accumulatedContent = dataFile?.Content ?? string.Empty; + string accumulatedLimited = limitedFile?.Content ?? string.Empty; + string lastFullSha = dataFile?.Sha; + string lastLimitedSha = limitedFile?.Sha; + int updatedRuns = 0; + + foreach (BuildExtensions.WorkflowRunInfo run in runs) + { + AbsolutePath runDirectory = ArtifactsDirectory / "SeedHistory" / run.Id.ToString(); + runDirectory.CreateOrCleanDirectory(); + try + { + await BuildExtensions.DownloadArtifactsFromRunStartingWith(run.Id, "Benchmarks-", + runDirectory, GithubToken); + } + catch (Exception ex) + { + Log.Warning(ex, "Skipping run #{RunId} ({Sha}): could not download artifacts.", run.Id, + run.Sha?.Substring(0, 8)); + continue; + } + + List benchmarkReports = LoadBenchmarkJsonReports(runDirectory / "Benchmarks" / "results"); + if (benchmarkReports.Count == 0) + { + Log.Information("Run #{RunId} ({Sha}) had no benchmark JSON reports - skipping.", run.Id, + run.Sha?.Substring(0, 8)); + continue; + } + + PageBenchmarkReportGenerator.CommitInfo commitInfo = new( + run.Sha, + run.Author ?? "unknown", + run.Date ?? string.Empty, + run.Message ?? string.Empty); + + (string updated, string limited) = PageBenchmarkReportGenerator.Append( + commitInfo, + accumulatedContent, + benchmarkReports, + BenchmarkLimit); + + if (string.IsNullOrWhiteSpace(updated)) + { + Log.Information("Run #{RunId} ({Sha}) already recorded - skipping.", run.Id, + run.Sha?.Substring(0, 8)); + continue; + } + + accumulatedContent = updated; + accumulatedLimited = limited; + updatedRuns++; + Log.Information("Seeded run #{RunId} ({Sha}): {Message}", run.Id, + run.Sha?.Substring(0, 8), commitInfo.Message); + } + + if (updatedRuns == 0) + { + Log.Information("No new benchmark data to seed."); + return; + } + + string commitMessage = $"Seed historic benchmark data ({updatedRuns} runs)"; + await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, + accumulatedContent, lastFullSha, GithubToken); + await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage, + accumulatedLimited, lastLimitedSha, GithubToken); + Log.Information("Seeded benchmark data from {Count} historical runs.", updatedRuns); + }); + + private static List LoadBenchmarkJsonReports(AbsolutePath resultsDirectory) + { + List reports = new(); + if (!Directory.Exists(resultsDirectory)) + { + return reports; + } + + foreach (string file in Directory.GetFiles(resultsDirectory, "*-report-full-compressed.json")) + { + reports.Add(File.ReadAllText(file)); + } + + return reports; + } + + private static PageBenchmarkReportGenerator.CommitInfo ReadCurrentCommitInfo() + { + Output[] lines = GitTasks.Git("log -1").ToArray(); + string commitId = null, author = null, date = null, message = null; + foreach (string line in lines.Select(x => x.Text)) + { + if (commitId == null && line.StartsWith("commit ")) + { + commitId = line.Substring("commit ".Length).Substring(0, 40); + continue; + } + + if (author == null && line.StartsWith("Author: ")) + { + author = line.Substring("Author: ".Length); + int index = author.IndexOf(" <", StringComparison.Ordinal); + if (index > 0) + { + author = author.Substring(0, index); + } + + continue; + } + + if (date == null && line.StartsWith("Date: ")) + { + date = line.Substring("Date: ".Length); + continue; + } + + if (commitId != null && author != null && date != null && !string.IsNullOrWhiteSpace(line)) + { + message = line.Trim(); + break; + } + } + + return new PageBenchmarkReportGenerator.CommitInfo(commitId, author, date, message); + } + async Task DownloadBaselineBenchmarks(AbsolutePath baselineDirectory) { long[] candidateRunIds = await BuildExtensions.FindRecentSuccessfulRunIds("build.yml", "main", 10, GithubToken); diff --git a/Pipeline/BuildExtensions.cs b/Pipeline/BuildExtensions.cs index cfc0e060..fbdd06e4 100644 --- a/Pipeline/BuildExtensions.cs +++ b/Pipeline/BuildExtensions.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Text; using System.Text.Json; using System.Threading.Tasks; using Nuke.Common.CI.GitHubActions; @@ -163,4 +164,181 @@ private static async Task DownloadArtifactsFromRun(string runId, Func + /// Ensures that the given branch exists in the repository. If it does not, creates it pointing at the head + /// of . Returns if the branch was newly created. + /// + public static async Task EnsureBranchExistsAsync(string branch, string sourceBranch, string githubToken) + { + using HttpClient client = CreateGithubClient(githubToken); + HttpResponseMessage exists = await client.GetAsync( + $"{RepositoryApiBaseUrl}/git/ref/heads/{Uri.EscapeDataString(branch)}"); + if (exists.IsSuccessStatusCode) + { + return false; + } + + HttpResponseMessage source = await client.GetAsync( + $"{RepositoryApiBaseUrl}/git/ref/heads/{Uri.EscapeDataString(sourceBranch)}"); + if (!source.IsSuccessStatusCode) + { + string errorContent = await source.Content.ReadAsStringAsync(); + throw new InvalidOperationException( + $"Could not read source branch '{sourceBranch}': {errorContent}"); + } + + string sourceContent = await source.Content.ReadAsStringAsync(); + using JsonDocument document = JsonDocument.Parse(sourceContent); + string sha = document.RootElement.GetProperty("object").GetProperty("sha").GetString(); + + string body = JsonSerializer.Serialize(new + { + @ref = $"refs/heads/{branch}", + sha, + }); + HttpResponseMessage create = await client.PostAsync($"{RepositoryApiBaseUrl}/git/refs", + new StringContent(body, Encoding.UTF8, "application/json")); + if (!create.IsSuccessStatusCode) + { + string errorContent = await create.Content.ReadAsStringAsync(); + throw new InvalidOperationException( + $"Could not create branch '{branch}' from '{sourceBranch}': {errorContent}"); + } + + Log.Information("Created branch '{Branch}' from '{SourceBranch}' at {Sha}.", branch, sourceBranch, + sha?.Substring(0, 8)); + return true; + } + + /// + /// Reads a file from the given using the GitHub contents API. + /// Returns if the file does not exist on that branch. + /// + public static async Task ReadBranchFileAsync(string path, string branch, string githubToken) + { + using HttpClient client = CreateGithubClient(githubToken); + HttpResponseMessage response = await client.GetAsync( + $"{RepositoryApiBaseUrl}/contents/{path}?ref={Uri.EscapeDataString(branch)}"); + if (!response.IsSuccessStatusCode) + { + return null; + } + + string responseContent = await response.Content.ReadAsStringAsync(); + using JsonDocument document = JsonDocument.Parse(responseContent); + string sha = document.RootElement.GetProperty("sha").GetString(); + string downloadUrl = + $"https://raw.githubusercontent.com/{Owner}/{Repo}/refs/heads/{branch}/{path}"; + using HttpClient rawClient = new(); + rawClient.DefaultRequestHeaders.UserAgent.ParseAdd(Repo); + string content = await rawClient.GetStringAsync(downloadUrl); + return new GithubFile(content, sha); + } + + /// + /// Writes a file to the given using the GitHub contents API. + /// + public static async Task WriteBranchFileAsync(string path, string branch, string commitMessage, string content, + string existingSha, string githubToken) + { + using HttpClient client = CreateGithubClient(githubToken); + GithubUpdateFile body = new(commitMessage, Base64Encode(content), existingSha, branch); + HttpResponseMessage response = await client.PutAsync( + $"{RepositoryApiBaseUrl}/contents/{path}", + new StringContent(JsonSerializer.Serialize(body), Encoding.UTF8, "application/json")); + if (response.IsSuccessStatusCode) + { + Log.Information("Updated {Path} on branch '{Branch}'.", path, branch); + } + else + { + string responseContent = await response.Content.ReadAsStringAsync(); + throw new InvalidOperationException( + $"Could not update '{path}' on branch '{branch}': {responseContent}"); + } + } + + /// + /// Lists successful runs of the given workflow on the given branch (newest first), returning the run id + /// plus the head commit metadata (sha, author, date, message). + /// + public static async Task> ListSuccessfulRunsAsync(string workflowFileName, string branch, + int count, string githubToken) + { + using HttpClient client = CreateGithubClient(githubToken); + string url = $"{RepositoryApiBaseUrl}/actions/workflows/{Uri.EscapeDataString(workflowFileName)}/runs" + + $"?status=success&branch={Uri.EscapeDataString(branch)}&per_page={count}"; + HttpResponseMessage response = await client.GetAsync(url); + string responseContent = await response.Content.ReadAsStringAsync(); + if (!response.IsSuccessStatusCode) + { + Log.Warning( + $"Could not list recent runs for workflow '{workflowFileName}' on branch '{branch}': {responseContent}"); + return new List(); + } + + List result = new(); + using JsonDocument document = JsonDocument.Parse(responseContent); + foreach (JsonElement run in document.RootElement.GetProperty("workflow_runs").EnumerateArray()) + { + long id = run.GetProperty("id").GetInt64(); + string sha = run.GetProperty("head_sha").GetString(); + string headBranch = run.TryGetProperty("head_branch", out JsonElement hb) ? hb.GetString() : null; + string createdAt = run.TryGetProperty("created_at", out JsonElement ca) ? ca.GetString() : null; + string author = null; + string message = null; + string commitDate = createdAt; + if (run.TryGetProperty("head_commit", out JsonElement hc) && hc.ValueKind == JsonValueKind.Object) + { + if (hc.TryGetProperty("message", out JsonElement m)) + { + message = m.GetString()?.Split('\n')[0]?.Trim(); + } + + if (hc.TryGetProperty("timestamp", out JsonElement ts)) + { + commitDate = ts.GetString() ?? createdAt; + } + + if (hc.TryGetProperty("author", out JsonElement aut) && aut.ValueKind == JsonValueKind.Object && + aut.TryGetProperty("name", out JsonElement an)) + { + author = an.GetString(); + } + } + + result.Add(new WorkflowRunInfo(id, sha, headBranch, author, commitDate, message)); + } + + return result; + } + + private static HttpClient CreateGithubClient(string githubToken) + { + HttpClient client = new(); + client.DefaultRequestHeaders.UserAgent.ParseAdd(Repo); + if (!string.IsNullOrEmpty(githubToken)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", githubToken); + } + + return client; + } + + private static string Base64Encode(string plainText) + { + byte[] plainTextBytes = Encoding.UTF8.GetBytes(plainText); + return Convert.ToBase64String(plainTextBytes); + } + + public record GithubFile(string Content, string Sha); + + public record WorkflowRunInfo(long Id, string Sha, string Branch, string Author, string Date, string Message); + + // ReSharper disable InconsistentNaming + // ReSharper disable NotAccessedPositionalProperty.Local + private record GithubUpdateFile(string message, string content, string sha, string branch); + // ReSharper restore NotAccessedPositionalProperty.Local + // ReSharper restore InconsistentNaming } diff --git a/Pipeline/PageBenchmarkReportGenerator.cs b/Pipeline/PageBenchmarkReportGenerator.cs new file mode 100644 index 00000000..033e272a --- /dev/null +++ b/Pipeline/PageBenchmarkReportGenerator.cs @@ -0,0 +1,316 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using Serilog; + +// ReSharper disable CollectionNeverQueried.Local +// ReSharper disable UnusedMember.Local +// ReSharper disable UnusedAutoPropertyAccessor.Local +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Local + +namespace Build; + +/// +/// Aggregates BenchmarkDotNet JSON reports into the data.js / limited-data.js files consumed +/// by the documentation site (). +/// +/// +/// The structure intentionally mirrors aweXpect's PageBenchmarkReportGenerator so the same docs-site +/// renderer can pick up Mockolate's data without per-project tweaks. The differences are: +/// +/// The set of compared libraries (Mockolate plus Moq/NSubstitute/FakeItEasy/TUnitMocks/Imposter). +/// Chart keys include BenchmarkDotNet parameter sets (e.g. Method (N=1)) so that each +/// [Params] combination becomes its own chart. +/// +/// +public class PageBenchmarkReportGenerator +{ + private const string FilePrefix = "window.BENCHMARK_DATA = "; + + internal const string BaselineLibrary = "Mockolate"; + + private static readonly JsonSerializerOptions BenchmarkSerializerOptions = new() + { + WriteIndented = true, + }; + + public static (string FullContent, string LimitedContent) Append(CommitInfo commitInfo, + string currentFileContent, List benchmarkReportsContents, int limit) + { + PageReportData pageReport = ParseOrCreate(currentFileContent); + + if (pageReport.Values.Any(r => r.Commits.Any(c => c.Sha == commitInfo.Sha))) + { + Log.Warning( + "The benchmark already has data for {Sha}: {Message} by {Author} on {Date}", + commitInfo.Sha, commitInfo.Message, commitInfo.Author, commitInfo.Date); + return (null, null); + } + + Log.Debug( + "Updating benchmark report for {Sha}: {Message} by {Author} on {Date}", + commitInfo.Sha, commitInfo.Message, commitInfo.Author, commitInfo.Date); + + foreach (string benchmarkReportContent in benchmarkReportsContents) + { + BenchmarkReport benchmarkReport = JsonSerializer.Deserialize(benchmarkReportContent); + if (!pageReport.Append(commitInfo, benchmarkReport)) + { + throw new NotSupportedException("The new benchmark data is incorrect"); + } + } + + string newFileContent = + $"{FilePrefix}{JsonSerializer.Serialize(pageReport, BenchmarkSerializerOptions)}"; + string limitedFileContent = + $"{FilePrefix}{JsonSerializer.Serialize(pageReport.Limit(limit), BenchmarkSerializerOptions)}"; + return (newFileContent, limitedFileContent); + } + + internal static PageReportData ParseOrCreate(string currentFileContent) + { + if (string.IsNullOrWhiteSpace(currentFileContent)) + { + return new PageReportData(); + } + + if (!currentFileContent.StartsWith(FilePrefix)) + { + throw new NotSupportedException($"The benchmark data file is incorrect (does not start with {FilePrefix})"); + } + + return JsonSerializer.Deserialize(currentFileContent.Substring(FilePrefix.Length)) + ?? new PageReportData(); + } + + internal sealed class PageReportData : Dictionary + { + public bool Append(CommitInfo commitInfo, BenchmarkReport benchmarkReport) + { + HashSet chartsTouchedByBaseline = new(); + foreach (BenchmarkReport.Benchmark benchmark in benchmarkReport.Benchmarks) + { + if (!Append(commitInfo, benchmark, chartsTouchedByBaseline)) + { + return false; + } + } + + return true; + } + + private bool Append(CommitInfo commitInfo, BenchmarkReport.Benchmark benchmark, + HashSet chartsTouchedByBaseline) + { + if (!ParseMethod(benchmark.Method, out string scenario, out string library)) + { + return false; + } + + if (!IsIncluded(library)) + { + return true; + } + + string chartKey = BuildChartKey(scenario, benchmark.Parameters); + if (!TryGetValue(chartKey, out PageReport pageReport)) + { + pageReport = new PageReport(); + this[chartKey] = pageReport; + } + + if (library == BaselineLibrary && chartsTouchedByBaseline.Add(chartKey)) + { + pageReport.Commits.Add(commitInfo); + pageReport.Labels.Add(commitInfo.Sha.Substring(0, 8)); + } + + AppendTimeDataset(benchmark, pageReport, library); + AppendMemoryDataset(benchmark, pageReport, library); + + return true; + } + + private static string BuildChartKey(string scenario, string parameters) + => string.IsNullOrWhiteSpace(parameters) ? scenario : $"{scenario} ({parameters})"; + + private static void AppendMemoryDataset(BenchmarkReport.Benchmark benchmark, PageReport pageReport, + string library) + { + PageReport.Dataset memoryDataset = pageReport.Datasets.FirstOrDefault(x + => x.Label.StartsWith(library, StringComparison.OrdinalIgnoreCase) && x.YAxisId == "y1"); + if (memoryDataset == null) + { + memoryDataset = new PageReport.Dataset + { + Label = $"{library} memory", + Unit = "b", + PointStyle = "triangle", + BorderDash = [5, 5,], + YAxisId = "y1", + BackgroundColor = GetColor(library), + BorderColor = GetColor(library), + Data = new List(), + }; + pageReport.Datasets.Add(memoryDataset); + } + + memoryDataset.Data.Add(benchmark.Metrics + .Where(x => x.Descriptor.Id == "Allocated Memory") + .Select(x => x.Value) + .FirstOrDefault(double.NaN)); + } + + private static void AppendTimeDataset(BenchmarkReport.Benchmark benchmark, PageReport pageReport, string library) + { + PageReport.Dataset timeDataset = pageReport.Datasets.FirstOrDefault(x + => x.Label.StartsWith(library, StringComparison.OrdinalIgnoreCase) && x.YAxisId == "y"); + if (timeDataset == null) + { + timeDataset = new PageReport.Dataset + { + Label = $"{library} time", + Unit = "ns", + PointStyle = "circle", + YAxisId = "y", + BackgroundColor = GetColor(library), + BorderColor = GetColor(library), + Data = new List(), + }; + pageReport.Datasets.Add(timeDataset); + } + + timeDataset.Data.Add(benchmark.Statistics.Mean); + } + + private static bool IsIncluded(string library) + => library is "Mockolate" or "Moq" or "NSubstitute" or "FakeItEasy" or "TUnitMocks" or "Imposter"; + + private static string GetColor(string library) + => library switch + { + "Mockolate" => "#63A2AC", + "Moq" => "#A052B0", + "NSubstitute" => "#5E2750", + "FakeItEasy" => "#4A6FA5", + "TUnitMocks" => "#FF8C00", + "Imposter" => "#E84393", + _ => "#888888", + }; + + private static bool ParseMethod(string method, out string scenario, out string library) + { + int index = method.LastIndexOf('_'); + if (index <= 0) + { + scenario = null; + library = null; + return false; + } + + scenario = method.Substring(0, index); + library = method.Substring(index + 1); + return true; + } + + public PageReportData Limit(int limit) + { + PageReportData pageReportData = new(); + foreach ((string key, PageReport pageReport) in this) + { + pageReportData[key] = pageReport.Limit(limit); + } + + return pageReportData; + } + } + + public class CommitInfo(string sha, string author, string date, string message) + { + [JsonPropertyName("sha")] public string Sha { get; } = sha; + [JsonPropertyName("author")] public string Author { get; } = author; + [JsonPropertyName("date")] public string Date { get; } = date; + [JsonPropertyName("message")] public string Message { get; } = message; + } + + internal sealed class PageReport + { + [JsonPropertyName("commits")] public List Commits { get; init; } = new(); + [JsonPropertyName("labels")] public List Labels { get; init; } = new(); + + [JsonPropertyName("datasets")] public List Datasets { get; init; } = new(); + + public PageReport Limit(int limit) + => new() + { + Commits = Commits.TakeLast(limit).ToList(), + Labels = Labels.TakeLast(limit).ToList(), + Datasets = Datasets.Select(dataset => dataset.Limit(limit)).ToList(), + }; + + public class Dataset + { + [JsonPropertyName("label")] public string Label { get; init; } + [JsonPropertyName("unit")] public string Unit { get; set; } + + [JsonPropertyName("data")] public List Data { get; init; } + + [JsonPropertyName("borderColor")] public string BorderColor { get; set; } + + [JsonPropertyName("backgroundColor")] public string BackgroundColor { get; set; } + + [JsonPropertyName("yAxisID")] public string YAxisId { get; init; } + + [JsonPropertyName("borderDash")] public int[] BorderDash { get; set; } = []; + + [JsonPropertyName("pointStyle")] public string PointStyle { get; set; } + + public Dataset Limit(int limit) + => new() + { + Label = Label, + Unit = Unit, + Data = Data.TakeLast(limit).ToList(), + BorderColor = BorderColor, + BackgroundColor = BackgroundColor, + YAxisId = YAxisId, + BorderDash = BorderDash, + PointStyle = PointStyle, + }; + } + } + + internal sealed class BenchmarkReport + { + public Benchmark[] Benchmarks { get; init; } + + public class Benchmark + { + public string Type { get; init; } + public string Method { get; init; } + public string Parameters { get; init; } + public BenchmarkStatistics Statistics { get; init; } + public BenchmarkMetrics[] Metrics { get; init; } + } + + public class BenchmarkStatistics + { + public double Mean { get; init; } + } + + public class BenchmarkMetrics + { + public double Value { get; init; } + public BenchmarkMetricDescriptor Descriptor { get; init; } + } + + public class BenchmarkMetricDescriptor + { + public string Id { get; init; } + public string DisplayName { get; init; } + public string Unit { get; init; } + } + } +} From c5acd3866d54be350abdee66d4a9e736184e0baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 3 May 2026 14:42:17 +0200 Subject: [PATCH 2/3] refactor: drop bootstrap-only benchmark helpers Remove EnsureBranchExistsAsync, ParseOrCreate, BenchmarkSeedHistory and their supporting types (ListSuccessfulRunsAsync, WorkflowRunInfo, BenchmarkSeedRunLimit parameter). The 'benchmarks' branch and data files have already been seeded, so PublishBenchmarkReport can now assume both exist - matching the aweXpect surface. If the branch or data file ever goes missing, PublishBenchmarkReport will fail loudly (NRE on dataFile.Content / NotSupportedException on the missing window.BENCHMARK_DATA prefix), which is the same behaviour as aweXpect's BenchmarkReport target. --- .nuke/build.schema.json | 6 -- Pipeline/Build.Benchmarks.cs | 106 +---------------------- Pipeline/BuildExtensions.cs | 103 ---------------------- Pipeline/PageBenchmarkReportGenerator.cs | 24 ++--- 4 files changed, 10 insertions(+), 229 deletions(-) diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index 4654f133..42be27ab 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -29,7 +29,6 @@ "BenchmarkDotNet", "BenchmarkResult", "Benchmarks", - "BenchmarkSeedHistory", "CalculateNugetVersion", "Clean", "CodeAnalysis", @@ -124,11 +123,6 @@ "type": "string", "description": "Filter for BenchmarkDotNet - Default is '*'" }, - "BenchmarkSeedRunLimit": { - "type": "integer", - "description": "Maximum number of historic Build runs to seed via 'BenchmarkSeedHistory' - Default is 30", - "format": "int32" - }, "Configuration": { "type": "string", "description": "Configuration to build - Default is 'Debug' (local) or 'Release' (server)", diff --git a/Pipeline/Build.Benchmarks.cs b/Pipeline/Build.Benchmarks.cs index 3b93aefc..f70e688e 100644 --- a/Pipeline/Build.Benchmarks.cs +++ b/Pipeline/Build.Benchmarks.cs @@ -25,9 +25,6 @@ partial class Build [Parameter("Filter for BenchmarkDotNet - Default is '*'")] readonly string BenchmarkFilter = "*"; - [Parameter("Maximum number of historic Build runs to seed via 'BenchmarkSeedHistory' - Default is 30")] - readonly int BenchmarkSeedRunLimit = 30; - Target BenchmarkDotNet => _ => _ .Executes(() => { @@ -158,8 +155,6 @@ partial class Build "Appending benchmark data for commit {Sha} ({Author}, {Date}): {Message}", commitInfo.Sha, commitInfo.Author, commitInfo.Date, commitInfo.Message); - await BuildExtensions.EnsureBranchExistsAsync(BenchmarkBranch, "main", GithubToken); - BuildExtensions.GithubFile dataFile = await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken); BuildExtensions.GithubFile limitedFile = @@ -167,7 +162,7 @@ partial class Build (string updated, string limited) = PageBenchmarkReportGenerator.Append( commitInfo, - dataFile?.Content ?? string.Empty, + dataFile.Content, benchmarkReports, BenchmarkLimit); @@ -180,104 +175,9 @@ partial class Build string commitMessage = $"Update benchmark for {commitInfo.Sha.Substring(0, 8)}: {commitInfo.Message} by {commitInfo.Author}"; await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, updated, - dataFile?.Sha, GithubToken); - await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage, - limited, limitedFile?.Sha, GithubToken); - }); - - Target BenchmarkSeedHistory => _ => _ - .Description("Seeds the benchmark data files on the 'benchmarks' branch with results from past " + - "successful 'Build' workflow runs on main, by downloading their Benchmarks-* artifacts. " + - "One-shot tool; safe to re-run (commits already recorded are skipped).") - .Requires(() => GithubToken) - .Executes(async () => - { - List runs = await BuildExtensions.ListSuccessfulRunsAsync( - "build.yml", "main", BenchmarkSeedRunLimit, GithubToken); - if (runs.Count == 0) - { - Log.Warning("No successful 'Build' runs found on main - nothing to seed."); - return; - } - - runs.Reverse(); - Log.Information("Seeding from {Count} historical runs (oldest first).", runs.Count); - - await BuildExtensions.EnsureBranchExistsAsync(BenchmarkBranch, "main", GithubToken); - - BuildExtensions.GithubFile dataFile = - await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken); - BuildExtensions.GithubFile limitedFile = - await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken); - - string accumulatedContent = dataFile?.Content ?? string.Empty; - string accumulatedLimited = limitedFile?.Content ?? string.Empty; - string lastFullSha = dataFile?.Sha; - string lastLimitedSha = limitedFile?.Sha; - int updatedRuns = 0; - - foreach (BuildExtensions.WorkflowRunInfo run in runs) - { - AbsolutePath runDirectory = ArtifactsDirectory / "SeedHistory" / run.Id.ToString(); - runDirectory.CreateOrCleanDirectory(); - try - { - await BuildExtensions.DownloadArtifactsFromRunStartingWith(run.Id, "Benchmarks-", - runDirectory, GithubToken); - } - catch (Exception ex) - { - Log.Warning(ex, "Skipping run #{RunId} ({Sha}): could not download artifacts.", run.Id, - run.Sha?.Substring(0, 8)); - continue; - } - - List benchmarkReports = LoadBenchmarkJsonReports(runDirectory / "Benchmarks" / "results"); - if (benchmarkReports.Count == 0) - { - Log.Information("Run #{RunId} ({Sha}) had no benchmark JSON reports - skipping.", run.Id, - run.Sha?.Substring(0, 8)); - continue; - } - - PageBenchmarkReportGenerator.CommitInfo commitInfo = new( - run.Sha, - run.Author ?? "unknown", - run.Date ?? string.Empty, - run.Message ?? string.Empty); - - (string updated, string limited) = PageBenchmarkReportGenerator.Append( - commitInfo, - accumulatedContent, - benchmarkReports, - BenchmarkLimit); - - if (string.IsNullOrWhiteSpace(updated)) - { - Log.Information("Run #{RunId} ({Sha}) already recorded - skipping.", run.Id, - run.Sha?.Substring(0, 8)); - continue; - } - - accumulatedContent = updated; - accumulatedLimited = limited; - updatedRuns++; - Log.Information("Seeded run #{RunId} ({Sha}): {Message}", run.Id, - run.Sha?.Substring(0, 8), commitInfo.Message); - } - - if (updatedRuns == 0) - { - Log.Information("No new benchmark data to seed."); - return; - } - - string commitMessage = $"Seed historic benchmark data ({updatedRuns} runs)"; - await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, - accumulatedContent, lastFullSha, GithubToken); + dataFile.Sha, GithubToken); await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage, - accumulatedLimited, lastLimitedSha, GithubToken); - Log.Information("Seeded benchmark data from {Count} historical runs.", updatedRuns); + limited, limitedFile.Sha, GithubToken); }); private static List LoadBenchmarkJsonReports(AbsolutePath resultsDirectory) diff --git a/Pipeline/BuildExtensions.cs b/Pipeline/BuildExtensions.cs index fbdd06e4..7e6f762e 100644 --- a/Pipeline/BuildExtensions.cs +++ b/Pipeline/BuildExtensions.cs @@ -165,52 +165,6 @@ private static async Task DownloadArtifactsFromRun(string runId, Func - /// Ensures that the given branch exists in the repository. If it does not, creates it pointing at the head - /// of . Returns if the branch was newly created. - /// - public static async Task EnsureBranchExistsAsync(string branch, string sourceBranch, string githubToken) - { - using HttpClient client = CreateGithubClient(githubToken); - HttpResponseMessage exists = await client.GetAsync( - $"{RepositoryApiBaseUrl}/git/ref/heads/{Uri.EscapeDataString(branch)}"); - if (exists.IsSuccessStatusCode) - { - return false; - } - - HttpResponseMessage source = await client.GetAsync( - $"{RepositoryApiBaseUrl}/git/ref/heads/{Uri.EscapeDataString(sourceBranch)}"); - if (!source.IsSuccessStatusCode) - { - string errorContent = await source.Content.ReadAsStringAsync(); - throw new InvalidOperationException( - $"Could not read source branch '{sourceBranch}': {errorContent}"); - } - - string sourceContent = await source.Content.ReadAsStringAsync(); - using JsonDocument document = JsonDocument.Parse(sourceContent); - string sha = document.RootElement.GetProperty("object").GetProperty("sha").GetString(); - - string body = JsonSerializer.Serialize(new - { - @ref = $"refs/heads/{branch}", - sha, - }); - HttpResponseMessage create = await client.PostAsync($"{RepositoryApiBaseUrl}/git/refs", - new StringContent(body, Encoding.UTF8, "application/json")); - if (!create.IsSuccessStatusCode) - { - string errorContent = await create.Content.ReadAsStringAsync(); - throw new InvalidOperationException( - $"Could not create branch '{branch}' from '{sourceBranch}': {errorContent}"); - } - - Log.Information("Created branch '{Branch}' from '{SourceBranch}' at {Sha}.", branch, sourceBranch, - sha?.Substring(0, 8)); - return true; - } - /// /// Reads a file from the given using the GitHub contents API. /// Returns if the file does not exist on that branch. @@ -259,61 +213,6 @@ public static async Task WriteBranchFileAsync(string path, string branch, string } } - /// - /// Lists successful runs of the given workflow on the given branch (newest first), returning the run id - /// plus the head commit metadata (sha, author, date, message). - /// - public static async Task> ListSuccessfulRunsAsync(string workflowFileName, string branch, - int count, string githubToken) - { - using HttpClient client = CreateGithubClient(githubToken); - string url = $"{RepositoryApiBaseUrl}/actions/workflows/{Uri.EscapeDataString(workflowFileName)}/runs" + - $"?status=success&branch={Uri.EscapeDataString(branch)}&per_page={count}"; - HttpResponseMessage response = await client.GetAsync(url); - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) - { - Log.Warning( - $"Could not list recent runs for workflow '{workflowFileName}' on branch '{branch}': {responseContent}"); - return new List(); - } - - List result = new(); - using JsonDocument document = JsonDocument.Parse(responseContent); - foreach (JsonElement run in document.RootElement.GetProperty("workflow_runs").EnumerateArray()) - { - long id = run.GetProperty("id").GetInt64(); - string sha = run.GetProperty("head_sha").GetString(); - string headBranch = run.TryGetProperty("head_branch", out JsonElement hb) ? hb.GetString() : null; - string createdAt = run.TryGetProperty("created_at", out JsonElement ca) ? ca.GetString() : null; - string author = null; - string message = null; - string commitDate = createdAt; - if (run.TryGetProperty("head_commit", out JsonElement hc) && hc.ValueKind == JsonValueKind.Object) - { - if (hc.TryGetProperty("message", out JsonElement m)) - { - message = m.GetString()?.Split('\n')[0]?.Trim(); - } - - if (hc.TryGetProperty("timestamp", out JsonElement ts)) - { - commitDate = ts.GetString() ?? createdAt; - } - - if (hc.TryGetProperty("author", out JsonElement aut) && aut.ValueKind == JsonValueKind.Object && - aut.TryGetProperty("name", out JsonElement an)) - { - author = an.GetString(); - } - } - - result.Add(new WorkflowRunInfo(id, sha, headBranch, author, commitDate, message)); - } - - return result; - } - private static HttpClient CreateGithubClient(string githubToken) { HttpClient client = new(); @@ -334,8 +233,6 @@ private static string Base64Encode(string plainText) public record GithubFile(string Content, string Sha); - public record WorkflowRunInfo(long Id, string Sha, string Branch, string Author, string Date, string Message); - // ReSharper disable InconsistentNaming // ReSharper disable NotAccessedPositionalProperty.Local private record GithubUpdateFile(string message, string content, string sha, string branch); diff --git a/Pipeline/PageBenchmarkReportGenerator.cs b/Pipeline/PageBenchmarkReportGenerator.cs index 033e272a..9b090db6 100644 --- a/Pipeline/PageBenchmarkReportGenerator.cs +++ b/Pipeline/PageBenchmarkReportGenerator.cs @@ -39,7 +39,13 @@ public class PageBenchmarkReportGenerator public static (string FullContent, string LimitedContent) Append(CommitInfo commitInfo, string currentFileContent, List benchmarkReportsContents, int limit) { - PageReportData pageReport = ParseOrCreate(currentFileContent); + if (!currentFileContent.StartsWith(FilePrefix)) + { + throw new NotSupportedException($"The benchmark data file is incorrect (does not start with {FilePrefix})"); + } + + PageReportData pageReport = + JsonSerializer.Deserialize(currentFileContent.Substring(FilePrefix.Length)); if (pageReport.Values.Any(r => r.Commits.Any(c => c.Sha == commitInfo.Sha))) { @@ -69,22 +75,6 @@ public static (string FullContent, string LimitedContent) Append(CommitInfo comm return (newFileContent, limitedFileContent); } - internal static PageReportData ParseOrCreate(string currentFileContent) - { - if (string.IsNullOrWhiteSpace(currentFileContent)) - { - return new PageReportData(); - } - - if (!currentFileContent.StartsWith(FilePrefix)) - { - throw new NotSupportedException($"The benchmark data file is incorrect (does not start with {FilePrefix})"); - } - - return JsonSerializer.Deserialize(currentFileContent.Substring(FilePrefix.Length)) - ?? new PageReportData(); - } - internal sealed class PageReportData : Dictionary { public bool Append(CommitInfo commitInfo, BenchmarkReport benchmarkReport) From 528d109ed19df359009f95ff0da5cdc4951cb6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 3 May 2026 14:44:16 +0200 Subject: [PATCH 3/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba6d3b44..89add602 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,6 +109,7 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: + actions: read contents: write env: DOTNET_NOLOGO: true