From 30634e4a99b5ec3fd28a0c1724702bc26203f82f Mon Sep 17 00:00:00 2001 From: Eugene Auduchinok Date: Wed, 3 Jun 2026 17:05:01 +0200 Subject: [PATCH] Unify baseline checking behind a single checkBaseline helper Introduce checkBaseline/checkBaselineWith in the Compiler test module and route the syntax-tree tests, surface-area verification, and the .err.bsl / .il.bsl / .vsbsl component-test baselines through it, replacing each site's own shouldUpdateBaselines check and ad-hoc file writes. The helper reads the baseline file, compares via a pluggable comparer (default exact equality; surface area and error baselines pass a set-based comparer, IL passes an ILChecker-based one), and on mismatch writes the produced content to an output file or updates the baseline when TEST_UPDATE_BSL is set. The output file extension is replaced for .bsl (foo.bsl -> foo.out) and appended otherwise (neg78.vsbsl -> neg78.vsbsl.out) so the .bsl/.vsbsl baselines a single test can have no longer collide. Removes the now-redundant updateBaseLineIfEnvironmentSaysSo, createBaselineErrors, convenienceBaselineInstructions, and assertBaseline, and drops the unused BaselineFile.FilePath field. This also fixes the IL baseline update path, which previously copied the previous run's IL to the baseline because the copy ran before the fresh actual was written. The SyntaxTree-local .gitignore (*.actual) is removed since those tests now write .out, covered by the global *.out rule. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../SurfaceArea.fs | 3 +- .../SyntaxTreeTests.fs | 20 +-- tests/FSharp.Core.UnitTests/SurfaceArea.fs | 3 +- tests/FSharp.Test.Utilities/Compiler.fs | 121 ++++++++---------- tests/FSharp.Test.Utilities/SurfaceArea.fs | 38 ++---- tests/service/data/SyntaxTree/.gitignore | 1 - 6 files changed, 64 insertions(+), 122 deletions(-) delete mode 100644 tests/service/data/SyntaxTree/.gitignore diff --git a/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs b/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs index 2a8e963796e..1656e11a1a7 100644 --- a/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs +++ b/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs @@ -28,5 +28,4 @@ type SurfaceAreaTest() = Assembly.LoadFrom path let baseline = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Compiler.Service.SurfaceArea.{platform}.bsl") - let outFileName = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Compiler.Service.SurfaceArea.{platform}.out") - FSharp.Test.SurfaceArea.verify assembly baseline outFileName + FSharp.Test.SurfaceArea.verify assembly baseline diff --git a/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs b/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs index 30885e7661c..9c7559e90a3 100644 --- a/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs @@ -178,25 +178,7 @@ let ParseFile fileName = |> normalize |> sprintf "%s\n" - let bslPath = $"{fullPath}.bsl" - let actualPath = $"{fullPath}.actual" - - let expected = - if File.Exists bslPath then - File.ReadAllText bslPath |> normalize - else - "No baseline was found" - - if expected = actual then - File.Delete(actualPath) - else - if shouldUpdateBaselines then - File.Delete(actualPath) - File.WriteAllText(bslPath, actual) - else - File.WriteAllText(actualPath, actual) - - Assert.Equal(expected, actual) + checkBaseline actual $"{fullPath}.bsl" // Run type checker to assert that it doesn't fail with the tree produced by the parser CompilerAssert.ParseAndTypeCheck([|"--langversion:preview"|], fileName, contents) |> ignore diff --git a/tests/FSharp.Core.UnitTests/SurfaceArea.fs b/tests/FSharp.Core.UnitTests/SurfaceArea.fs index 869a3bd0ab7..8adf6e5f512 100644 --- a/tests/FSharp.Core.UnitTests/SurfaceArea.fs +++ b/tests/FSharp.Core.UnitTests/SurfaceArea.fs @@ -38,5 +38,4 @@ type SurfaceAreaTest() = #endif let assembly = typeof.Assembly let baseline = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Core.SurfaceArea.{platform}.{flavor}.bsl") - let outFileName = Path.Combine(Path.GetDirectoryName(assembly.Location), $"FSharp.Core.SurfaceArea.{platform}.{flavor}.out") - SurfaceArea.verify assembly baseline outFileName + SurfaceArea.verify assembly baseline diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index 914741559a2..d5ae4a34a27 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -30,13 +30,54 @@ module rec Compiler = let shouldUpdateBaselines = Environment.GetEnvironmentVariable("TEST_UPDATE_BSL") <> null + let private baselineFailureMessage (expectedFile: string) (outFile: string) (diff: string) = + $"""Baseline mismatch for {expectedFile} +to update the baseline: +$ cp {outFile} {expectedFile} +to compare: +$ code --diff {outFile} {expectedFile} +(or set TEST_UPDATE_BSL=1 and re-run to update the baseline automatically) +{diff}""" + + let private baselineOutputFile (expectedFile: string) = + if Path.GetExtension(expectedFile) = ".bsl" then + Path.ChangeExtension(expectedFile, ".out") + else + expectedFile + ".out" + + let checkBaselineWith (compare: string -> string -> string option) (expected: string) (expectedFile: string) = + let outFile = baselineOutputFile expectedFile + let baselineContent = + if FileSystem.FileExistsShim expectedFile then File.ReadAllText expectedFile else "" + let diff = compare baselineContent expected + + match diff with + | None -> + if FileSystem.FileExistsShim outFile then + FileSystem.FileDeleteShim outFile + | Some diff -> + if shouldUpdateBaselines then + if FileSystem.FileExistsShim outFile then + FileSystem.FileDeleteShim outFile + File.WriteAllText(expectedFile, expected) + else + File.WriteAllText(outFile, expected) + + Assert.True(false, baselineFailureMessage expectedFile outFile diff) + + let checkBaseline (expected: string) (expectedFile: string) = + let compare fileContent produced = + let e = normalizeNewlines fileContent + let a = normalizeNewlines produced + if e = a then None else Some $"Expected:\n{e}\nActual:\n{a}" + checkBaselineWith compare expected expectedFile + [] type SourceUtilities () = static member getCurrentMethodName([] memberName: string) = memberName type BaselineFile = { - FilePath: string BslSource: string Content: string option } @@ -251,8 +292,6 @@ module rec Compiler = | Some s -> s | None -> sourceFilePath + sourceBaselineSuffix + ".il.bsl" - let fsOutFilePath = normalizePathSeparator (Path.ChangeExtension(outputDirectoryPath ++ filename, ".err")) - let ilOutFilePath = normalizePathSeparator (Path.ChangeExtension(outputDirectoryPath ++ filename, ".il")) let fsBslSource = readFileOrDefault fsBslFilePath let ilBslSource = readFileOrDefault ilBslFilePath @@ -262,8 +301,8 @@ module rec Compiler = Some { SourceFilename = Some sourceFilePath - FSBaseline = { FilePath = fsOutFilePath; BslSource = fsBslFilePath; Content = fsBslSource } - ILBaseline = { FilePath = ilOutFilePath; BslSource = ilBslFilePath; Content = ilBslSource } + FSBaseline = { BslSource = fsBslFilePath; Content = fsBslSource } + ILBaseline = { BslSource = ilBslFilePath; Content = ilBslSource } } Options = Compiler.defaultOptions OutputType = Library @@ -1220,36 +1259,6 @@ module rec Compiler = | _ -> failwith "FSI running only supports F#." - let convenienceBaselineInstructions baseline expected actual = - $"""to update baseline: -$ cp {baseline.FilePath} {baseline.BslSource} -to compare baseline: -$ code --diff {baseline.FilePath} {baseline.BslSource} -Expected: -{expected} -Actual: -{actual}""" - let updateBaseLineIfEnvironmentSaysSo baseline = - if shouldUpdateBaselines then - if FileSystem.FileExistsShim baseline.FilePath then - FileSystem.CopyShim(baseline.FilePath, baseline.BslSource, true) - - let assertBaseline expected actual baseline fOnFail = - if expected <> actual then - fOnFail() - updateBaseLineIfEnvironmentSaysSo baseline - createBaselineErrors baseline actual - Assert.True((expected = actual), convenienceBaselineInstructions baseline expected actual) - elif FileSystem.FileExistsShim baseline.FilePath then - FileSystem.FileDeleteShim baseline.FilePath - - - let private createBaselineErrors (baselineFile: BaselineFile) (actualErrors: string) : unit = - printfn $"creating baseline error file for convenience: {baselineFile.FilePath}, expected: {baselineFile.BslSource}" - let file = FileSystem.OpenFileForWriteShim(baselineFile.FilePath) - file.SetLength(0) - file.WriteAllText(actualErrors) - /// Turn our ErrorInfo back into a genuine FSharpDiagnostic let private toFSharpDiagnostic (ei: ErrorInfo) : FSharpDiagnostic = @@ -1309,19 +1318,8 @@ Actual: match o.Compilation with | FS fs -> fs | _ -> failwith "verifyBaseline only supports F#" - let expected = - fsSource.Baseline.Value.FSBaseline.Content - |> Option.defaultValue "" - |> normalizeNewlines - // 4) Compare or update - if expected <> formattedActual then - // same update mechanism you already have: - fsSource.CreateOutputDirectory() - createBaselineErrors fsSource.Baseline.Value.FSBaseline formattedActual - updateBaseLineIfEnvironmentSaysSo fsSource.Baseline.Value.FSBaseline - let msg = convenienceBaselineInstructions fsSource.Baseline.Value.FSBaseline expected formattedActual - Assert.True(false, msg) + checkBaseline formattedActual fsSource.Baseline.Value.FSBaseline.BslSource // 5) Return the original result for fluent chaining cResult @@ -1387,14 +1385,8 @@ Actual: | None -> String.Empty let success, errorMsg, actualIL = ILChecker.verifyILAndReturnActual [] p [expectedIL] - if not success then - // Failed try update baselines if required - // If we are here then the il file has been produced we can write it back to the baseline location - // if the environment variable TEST_UPDATE_BSL has been set - updateBaseLineIfEnvironmentSaysSo baseline.ILBaseline - createBaselineErrors baseline.ILBaseline actualIL - let errorMsg = (convenienceBaselineInstructions baseline.ILBaseline expectedIL actualIL) + errorMsg - Assert.Fail(errorMsg) + let compare _ _ = if success then None else Some errorMsg + checkBaselineWith compare actualIL baseline.ILBaseline.BslSource let verifyILBaseline (compilationResult: CompilationResult) : CompilationResult = match compilationResult with @@ -2011,22 +2003,9 @@ Actual: |> String.Concat let withResultsMatchingFile (path:string) (result:CompilationResult) = - let expectedContent = File.ReadAllText(path) |> normalizeNewLines - let actualErrors = renderToString result - - match Assert.shouldBeSameMultilineStringSets expectedContent actualErrors with - | None -> () - | Some diff -> - if shouldUpdateBaselines then - File.WriteAllText(path, actualErrors) - - printfn $"{Path.GetFullPath path} \n {diff}" - printfn "==========================EXPECTED===========================" - printfn "%s" expectedContent - printfn "===========================ACTUAL============================" - printfn "%s" actualErrors - Assert.True(String.IsNullOrEmpty(diff), path) - + let compare fileContent produced = + Assert.shouldBeSameMultilineStringSets (normalizeNewLines fileContent) produced + checkBaselineWith compare (renderToString result) path result let checkCodes (expected: int list) (selector: CompilationOutput -> ErrorInfo list) (result: CompilationResult) : CompilationResult = diff --git a/tests/FSharp.Test.Utilities/SurfaceArea.fs b/tests/FSharp.Test.Utilities/SurfaceArea.fs index 99b7838507b..7e60103175b 100644 --- a/tests/FSharp.Test.Utilities/SurfaceArea.fs +++ b/tests/FSharp.Test.Utilities/SurfaceArea.fs @@ -42,42 +42,26 @@ module FSharp.Test.SurfaceArea |] assembly, actual - let private appendNewLine str = str + System.Environment.NewLine - // verify public surface area matches expected, handles baseline update when TEST_UPDATE_BSL is set - let verify assembly baselinePath outFilePath : unit = - let expected = - File.ReadAllLines(baselinePath) - |> String.concat System.Environment.NewLine - |> appendNewLine - + let verify assembly baselinePath : unit = let normalize (s:string) = Regex.Replace(s, "(\\r\\n|\\n|\\r)+", Environment.NewLine).Trim() let asm, actualNotNormalized = getSurfaceAreaForAssembly (assembly) - let actual = - actualNotNormalized - |> Seq.map normalize + let actual = + actualNotNormalized + |> Seq.map normalize |> Seq.filter (String.IsNullOrWhiteSpace >> not) |> Seq.sort |> String.concat Environment.NewLine - let expected = normalize expected - - match Assert.shouldBeSameMultilineStringSets expected actual with - | None -> - File.Delete(outFilePath) - - | Some diff -> - if shouldUpdateBaselines then - File.Delete(outFilePath) - File.WriteAllText(baselinePath, actual) - else - File.WriteAllText(outFilePath, actual) - - let msg = $"""Assembly: %A{asm} + let compare fileContent produced = + match Assert.shouldBeSameMultilineStringSets (normalize fileContent) produced with + | None -> None + | Some diff -> + Some $"""Assembly: %A{asm} Expected and actual surface area don't match. To see the delta, run: - windiff {baselinePath} {outFilePath} + windiff {baselinePath} {Path.ChangeExtension(baselinePath, ".out")} {diff}""" - failwith msg + checkBaselineWith compare actual baselinePath diff --git a/tests/service/data/SyntaxTree/.gitignore b/tests/service/data/SyntaxTree/.gitignore deleted file mode 100644 index 5842cb6df9b..00000000000 --- a/tests/service/data/SyntaxTree/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.actual