diff --git a/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs b/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs index 2a8e963796e..1656e11a1a7 100644 --- a/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs +++ b/tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs @@ -28,5 +28,4 @@ type SurfaceAreaTest() = Assembly.LoadFrom path let baseline = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Compiler.Service.SurfaceArea.{platform}.bsl") - let outFileName = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Compiler.Service.SurfaceArea.{platform}.out") - FSharp.Test.SurfaceArea.verify assembly baseline outFileName + FSharp.Test.SurfaceArea.verify assembly baseline diff --git a/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs b/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs index 30885e7661c..9c7559e90a3 100644 --- a/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs @@ -178,25 +178,7 @@ let ParseFile fileName = |> normalize |> sprintf "%s\n" - let bslPath = $"{fullPath}.bsl" - let actualPath = $"{fullPath}.actual" - - let expected = - if File.Exists bslPath then - File.ReadAllText bslPath |> normalize - else - "No baseline was found" - - if expected = actual then - File.Delete(actualPath) - else - if shouldUpdateBaselines then - File.Delete(actualPath) - File.WriteAllText(bslPath, actual) - else - File.WriteAllText(actualPath, actual) - - Assert.Equal(expected, actual) + checkBaseline actual $"{fullPath}.bsl" // Run type checker to assert that it doesn't fail with the tree produced by the parser CompilerAssert.ParseAndTypeCheck([|"--langversion:preview"|], fileName, contents) |> ignore diff --git a/tests/FSharp.Core.UnitTests/SurfaceArea.fs b/tests/FSharp.Core.UnitTests/SurfaceArea.fs index 869a3bd0ab7..8adf6e5f512 100644 --- a/tests/FSharp.Core.UnitTests/SurfaceArea.fs +++ b/tests/FSharp.Core.UnitTests/SurfaceArea.fs @@ -38,5 +38,4 @@ type SurfaceAreaTest() = #endif let assembly = typeof.Assembly let baseline = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Core.SurfaceArea.{platform}.{flavor}.bsl") - let outFileName = Path.Combine(Path.GetDirectoryName(assembly.Location), $"FSharp.Core.SurfaceArea.{platform}.{flavor}.out") - SurfaceArea.verify assembly baseline outFileName + SurfaceArea.verify assembly baseline diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index 914741559a2..d5ae4a34a27 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -30,13 +30,54 @@ module rec Compiler = let shouldUpdateBaselines = Environment.GetEnvironmentVariable("TEST_UPDATE_BSL") <> null + let private baselineFailureMessage (expectedFile: string) (outFile: string) (diff: string) = + $"""Baseline mismatch for {expectedFile} +to update the baseline: +$ cp {outFile} {expectedFile} +to compare: +$ code --diff {outFile} {expectedFile} +(or set TEST_UPDATE_BSL=1 and re-run to update the baseline automatically) +{diff}""" + + let private baselineOutputFile (expectedFile: string) = + if Path.GetExtension(expectedFile) = ".bsl" then + Path.ChangeExtension(expectedFile, ".out") + else + expectedFile + ".out" + + let checkBaselineWith (compare: string -> string -> string option) (expected: string) (expectedFile: string) = + let outFile = baselineOutputFile expectedFile + let baselineContent = + if FileSystem.FileExistsShim expectedFile then File.ReadAllText expectedFile else "" + let diff = compare baselineContent expected + + match diff with + | None -> + if FileSystem.FileExistsShim outFile then + FileSystem.FileDeleteShim outFile + | Some diff -> + if shouldUpdateBaselines then + if FileSystem.FileExistsShim outFile then + FileSystem.FileDeleteShim outFile + File.WriteAllText(expectedFile, expected) + else + File.WriteAllText(outFile, expected) + + Assert.True(false, baselineFailureMessage expectedFile outFile diff) + + let checkBaseline (expected: string) (expectedFile: string) = + let compare fileContent produced = + let e = normalizeNewlines fileContent + let a = normalizeNewlines produced + if e = a then None else Some $"Expected:\n{e}\nActual:\n{a}" + checkBaselineWith compare expected expectedFile + [] type SourceUtilities () = static member getCurrentMethodName([] memberName: string) = memberName type BaselineFile = { - FilePath: string BslSource: string Content: string option } @@ -251,8 +292,6 @@ module rec Compiler = | Some s -> s | None -> sourceFilePath + sourceBaselineSuffix + ".il.bsl" - let fsOutFilePath = normalizePathSeparator (Path.ChangeExtension(outputDirectoryPath ++ filename, ".err")) - let ilOutFilePath = normalizePathSeparator (Path.ChangeExtension(outputDirectoryPath ++ filename, ".il")) let fsBslSource = readFileOrDefault fsBslFilePath let ilBslSource = readFileOrDefault ilBslFilePath @@ -262,8 +301,8 @@ module rec Compiler = Some { SourceFilename = Some sourceFilePath - FSBaseline = { FilePath = fsOutFilePath; BslSource = fsBslFilePath; Content = fsBslSource } - ILBaseline = { FilePath = ilOutFilePath; BslSource = ilBslFilePath; Content = ilBslSource } + FSBaseline = { BslSource = fsBslFilePath; Content = fsBslSource } + ILBaseline = { BslSource = ilBslFilePath; Content = ilBslSource } } Options = Compiler.defaultOptions OutputType = Library @@ -1220,36 +1259,6 @@ module rec Compiler = | _ -> failwith "FSI running only supports F#." - let convenienceBaselineInstructions baseline expected actual = - $"""to update baseline: -$ cp {baseline.FilePath} {baseline.BslSource} -to compare baseline: -$ code --diff {baseline.FilePath} {baseline.BslSource} -Expected: -{expected} -Actual: -{actual}""" - let updateBaseLineIfEnvironmentSaysSo baseline = - if shouldUpdateBaselines then - if FileSystem.FileExistsShim baseline.FilePath then - FileSystem.CopyShim(baseline.FilePath, baseline.BslSource, true) - - let assertBaseline expected actual baseline fOnFail = - if expected <> actual then - fOnFail() - updateBaseLineIfEnvironmentSaysSo baseline - createBaselineErrors baseline actual - Assert.True((expected = actual), convenienceBaselineInstructions baseline expected actual) - elif FileSystem.FileExistsShim baseline.FilePath then - FileSystem.FileDeleteShim baseline.FilePath - - - let private createBaselineErrors (baselineFile: BaselineFile) (actualErrors: string) : unit = - printfn $"creating baseline error file for convenience: {baselineFile.FilePath}, expected: {baselineFile.BslSource}" - let file = FileSystem.OpenFileForWriteShim(baselineFile.FilePath) - file.SetLength(0) - file.WriteAllText(actualErrors) - /// Turn our ErrorInfo back into a genuine FSharpDiagnostic let private toFSharpDiagnostic (ei: ErrorInfo) : FSharpDiagnostic = @@ -1309,19 +1318,8 @@ Actual: match o.Compilation with | FS fs -> fs | _ -> failwith "verifyBaseline only supports F#" - let expected = - fsSource.Baseline.Value.FSBaseline.Content - |> Option.defaultValue "" - |> normalizeNewlines - // 4) Compare or update - if expected <> formattedActual then - // same update mechanism you already have: - fsSource.CreateOutputDirectory() - createBaselineErrors fsSource.Baseline.Value.FSBaseline formattedActual - updateBaseLineIfEnvironmentSaysSo fsSource.Baseline.Value.FSBaseline - let msg = convenienceBaselineInstructions fsSource.Baseline.Value.FSBaseline expected formattedActual - Assert.True(false, msg) + checkBaseline formattedActual fsSource.Baseline.Value.FSBaseline.BslSource // 5) Return the original result for fluent chaining cResult @@ -1387,14 +1385,8 @@ Actual: | None -> String.Empty let success, errorMsg, actualIL = ILChecker.verifyILAndReturnActual [] p [expectedIL] - if not success then - // Failed try update baselines if required - // If we are here then the il file has been produced we can write it back to the baseline location - // if the environment variable TEST_UPDATE_BSL has been set - updateBaseLineIfEnvironmentSaysSo baseline.ILBaseline - createBaselineErrors baseline.ILBaseline actualIL - let errorMsg = (convenienceBaselineInstructions baseline.ILBaseline expectedIL actualIL) + errorMsg - Assert.Fail(errorMsg) + let compare _ _ = if success then None else Some errorMsg + checkBaselineWith compare actualIL baseline.ILBaseline.BslSource let verifyILBaseline (compilationResult: CompilationResult) : CompilationResult = match compilationResult with @@ -2011,22 +2003,9 @@ Actual: |> String.Concat let withResultsMatchingFile (path:string) (result:CompilationResult) = - let expectedContent = File.ReadAllText(path) |> normalizeNewLines - let actualErrors = renderToString result - - match Assert.shouldBeSameMultilineStringSets expectedContent actualErrors with - | None -> () - | Some diff -> - if shouldUpdateBaselines then - File.WriteAllText(path, actualErrors) - - printfn $"{Path.GetFullPath path} \n {diff}" - printfn "==========================EXPECTED===========================" - printfn "%s" expectedContent - printfn "===========================ACTUAL============================" - printfn "%s" actualErrors - Assert.True(String.IsNullOrEmpty(diff), path) - + let compare fileContent produced = + Assert.shouldBeSameMultilineStringSets (normalizeNewLines fileContent) produced + checkBaselineWith compare (renderToString result) path result let checkCodes (expected: int list) (selector: CompilationOutput -> ErrorInfo list) (result: CompilationResult) : CompilationResult = diff --git a/tests/FSharp.Test.Utilities/SurfaceArea.fs b/tests/FSharp.Test.Utilities/SurfaceArea.fs index 99b7838507b..7e60103175b 100644 --- a/tests/FSharp.Test.Utilities/SurfaceArea.fs +++ b/tests/FSharp.Test.Utilities/SurfaceArea.fs @@ -42,42 +42,26 @@ module FSharp.Test.SurfaceArea |] assembly, actual - let private appendNewLine str = str + System.Environment.NewLine - // verify public surface area matches expected, handles baseline update when TEST_UPDATE_BSL is set - let verify assembly baselinePath outFilePath : unit = - let expected = - File.ReadAllLines(baselinePath) - |> String.concat System.Environment.NewLine - |> appendNewLine - + let verify assembly baselinePath : unit = let normalize (s:string) = Regex.Replace(s, "(\\r\\n|\\n|\\r)+", Environment.NewLine).Trim() let asm, actualNotNormalized = getSurfaceAreaForAssembly (assembly) - let actual = - actualNotNormalized - |> Seq.map normalize + let actual = + actualNotNormalized + |> Seq.map normalize |> Seq.filter (String.IsNullOrWhiteSpace >> not) |> Seq.sort |> String.concat Environment.NewLine - let expected = normalize expected - - match Assert.shouldBeSameMultilineStringSets expected actual with - | None -> - File.Delete(outFilePath) - - | Some diff -> - if shouldUpdateBaselines then - File.Delete(outFilePath) - File.WriteAllText(baselinePath, actual) - else - File.WriteAllText(outFilePath, actual) - - let msg = $"""Assembly: %A{asm} + let compare fileContent produced = + match Assert.shouldBeSameMultilineStringSets (normalize fileContent) produced with + | None -> None + | Some diff -> + Some $"""Assembly: %A{asm} Expected and actual surface area don't match. To see the delta, run: - windiff {baselinePath} {outFilePath} + windiff {baselinePath} {Path.ChangeExtension(baselinePath, ".out")} {diff}""" - failwith msg + checkBaselineWith compare actual baselinePath diff --git a/tests/service/data/SyntaxTree/.gitignore b/tests/service/data/SyntaxTree/.gitignore deleted file mode 100644 index 5842cb6df9b..00000000000 --- a/tests/service/data/SyntaxTree/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.actual