From 7e8f98e141a13d3802f2b56d1ae3e2d494d2c0b3 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Wed, 27 Aug 2025 23:54:00 +0000 Subject: [PATCH 1/2] Unify ResultChecker logic Ports some changes to the ResultChecker logic from ILLink over to the ILC test infra. Includes https://github.com/dotnet/runtime/commit/80414e96ef5b5e110c52cd92265fb3a348e56a8b, and some other small fixes. --- .../TestCasesRunner/ResultChecker.cs | 77 +++++++++++-------- .../TestCasesRunner/TrimmingTestLogger.cs | 5 +- 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs b/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs index 673df2e9a11500..e2ca44d04ce042 100644 --- a/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs +++ b/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; @@ -198,8 +199,11 @@ protected virtual void InitialChecking(TrimmedTestCaseResult testResult, Assembl private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogger logger, bool checkRemainingErrors) { - List loggedMessages = logger.GetLoggedMessages(); + ImmutableArray allMessages = logger.GetLoggedMessages(); + List unmatchedMessages = [.. allMessages]; List<(ICustomAttributeProvider, CustomAttribute)> expectedNoWarningsAttributes = new(); + List missingMessageWarnings = []; + List unexpectedMessageWarnings = []; foreach (var attrProvider in GetAttributeProviders(original)) { foreach (var attr in attrProvider.CustomAttributes) @@ -216,33 +220,29 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge List matchedMessages; if ((bool)attr.ConstructorArguments[1].Value) - matchedMessages = loggedMessages.Where(m => Regex.IsMatch(m.ToString(), expectedMessage)).ToList(); + matchedMessages = unmatchedMessages.Where(m => Regex.IsMatch(m.ToString(), expectedMessage)).ToList(); else - matchedMessages = loggedMessages.Where(m => MessageTextContains(m.ToString(), expectedMessage)).ToList(); - Assert.True( - matchedMessages.Count > 0, - $"Expected to find logged message matching `{expectedMessage}`, but no such message was found.{Environment.NewLine}Logged messages:{Environment.NewLine}{string.Join(Environment.NewLine, loggedMessages)}"); + matchedMessages = unmatchedMessages.Where(m => MessageTextContains(m.ToString(), expectedMessage)).ToList(); + if (matchedMessages.Count == 0) + missingMessageWarnings.Add($"Expected to find logged message matching `{expectedMessage}`, but no such message was found.{Environment.NewLine}"); foreach (var matchedMessage in matchedMessages) - loggedMessages.Remove(matchedMessage); + unmatchedMessages.Remove(matchedMessage); } break; case nameof(LogDoesNotContainAttribute): { var unexpectedMessage = (string)attr.ConstructorArguments[0].Value; - foreach (var loggedMessage in loggedMessages) + foreach (var loggedMessage in unmatchedMessages) { - var isLogged = () => - { - if ((bool)attr.ConstructorArguments[1].Value) - return !Regex.IsMatch(loggedMessage.ToString(), unexpectedMessage); - return !MessageTextContains(loggedMessage.ToString(), unexpectedMessage); - }; - - Assert.True( - isLogged(), - $"Expected to not find logged message matching `{unexpectedMessage}`, but found:{Environment.NewLine}{loggedMessage}{Environment.NewLine}Logged messages:{Environment.NewLine}{string.Join(Environment.NewLine, loggedMessages)}"); + bool isRegex = (bool)attr.ConstructorArguments[1].Value; + bool foundMatch = isRegex + ? Regex.IsMatch(loggedMessage.ToString(), unexpectedMessage) + : loggedMessage.ToString().Contains(unexpectedMessage); + + if (foundMatch) + unexpectedMessageWarnings.Add($"Expected to not find logged message matching `{unexpectedMessage}`, but found:{Environment.NewLine}{loggedMessage}"); } } break; @@ -281,7 +281,7 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge string? expectedOrigin = null; bool expectedWarningFound = false; - foreach (var loggedMessage in loggedMessages) + foreach (var loggedMessage in unmatchedMessages) { if (loggedMessage.Category != MessageCategory.Warning || loggedMessage.Code != expectedWarningCodeNumber) continue; @@ -350,7 +350,7 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge actualName?.Contains("<" + expectedMember.Name + ">") == true) { expectedWarningFound = true; - loggedMessages.Remove(loggedMessage); + unmatchedMessages.Remove(loggedMessage); break; } if (actualName?.StartsWith(expectedTypeName) == true) @@ -359,14 +359,14 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge (expectedMember is FieldDefinition || expectedMember is PropertyDefinition)) { expectedWarningFound = true; - loggedMessages.Remove(loggedMessage); + unmatchedMessages.Remove(loggedMessage); break; } if (methodDesc.IsConstructor && (expectedMember is FieldDefinition || expectedMember is PropertyDefinition || new AssemblyQualifiedToken(methodDesc.OwningType).Equals(new AssemblyQualifiedToken(expectedMember)))) { expectedWarningFound = true; - loggedMessages.Remove(loggedMessage); + unmatchedMessages.Remove(loggedMessage); break; } } @@ -377,7 +377,7 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge if (NameUtils.GetActualOriginDisplayName(methodDesc) == "Program.
$(String[])") { expectedWarningFound = true; - loggedMessages.Remove(loggedMessage); + unmatchedMessages.Remove(loggedMessage); break; } } @@ -388,14 +388,14 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge if (LogMessageHasSameOriginMember(loggedMessage, attrProvider)) { expectedWarningFound = true; - loggedMessages.Remove(loggedMessage); + unmatchedMessages.Remove(loggedMessage); break; } continue; } expectedWarningFound = true; - loggedMessages.Remove(loggedMessage); + unmatchedMessages.Remove(loggedMessage); break; } @@ -403,11 +403,11 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge ? NameUtils.GetExpectedOriginDisplayName(attrProvider) + ": " : ""; - Assert.True(expectedWarningFound, - $"Expected to find warning: {(fileName != null ? fileName + (sourceLine != null ? $"({sourceLine},{sourceColumn})" : "") + ": " : "")}" + + if (!expectedWarningFound) + missingMessageWarnings.Add($"Expected to find warning: {(fileName != null ? fileName + (sourceLine != null ? $"({sourceLine},{sourceColumn})" : "") + ": " : "")}" + $"warning {expectedWarningCode}: {expectedOriginString}" + $"and message containing {string.Join(" ", expectedMessageContains.Select(m => "'" + m + "'"))}, " + - $"but no such message was found.{Environment.NewLine}Logged messages:{Environment.NewLine}{string.Join(Environment.NewLine, loggedMessages)}"); + $"but no such message was found"); } break; @@ -431,7 +431,7 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge int? unexpectedWarningCodeNumber = unexpectedWarningCode == null ? null : int.Parse(unexpectedWarningCode.Substring(2)); MessageContainer? unexpectedWarningMessage = null; - foreach (var mc in logger.GetLoggedMessages()) + foreach (var mc in unmatchedMessages) { if (mc.Category != MessageCategory.Warning) continue; @@ -443,17 +443,30 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge if (attrProvider is IMemberDefinition attrMember && (mc.Origin?.MemberDefinition is TypeSystemEntity member) && member.ToString()?.Contains(attrMember.FullName) != true) continue; - unexpectedWarningMessage = mc; - break; + unexpectedMessageWarnings.Add($"Unexpected warning found: {mc}"); } Assert.False(unexpectedWarningMessage.HasValue, $"Unexpected warning found: {unexpectedWarningMessage}"); } + if (missingMessageWarnings.Any()) + { + missingMessageWarnings.Add("Unmatched Messages:" + Environment.NewLine); + missingMessageWarnings.AddRange(unmatchedMessages.Select(m => m.ToString())); + missingMessageWarnings.Add(Environment.NewLine + "All Messages:" + Environment.NewLine); + missingMessageWarnings.AddRange(allMessages.Select(m => m.ToString())); + Assert.Fail(string.Join(Environment.NewLine, missingMessageWarnings)); + } + + if (unexpectedMessageWarnings.Any()) + { + Assert.Fail(string.Join(Environment.NewLine, unexpectedMessageWarnings)); + } + if (checkRemainingErrors) { - var remainingErrors = loggedMessages.Where(m => Regex.IsMatch(m.ToString(), @".*(error | warning): \d{4}.*")); + var remainingErrors = unmatchedMessages.Where(m => Regex.IsMatch(m.ToString(), @".*(error | warning): \d{4}.*")); Assert.False(remainingErrors.Any(), $"Found unexpected errors:{Environment.NewLine}{string.Join(Environment.NewLine, remainingErrors)}"); } diff --git a/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/TrimmingTestLogger.cs b/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/TrimmingTestLogger.cs index 3bc19c3eef01a2..da5f223229c1d1 100644 --- a/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/TrimmingTestLogger.cs +++ b/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/TrimmingTestLogger.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using ILCompiler; using ILCompiler.Logging; @@ -24,9 +25,9 @@ public TrimmingTestLogger() public TextWriter Writer => _infoWriter; - public List GetLoggedMessages() + public ImmutableArray GetLoggedMessages() { - return _messageContainers; + return _messageContainers.ToImmutableArray(); } public void WriteError(MessageContainer error) From 5bd354f68697d1c31a289befbdb0889a6ece5e24 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 28 Aug 2025 00:01:39 +0000 Subject: [PATCH 2/2] Clean up --- .../TestCasesRunner/ResultChecker.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs b/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs index e2ca44d04ce042..5b93e9eff94571 100644 --- a/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs +++ b/src/coreclr/tools/aot/ILCompiler.Trimming.Tests/TestCasesRunner/ResultChecker.cs @@ -430,7 +430,6 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge int? unexpectedWarningCodeNumber = unexpectedWarningCode == null ? null : int.Parse(unexpectedWarningCode.Substring(2)); - MessageContainer? unexpectedWarningMessage = null; foreach (var mc in unmatchedMessages) { if (mc.Category != MessageCategory.Warning) @@ -445,9 +444,6 @@ private void VerifyLoggedMessages(AssemblyDefinition original, TrimmingTestLogge unexpectedMessageWarnings.Add($"Unexpected warning found: {mc}"); } - - Assert.False(unexpectedWarningMessage.HasValue, - $"Unexpected warning found: {unexpectedWarningMessage}"); } if (missingMessageWarnings.Any())