From d0f0743d7061722330d392f75e66ba5091a197d4 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 3 Apr 2020 17:41:39 -0400 Subject: [PATCH 1/6] [Harness] Move to create a more general TestTasks class to be reused. Some of the Jenkins test tasks are very useful and do A LOT of stuff. So we try to generalize the base class, to later be able to share the most usebul ones in the shared lib. --- tests/xharness/Jenkins/Jenkins.cs | 33 +- .../TestTasks/AggregatedRunSimulatorTask.cs | 2 +- .../Jenkins/TestTasks/BuildProjectTask.cs | 1 + .../Jenkins/TestTasks/BuildToolTask.cs | 2 +- .../Jenkins/TestTasks/JenkinsTestTask.cs | 88 +++++ tests/xharness/Jenkins/TestTasks/MakeTask.cs | 2 +- .../Jenkins/TestTasks/RunSimulatorTask.cs | 1 + .../xharness/Jenkins/TestTasks/RunTestTask.cs | 2 +- tests/xharness/TestProject.cs | 4 +- tests/xharness/TestTasks/IAcquiredResource.cs | 7 + .../{Jenkins => TestTasks}/Resource.cs | 7 +- .../{Jenkins => TestTasks}/Resources.cs | 2 +- .../{Jenkins => }/TestTasks/TestTask.cs | 370 +++++++----------- tests/xharness/xharness.csproj | 9 +- 14 files changed, 278 insertions(+), 252 deletions(-) create mode 100644 tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs create mode 100644 tests/xharness/TestTasks/IAcquiredResource.cs rename tests/xharness/{Jenkins => TestTasks}/Resource.cs (96%) rename tests/xharness/{Jenkins => TestTasks}/Resources.cs (97%) rename tests/xharness/{Jenkins => }/TestTasks/TestTask.cs (58%) diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 9c39923aae28..811ccea5dce9 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -13,6 +13,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Collections; using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; +using Xharness.TestTasks; namespace Xharness.Jenkins { public class Jenkins @@ -77,7 +78,7 @@ public ILogs Logs { } } - List Tasks = new List (); + List Tasks = new List (); Dictionary DependencyTasks = new Dictionary (); internal static Resource DesktopResource = new Resource ("Desktop", Environment.ProcessorCount); @@ -503,7 +504,7 @@ IEnumerable CreateTestVariations (IEnumerable tests, Func> CreateRunSimulatorTasksAsync () + async Task> CreateRunSimulatorTasksAsync () { var runSimulatorTasks = new List (); @@ -569,7 +570,7 @@ async Task> CreateRunSimulatorTasksAsync () return rv; } - Task> CreateRunDeviceTasksAsync () + Task> CreateRunDeviceTasksAsync () { var rv = new List (); var projectTasks = new List (); @@ -663,7 +664,7 @@ Task> CreateRunDeviceTasksAsync () rv.AddRange (projectTasks); } - return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast () ?? test.Candidates))); + return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast () ?? test.Candidates))); } static string AddSuffixToPath (string path, string suffix) @@ -1372,7 +1373,7 @@ Task RunTestServer () try { var allTasks = Tasks.SelectMany ((v) => { - var rv = new List (); + var rv = new List (); var runsim = v as AggregatedRunSimulatorTask; if (runsim != null) rv.AddRange (runsim.Tasks); @@ -1380,9 +1381,9 @@ Task RunTestServer () return rv; }); - IEnumerable find_tasks (StreamWriter writer, string ids) + IEnumerable find_tasks (StreamWriter writer, string ids) { - IEnumerable tasks; + IEnumerable tasks; switch (request.Url.Query) { case "?all": tasks = Tasks; @@ -1395,10 +1396,10 @@ IEnumerable find_tasks (StreamWriter writer, string ids) break; case "?": writer.WriteLine ("No tasks specified"); - return Array.Empty (); + return Array.Empty (); default: var id_inputs = ids.Substring (1).Split (','); - var rv = new List (id_inputs.Length); + var rv = new List (id_inputs.Length); foreach (var id_input in id_inputs) { if (int.TryParse (id_input, out var id)) { var task = Tasks.FirstOrDefault ((t) => t.ID == id); @@ -1678,7 +1679,7 @@ IEnumerable find_tasks (StreamWriter writer, string ids) return tcs.Task; } - string GetTestColor (IEnumerable tests) + string GetTestColor (IEnumerable tests) { if (!tests.Any ()) return "black"; @@ -1708,7 +1709,7 @@ string GetTestColor (IEnumerable tests) return "black"; } - string GetTestColor (TestTask test) + string GetTestColor (JenkinsTestTask test) { if (test.NotStarted) { return "black"; @@ -1862,7 +1863,7 @@ void GenerateReportImpl (Stream stream, StreamWriter markdown_summary = null) throw new NotImplementedException (); } - var allTasks = new List (); + var allTasks = new List (); if (!populating) { allTasks.AddRange (allExecuteTasks); allTasks.AddRange (allSimulatorTasks); @@ -2116,7 +2117,7 @@ void GenerateReportImpl (Stream stream, StreamWriter markdown_summary = null) writer.WriteLine ("
"); writer.WriteLine ("
"); - var orderedTasks = allTasks.GroupBy ((TestTask v) => v.TestName); + var orderedTasks = allTasks.GroupBy ((JenkinsTestTask v) => v.TestName); if (IsServerMode) { // In server mode don't take into account anything that can change during a test run @@ -2405,7 +2406,7 @@ void GenerateReportImpl (Stream stream, StreamWriter markdown_summary = null) writer.WriteLine ("
"); if (failedTests.Count () == 0) { foreach (var group in failedTests.GroupBy ((v) => v.TestName)) { - var enumerableGroup = group as IEnumerable; + var enumerableGroup = group as IEnumerable; if (enumerableGroup != null) { writer.WriteLine ("{0} ({1})
", group.Key, string.Join (", ", enumerableGroup.Select ((v) => string.Format ("{1}", GetTestColor (v), string.IsNullOrEmpty (v.Mode) ? v.ExecutionResult.ToString () : v.Mode)).ToArray ()), group.Key.Replace (' ', '-')); continue; @@ -2468,11 +2469,11 @@ static string LinkEncode (string path) return System.Web.HttpUtility.UrlEncode (path).Replace ("%2f", "/").Replace ("+", "%20"); } - string RenderTextStates (IEnumerable tests) + string RenderTextStates (IEnumerable tests) { // Create a collection of all non-ignored tests in the group (unless all tests were ignored). var allIgnored = tests.All ((v) => v.ExecutionResult == TestExecutingResult.Ignored); - IEnumerable relevantGroup; + IEnumerable relevantGroup; if (allIgnored) { relevantGroup = tests; } else { diff --git a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs index 1b42f64d6f04..9a06e382a664 100644 --- a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs @@ -9,7 +9,7 @@ namespace Xharness.Jenkins.TestTasks { // This class groups simulator run tasks according to the // simulator they'll run from, so that we minimize switching // between different simulators (which is slow). - class AggregatedRunSimulatorTask : TestTask + class AggregatedRunSimulatorTask : JenkinsTestTask { public IEnumerable Tasks; diff --git a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs index 10ae5470980d..7a99b315b35b 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs @@ -8,6 +8,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Execution; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; +using Xharness.TestTasks; namespace Xharness.Jenkins.TestTasks { abstract class BuildProjectTask : BuildToolTask diff --git a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs index 2211c59764c3..d3181f2f4b83 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs @@ -4,7 +4,7 @@ namespace Xharness.Jenkins.TestTasks { - public abstract class BuildToolTask : TestTask + public abstract class BuildToolTask : JenkinsTestTask { protected readonly IProcessManager ProcessManager; diff --git a/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs b/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs new file mode 100644 index 000000000000..2ec6c1987ad1 --- /dev/null +++ b/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs @@ -0,0 +1,88 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.DotNet.XHarness.iOS.Shared.Logging; +using Xharness.TestTasks; + +namespace Xharness.Jenkins.TestTasks { + public abstract class JenkinsTestTask : Xharness.TestTasks.TestTasks + { + public Jenkins Jenkins; + public Harness Harness { get { return Jenkins.Harness; } } + + + public override string LogDirectory { + get { + var rv = Path.Combine (Jenkins.LogDirectory, TestName, ID.ToString ()); + Directory.CreateDirectory (rv); + return rv; + } + } + + public override void GenerateReport () => Jenkins.GenerateReport (); + + protected override void WriteLineToRunnerLog (string message) => Harness.HarnessLog.WriteLine (message); + + protected override void SetEnvironmentVariables (Process process) + { + var xcodeRoot = Harness.XcodeRoot; + + switch (Platform) { + case TestPlatform.iOS: + case TestPlatform.iOS_Unified: + case TestPlatform.iOS_Unified32: + case TestPlatform.iOS_Unified64: + case TestPlatform.iOS_TodayExtension64: + case TestPlatform.tvOS: + case TestPlatform.watchOS: + case TestPlatform.watchOS_32: + case TestPlatform.watchOS_64_32: + process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; + process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); + process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); + break; + case TestPlatform.Mac: + case TestPlatform.Mac_Modern: + case TestPlatform.Mac_Full: + case TestPlatform.Mac_System: + process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; + process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); + process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); + process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + break; + case TestPlatform.All: + // Don't set: + // MSBuildExtensionsPath + // TargetFrameworkFallbackSearchPaths + // because these values used by both XM and XI and we can't set it to two different values at the same time. + // Any test that depends on these values should not be using 'TestPlatform.All' + process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; + process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + break; + default: + throw new NotImplementedException (); + } + + foreach (var kvp in Environment) + process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value; + } + + + protected override void LogEvent (ILog log, string text, params object [] args) + { + base.LogEvent (log, text, args); + Jenkins.MainLog.WriteLine (text, args); + } + + protected override Task NotifyAndAcquireDesktopResourceAsync () + { + return NotifyBlockingWaitAsync (SupportsParallelExecution ? Jenkins.DesktopResource.AcquireConcurrentAsync () : Jenkins.DesktopResource.AcquireExclusiveAsync ()); + } + + } +} diff --git a/tests/xharness/Jenkins/TestTasks/MakeTask.cs b/tests/xharness/Jenkins/TestTasks/MakeTask.cs index ec563feb3682..26455ca114d2 100644 --- a/tests/xharness/Jenkins/TestTasks/MakeTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MakeTask.cs @@ -39,7 +39,7 @@ protected override async Task ExecuteAsync () } } using (var reader = log.GetReader ()) - AddWrenchLogFiles (reader); + AddCILogFiles (reader); Jenkins.MainLog.WriteLine ("Made {0} ({1})", TestName, Mode); } } diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs index b46252cf9445..3b8737c8649f 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs @@ -9,6 +9,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Listeners; using Microsoft.DotNet.XHarness.iOS.Shared.Collections; using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; +using Xharness.TestTasks; namespace Xharness.Jenkins.TestTasks { class RunSimulatorTask : RunXITask diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index e933470aabfb..37b48d222e75 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -10,7 +10,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; namespace Xharness.Jenkins.TestTasks { - internal abstract class RunTestTask : TestTask + internal abstract class RunTestTask : JenkinsTestTask { protected IProcessManager ProcessManager { get; } IResultParser ResultParser { get; } = new XmlResultParser (); diff --git a/tests/xharness/TestProject.cs b/tests/xharness/TestProject.cs index cb01f1bc366c..6062da0783c2 100644 --- a/tests/xharness/TestProject.cs +++ b/tests/xharness/TestProject.cs @@ -112,14 +112,14 @@ public virtual TestProject Clone () return rv; } - internal async Task CreateCloneAsync (TestTask test) + internal async Task CreateCloneAsync (JenkinsTestTask test) { var rv = Clone (); await rv.CreateCopyAsync (test); return rv; } - internal async Task CreateCopyAsync (TestTask test = null) + internal async Task CreateCopyAsync (JenkinsTestTask test = null) { var directory = DirectoryUtilities.CreateTemporaryDirectory (test?.TestName ?? System.IO.Path.GetFileNameWithoutExtension (Path)); Directory.CreateDirectory (directory); diff --git a/tests/xharness/TestTasks/IAcquiredResource.cs b/tests/xharness/TestTasks/IAcquiredResource.cs new file mode 100644 index 000000000000..4f26038be74d --- /dev/null +++ b/tests/xharness/TestTasks/IAcquiredResource.cs @@ -0,0 +1,7 @@ +using System; +namespace Xharness.TestTasks { + public interface IAcquiredResource : IDisposable + { + Resource Resource { get; } + } +} diff --git a/tests/xharness/Jenkins/Resource.cs b/tests/xharness/TestTasks/Resource.cs similarity index 96% rename from tests/xharness/Jenkins/Resource.cs rename to tests/xharness/TestTasks/Resource.cs index 96ce94394e91..fb667b5393e9 100644 --- a/tests/xharness/Jenkins/Resource.cs +++ b/tests/xharness/TestTasks/Resource.cs @@ -2,7 +2,7 @@ using System.Collections.Concurrent; using System.Threading.Tasks; -namespace Xharness.Jenkins +namespace Xharness.TestTasks { // This is a very simple class to manage the general concept of 'resource'. // Performance isn't important, so this is very simple. @@ -90,9 +90,4 @@ void IDisposable.Dispose () public Resource Resource { get; } } } - - public interface IAcquiredResource : IDisposable - { - Resource Resource { get; } - } } diff --git a/tests/xharness/Jenkins/Resources.cs b/tests/xharness/TestTasks/Resources.cs similarity index 97% rename from tests/xharness/Jenkins/Resources.cs rename to tests/xharness/TestTasks/Resources.cs index c63ee5c04498..6e12a254f029 100644 --- a/tests/xharness/Jenkins/Resources.cs +++ b/tests/xharness/TestTasks/Resources.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Threading.Tasks; -namespace Xharness.Jenkins +namespace Xharness.TestTasks { class Resources { diff --git a/tests/xharness/Jenkins/TestTasks/TestTask.cs b/tests/xharness/TestTasks/TestTask.cs similarity index 58% rename from tests/xharness/Jenkins/TestTasks/TestTask.cs rename to tests/xharness/TestTasks/TestTask.cs index 2836e4ccdab4..6088489f6be0 100644 --- a/tests/xharness/Jenkins/TestTasks/TestTask.cs +++ b/tests/xharness/TestTasks/TestTask.cs @@ -10,70 +10,61 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; -namespace Xharness.Jenkins.TestTasks { - public abstract class TestTask +namespace Xharness.TestTasks { + public abstract class TestTasks { static int counter; - public readonly int ID; + static DriveInfo RootDrive; - bool? supports_parallel_execution; + #region Public vars - public Jenkins Jenkins; - public Harness Harness { get { return Jenkins.Harness; } } - public TestProject TestProject; - public string ProjectFile { get { return TestProject?.Path; } } + public readonly int ID; + public bool BuildOnly; + public string KnownFailure; public string ProjectConfiguration; public string ProjectPlatform; public Dictionary Environment = new Dictionary (); - public Func Dependency; // a task that's feteched and awaited before this task's ExecuteAsync method public Task InitialTask; // a task that's executed before this task's ExecuteAsync method. public Task CompletedTask; // a task that's executed after this task's ExecuteAsync method. + public TestProject TestProject; + public List Resources = new List (); - public bool BuildOnly; - public string KnownFailure; + #endregion - public TestTask () - { - ID = Interlocked.Increment (ref counter); - } + #region Properties - // VerifyRun is called in RunInternalAsync/ExecuteAsync to verify that the task can be executed/run. - // Typically used to fail tasks that don't have an available device, or if there's not enough disk space. - public virtual Task VerifyRunAsync () - { - return VerifyDiskSpaceAsync (); - } + protected static string Timestamp => Helpers.Timestamp; + public string ProjectFile => TestProject?.Path; + public bool HasCustomTestName => test_name != null; + public TestPlatform Platform { get; set; } - static DriveInfo RootDrive; - protected Task VerifyDiskSpaceAsync () - { - if (Finished) - return Task.CompletedTask; + public bool NotStarted => (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted; + public bool InProgress => (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress; + public bool Waiting => (ExecutionResult & TestExecutingResult.Waiting) == TestExecutingResult.Waiting; + public bool Finished => (ExecutionResult & TestExecutingResult.Finished) == TestExecutingResult.Finished; - if (RootDrive == null) - RootDrive = new DriveInfo ("/"); - var afs = RootDrive.AvailableFreeSpace; - const long minSpaceRequirement = 1024 * 1024 * 1024; /* 1 GB */ - if (afs < minSpaceRequirement) { - FailureMessage = $"Not enough space on the root drive '{RootDrive.Name}': {afs / (1024.0 * 1024):#.##} MB left of {minSpaceRequirement / (1024.0 * 1024):#.##} MB required"; - ExecutionResult = TestExecutingResult.Failed; + public bool Building => (ExecutionResult & TestExecutingResult.Building) == TestExecutingResult.Building; + public bool Built => (ExecutionResult & TestExecutingResult.Built) == TestExecutingResult.Built; + public bool Running => (ExecutionResult & TestExecutingResult.Running) == TestExecutingResult.Running; + + public bool BuildSucceeded => (ExecutionResult & TestExecutingResult.BuildSucceeded) == TestExecutingResult.BuildSucceeded; + public bool Succeeded => (ExecutionResult & TestExecutingResult.Succeeded) == TestExecutingResult.Succeeded; + public bool Failed => (ExecutionResult & TestExecutingResult.Failed) == TestExecutingResult.Failed; + public bool Ignored { + get => ExecutionResult == TestExecutingResult.Ignored; + set { + if (ExecutionResult != TestExecutingResult.NotStarted && ExecutionResult != TestExecutingResult.Ignored) + throw new InvalidOperationException (); + ExecutionResult = value ? TestExecutingResult.Ignored : TestExecutingResult.NotStarted; } - return Task.CompletedTask; } + public bool DeviceNotFound => ExecutionResult == TestExecutingResult.DeviceNotFound; - public void CloneTestProject (TestProject project) - { - // Don't build in the original project directory - // We can build multiple projects in parallel, and if some of those - // projects have the same project dependencies, then we may end up - // building the same (dependent) project simultaneously (and they can - // stomp on eachother). - // So we clone the project file to a separate directory and build there instead. - // This is done asynchronously to speed to the initial test load. - TestProject = project.Clone (); - InitialTask = TestProject.CreateCopyAsync (); - } + public bool Crashed => (ExecutionResult & TestExecutingResult.Crashed) == TestExecutingResult.Crashed; + public bool TimedOut => (ExecutionResult & TestExecutingResult.TimedOut) == TestExecutingResult.TimedOut; + public bool BuildFailure => (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure; + public bool HarnessException => (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException; protected Stopwatch duration = new Stopwatch (); public TimeSpan Duration { @@ -82,19 +73,6 @@ public TimeSpan Duration { } } - protected Stopwatch waitingDuration = new Stopwatch (); - public TimeSpan WaitingDuration => waitingDuration.Elapsed; - - TestExecutingResult execution_result; - public virtual TestExecutingResult ExecutionResult { - get { - return execution_result; - } - set { - execution_result = value; - } - } - string failure_message; public string FailureMessage { get { return failure_message; } @@ -104,48 +82,79 @@ public string FailureMessage { } } - public virtual string ProgressMessage { get; } - public bool NotStarted { get { return (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted; } } - public bool InProgress { get { return (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress; } } - public bool Waiting { get { return (ExecutionResult & TestExecutingResult.Waiting) == TestExecutingResult.Waiting; } } - public bool Finished { get { return (ExecutionResult & TestExecutingResult.Finished) == TestExecutingResult.Finished; } } + ILog test_log; + public ILog MainLog { + get { + if (test_log == null) + test_log = Logs.Create ($"main-{Timestamp}.log", "Main log"); + return test_log; + } + } - public bool Building { get { return (ExecutionResult & TestExecutingResult.Building) == TestExecutingResult.Building; } } - public bool Built { get { return (ExecutionResult & TestExecutingResult.Built) == TestExecutingResult.Built; } } - public bool Running { get { return (ExecutionResult & TestExecutingResult.Running) == TestExecutingResult.Running; } } + ILogs logs; + public ILogs Logs { + get { + return logs ?? (logs = new Logs (LogDirectory)); + } + } - public bool BuildSucceeded { get { return (ExecutionResult & TestExecutingResult.BuildSucceeded) == TestExecutingResult.BuildSucceeded; } } - public bool Succeeded { get { return (ExecutionResult & TestExecutingResult.Succeeded) == TestExecutingResult.Succeeded; } } - public bool Failed { get { return (ExecutionResult & TestExecutingResult.Failed) == TestExecutingResult.Failed; } } - public bool Ignored { - get { return ExecutionResult == TestExecutingResult.Ignored; } - set { - if (ExecutionResult != TestExecutingResult.NotStarted && ExecutionResult != TestExecutingResult.Ignored) - throw new InvalidOperationException (); - ExecutionResult = value ? TestExecutingResult.Ignored : TestExecutingResult.NotStarted; + IEnumerable referencedNunitAndXunitTestAssemblies; + public IEnumerable ReferencedNunitAndXunitTestAssemblies { + get { + if (referencedNunitAndXunitTestAssemblies != null) + return referencedNunitAndXunitTestAssemblies; + + if (TestName.Contains ("BCL tests group")) { // avoid loading unrelated projects + if (!File.Exists (ProjectFile)) + return Enumerable.Empty (); + + var csproj = new XmlDocument (); + try { + csproj.LoadWithoutNetworkAccess (ProjectFile.Replace ("\\", "/")); + referencedNunitAndXunitTestAssemblies = csproj.GetNunitAndXunitTestReferences (); + } catch (Exception e) { + referencedNunitAndXunitTestAssemblies = new string [] { $"Exception: {e.Message}", $"Filename: {ProjectFile}" }; + } + } else { + referencedNunitAndXunitTestAssemblies = Enumerable.Empty (); + } + return referencedNunitAndXunitTestAssemblies; } } - public bool DeviceNotFound { get { return ExecutionResult == TestExecutingResult.DeviceNotFound; } } - public bool Crashed { get { return (ExecutionResult & TestExecutingResult.Crashed) == TestExecutingResult.Crashed; } } - public bool TimedOut { get { return (ExecutionResult & TestExecutingResult.TimedOut) == TestExecutingResult.TimedOut; } } - public bool BuildFailure { get { return (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure; } } - public bool HarnessException { get { return (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException; } } + #endregion + + #region Abstract + + public abstract void GenerateReport (); + public abstract string LogDirectory { get; } + protected abstract Task ExecuteAsync (); + protected abstract void SetEnvironmentVariables (Process process); + protected abstract Task NotifyAndAcquireDesktopResourceAsync (); + protected abstract void WriteLineToRunnerLog (string message); + #endregion + + #region Virtual + + public virtual string ProgressMessage { get; } public virtual string Mode { get; set; } public virtual string Variation { get; set; } - protected static string Timestamp { - get { - return Helpers.Timestamp; - } + + bool? supports_parallel_execution; + public virtual bool SupportsParallelExecution { + get => supports_parallel_execution ?? true; + set => supports_parallel_execution = value; } - public bool HasCustomTestName { - get { - return test_name != null; - } + public virtual IEnumerable AggregatedLogs => Logs; + + TestExecutingResult execution_result; + public virtual TestExecutingResult ExecutionResult { + get => execution_result; + set => execution_result = value; } string test_name; @@ -185,64 +194,67 @@ public virtual string TestName { } } - public TestPlatform Platform { get; set; } + protected virtual void PropagateResults () { } - public List Resources = new List (); + protected virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args); - ILog test_log; - public ILog MainLog { - get { - if (test_log == null) - test_log = Logs.Create ($"main-{Timestamp}.log", "Main log"); - return test_log; - } + public virtual void Reset () + { + test_log = null; + failure_message = null; + logs = null; + duration.Reset (); + execution_result = TestExecutingResult.NotStarted; + execute_task = null; } - public virtual IEnumerable AggregatedLogs { - get { - return Logs; - } - } - public string LogDirectory { - get { - var rv = Path.Combine (Jenkins.LogDirectory, TestName, ID.ToString ()); - Directory.CreateDirectory (rv); - return rv; - } - } + #endregion - ILogs logs; - public ILogs Logs { - get { - return logs ?? (logs = new Logs (LogDirectory)); - } + public TestTasks () + { + ID = Interlocked.Increment (ref counter); } - IEnumerable referencedNunitAndXunitTestAssemblies; - public IEnumerable ReferencedNunitAndXunitTestAssemblies { - get { - if (referencedNunitAndXunitTestAssemblies != null) - return referencedNunitAndXunitTestAssemblies; + // VerifyRun is called in RunInternalAsync/ExecuteAsync to verify that the task can be executed/run. + // Typically used to fail tasks that don't have an available device, or if there's not enough disk space. + public virtual Task VerifyRunAsync () + { + return VerifyDiskSpaceAsync (); + } - if (TestName.Contains ("BCL tests group")) { // avoid loading unrelated projects - if (!File.Exists (ProjectFile)) - return Enumerable.Empty (); + protected Task VerifyDiskSpaceAsync () + { + if (Finished) + return Task.CompletedTask; - var csproj = new XmlDocument (); - try { - csproj.LoadWithoutNetworkAccess (ProjectFile.Replace ("\\", "/")); - referencedNunitAndXunitTestAssemblies = csproj.GetNunitAndXunitTestReferences (); - } catch (Exception e) { - referencedNunitAndXunitTestAssemblies = new string [] { $"Exception: {e.Message}", $"Filename: {ProjectFile}" }; - } - } else { - referencedNunitAndXunitTestAssemblies = Enumerable.Empty (); - } - return referencedNunitAndXunitTestAssemblies; + if (RootDrive == null) + RootDrive = new DriveInfo ("/"); + var afs = RootDrive.AvailableFreeSpace; + const long minSpaceRequirement = 1024 * 1024 * 1024; /* 1 GB */ + if (afs < minSpaceRequirement) { + FailureMessage = $"Not enough space on the root drive '{RootDrive.Name}': {afs / (1024.0 * 1024):#.##} MB left of {minSpaceRequirement / (1024.0 * 1024):#.##} MB required"; + ExecutionResult = TestExecutingResult.Failed; } + return Task.CompletedTask; } + public void CloneTestProject (TestProject project) + { + // Don't build in the original project directory + // We can build multiple projects in parallel, and if some of those + // projects have the same project dependencies, then we may end up + // building the same (dependent) project simultaneously (and they can + // stomp on eachother). + // So we clone the project file to a separate directory and build there instead. + // This is done asynchronously to speed to the initial test load. + TestProject = project.Clone (); + InitialTask = TestProject.CreateCopyAsync (); + } + + protected Stopwatch waitingDuration = new Stopwatch (); + public TimeSpan WaitingDuration => waitingDuration.Elapsed; + Task execute_task; async Task RunInternalAsync () { @@ -288,21 +300,7 @@ async Task RunInternalAsync () duration.Stop (); } - Jenkins.GenerateReport (); - } - - protected virtual void PropagateResults () - { - } - - public virtual void Reset () - { - test_log = null; - failure_message = null; - logs = null; - duration.Reset (); - execution_result = TestExecutingResult.NotStarted; - execute_task = null; + GenerateReport (); } public Task RunAsync () @@ -312,62 +310,14 @@ public Task RunAsync () return execute_task; } - protected abstract Task ExecuteAsync (); public override string ToString () { return ExecutionResult.ToString (); } - protected void SetEnvironmentVariables (Process process) - { - var xcodeRoot = Harness.XcodeRoot; - - switch (Platform) { - case TestPlatform.iOS: - case TestPlatform.iOS_Unified: - case TestPlatform.iOS_Unified32: - case TestPlatform.iOS_Unified64: - case TestPlatform.iOS_TodayExtension64: - case TestPlatform.tvOS: - case TestPlatform.watchOS: - case TestPlatform.watchOS_32: - case TestPlatform.watchOS_64_32: - process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; - process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); - process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); - break; - case TestPlatform.Mac: - case TestPlatform.Mac_Modern: - case TestPlatform.Mac_Full: - case TestPlatform.Mac_System: - process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; - process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); - process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); - process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - break; - case TestPlatform.All: - // Don't set: - // MSBuildExtensionsPath - // TargetFrameworkFallbackSearchPaths - // because these values used by both XM and XI and we can't set it to two different values at the same time. - // Any test that depends on these values should not be using 'TestPlatform.All' - process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; - process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - break; - default: - throw new NotImplementedException (); - } - foreach (var kvp in Environment) - process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value; - } - - protected void AddWrenchLogFiles (StreamReader stream) + protected void AddCILogFiles (StreamReader stream) { string line; while ((line = stream.ReadLine ()) != null) { @@ -385,18 +335,12 @@ protected void AddWrenchLogFiles (StreamReader stream) Logs.AddFile (src); break; default: - Harness.HarnessLog.WriteLine ("Unknown @MonkeyWrench command in {0}: {1}", TestName, name); + WriteLineToRunnerLog ($"Unknown @MonkeyWrench command in {TestName}: {name}"); break; } } } - protected void LogEvent (ILog log, string text, params object [] args) - { - Jenkins.MainLog.WriteLine (text, args); - log.WriteLine (text, args); - } - public string GuessFailureReason (ILog log) { try { @@ -410,7 +354,7 @@ public string GuessFailureReason (ILog log) } } } catch (Exception e) { - Harness.Log ("Failed to guess failure reason: {0}", e.Message); + WriteLineToRunnerLog ($"Failed to guess failure reason: {e.Message}"); } return null; @@ -434,20 +378,6 @@ public async Task NotifyBlockingWaitAsync (Task NotifyAndAcquireDesktopResourceAsync () - { - return NotifyBlockingWaitAsync (SupportsParallelExecution ? Jenkins.DesktopResource.AcquireConcurrentAsync () : Jenkins.DesktopResource.AcquireExclusiveAsync ()); - } - class BlockingWait : IAcquiredResource, IDisposable { public IAcquiredResource Wrapped; diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 15c003c92fbc..c34dbd3cfa24 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -81,8 +81,6 @@ - - @@ -98,7 +96,7 @@ - + @@ -127,6 +125,10 @@ + + + + @@ -238,6 +240,7 @@ + \ No newline at end of file From 622ba1eece322f37c74fd91f98c04f0ba6e5607b Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sat, 4 Apr 2020 11:36:24 -0400 Subject: [PATCH 2/6] [Harness] Move BuildToolTask to a more general namespace. Move the task and use composition so that we can reuse the code. This will later allow other projects to use the class without the need of Jenkins or Harness and just implement the base class. --- tests/xharness/AppRunner.cs | 2 +- .../Jenkins/TestTasks/BuildProjectTask.cs | 2 +- .../Jenkins/TestTasks/BuildToolTask.cs | 40 ++++++++++++------- .../Jenkins/TestTasks/MacExecuteTask.cs | 2 +- tests/xharness/Jenkins/TestTasks/MakeTask.cs | 2 +- .../Jenkins/TestTasks/NUnitExecuteTask.cs | 2 +- .../xharness/Jenkins/TestTasks/RunTestTask.cs | 3 +- tests/xharness/TestTasks/BuildToolTask.cs | 36 +++++++++++++++++ tests/xharness/TestTasks/TestTask.cs | 2 +- tests/xharness/xharness.csproj | 1 + 10 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 tests/xharness/TestTasks/BuildToolTask.cs diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 2860a248ccc2..9a683605ab7c 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -441,4 +441,4 @@ await testReporter.CollectSimulatorResult ( return testReporter.Success.Value ? 0 : 1; } } -} \ No newline at end of file +} diff --git a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs index 7a99b315b35b..d0b76654bc38 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; diff --git a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs index d3181f2f4b83..15d7a9fe5d54 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs @@ -1,30 +1,40 @@ -using System; -using System.Threading.Tasks; +using System.Threading.Tasks; using Microsoft.DotNet.XHarness.iOS.Shared.Execution; namespace Xharness.Jenkins.TestTasks { public abstract class BuildToolTask : JenkinsTestTask { - protected readonly IProcessManager ProcessManager; + readonly Xharness.TestTasks.BuildToolTask buildToolTask; - public bool SpecifyPlatform = true; - public bool SpecifyConfiguration = true; + public IProcessManager ProcessManager => buildToolTask.ProcessManager; - protected BuildToolTask (IProcessManager processManager) - { - ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + public bool SpecifyPlatform { + get => buildToolTask.SpecifyPlatform; + set => buildToolTask.SpecifyPlatform = value; } - public override string Mode { - get { return Platform.ToString (); } - set { throw new NotSupportedException (); } + public bool SpecifyConfiguration { + get => buildToolTask.SpecifyConfiguration; + set => buildToolTask.SpecifyConfiguration = value; + } + + protected BuildToolTask (IProcessManager processManager) + => buildToolTask = new Xharness.TestTasks.BuildToolTask (processManager); + + public override TestPlatform Platform { + get => base.Platform; + set { + base.Platform = value; + buildToolTask.Platform = value; + } } - public virtual Task CleanAsync () - { - Console.WriteLine ("Clean is not implemented for {0}", GetType ().Name); - return Task.CompletedTask; + public override string Mode { + get => buildToolTask.Mode; + set => buildToolTask.Mode = value; } + + public virtual Task CleanAsync () => buildToolTask.CleanAsync (); } } diff --git a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs index 7588b0732081..7e513ea2044a 100644 --- a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; diff --git a/tests/xharness/Jenkins/TestTasks/MakeTask.cs b/tests/xharness/Jenkins/TestTasks/MakeTask.cs index 26455ca114d2..b00d7dd1747d 100644 --- a/tests/xharness/Jenkins/TestTasks/MakeTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MakeTask.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.iOS.Shared; diff --git a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs index abc1d3110873..b79d3998c709 100644 --- a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index 37b48d222e75..7ed29ee3c3e2 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -10,8 +10,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; namespace Xharness.Jenkins.TestTasks { - internal abstract class RunTestTask : JenkinsTestTask - { + internal abstract class RunTestTask : JenkinsTestTask { protected IProcessManager ProcessManager { get; } IResultParser ResultParser { get; } = new XmlResultParser (); diff --git a/tests/xharness/TestTasks/BuildToolTask.cs b/tests/xharness/TestTasks/BuildToolTask.cs new file mode 100644 index 000000000000..d0edf6656840 --- /dev/null +++ b/tests/xharness/TestTasks/BuildToolTask.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading.Tasks; +using Microsoft.DotNet.XHarness.iOS.Shared.Execution; + +namespace Xharness.TestTasks { + + public class BuildToolTask + { + public IProcessManager ProcessManager { get; private set; } + public TestPlatform Platform { get; set; } + + public bool SpecifyPlatform { get; set; } = true; + public bool SpecifyConfiguration { get; set; } = true; + + public BuildToolTask (IProcessManager processManager) + { + ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + } + + public BuildToolTask (IProcessManager processManager, TestPlatform platform) : this (processManager) + { + Platform = platform; + } + + public virtual string Mode { + get { return Platform.ToString (); } + set { throw new NotSupportedException (); } + } + + public virtual Task CleanAsync () + { + Console.WriteLine ("Clean is not implemented for {0}", GetType ().Name); + return Task.CompletedTask; + } + } +} diff --git a/tests/xharness/TestTasks/TestTask.cs b/tests/xharness/TestTasks/TestTask.cs index 6088489f6be0..8a2bdf6c0483 100644 --- a/tests/xharness/TestTasks/TestTask.cs +++ b/tests/xharness/TestTasks/TestTask.cs @@ -37,7 +37,6 @@ public abstract class TestTasks protected static string Timestamp => Helpers.Timestamp; public string ProjectFile => TestProject?.Path; public bool HasCustomTestName => test_name != null; - public TestPlatform Platform { get; set; } public bool NotStarted => (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted; public bool InProgress => (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress; @@ -138,6 +137,7 @@ public IEnumerable ReferencedNunitAndXunitTestAssemblies { #region Virtual + public virtual TestPlatform Platform { get; set; } public virtual string ProgressMessage { get; } public virtual string Mode { get; set; } public virtual string Variation { get; set; } diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index c34dbd3cfa24..ca532729be88 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -129,6 +129,7 @@ + From 058baa7e5845179b44c17c989fcb11904a5ccd9a Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sun, 5 Apr 2020 18:54:23 -0400 Subject: [PATCH 3/6] [Harness] Generalize the BuildProjectTask. Use compopsition to generalize the BuildProjectTask to later reuse it in the CLI. --- tests/xharness/Jenkins/Jenkins.cs | 91 +++++-------- .../TestTasks/AggregatedRunSimulatorTask.cs | 2 +- .../Jenkins/TestTasks/BuildProjectTask.cs | 106 ++------------- .../Jenkins/TestTasks/BuildToolTask.cs | 18 ++- .../Jenkins/TestTasks/DotNetBuildTask.cs | 2 +- .../Jenkins/TestTasks/JenkinsTestTask.cs | 12 +- .../xharness/Jenkins/TestTasks/MSBuildTask.cs | 2 +- tests/xharness/Jenkins/TestTasks/MakeTask.cs | 2 +- .../xharness/Jenkins/TestTasks/RunTestTask.cs | 3 +- .../Logging/IEventLogger.cs | 5 + tests/xharness/TestTasks/BuildProjectTask.cs | 121 ++++++++++++++++++ tests/xharness/TestTasks/BuildToolTask.cs | 2 + tests/xharness/TestTasks/IEnvManager.cs | 12 ++ tests/xharness/TestTasks/IResourceManager.cs | 12 ++ tests/xharness/TestTasks/TestTask.cs | 9 +- .../Xharness.Tests/Tests/AppRunnerTests.cs | 2 - tests/xharness/xharness.csproj | 3 + 17 files changed, 237 insertions(+), 167 deletions(-) create mode 100644 tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/Logging/IEventLogger.cs create mode 100644 tests/xharness/TestTasks/BuildProjectTask.cs create mode 100644 tests/xharness/TestTasks/IEnvManager.cs create mode 100644 tests/xharness/TestTasks/IResourceManager.cs diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 811ccea5dce9..21b31aa30070 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -16,7 +16,7 @@ using Xharness.TestTasks; namespace Xharness.Jenkins { - public class Jenkins + public class Jenkins : IResourceManager { readonly ISimulatorsLoader simulators; readonly IDeviceLoader devices; @@ -81,8 +81,8 @@ public ILogs Logs { List Tasks = new List (); Dictionary DependencyTasks = new Dictionary (); - internal static Resource DesktopResource = new Resource ("Desktop", Environment.ProcessorCount); - internal static Resource NugetResource = new Resource ("Nuget", 1); // nuget is not parallel-safe :( + public Resource DesktopResource { get; } = new Resource ("Desktop", Environment.ProcessorCount); + public Resource NugetResource { get; } = new Resource ("Nuget", 1); // nuget is not parallel-safe :( static Dictionary device_resources = new Dictionary (); internal static Resources GetDeviceResources (IEnumerable devices) @@ -482,9 +482,7 @@ IEnumerable CreateTestVariations (IEnumerable tests, Func> CreateRunSimulatorTasksAsync () configurations = new string [] { "Debug" }; foreach (var config in configurations) { foreach (var pair in ps) { - var derived = new MSBuildTask (processManager) { - Jenkins = this, + var derived = new MSBuildTask (jenkins: this, testProject: project, processManager: processManager) { ProjectConfiguration = config, ProjectPlatform = "iPhoneSimulator", Platform = pair.Item2, @@ -562,8 +559,7 @@ async Task> CreateRunSimulatorTasksAsync () var rv = new List (); foreach (var taskGroup in testVariations.GroupBy ((RunSimulatorTask task) => task.Device?.UDID ?? task.Candidates.ToString ())) { - rv.Add (new AggregatedRunSimulatorTask (taskGroup) { - Jenkins = this, + rv.Add (new AggregatedRunSimulatorTask (jenkins: this, tasks: taskGroup) { TestName = $"Tests for {taskGroup.Key}", }); } @@ -585,8 +581,7 @@ Task> CreateRunDeviceTasksAsync () projectTasks.Clear (); if (!project.SkipiOSVariation) { - var build64 = new MSBuildTask (processManager) { - Jenkins = this, + var build64 = new MSBuildTask (jenkins: this, testProject: project, processManager: processManager) { ProjectConfiguration = "Debug64", ProjectPlatform = "iPhone", Platform = TestPlatform.iOS_Unified64, @@ -595,8 +590,7 @@ Task> CreateRunDeviceTasksAsync () build64.CloneTestProject (project); projectTasks.Add (new RunDeviceTask (devices, build64, processManager, devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !IncludeiOS64 }); - var build32 = new MSBuildTask (processManager) { - Jenkins = this, + var build32 = new MSBuildTask (jenkins: this, testProject: project, processManager: processManager) { ProjectConfiguration = project.Name != "dont link" ? "Debug32" : "Release32", ProjectPlatform = "iPhone", Platform = TestPlatform.iOS_Unified32, @@ -606,8 +600,7 @@ Task> CreateRunDeviceTasksAsync () projectTasks.Add (new RunDeviceTask (devices, build32, processManager, devices.Connected32BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !IncludeiOS32 }); var todayProject = project.AsTodayExtensionProject (); - var buildToday = new MSBuildTask (processManager) { - Jenkins = this, + var buildToday = new MSBuildTask (jenkins: this, testProject: todayProject, processManager: processManager) { ProjectConfiguration = "Debug64", ProjectPlatform = "iPhone", Platform = TestPlatform.iOS_TodayExtension64, @@ -619,8 +612,7 @@ Task> CreateRunDeviceTasksAsync () if (!project.SkiptvOSVariation) { var tvOSProject = project.AsTvOSProject (); - var buildTV = new MSBuildTask (processManager) { - Jenkins = this, + var buildTV = new MSBuildTask (jenkins: this, testProject: tvOSProject, processManager: processManager) { ProjectConfiguration = "Debug", ProjectPlatform = "iPhone", Platform = TestPlatform.tvOS, @@ -633,8 +625,7 @@ Task> CreateRunDeviceTasksAsync () if (!project.SkipwatchOSVariation) { var watchOSProject = project.AsWatchOSProject (); if (!project.SkipwatchOS32Variation) { - var buildWatch32 = new MSBuildTask (processManager) { - Jenkins = this, + var buildWatch32 = new MSBuildTask (jenkins: this, testProject: watchOSProject, processManager: processManager) { ProjectConfiguration = "Debug32", ProjectPlatform = "iPhone", Platform = TestPlatform.watchOS_32, @@ -645,8 +636,7 @@ Task> CreateRunDeviceTasksAsync () } if (!project.SkipwatchOSARM64_32Variation) { - var buildWatch64_32 = new MSBuildTask (processManager) { - Jenkins = this, + var buildWatch64_32 = new MSBuildTask (jenkins: this, testProject: watchOSProject, processManager: processManager) { ProjectConfiguration = "Release64_32", // We don't support Debug for ARM64_32 yet. ProjectPlatform = "iPhone", Platform = TestPlatform.watchOS_64_32, @@ -941,10 +931,9 @@ Task PopulateTasksAsync () var crashReportSnapshotFactory = new CrashSnapshotReporterFactory (processManager); - var buildiOSMSBuild_net461 = new MSBuildTask (processManager) + var net461Project = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))); + var buildiOSMSBuild_net461 = new MSBuildTask (jenkins: this, testProject: net461Project, processManager: processManager) { - Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = true, ProjectConfiguration = "Debug-net461", @@ -955,7 +944,7 @@ Task PopulateTasksAsync () var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), - TestProject = buildiOSMSBuild_net461.TestProject, + TestProject = net461Project, ProjectConfiguration = "Debug-net461", Platform = TestPlatform.iOS, TestName = "MSBuild tests", @@ -966,9 +955,8 @@ Task PopulateTasksAsync () }; Tasks.Add (nunitExecutioniOSMSBuild_net461); - var buildiOSMSBuild_netstandard2 = new MSBuildTask (processManager) { - Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), + var netstandard2Project = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))); + var buildiOSMSBuild_netstandard2 = new MSBuildTask (jenkins: this, testProject: netstandard2Project, processManager: processManager) { SpecifyPlatform = false, SpecifyConfiguration = true, ProjectConfiguration = "Debug-netstandard2.0", @@ -978,7 +966,7 @@ Task PopulateTasksAsync () }; var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), - TestProject = buildiOSMSBuild_netstandard2.TestProject, + TestProject = netstandard2Project, ProjectConfiguration = "Debug-netstandard2.0", Platform = TestPlatform.iOS, TestName = "MSBuild tests", @@ -989,10 +977,9 @@ Task PopulateTasksAsync () }; Tasks.Add (nunitExecutioniOSMSBuild_netstandard2); - var buildInstallSources = new MSBuildTask (processManager) + var installSourcesProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "InstallSourcesTests.csproj"))); + var buildInstallSources = new MSBuildTask (jenkins: this, testProject: installSourcesProject, processManager: processManager) { - Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "InstallSourcesTests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, @@ -1001,7 +988,7 @@ Task PopulateTasksAsync () var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), - TestProject = buildInstallSources.TestProject, + TestProject = installSourcesProject, Platform = TestPlatform.iOS, TestName = "Install Sources tests", Mode = "iOS", @@ -1040,10 +1027,9 @@ Task PopulateTasksAsync () throw new NotImplementedException (project.TargetFrameworkFlavors.ToString ()); } foreach (var config in configurations) { - MSBuildTask build = new MSBuildTask (processManager); + MSBuildTask build = new MSBuildTask (jenkins: this, testProject: project, processManager: processManager); build.Platform = platform; build.CloneTestProject (project); - build.Jenkins = this; build.SolutionPath = project.SolutionPath; build.ProjectConfiguration = config; build.ProjectPlatform = project.Platform; @@ -1083,9 +1069,8 @@ Task PopulateTasksAsync () } } - var buildMTouch = new MakeTask (processManager) + var buildMTouch = new MakeTask (jenkins: this, processManager: processManager) { - Jenkins = this, TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.sln"))), SpecifyPlatform = false, SpecifyConfiguration = false, @@ -1105,8 +1090,7 @@ Task PopulateTasksAsync () }; Tasks.Add (nunitExecutionMTouch); - var buildGenerator = new MakeTask (processManager) { - Jenkins = this, + var buildGenerator = new MakeTask (jenkins: this, processManager: processManager) { TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "src", "generator.sln"))), SpecifyPlatform = false, SpecifyConfiguration = false, @@ -1125,15 +1109,15 @@ Task PopulateTasksAsync () }; Tasks.Add (runGenerator); - var buildDotNetGenerator = new DotNetBuildTask (processManager) { - Jenkins = this, + var buildDotNetGeneratorProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bgen", "bgen-tests.csproj"))); + var buildDotNetGenerator = new DotNetBuildTask (jenkins: this, testProject: buildDotNetGeneratorProject, processManager: processManager) { TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bgen", "bgen-tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, }; var runDotNetGenerator = new DotNetTestTask (buildDotNetGenerator, processManager) { - TestProject = buildDotNetGenerator.TestProject, + TestProject = buildDotNetGeneratorProject, Platform = TestPlatform.iOS, TestName = "Generator tests", Mode = ".NET", @@ -1141,9 +1125,8 @@ Task PopulateTasksAsync () }; Tasks.Add (runDotNetGenerator); - var run_mmp = new MakeTask (processManager) + var run_mmp = new MakeTask (jenkins: this, processManager: processManager) { - Jenkins = this, Platform = TestPlatform.Mac, TestName = "MMP Regression Tests", Target = "all", // -j" + Environment.ProcessorCount, @@ -1160,9 +1143,8 @@ Task PopulateTasksAsync () run_mmp.Environment.Add ("BUILD_REVISION", "jenkins"); // This will print "@MonkeyWrench: AddFile: " lines, which we can use to get the log filenames. Tasks.Add (run_mmp); - var runMacBindingProject = new MakeTask (processManager) + var runMacBindingProject = new MakeTask (jenkins: this, processManager: processManager) { - Jenkins = this, Platform = TestPlatform.Mac, TestName = "Mac Binding Projects", Target = "all", @@ -1172,8 +1154,7 @@ Task PopulateTasksAsync () }; Tasks.Add (runMacBindingProject); - var buildXtroTests = new MakeTask (processManager) { - Jenkins = this, + var buildXtroTests = new MakeTask (jenkins: this, processManager: processManager) { Platform = TestPlatform.All, TestName = "Xtro", Target = "wrench", @@ -1182,7 +1163,6 @@ Task PopulateTasksAsync () Timeout = TimeSpan.FromMinutes (15), }; var runXtroReporter = new RunXtroTask (buildXtroTests, processManager, crashReportSnapshotFactory) { - Jenkins = this, Platform = TestPlatform.Mac, TestName = buildXtroTests.TestName, Ignored = buildXtroTests.Ignored, @@ -1190,8 +1170,7 @@ Task PopulateTasksAsync () }; Tasks.Add (runXtroReporter); - var buildCecilTests = new MakeTask (processManager) { - Jenkins = this, + var buildCecilTests = new MakeTask (jenkins: this, processManager: processManager) { Platform = TestPlatform.All, TestName = "Cecil", Target = "build", @@ -1210,8 +1189,7 @@ Task PopulateTasksAsync () }; Tasks.Add (runCecilTests); - var runDocsTests = new MakeTask (processManager) { - Jenkins = this, + var runDocsTests = new MakeTask (jenkins: this, processManager: processManager) { Platform = TestPlatform.All, TestName = "Documentation", Target = "wrench-docs", @@ -1221,9 +1199,8 @@ Task PopulateTasksAsync () }; Tasks.Add (runDocsTests); - var buildSampleTests = new MSBuildTask (processManager) { - Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.sln"))), + var buildSampleTestsProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.sln"))); + var buildSampleTests = new MSBuildTask (jenkins: this, testProject: buildSampleTestsProject, processManager: processManager) { SpecifyPlatform = false, Platform = TestPlatform.All, ProjectConfiguration = "Debug", diff --git a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs index 9a06e382a664..da51fa875540 100644 --- a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs @@ -20,7 +20,7 @@ class AggregatedRunSimulatorTask : JenkinsTestTask Stopwatch run_timer = new Stopwatch (); public TimeSpan RunDuration { get { return run_timer.Elapsed; } } - public AggregatedRunSimulatorTask (IEnumerable tasks) + public AggregatedRunSimulatorTask (Jenkins jenkins, IEnumerable tasks) : base (jenkins) { this.Tasks = tasks; } diff --git a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs index d0b76654bc38..e7cbbd5651dc 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs @@ -1,116 +1,34 @@ using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Threading.Tasks; -using System.Xml; -using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Execution; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; -using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; using Xharness.TestTasks; namespace Xharness.Jenkins.TestTasks { abstract class BuildProjectTask : BuildToolTask { - public string SolutionPath; - protected BuildProjectTask (IProcessManager processManager) : base (processManager) - { + public string SolutionPath { + get => buildProjectTask.SolutionPath; + set => buildProjectTask.SolutionPath = value; } - public bool RestoreNugets { - get { - return TestProject.RestoreNugetsInProject || !string.IsNullOrEmpty (SolutionPath); - } - } + Xharness.TestTasks.BuildProjectTask buildProjectTask; - public override bool SupportsParallelExecution { - get { - return Platform.ToString ().StartsWith ("Mac", StringComparison.Ordinal); - } - } - async Task RestoreNugetsAsync (string projectPath, ILog log, bool useXIBuild = false) + protected BuildProjectTask (Jenkins jenkins, TestProject testProject, IProcessManager processManager) : base (jenkins, processManager) { - using (var resource = await Jenkins.NugetResource.AcquireExclusiveAsync ()) { - // we do not want to use xibuild on solutions, we will have some failures with Mac Full - var isSolution = projectPath.EndsWith (".sln", StringComparison.Ordinal); - if (!File.Exists (projectPath)) - throw new FileNotFoundException ("Could not find the solution whose nugets to restore.", projectPath); - - using (var nuget = new Process ()) { - nuget.StartInfo.FileName = useXIBuild && !isSolution ? Harness.XIBuildPath : - "/Library/Frameworks/Mono.framework/Versions/Current/Commands/nuget"; - var args = new List (); - args.Add ((useXIBuild && !isSolution ? "/" : "") + "restore"); // diff param depending on the tool - args.Add (projectPath); - if (useXIBuild && !isSolution) - args.Add ("/verbosity:detailed"); - else { - args.Add ("-verbosity"); - args.Add ("detailed"); - } - nuget.StartInfo.Arguments = StringUtils.FormatArguments (args); - SetEnvironmentVariables (nuget); - LogEvent (log, "Restoring nugets for {0} ({1}) on path {2}", TestName, Mode, projectPath); - - var timeout = TimeSpan.FromMinutes (15); - var result = await ProcessManager.RunAsync (nuget, log, timeout); - if (result.TimedOut) { - log.WriteLine ("Nuget restore timed out after {0} seconds.", timeout.TotalSeconds); - return TestExecutingResult.TimedOut; - } else if (!result.Succeeded) { - return TestExecutingResult.Failed; - } - } - - LogEvent (log, "Restoring nugets completed for {0} ({1}) on path {2}", TestName, Mode, projectPath); - return TestExecutingResult.Succeeded; - } + TestProject = testProject ?? throw new ArgumentNullException (nameof (testProject)); + buildProjectTask = new Xharness.TestTasks.BuildProjectTask (TestProject, processManager, Jenkins, this, this); } - List GetNestedReferenceProjects (string csproj) - { - if (!File.Exists (csproj)) - throw new FileNotFoundException ("Could not find the project whose reference projects needed to be found.", csproj); - var result = new List (); - var doc = new XmlDocument (); - doc.Load (csproj.Replace ("\\", "/")); - foreach (var referenceProject in doc.GetProjectReferences ()) { - var fixPath = referenceProject.Replace ("\\", "/"); // do the replace in case we use win paths - result.Add (fixPath); - // get all possible references - result.AddRange (GetNestedReferenceProjects (fixPath)); - } - return result; - } + public bool RestoreNugets => buildProjectTask.RestoreNugets; + + public override bool SupportsParallelExecution => buildProjectTask.SupportsParallelExecution; // This method must be called with the desktop resource acquired // (which is why it takes an IAcquiredResources as a parameter without using it in the function itself). - protected async Task RestoreNugetsAsync (ILog log, IAcquiredResource resource, bool useXIBuild = false) - { - if (!RestoreNugets) - return; - - if (!File.Exists (SolutionPath ?? TestProject.Path)) - throw new FileNotFoundException ("Could not find the solution whose nugets to restore.", SolutionPath ?? TestProject.Path); - - // might happen that the project does contain reference projects with nugets, grab the reference projects and ensure - // thast they have the nugets restored (usually, watch os test projects - if (SolutionPath == null) { - var references = GetNestedReferenceProjects (TestProject.Path); - foreach (var referenceProject in references) { - var execResult = await RestoreNugetsAsync (referenceProject, log, useXIBuild); // do the replace in case we use win paths - if (execResult == TestExecutingResult.TimedOut) { - ExecutionResult = execResult; - return; - } - } - } - - // restore for the main project/solution] - ExecutionResult = await RestoreNugetsAsync (SolutionPath ?? TestProject.Path, log, useXIBuild); - } + protected async Task RestoreNugetsAsync (ILog log, IAcquiredResource resource, bool useXIBuild = false) => + ExecutionResult = await buildProjectTask.RestoreNugetsAsync (log, resource, useXIBuild); } } diff --git a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs index 15d7a9fe5d54..d5e2a8210979 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs @@ -9,6 +9,14 @@ public abstract class BuildToolTask : JenkinsTestTask public IProcessManager ProcessManager => buildToolTask.ProcessManager; + public override string TestName { + get => base.TestName; + set { + base.TestName = value; + buildToolTask.TestName = value; + } + } + public bool SpecifyPlatform { get => buildToolTask.SpecifyPlatform; set => buildToolTask.SpecifyPlatform = value; @@ -19,7 +27,15 @@ public bool SpecifyConfiguration { set => buildToolTask.SpecifyConfiguration = value; } - protected BuildToolTask (IProcessManager processManager) + public override TestProject TestProject { + get => base.TestProject; + set { + base.TestProject = value; + buildToolTask.TestProject = value; + } + } + + protected BuildToolTask (Jenkins jenkins, IProcessManager processManager) : base (jenkins) => buildToolTask = new Xharness.TestTasks.BuildToolTask (processManager); public override TestPlatform Platform { diff --git a/tests/xharness/Jenkins/TestTasks/DotNetBuildTask.cs b/tests/xharness/Jenkins/TestTasks/DotNetBuildTask.cs index cc96e230f4f7..a0670096d573 100644 --- a/tests/xharness/Jenkins/TestTasks/DotNetBuildTask.cs +++ b/tests/xharness/Jenkins/TestTasks/DotNetBuildTask.cs @@ -4,7 +4,7 @@ namespace Xharness.Jenkins.TestTasks { class DotNetBuildTask : MSBuildTask { - public DotNetBuildTask (IProcessManager processManager) : base (processManager) + public DotNetBuildTask (Jenkins jenkins, TestProject testProject, IProcessManager processManager) : base (jenkins, testProject, processManager) { SetDotNetEnvironmentVariables (Environment); } diff --git a/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs b/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs index 2ec6c1987ad1..0f7cd7d0de33 100644 --- a/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs @@ -8,9 +8,10 @@ namespace Xharness.Jenkins.TestTasks { public abstract class JenkinsTestTask : Xharness.TestTasks.TestTasks { - public Jenkins Jenkins; + public Jenkins Jenkins { get; private set; } public Harness Harness { get { return Jenkins.Harness; } } + public override IResourceManager ResourceManager => Jenkins; public override string LogDirectory { get { @@ -20,11 +21,16 @@ public override string LogDirectory { } } + public JenkinsTestTask (Jenkins jenkins) : base () + { + Jenkins = jenkins ?? throw new ArgumentNullException (nameof (jenkins)); + } + public override void GenerateReport () => Jenkins.GenerateReport (); protected override void WriteLineToRunnerLog (string message) => Harness.HarnessLog.WriteLine (message); - protected override void SetEnvironmentVariables (Process process) + public override void SetEnvironmentVariables (Process process) { var xcodeRoot = Harness.XcodeRoot; @@ -73,7 +79,7 @@ protected override void SetEnvironmentVariables (Process process) } - protected override void LogEvent (ILog log, string text, params object [] args) + public override void LogEvent (ILog log, string text, params object [] args) { base.LogEvent (log, text, args); Jenkins.MainLog.WriteLine (text, args); diff --git a/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs b/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs index b1c92b29077d..78a263bd0c5f 100644 --- a/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs @@ -32,7 +32,7 @@ protected virtual List ToolArguments { } } - public MSBuildTask (IProcessManager processManager) : base (processManager) + public MSBuildTask (Jenkins jenkins, TestProject testProject, IProcessManager processManager) : base (jenkins, testProject, processManager) { } diff --git a/tests/xharness/Jenkins/TestTasks/MakeTask.cs b/tests/xharness/Jenkins/TestTasks/MakeTask.cs index b00d7dd1747d..f426594f2299 100644 --- a/tests/xharness/Jenkins/TestTasks/MakeTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MakeTask.cs @@ -12,7 +12,7 @@ class MakeTask : BuildToolTask public string WorkingDirectory; public TimeSpan Timeout = TimeSpan.FromMinutes (5); - public MakeTask (IProcessManager processManager) : base (processManager) + public MakeTask (Jenkins jenkins, IProcessManager processManager) : base (jenkins, processManager) { } diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index 7ed29ee3c3e2..1ee21a91e6bb 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -19,12 +19,11 @@ internal abstract class RunTestTask : JenkinsTestTask { public double TimeoutMultiplier { get; set; } = 1; public string WorkingDirectory; - public RunTestTask (BuildToolTask build_task, IProcessManager processManager) + public RunTestTask (BuildToolTask build_task, IProcessManager processManager) : base (build_task.Jenkins) { this.BuildTask = build_task; this.ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); - Jenkins = build_task.Jenkins; TestProject = build_task.TestProject; Platform = build_task.Platform; ProjectPlatform = build_task.ProjectPlatform; diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/Logging/IEventLogger.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/Logging/IEventLogger.cs new file mode 100644 index 000000000000..bccb7189a49b --- /dev/null +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/Logging/IEventLogger.cs @@ -0,0 +1,5 @@ +namespace Microsoft.DotNet.XHarness.iOS.Shared.Logging { + public interface IEventLogger { + public void LogEvent (ILog log, string text, params object [] args); + } +} diff --git a/tests/xharness/TestTasks/BuildProjectTask.cs b/tests/xharness/TestTasks/BuildProjectTask.cs new file mode 100644 index 000000000000..f05a10e7dddf --- /dev/null +++ b/tests/xharness/TestTasks/BuildProjectTask.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using System.Xml; +using Microsoft.DotNet.XHarness.iOS.Shared; +using Microsoft.DotNet.XHarness.iOS.Shared.Execution; +using Microsoft.DotNet.XHarness.iOS.Shared.Logging; +using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; + +namespace Xharness.TestTasks { + public class BuildProjectTask : BuildToolTask { + public IResourceManager ResourceManager { get; set; } + public IEnvManager EnviromentManager { get; set; } + public IEventLogger EventLogger { get; set; } + + public string SolutionPath { get; set; } + + public BuildProjectTask (TestProject project, IProcessManager processManager, IResourceManager resourceManager, IEventLogger eventLogger, IEnvManager envManager) : base (processManager) + { + TestProject = project ?? throw new ArgumentNullException (nameof (project)); + ResourceManager = resourceManager ?? throw new ArgumentNullException (nameof (resourceManager)); + EventLogger = eventLogger ?? throw new ArgumentNullException (nameof (eventLogger)); + EnviromentManager = envManager ?? throw new ArgumentNullException (nameof (envManager)); + } + + public bool RestoreNugets { + get { + return TestProject.RestoreNugetsInProject || !string.IsNullOrEmpty (SolutionPath); + } + } + + public bool SupportsParallelExecution { + get { + return Platform.ToString ().StartsWith ("Mac", StringComparison.Ordinal); + } + } + + async Task RestoreNugetsAsync (string projectPath, ILog log, bool useXIBuild = false) + { + using (var resource = await ResourceManager.NugetResource.AcquireExclusiveAsync ()) { + // we do not want to use xibuild on solutions, we will have some failures with Mac Full + var isSolution = projectPath.EndsWith (".sln", StringComparison.Ordinal); + if (!File.Exists (projectPath)) + throw new FileNotFoundException ("Could not find the solution whose nugets to restore.", projectPath); + + using (var nuget = new Process ()) { + nuget.StartInfo.FileName = useXIBuild && !isSolution ? Harness.XIBuildPath : + "/Library/Frameworks/Mono.framework/Versions/Current/Commands/nuget"; + var args = new List (); + args.Add ((useXIBuild && !isSolution ? "/" : "") + "restore"); // diff param depending on the tool + args.Add (projectPath); + if (useXIBuild && !isSolution) + args.Add ("/verbosity:detailed"); + else { + args.Add ("-verbosity"); + args.Add ("detailed"); + } + nuget.StartInfo.Arguments = StringUtils.FormatArguments (args); + EnviromentManager.SetEnvironmentVariables (nuget); + EventLogger.LogEvent (log, "Restoring nugets for {0} ({1}) on path {2}", TestName, Mode, projectPath); + + var timeout = TimeSpan.FromMinutes (15); + var result = await ProcessManager.RunAsync (nuget, log, timeout); + if (result.TimedOut) { + log.WriteLine ("Nuget restore timed out after {0} seconds.", timeout.TotalSeconds); + return TestExecutingResult.TimedOut; + } else if (!result.Succeeded) { + return TestExecutingResult.Failed; + } + } + + EventLogger.LogEvent (log, "Restoring nugets completed for {0} ({1}) on path {2}", TestName, Mode, projectPath); + return TestExecutingResult.Succeeded; + } + } + + List GetNestedReferenceProjects (string csproj) + { + if (!File.Exists (csproj)) + throw new FileNotFoundException ("Could not find the project whose reference projects needed to be found.", csproj); + var result = new List (); + var doc = new XmlDocument (); + doc.Load (csproj.Replace ("\\", "/")); + foreach (var referenceProject in doc.GetProjectReferences ()) { + var fixPath = referenceProject.Replace ("\\", "/"); // do the replace in case we use win paths + result.Add (fixPath); + // get all possible references + result.AddRange (GetNestedReferenceProjects (fixPath)); + } + return result; + } + + // This method must be called with the desktop resource acquired + // (which is why it takes an IAcquiredResources as a parameter without using it in the function itself). + public async Task RestoreNugetsAsync (ILog log, IAcquiredResource resource, bool useXIBuild = false) + { + if (!RestoreNugets) + return TestExecutingResult.Ignored; + + if (!File.Exists (SolutionPath ?? TestProject.Path)) + throw new FileNotFoundException ("Could not find the solution whose nugets to restore.", SolutionPath ?? TestProject.Path); + + // might happen that the project does contain reference projects with nugets, grab the reference projects and ensure + // thast they have the nugets restored (usually, watch os test projects + if (SolutionPath == null) { + var references = GetNestedReferenceProjects (TestProject.Path); + foreach (var referenceProject in references) { + var execResult = await RestoreNugetsAsync (referenceProject, log, useXIBuild); // do the replace in case we use win paths + if (execResult == TestExecutingResult.TimedOut) { + return execResult; + } + } + } + + // restore for the main project/solution] + return await RestoreNugetsAsync (SolutionPath ?? TestProject.Path, log, useXIBuild); + } + } +} diff --git a/tests/xharness/TestTasks/BuildToolTask.cs b/tests/xharness/TestTasks/BuildToolTask.cs index d0edf6656840..16f5efa37bbf 100644 --- a/tests/xharness/TestTasks/BuildToolTask.cs +++ b/tests/xharness/TestTasks/BuildToolTask.cs @@ -6,8 +6,10 @@ namespace Xharness.TestTasks { public class BuildToolTask { + public string TestName { get; set; } public IProcessManager ProcessManager { get; private set; } public TestPlatform Platform { get; set; } + public TestProject TestProject { get; set; } public bool SpecifyPlatform { get; set; } = true; public bool SpecifyConfiguration { get; set; } = true; diff --git a/tests/xharness/TestTasks/IEnvManager.cs b/tests/xharness/TestTasks/IEnvManager.cs new file mode 100644 index 000000000000..71fa96b08448 --- /dev/null +++ b/tests/xharness/TestTasks/IEnvManager.cs @@ -0,0 +1,12 @@ +using System; +using System.Diagnostics; + +namespace Xharness.TestTasks { + /// + /// Knows how to handle the different enviroment variables to be used with the processes executed by xharness and + /// the different tests tasks. + /// + public interface IEnvManager { + void SetEnvironmentVariables (Process process); + } +} diff --git a/tests/xharness/TestTasks/IResourceManager.cs b/tests/xharness/TestTasks/IResourceManager.cs new file mode 100644 index 000000000000..9871d1421f5b --- /dev/null +++ b/tests/xharness/TestTasks/IResourceManager.cs @@ -0,0 +1,12 @@ +namespace Xharness.TestTasks { + + /// + /// Managed the resorces that can be used by the different tests tasks so that when ran async they do not step + /// on eachother. + /// + public interface IResourceManager + { + Resource DesktopResource { get; } + Resource NugetResource { get; } + } +} diff --git a/tests/xharness/TestTasks/TestTask.cs b/tests/xharness/TestTasks/TestTask.cs index 8a2bdf6c0483..c12caaacc969 100644 --- a/tests/xharness/TestTasks/TestTask.cs +++ b/tests/xharness/TestTasks/TestTask.cs @@ -11,7 +11,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; namespace Xharness.TestTasks { - public abstract class TestTasks + public abstract class TestTasks : IEnvManager, IEventLogger { static int counter; static DriveInfo RootDrive; @@ -27,7 +27,6 @@ public abstract class TestTasks public Func Dependency; // a task that's feteched and awaited before this task's ExecuteAsync method public Task InitialTask; // a task that's executed before this task's ExecuteAsync method. public Task CompletedTask; // a task that's executed after this task's ExecuteAsync method. - public TestProject TestProject; public List Resources = new List (); #endregion @@ -126,10 +125,11 @@ public IEnumerable ReferencedNunitAndXunitTestAssemblies { #region Abstract + public abstract IResourceManager ResourceManager { get; } public abstract void GenerateReport (); public abstract string LogDirectory { get; } + public abstract void SetEnvironmentVariables (Process process); protected abstract Task ExecuteAsync (); - protected abstract void SetEnvironmentVariables (Process process); protected abstract Task NotifyAndAcquireDesktopResourceAsync (); protected abstract void WriteLineToRunnerLog (string message); @@ -137,6 +137,7 @@ public IEnumerable ReferencedNunitAndXunitTestAssemblies { #region Virtual + public virtual TestProject TestProject { get; set; } public virtual TestPlatform Platform { get; set; } public virtual string ProgressMessage { get; } public virtual string Mode { get; set; } @@ -196,7 +197,7 @@ public virtual string TestName { protected virtual void PropagateResults () { } - protected virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args); + public virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args); public virtual void Reset () { diff --git a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs index 211cf90518de..d0095f55e22c 100644 --- a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs @@ -13,8 +13,6 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; using Microsoft.DotNet.XHarness.iOS.Shared; -using Microsoft.DotNet.XHarness.iOS.Shared.Execution; -using Microsoft.DotNet.XHarness.iOS.Shared.Execution.Mlaunch; using Microsoft.DotNet.XHarness.iOS.Shared.Listeners; using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index ca532729be88..8f33c4c5ddf3 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -130,6 +130,9 @@ + + + From 88385599b7b485c493f15957d7f3a3995a0102c9 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 6 Apr 2020 09:53:29 -0400 Subject: [PATCH 4/6] Update tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs Co-Authored-By: Rolf Bjarne Kvinge --- tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs b/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs index 2ec6c1987ad1..902f4005a993 100644 --- a/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/JenkinsTestTask.cs @@ -6,7 +6,7 @@ using Xharness.TestTasks; namespace Xharness.Jenkins.TestTasks { - public abstract class JenkinsTestTask : Xharness.TestTasks.TestTasks + public abstract class AppleTestTask : Xharness.TestTasks.TestTasks { public Jenkins Jenkins; public Harness Harness { get { return Jenkins.Harness; } } From e221475f6f896b01b42d42f67e3b50524a6f2078 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 6 Apr 2020 16:11:12 -0400 Subject: [PATCH 5/6] Ensure we do the nuget restore in the clone. --- tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs | 12 ++++++++++-- tests/xharness/TestTasks/BuildProjectTask.cs | 3 +-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs index e7cbbd5651dc..0d1794cc4e9b 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs @@ -15,11 +15,19 @@ public string SolutionPath { Xharness.TestTasks.BuildProjectTask buildProjectTask; + public override TestProject TestProject { + get => base.TestProject; + set { + base.TestProject = value; + buildProjectTask.TestProject = value; + } + } + protected BuildProjectTask (Jenkins jenkins, TestProject testProject, IProcessManager processManager) : base (jenkins, processManager) { + buildProjectTask = new Xharness.TestTasks.BuildProjectTask (processManager, Jenkins, this, this); TestProject = testProject ?? throw new ArgumentNullException (nameof (testProject)); - buildProjectTask = new Xharness.TestTasks.BuildProjectTask (TestProject, processManager, Jenkins, this, this); } public bool RestoreNugets => buildProjectTask.RestoreNugets; @@ -28,7 +36,7 @@ protected BuildProjectTask (Jenkins jenkins, TestProject testProject, IProcessMa // This method must be called with the desktop resource acquired // (which is why it takes an IAcquiredResources as a parameter without using it in the function itself). - protected async Task RestoreNugetsAsync (ILog log, IAcquiredResource resource, bool useXIBuild = false) => + protected async Task RestoreNugetsAsync (ILog log, IAcquiredResource resource, bool useXIBuild = false) => ExecutionResult = await buildProjectTask.RestoreNugetsAsync (log, resource, useXIBuild); } } diff --git a/tests/xharness/TestTasks/BuildProjectTask.cs b/tests/xharness/TestTasks/BuildProjectTask.cs index f05a10e7dddf..a8e794e79f13 100644 --- a/tests/xharness/TestTasks/BuildProjectTask.cs +++ b/tests/xharness/TestTasks/BuildProjectTask.cs @@ -17,9 +17,8 @@ public class BuildProjectTask : BuildToolTask { public string SolutionPath { get; set; } - public BuildProjectTask (TestProject project, IProcessManager processManager, IResourceManager resourceManager, IEventLogger eventLogger, IEnvManager envManager) : base (processManager) + public BuildProjectTask (IProcessManager processManager, IResourceManager resourceManager, IEventLogger eventLogger, IEnvManager envManager) : base (processManager) { - TestProject = project ?? throw new ArgumentNullException (nameof (project)); ResourceManager = resourceManager ?? throw new ArgumentNullException (nameof (resourceManager)); EventLogger = eventLogger ?? throw new ArgumentNullException (nameof (eventLogger)); EnviromentManager = envManager ?? throw new ArgumentNullException (nameof (envManager)); From 66753dfabf2ef26adfcbcf3963c9054c0a7af638 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 8 Apr 2020 15:12:11 -0400 Subject: [PATCH 6/6] Fix some conflicts. --- tests/xharness/TestTasks/BuildToolTask.cs | 4 ++-- tests/xharness/TestTasks/TestTask.cs | 4 ++-- tests/xharness/xharness.csproj | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/xharness/TestTasks/BuildToolTask.cs b/tests/xharness/TestTasks/BuildToolTask.cs index 16f5efa37bbf..bf4cac65566c 100644 --- a/tests/xharness/TestTasks/BuildToolTask.cs +++ b/tests/xharness/TestTasks/BuildToolTask.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.iOS.Shared.Execution; @@ -7,7 +7,7 @@ namespace Xharness.TestTasks { public class BuildToolTask { public string TestName { get; set; } - public IProcessManager ProcessManager { get; private set; } + public IProcessManager ProcessManager { get; } public TestPlatform Platform { get; set; } public TestProject TestProject { get; set; } diff --git a/tests/xharness/TestTasks/TestTask.cs b/tests/xharness/TestTasks/TestTask.cs index c12caaacc969..0d1df8d3d950 100644 --- a/tests/xharness/TestTasks/TestTask.cs +++ b/tests/xharness/TestTasks/TestTask.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -197,7 +197,7 @@ public virtual string TestName { protected virtual void PropagateResults () { } - public virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args); + protected virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args); public virtual void Reset () { diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index bbe1e1121500..6f83acc7a58d 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -1,4 +1,4 @@ - + Debug @@ -76,10 +76,10 @@ + - @@ -101,6 +101,7 @@ + @@ -125,7 +126,7 @@ - + SdkVersions.cs @@ -145,4 +146,4 @@ - \ No newline at end of file +