Skip to content

Disable known failing device tests using xharness #4884

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chamons merged 9 commits into from
Oct 3, 2018
Merged

Disable known failing device tests using xharness #4884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0402d1d
Disable device tests using xharness
chamons Sep 27, 2018
65990b9
Refactor to build only not skipping tests
chamons Oct 1, 2018
e721a46
Cleanup
chamons Oct 1, 2018
1954361
More cleanup
chamons Oct 1, 2018
11bb900
Actually force extensions to be build only
chamons Oct 1, 2018
d8053bc
Actually make BuildOnly do what it should
chamons Oct 2, 2018
222476d
Remove unnecessary bit
chamons Oct 2, 2018
ead4806
Remove dead code
chamons Oct 2, 2018
17281bb
Remove unnecessary diffs
chamons Oct 2, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions tests/xharness/Harness.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,19 +329,20 @@ void AutoConfigureIOS ()
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "bcl-test/mscorlib/mscorlib-0.csproj")), false));
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "bcl-test/mscorlib/mscorlib-1.csproj")), false));
foreach (var p in test_suites)
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".csproj"))));
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".csproj"))) { Name = p });
foreach (var p in fsharp_test_suites)
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".fsproj"))));
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".fsproj"))) { Name = p });
foreach (var p in library_projects)
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".csproj")), false));
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".csproj")), false) { Name = p });
foreach (var p in fsharp_library_projects)
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".fsproj")), false));
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".fsproj")), false) { Name = p });

foreach (var p in bcl_suites) {
BCLTestInfo bclTestInfo = new BCLTestInfo (this, p);
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "bcl-test/" + p + "/" + p + ".csproj"))) {
SkipwatchOSVariation = bcl_skip_watchos.Contains (p),
BCLInfo = bclTestInfo
BCLInfo = bclTestInfo,
Name = p
});
}

Expand Down
29 changes: 24 additions & 5 deletions tests/xharness/Jenkins.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public class Jenkins
public bool IncludeMac32 = true;
public bool IncludeiOS = true;
public bool IncludeiOSExtensions;
public bool ForceExtensionBuildOnly;
public bool IncludetvOS = true;
public bool IncludewatchOS = true;
public bool IncludeMmpTest;
Expand Down Expand Up @@ -385,7 +386,7 @@ IEnumerable<TestTask> CreateRunDeviceTasks ()
TestName = project.Name,
};
build64.CloneTestProject (project);
rv.Add (new RunDeviceTask (build64, Devices.ConnectedDevices.Where ((dev) => dev.DevicePlatform == DevicePlatform.iOS && dev.Supports64Bit)) { Ignored = ignored || !IncludeiOS });
rv.Add (new RunDeviceTask (build64, Devices.Connected64BitIOS) { Ignored = ignored || !IncludeiOS, BuildOnly = project.BuildOnly });

var build32 = new XBuildTask {
Jenkins = this,
Expand All @@ -395,7 +396,7 @@ IEnumerable<TestTask> CreateRunDeviceTasks ()
TestName = project.Name,
};
build32.CloneTestProject (project);
rv.Add (new RunDeviceTask (build32, Devices.ConnectedDevices.Where ((dev) => dev.DevicePlatform == DevicePlatform.iOS && dev.Supports32Bit)) { Ignored = ignored || !IncludeiOS });
rv.Add (new RunDeviceTask (build32, Devices.Connected32BitIOS) { Ignored = ignored || !IncludeiOS, BuildOnly = project.BuildOnly });

var todayProject = project.AsTodayExtensionProject ();
var buildToday = new XBuildTask {
Expand All @@ -406,7 +407,7 @@ IEnumerable<TestTask> CreateRunDeviceTasks ()
TestName = project.Name,
};
buildToday.CloneTestProject (todayProject);
rv.Add (new RunDeviceTask (buildToday, Devices.ConnectedDevices.Where ((dev) => dev.DevicePlatform == DevicePlatform.iOS && dev.Supports64Bit)) { Ignored = ignored || !IncludeiOSExtensions });
rv.Add (new RunDeviceTask (buildToday, Devices.Connected64BitIOS) { Ignored = ignored || !IncludeiOSExtensions, BuildOnly = project.BuildOnly || ForceExtensionBuildOnly });
}

if (!project.SkiptvOSVariation) {
Expand All @@ -419,7 +420,7 @@ IEnumerable<TestTask> CreateRunDeviceTasks ()
TestName = project.Name,
};
buildTV.CloneTestProject (tvOSProject);
rv.Add (new RunDeviceTask (buildTV, Devices.ConnectedDevices.Where ((dev) => dev.DevicePlatform == DevicePlatform.tvOS)) { Ignored = ignored || !IncludetvOS });
rv.Add (new RunDeviceTask (buildTV, Devices.ConnectedTV) { Ignored = ignored || !IncludetvOS, BuildOnly = project.BuildOnly });
}

if (!project.SkipwatchOSVariation) {
Expand All @@ -432,7 +433,7 @@ IEnumerable<TestTask> CreateRunDeviceTasks ()
TestName = project.Name,
};
buildWatch.CloneTestProject (watchOSProject);
rv.Add (new RunDeviceTask (buildWatch, Devices.ConnectedDevices.Where ((dev) => dev.DevicePlatform == DevicePlatform.watchOS)) { Ignored = ignored || !IncludewatchOS });
rv.Add (new RunDeviceTask (buildWatch, Devices.ConnectedWatch) { Ignored = ignored || !IncludewatchOS, BuildOnly = project.BuildOnly });
}
}

Expand Down Expand Up @@ -460,6 +461,8 @@ void SelectTests ()
// whatever we did automatically.
SelectTestsByLabel (pull_request);

DisableKnownFailingDeviceTests ();

if (!Harness.INCLUDE_IOS) {
MainLog.WriteLine ("The iOS build is diabled, so any iOS tests will be disabled as well.");
IncludeiOS = false;
Expand All @@ -481,6 +484,16 @@ void SelectTests ()
}
}

void DisableKnownFailingDeviceTests ()
{
// https://github.com/xamarin/maccore/issues/1008
ForceExtensionBuildOnly = true;

// https://github.com/xamarin/maccore/issues/1011
foreach (var mono in Harness.IOSTestProjects.Where (x => x.Name == "mini"))
mono.BuildOnly = true;
}

void SelectTestsByModifiedFiles (int pull_request)
{
var files = GitHub.GetModifiedFiles (Harness, pull_request);
Expand Down Expand Up @@ -3083,6 +3096,7 @@ protected override async Task RunTestAsync ()
abstract class RunTestTask : TestTask
{
public readonly BuildToolTask BuildTask;
public bool BuildOnly;

public RunTestTask (BuildToolTask build_task)
{
Expand Down Expand Up @@ -3150,6 +3164,11 @@ protected override async Task ExecuteAsync ()
if (!await BuildAsync ())
return;

if (BuildOnly) {
ExecutionResult = TestExecutingResult.Succeeded;
return;
}

ExecutionResult = TestExecutingResult.Running;
duration.Restart (); // don't count the build time.
await RunTestAsync ();
Expand Down
11 changes: 6 additions & 5 deletions tests/xharness/Simulators.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -646,11 +646,12 @@ public class Devices
bool loaded;

BlockingEnumerableCollection<Device> connected_devices = new BlockingEnumerableCollection<Device> ();
public IEnumerable<Device> ConnectedDevices {
get {
return connected_devices;
}
}

public IEnumerable<Device> ConnectedDevices => connected_devices;
public IEnumerable<Device> Connected64BitIOS => connected_devices.Where (x => x.DevicePlatform == DevicePlatform.iOS && x.Supports64Bit);
public IEnumerable<Device> Connected32BitIOS => connected_devices.Where (x => x.DevicePlatform == DevicePlatform.iOS && x.Supports32Bit);
public IEnumerable<Device> ConnectedTV => connected_devices.Where (x => x.DevicePlatform == DevicePlatform.tvOS);
public IEnumerable<Device> ConnectedWatch => connected_devices.Where (x => x.DevicePlatform == DevicePlatform.watchOS);

public async Task LoadAsync (Log log, bool extra_data = false, bool removed_locked = false, bool force = false)
{
Expand Down
1 change: 1 addition & 0 deletions tests/xharness/TestProject.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ public class iOSTestProject : TestProject
public bool SkipiOSVariation;
public bool SkipwatchOSVariation;
public bool SkiptvOSVariation;
public bool BuildOnly;

// Optional
public BCLTestInfo BCLInfo { get; set; }
Expand Down