PR to see what works needs to be done to merge matrix-testing into master

jenkins/add-commit-comment.sh

@@ -87,6 +87,8 @@ if test -n "$VERBOSE"; then
 	sed 's/^/    /' "$JSONFILE";
 fi
 
+cat $JSONFILE
+
 if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/commits/$HASH/comments" > "$LOGFILE" 2>&1; then
 	echo "Failed to add commit message."
 	echo "curl output:"

jenkins/add-commit-status.sh

@@ -128,6 +128,8 @@ trap cleanup EXIT
 	printf "}\n"
 ) > "$JSONFILE"
 
+cat $JSONFILE
+
 if test -n "$VERBOSE"; then
 	echo "JSON file:"
 	sed 's/^/    /' "$JSONFILE";

jenkins/vsts-device-tests-set-status.sh

@@ -89,7 +89,7 @@ if test -z "$START"; then
 
 	printf "%s%s on [Azure DevOps](%s)($DEVICE_TYPE)%s %s\\n\\n" "$RESULT_EMOJII" "$DESCRIPTION" "$VSTS_BUILD_URL" "$HTML_REPORT" "$RESULT_EMOJII" >> "$MESSAGE_FILE"
 
-	FILE=$PWD/tests/TestSummary.md
+	FILE=$PWD/TestSummary.md
 	if ! test -f "$FILE"; then
 		printf "🔥 Tests failed catastrophically on $DEVICE_TYPE  (no summary found)\\n" >> "$MESSAGE_FILE"
 	else

tests/Makefile

@@ -382,7 +382,7 @@ runner: xharness/xharness.exe
 # installed on the system.
 vsts-device-tests: xharness/xharness.exe
 	$(MAKE) -C $(TOP)/builds .stamp-mono-ios-sdk-destdir download -j
-	$(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --autoconf --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests,run-bcl-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND)
+	$(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --autoconf --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND)
 
 ifdef ENABLE_XAMARIN
 wrench-launch-external wrench-report-external:

tests/xharness/AppRunner.cs

@@ -373,6 +373,7 @@ public bool EnsureCleanSimulatorState {
 						var logFiles = new List<string> ();
 						// add our logs AND the logs of the previous task, which is the build task
 						logFiles.AddRange (Directory.GetFiles (Logs.Directory));
+						logFiles.Add (Harness.LogFile);
 						if (BuildTask != null) // when using the run command, we do not have a build task, ergo, there are no logs to add.
 							logFiles.AddRange (Directory.GetFiles (BuildTask.LogDirectory));
 						// add the attachments and write in the new filename

tests/xharness/TestProject.cs

@@ -98,7 +98,6 @@ public bool IsBclTest {
 
 		public bool IsBclxUnit => IsBclTest && (Name.Contains ("xUnit") || IsMscorlib);
 
-
 		public bool IsMscorlib => Name.Contains ("mscorlib");
 
 

tools/devops/beta-device-tests.yml

@@ -0,0 +1,19 @@
+# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending 
+# on the configuration to run different device tests.
+#
+# lame! cannot have the resources in the template.. :/
+resources:
+  repositories:
+  - repository: maccore
+    type: github
+    name: xamarin/maccore
+    ref: refs/heads/device-tests-yaml # should this be on a branch and not on master???
+    endpoint: xamarin
+
+# create parallel stages, this means that one single trigger to the pipeline will execute tvos and ios device tests
+stages:
+- template: templates/beta-stages.yml # all the smart stuff happens in the template, just needed to set the params
+  parameters:
+    pool: 'VSEng-Xamarin-Mac-Devices'
+    timeout: 1000
+    maxParallel: 5

tools/devops/device-tests.yml

@@ -1,5 +1,6 @@
-# iOS device pipeline, this file sets the different parameters required by the device-tests-common-yml, which contains all the steps and the interesting logic.
-
+# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending 
+# on the configuration to run different device tests.
+#
 # lame! cannot have the resources in the template.. :/
 resources:
   repositories:
@@ -9,12 +10,10 @@ resources:
     ref: refs/heads/device-tests-yaml # should this be on a branch and not on master???
     endpoint: xamarin
 
-jobs:
-- template: device-tests-common.yml # all the smart stuff happens in the template, just needed to set the params
+# create parallel stages, this means that one single trigger to the pipeline will execute tvos and ios device tests
+stages:
+- template: templates/stable-stages.yml # all the smart stuff happens in the template, just needed to set the params
   parameters:
-    name: 'macOS'
-    displayName: 'Run iOS Device Tests'
     pool: 'VSEng-Xamarin-Mac-Devices'
-    demands: 'ios'
-    statusContext: 'iOS-DDFun'
-    testsLabels: '--label=run-ios-64-tests'
+    timeout: 1000
+    maxParallel: 5

tools/devops/run-tests.sh

@@ -1,6 +1,8 @@
 #!/bin/bash -ex
 
-
+ls -l
+pwd
+ls -l xamarin-macios/
 if [[ "$SYSTEM_DEBUG" == "true" ]]; then
   DEBUG="-d"
 else
@@ -9,3 +11,6 @@ fi
 make $DEBUG -C xamarin-macios/builds download -j
 make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j
 make $DEBUG -C xamarin-macios/tests vsts-device-tests
+#echo "exit 100" > bar.sh
+#chmod a+x bar.sh
+#./bar.sh