From 0f53f01542a54ab593b2ad9c4f87bbeabcc9d6fe Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sat, 15 Feb 2020 00:43:40 -0500 Subject: [PATCH 01/98] [Harness] Use fullname for test-case in the xunit xslt. After reading the code of the parser used in VSTS I found out that name is not used, but fullname. That attribute is the one used in their xml parser to set the 'testCaseResultData.AutomatedTestName'. --- .../templates/common/TestRunner.xUnit/NUnit3Xml.xslt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/bcl-test/templates/common/TestRunner.xUnit/NUnit3Xml.xslt b/tests/bcl-test/templates/common/TestRunner.xUnit/NUnit3Xml.xslt index d9142b35923d..b264628fe113 100644 --- a/tests/bcl-test/templates/common/TestRunner.xUnit/NUnit3Xml.xslt +++ b/tests/bcl-test/templates/common/TestRunner.xUnit/NUnit3Xml.xslt @@ -7,7 +7,7 @@ - + @@ -147,6 +147,9 @@ + + + From 1495d6478d1e7788653808b8c3e11db83e396ad2 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sat, 15 Feb 2020 01:15:43 -0500 Subject: [PATCH 02/98] [Devops] Move to a 'matrix' pipeline to improve the device tests feedback Our bots spend most of the time compiling and deploying rather than executing tests. This means that a bot with an iOS device is most of the time not really running tests, but deploying them. With the old pipeline the average runtime to get feedback for all the device tests is around 6 to 7 hours. In order to improve the speed in which we get results the new pipeline spawns 5 different testing jobs and a last 6th job that is a fan in to report the completion to the monitoring person. The tests have been divided in the following way: 1. Xamarin tests, wich are all those tests that are not bcl tests or monotouch. 2. Monotouch tests, since it is a large set of tests. 3. All xUnit based BCL tests. 4. All NUnit based BCL tests. 5. mscorlib tests (all 3 splitted assemblies). In average the jobs individually take the following times: 1. Xamarin tests: 1h 30m 2. Monotouch tests: 45/50m 3. xUnit based BCL: 1h 30m 4. NUnit based BCL: 45 m 5. mscorlib 1h 30m In a perfect world with infinite bots, we can get device test results in under 2 hours, but in reality we might be getting around the 3 hours, which is half of the time atm. In order to make things easier for the future different templates in yml have been created under tools/devops/templates - tools/devops/templates/device-tests.yml Basic template that contains all the required steps needed to run the tests on device. This template takes two env vars. One which will contains the extra labels to pass to xharness and the context to be used to set the device tests status. - tools/devops/templates/job-matrix.yml Main template that allows to create pipelines with all the 5 jobs targeting different devices. The template takes a numer of parameters to set the targetd device and the capabilities of the agents. - tools/devops/templates/publish-results.yml Small helper template for posting messages. Once the 5 jobs are completed, a 6th job is kicked that does not need an iOS/tvOS device (we have more agents without capabilities). The jobs grabs the artifacts from the previous jos, unzipts them and performs the publication of the test results to vsts. Once completed it reports the result in a status to github. This job will execute no matter what were the results of the previous jobs (notice the always() condition). --- jenkins/vsts-device-tests-set-status.sh | 2 +- tests/Makefile | 2 +- tests/xharness/AppRunner.cs | 3 + tests/xharness/Jenkins.cs | 36 +++- tests/xharness/TestProject.cs | 8 + tools/devops/beta-device-tests.yml | 19 ++ tools/devops/cleanup.sh | 6 - tools/devops/device-tests-common.yml | 197 --------------------- tools/devops/device-tests.yml | 17 +- tools/devops/templates/beta-stages.yml | 76 ++++++++ tools/devops/templates/device-tests.yml | 159 +++++++++++++++++ tools/devops/templates/job-matrix.yml | 77 ++++++++ tools/devops/templates/publish-results.yml | 36 ++++ tools/devops/templates/stable-stages.yml | 95 ++++++++++ 14 files changed, 516 insertions(+), 217 deletions(-) create mode 100644 tools/devops/beta-device-tests.yml delete mode 100755 tools/devops/cleanup.sh delete mode 100644 tools/devops/device-tests-common.yml create mode 100644 tools/devops/templates/beta-stages.yml create mode 100644 tools/devops/templates/device-tests.yml create mode 100644 tools/devops/templates/job-matrix.yml create mode 100644 tools/devops/templates/publish-results.yml create mode 100644 tools/devops/templates/stable-stages.yml diff --git a/jenkins/vsts-device-tests-set-status.sh b/jenkins/vsts-device-tests-set-status.sh index afb69830fe1f..9ed60bf6d7b5 100755 --- a/jenkins/vsts-device-tests-set-status.sh +++ b/jenkins/vsts-device-tests-set-status.sh @@ -89,7 +89,7 @@ if test -z "$START"; then printf "%s%s on [Azure DevOps](%s)($DEVICE_TYPE)%s %s\\n\\n" "$RESULT_EMOJII" "$DESCRIPTION" "$VSTS_BUILD_URL" "$HTML_REPORT" "$RESULT_EMOJII" >> "$MESSAGE_FILE" - FILE=$PWD/tests/TestSummary.md + FILE=$PWD/TestSummary.md if ! test -f "$FILE"; then printf "🔥 Tests failed catastrophically on $DEVICE_TYPE (no summary found)\\n" >> "$MESSAGE_FILE" else diff --git a/tests/Makefile b/tests/Makefile index e13dc2b7218b..d875998baadf 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -382,7 +382,7 @@ runner: xharness/xharness.exe # installed on the system. vsts-device-tests: xharness/xharness.exe $(MAKE) -C $(TOP)/builds .stamp-mono-ios-sdk-destdir download -j - $(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --autoconf --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests,run-bcl-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND) + $(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --autoconf --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND) ifdef ENABLE_XAMARIN wrench-launch-external wrench-report-external: diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index c84f326ee0c4..fbd1cf056b68 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -373,6 +373,9 @@ public bool EnsureCleanSimulatorState { logs.AddRange (Directory.GetFiles (Logs.Directory)); logs.AddRange (Directory.GetFiles (BuildTask.LogDirectory)); // add the attachments and write in the new filename + // add a final prefix to the file name to make sure that the VSTS test uploaded just pick + // the final version, else we will upload tests more than once + newFilename = Path.Combine (Path.GetDirectoryName (newFilename), $"vsts-{Path.GetFileName (newFilename)}"); XmlResultParser.UpdateMissingData (path, newFilename, testRunName, logs); } else { // rename the path to the correct value diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index 3463e02b4c39..289b3194d559 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -40,6 +40,11 @@ public class Jenkins public bool IncludeXtro; public bool IncludeCecil; public bool IncludeDocs; + public bool IncludeNewBCL; + public bool IncludeOldBCL; + public bool IncludeMscorlib; + public bool IncludeXamarin = true; + public bool IncludeMonotouch = true; public bool CleanSuccessfulTestRuns = true; public bool UninstallTestApp = true; @@ -219,13 +224,31 @@ bool IsIncluded (TestProject project) { if (!project.IsExecutableProject) return false; + + if (project.IsBclTest) { + // logic is not that hard, lets start with the old blc, that is easy, it returns true + // if IncludeBCL || IncludeOldBCL + if (!project.IsNewBclTest) + return IncludeBcl || IncludeOldBCL; + // we now have to deal with the new bcls, that include the mscorlib variation, that + // is NEW && can be ignored, so lets split between all and the special case + if (project.IsMscorlib) { // not need to check if new since we check above + return IncludeMscorlib; + } else { + // simple case, is in if IncludeBCL || IncludeNew + return IncludeBcl || IncludeNewBCL; + } + + } - if (!IncludeBcl && project.IsBclTest) + if (IncludeMonotouch && project.IsMonotouch) + return true; + + if (!IncludeXamarin && !project.IsBclTest) return false; if (Harness.IncludeSystemPermissionTests == false && project.Name == "introspection") return false; - return true; } @@ -625,7 +648,8 @@ Task> CreateRunDeviceTasksAsync () foreach (var task in projectTasks) { task.TimeoutMultiplier = project.TimeoutMultiplier; task.BuildOnly |= project.BuildOnly; - task.Ignored |= ignored; + if (!task.Ignored && ignored) + task.Ignored = true; } rv.AddRange (projectTasks); } @@ -799,6 +823,9 @@ void SelectTestsByLabel (int pull_request) SetEnabled (labels, "mtouch", ref IncludeMtouch); SetEnabled (labels, "mmp", ref IncludeMmpTest); SetEnabled (labels, "bcl", ref IncludeBcl); + SetEnabled (labels, "new-bcl", ref IncludeNewBCL); + SetEnabled (labels, "old-bcl", ref IncludeOldBCL); + SetEnabled (labels, "mscorlib", ref IncludeMscorlib); SetEnabled (labels, "btouch", ref IncludeBtouch); SetEnabled (labels, "mac-binding-project", ref IncludeMacBindingProject); SetEnabled (labels, "ios-extensions", ref IncludeiOSExtensions); @@ -817,6 +844,9 @@ void SelectTestsByLabel (int pull_request) SetEnabled (labels, "mac", ref IncludeMac); SetEnabled (labels, "ios-msbuild", ref IncludeiOSMSBuild); SetEnabled (labels, "ios-simulator", ref IncludeSimulator); + SetEnabled (labels, "xamarin", ref IncludeXamarin); + SetEnabled (labels, "monotouch", ref IncludeMonotouch); + bool inc_permission_tests = false; if (SetEnabled (labels, "system-permission", ref inc_permission_tests)) Harness.IncludeSystemPermissionTests = inc_permission_tests; diff --git a/tests/xharness/TestProject.cs b/tests/xharness/TestProject.cs index b8d46c04e5db..3978c04957ba 100644 --- a/tests/xharness/TestProject.cs +++ b/tests/xharness/TestProject.cs @@ -94,6 +94,14 @@ public bool IsBclTest { } } + public bool IsMonotouch => Name.Contains ("monotouch"); + + public bool IsNewBclTest => IsBclTest && (Name.Contains ("xUnit") || IsMscorlib); + + + public bool IsMscorlib => Name.Contains ("mscorlib"); + + public virtual TestProject Clone () { TestProject rv = (TestProject) Activator.CreateInstance (GetType ()); diff --git a/tools/devops/beta-device-tests.yml b/tools/devops/beta-device-tests.yml new file mode 100644 index 000000000000..c1e84b3c7928 --- /dev/null +++ b/tools/devops/beta-device-tests.yml @@ -0,0 +1,19 @@ +# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending +# on the configuration to run different device tests. +# +# lame! cannot have the resources in the template.. :/ +resources: + repositories: + - repository: maccore + type: github + name: xamarin/maccore + ref: refs/heads/device-tests-yaml # should this be on a branch and not on master??? + endpoint: xamarin + +# create parallel stages, this means that one single trigger to the pipeline will execute tvos and ios device tests +stages: +- template: templates/beta-stages.yml # all the smart stuff happens in the template, just needed to set the params + parameters: + pool: 'VSEng-Xamarin-Mac-Devices' + timeout: 1000 + maxParallel: 5 diff --git a/tools/devops/cleanup.sh b/tools/devops/cleanup.sh deleted file mode 100755 index 090302f67b42..000000000000 --- a/tools/devops/cleanup.sh +++ /dev/null @@ -1,6 +0,0 @@ -sudo rm -rf /Applications/Visual\ Studio* -rm -rf ~/Library/Caches/VisualStudio -rm -rf ~/Library/Logs/VisualStudio -rm -rf ~/Library/VisualStudio -rm -rf ~/Library/Preferences/Xamarin/ -rm -rf ~/Library/Caches/com.xamarin.provisionator \ No newline at end of file diff --git a/tools/devops/device-tests-common.yml b/tools/devops/device-tests-common.yml deleted file mode 100644 index ef8bd716bc71..000000000000 --- a/tools/devops/device-tests-common.yml +++ /dev/null @@ -1,197 +0,0 @@ -# Xamarin -# -# The following template contains all the different steps that can be used to create a device tests pipeline in -# VSTS. The template accepts the following parameters: -# -# statusContext: Contains the context that will be used to set a status in a commit. The status can be used to indentify -# the source of the status, for example, iOS device tests, tvOS device tests, CI, etc.. -# testsLabels: Contains the labels that are going to be passed to the test runner to device which of the tests to be ran, that means -# that the caller can configure which tests to call, for example, just the ios 64b device tests. -# demands: The capabilities that the bots need. - -parameters: - -- name: name - type: string - default: 'macOS' # default - -- name: displayName - type: string - default: 'Run Device Tests' # default - -- name: pool - type: string - default: 'VSEng-Xamarin-Mac-Devices' # default - -- name: statusContext - type: string - default: 'iOS-DDFun' # default context, since we started dealing with iOS devices. - -- name: testsLabels - type: string - default: '--label=run-ios-64-tests' # default context, since we started dealing with iOS devices. - -- name: demands - type: string - default: 'ios' # default context, since we started dealing with iOS devices. - -jobs: -- job: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - timeoutInMinutes: 1000 - - pool: - name: ${{ parameters.pool }} - demands: ${{ parameters.demands }} - - steps: - - - checkout: self - - checkout: maccore - persistCredentials: true - - - bash: ./xamarin-macios/tools/devops/system-info.sh - displayName: System info - - - bash: cd xamarin-macios && ./configure --enable-xamarin - displayName: 'Enable Xamarin' - - - bash: | - set -x - set -e - - rm -f ~/Library/Caches/com.xamarin.provisionator/Provisions/*p12 - rm -f ~/Library/Caches/com.xamarin.provisionator/Provisions/*mobileprovision - - ./maccore/tools/install-qa-provisioning-profiles.sh -v - displayName: 'Add provisioning profiles' - env: - LOGIN_KEYCHAIN_PASSWORD: $(OSX_KEYCHAIN_PASS) - - - bash: | - set -x - set -e - - CONFIGURE_ARGS="--provision-xcode --provision-xamarin-studio --provision-mono --ignore-osx --ignore-autotools --ignore-cmake" - if grep ignore-7z xamarin-macios/system-dependencies.sh 2>&1 > /dev/null; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --provision-7z" - fi - if grep ignore-python3 xamarin-macios/system-dependencies.sh 2>&1 > /dev/null; then - CONFIGURE_ARGS="$CONFIGURE_ARGS --ignore-python3" - fi - ./xamarin-macios/system-dependencies.sh $CONFIGURE_ARGS - displayName: 'Provision dependencies' - timeoutInMinutes: 240 - - - bash: ./xamarin-macios/tools/devops/install-xi-pkg.sh - displayName: 'Install XI PKG' - timeoutInMinutes: 15 - - - bash: | - set -x - set +e # Don't care about errors here - - pkill -9 mlaunch - pkill -9 -f mono.*xharness.exe - pkill -9 -f ssh.*rsync.*xamarin-storage - - exit 0 - displayName: 'Process cleanup' - - - bash: | - security set-key-partition-list -S apple-tool:,apple: -s -k $(OSX_KEYCHAIN_PASS) login.keychain - displayName: 'Remove security UI-prompt (http://stackoverflow.com/a/40039594/183422)' - condition: succeededOrFailed() - - - bash: | - set -e - set -x - echo 1234567890 > ~/.mlaunch-verbosity - displayName: 'Make mlaunch verbose' - - - bash: | - set -e - set -x - sudo launchctl stop com.apple.usbmuxd - displayName: 'Fix device discovery (reset launchctl)' - - ### - ### Tell GitHub we're starting working on this commit - ### - - - bash: | - ./xamarin-macios/jenkins/vsts-device-tests-set-status.sh --start "--token=$(GitHub.Token)" "--device=$CONTEXT" - env: - CONTEXT: ${{ parameters.statusContext }} - displayName: Set pending GitHub status - continueOnError: true - condition: succeededOrFailed() - - ### - ### Run the device tests - ### - - - bash: ./xamarin-macios/tools/devops/run-tests.sh - env: - TESTS_EXTRA_ARGUMENTS: ${{ parameters.testsLabels }} - displayName: 'Run tests' - timeoutInMinutes: 600 - - ### - ### Report final results to GitHub - ### - - - bash: ./xamarin-macios/jenkins/vsts-device-tests-set-status.sh "--token=$(GitHub.Token)" "--device=$CONTEXT" - env: - CONTEXT: ${{ parameters.statusContext }} - displayName: Report results to GitHub as comment / status - continueOnError: true - condition: succeededOrFailed() - - - bash: ./xamarin-macios/tools/devops/add-summaries.sh - displayName: 'Add summaries' - continueOnError: true - condition: succeededOrFailed() - - - task: ArchiveFiles@1 - displayName: 'Archive HtmlReport' - inputs: - rootFolder: 'xamarin-macios/jenkins-results' - includeRootFolder: false - archiveFile: '$(Build.ArtifactStagingDirectory)/HtmlReport-$(Build.BuildId).zip' - continueOnError: true - condition: succeededOrFailed() - - ### - ### Upload the xml results to vsts. We have two types, nunit and xunit. We want both - ### - - - task: PublishTestResults@2 - displayName: 'Publish NUnit Device Test Results' - inputs: - testResultsFormat: NUnit - testResultsFiles: '**/nunit-test-*.xml' - failTaskOnFailedTests: true - continueOnError: true - condition: succeededOrFailed() - - ### - ### Push the HTML report to Azure DevOps (shows up in Summary tab as Build Artifact) - ### - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: HtmlReport' - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/HtmlReport-$(Build.BuildId).zip' - artifactName: HtmlReport - continueOnError: true - condition: succeededOrFailed() - - ### - ### Cleanup after us, not having that can lead to VSMac install issues - ### - - - bash: ./xamarin-macios/tools/devops/cleanup.sh - displayName: 'Cleanup' - continueOnError: true - condition: succeededOrFailed() diff --git a/tools/devops/device-tests.yml b/tools/devops/device-tests.yml index 93c601caeba1..38bfdc77cb1e 100644 --- a/tools/devops/device-tests.yml +++ b/tools/devops/device-tests.yml @@ -1,5 +1,6 @@ -# iOS device pipeline, this file sets the different parameters required by the device-tests-common-yml, which contains all the steps and the interesting logic. - +# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending +# on the configuration to run different device tests. +# # lame! cannot have the resources in the template.. :/ resources: repositories: @@ -9,12 +10,10 @@ resources: ref: refs/heads/device-tests-yaml # should this be on a branch and not on master??? endpoint: xamarin -jobs: -- template: device-tests-common.yml # all the smart stuff happens in the template, just needed to set the params +# create parallel stages, this means that one single trigger to the pipeline will execute tvos and ios device tests +stages: +- template: templates/stable-stages.yml # all the smart stuff happens in the template, just needed to set the params parameters: - name: 'macOS' - displayName: 'Run iOS Device Tests' pool: 'VSEng-Xamarin-Mac-Devices' - demands: 'ios' - statusContext: 'iOS-DDFun' - testsLabels: '--label=run-ios-64-tests' + timeout: 1000 + maxParallel: 5 diff --git a/tools/devops/templates/beta-stages.yml b/tools/devops/templates/beta-stages.yml new file mode 100644 index 000000000000..e1955e38dfbc --- /dev/null +++ b/tools/devops/templates/beta-stages.yml @@ -0,0 +1,76 @@ +# Template that contains the different stages to be executed in a pipeline +# +# The tamplate contains the following parameters +# +# include32b: Run tests on iOS 32b devices. +# useBetaDevices: Use beta devices instead of stable ones. +# pool: The pool/lab to use +# timeout: Timeout used in the different jobs. +# maxParallel: Max number of parallel jobs to do in a stage. +# +parameters: + +# set demands as possible parameters because we keep changing them from lab to lab :/ +- name: iOSBetaDemand + type: string + default: 'xismoke-beta' # old QA capability + +- name: tvOSBetaDemand + type: string + default: 'xitvos-beta' # old QA capability + +- name: pool + type: string + default: 'VSEng-Xamarin-Mac-Devices' # default ddfun pool + +- name: timeout + type: number + default: 1000 + +- name: maxParallel + type: number + default: 5 + +stages: +- stage: iOSDeviceTests + displayName: 'iOS Device Tests' + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel + jobs: + - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params + parameters: + demands: ${{ parameters.iOSBetaDemand }} + deviceLabel: 'run-ios-64-tests' + variation: "iOS" + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel}} + +- stage: tvOSDeviceTests + displayName: 'tvOS Device Tests' + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel + jobs: + - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params + parameters: + demands: ${{ parameters.tvOSBetaDemand }} + deviceLabel: 'run-tvos-tests' + variation: "tvOS" + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel }} + +- stage: monitoring + displayName: 'Device test monitoring' + dependsOn: + - iOSDeviceTests + - tvOSDeviceTests + + condition: always() # always notify of what happened. + + pool: + name: ${{ parameters.pool }} + demands: 'ios' # we should not have demands here, but the problem is that DDFun does not set all machines to be able to checout private repos :/ + + jobs: + - job: + steps: + - template: publish-results.yml # template contains the steps to get all test reports and upload them diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml new file mode 100644 index 000000000000..5b42443a1d1d --- /dev/null +++ b/tools/devops/templates/device-tests.yml @@ -0,0 +1,159 @@ +# Xamarin +# +# The following template contains all the different steps that can be used to create a device tests pipeline in +# VSTS. The template accepts the following parameters: +# +# demands: The capabilities that the bots need. +# +# Because we are using a matrix, the following env vars should be set in it: +# +# CONTEXT: The context that is used to set the status of the commit in github. +# TESTS_EXTRA_ARGUMENTS: the extra arguments passed to the runner to decide which tests need to be ran. + +steps: + +- checkout: self +- checkout: maccore + persistCredentials: true + +- bash: ./xamarin-macios/tools/devops/system-info.sh + displayName: System info + +- bash: cd xamarin-macios && ./configure --enable-xamarin + displayName: 'Enable Xamarin' + +- bash: | + set -x + set -e + + rm -f ~/Library/Caches/com.xamarin.provisionator/Provisions/*p12 + rm -f ~/Library/Caches/com.xamarin.provisionator/Provisions/*mobileprovision + + ./maccore/tools/install-qa-provisioning-profiles.sh -v + displayName: 'Add provisioning profiles' + env: + LOGIN_KEYCHAIN_PASSWORD: $(OSX_KEYCHAIN_PASS) + +- bash: | + set -x + set -e + + CONFIGURE_ARGS="--provision-xcode --provision-xamarin-studio --provision-mono --ignore-osx --ignore-autotools --ignore-cmake" + if grep ignore-7z xamarin-macios/system-dependencies.sh 2>&1 > /dev/null; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --provision-7z" + fi + if grep ignore-python3 xamarin-macios/system-dependencies.sh 2>&1 > /dev/null; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --ignore-python3" + fi + ./xamarin-macios/system-dependencies.sh $CONFIGURE_ARGS + if [ $? -eq 0 ]; then + echo "##vso[task.setvariable variable=ProvisioningProfiles]success" + else + echo "##vso[task.setvariable variable=ProvisioningProfiles]failure" + fi + displayName: 'Provision dependencies' + timeoutInMinutes: 240 + +- bash: ./xamarin-macios/tools/devops/install-xi-pkg.sh + displayName: 'Install XI PKG' + timeoutInMinutes: 15 + +- bash: | + set -x + set +e # Don't care about errors here + + pkill -9 mlaunch + pkill -9 -f mono.*xharness.exe + pkill -9 -f ssh.*rsync.*xamarin-storage + + exit 0 + displayName: 'Process cleanup' + +- bash: | + security set-key-partition-list -S apple-tool:,apple: -s -k $(OSX_KEYCHAIN_PASS) login.keychain + displayName: 'Remove security UI-prompt (http://stackoverflow.com/a/40039594/183422)' + condition: succeededOrFailed() + +- bash: | + set -e + set -x + echo 1234567890 > ~/.mlaunch-verbosity + displayName: 'Make mlaunch verbose' + +- bash: | + set -e + set -x + sudo launchctl stop com.apple.usbmuxd + displayName: 'Fix device discovery (reset launchctl)' + +### +### Run the device tests +### + +- bash: ./xamarin-macios/tools/devops/run-tests.sh + displayName: 'Run tests' + timeoutInMinutes: 600 + + +- bash: ./xamarin-macios/tools/devops/add-summaries.sh + displayName: 'Add summaries' + continueOnError: true + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + +- task: ArchiveFiles@1 + displayName: 'Archive HtmlReport' + inputs: + rootFolder: 'xamarin-macios/jenkins-results' + includeRootFolder: false + archiveFile: '$(Build.ArtifactStagingDirectory)/HtmlReport-$(Build.BuildId).zip' + continueOnError: true + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + +### +### Upload the xml results to vsts. We have two types, nunit and xunit. We want both +### + +- task: PublishTestResults@2 + displayName: 'Publish NUnit Device Test Results' + inputs: + testResultsFormat: NUnit + testResultsFiles: '**/vsts-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Test-Results-$(CONTEXT)-$(TESTS)' + continueOnError: true + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + +### +### Push the HTML report to Azure DevOps (shows up in Summary tab as Build Artifact) +### + +- task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: HtmlReport' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/HtmlReport-$(Build.BuildId).zip' + artifactName: HtmlReport-$(CONTEXT) + continueOnError: true + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + +- task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: TestSummary' + inputs: + targetPath: 'xamarin-macios/tests/TestSummary.md' + artifactName: TestSummary-$(CONTEXT) + continueOnError: true + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + +### +### Cleanup after us, not having that can lead to VSMac install issues +### + +- bash: | + sudo rm -rf /Applications/Visual\ Studio* + rm -rf ~/Library/Caches/VisualStudio + rm -rf ~/Library/Logs/VisualStudio + rm -rf ~/Library/VisualStudio + rm -rf ~/Library/Preferences/Xamarin/ + rm -rf ~/Library/Caches/com.xamarin.provisionator + displayName: 'Cleanup' + continueOnError: true + condition: always() # no matter what, includes cancellation diff --git a/tools/devops/templates/job-matrix.yml b/tools/devops/templates/job-matrix.yml new file mode 100644 index 000000000000..15ea14d19a8d --- /dev/null +++ b/tools/devops/templates/job-matrix.yml @@ -0,0 +1,77 @@ +# Common template that contains a matrix definition of the different jobs to be performed for device testing +# in xamarin-macios. The template defines a number of jobs that contains the project tests devided in groups, +# those are executed by different jobs in parallel (depending on agent availability). +# +# The template takes the following parameters: +# +# variation: Contains the string that will be used to identify the variation of the jobs executed. This is used to update the display names of the different jobs. +# pool: The agent pool to be used. +# demands: Contains the demands that the agents should have. This is used to state if the pipeline requires and iOS device, tvOS etc.. +# deviceLabel: Provide the device label used by xharness to decide what device type should be used in the tests execution. +# timeout: The timeout of the jobs in minutes. +# maxParallel: The amount of parallel jobs to be used + +parameters: +- name: variation + type: string + default: iOS + +- name: pool + type: string + default: 'VSEng-Xamarin-Mac-Devices' # default ddfun pool + +- name: demands + type: string + default: 'ios' # we default to ios devices + +- name: deviceLabel + type: string + default: 'run-ios-64-tests' # default to iOS 64b devices + +- name: timeout + type: number + default: 1000 + +- name: maxParallel + type: number + default: 5 + +jobs: + +### +### Run all the xamarin tests, not bcl or mscorlib +### + +- job: DDFun + displayName: 'Run ${{ parameters.variation }} Device Tests' + timeoutInMinutes: ${{ parameters.timeout }} + pool: + name: ${{ parameters.pool }} + demands: ${{ parameters.demands }} + + strategy: + maxParallel: ${{ parameters.maxParallel }} + matrix: + Xamarin: + CONTEXT: '${{ parameters.variation }}' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-xamarin-tests' # run all tests but the bcl ones + TESTS: 'Xamarin' + Monotouch: + CONTEXT: '${{ parameters.variation }}-monotouch-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-monotouch-tests' # run all tests but the bcl ones + TESTS: 'Monotouch' + xUnitBCL: + CONTEXT: '${{ parameters.variation }}-xUnit-BCL-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-new-bcl-tests,skip-mscorlib-tests' # run only the new bcl tests + TESTS: 'xUnitBCL' + nUnitBCL: + CONTEXT: '${{ parameters.variation }}-NUnit-BCL-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-old-bcl-tests' # run only the new bcl tests + TESTS: 'NUnitBCL' + Mscorlib: + CONTEXT: '${{ parameters.variation }}-mscorlib-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-mscorlib-tests' # run only the new bcl tests + TESTS: 'mscorlib' + + steps: + - template: device-tests.yml # all the smart stuff happens in the template, just needed to set the params diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml new file mode 100644 index 000000000000..29cf05c79dec --- /dev/null +++ b/tools/devops/templates/publish-results.yml @@ -0,0 +1,36 @@ +# Xamarin +# +# The following templates contains all the different steps that can be used to consume the tests results from a +# matrix job and will unzip the pipeline artifacts and upload the results to VSTS +# +# The template takes the following env vars +# +# CONTEXT: The context that is used to set the status of the commit. + +steps: +### +### Download all test summaries +### + +- task: DownloadPipelineArtifact@2 + displayName: Download test summaries + inputs: + patterns: '**/TestSummary*' + allowFailedBuilds: true + path: $(Pipeline.Workspace)/Summaries + +- bash: | + set -x + set -e + ls -la + cat $PIPELINE_WORKSPACE/Summaries/*.md >> ./TestSummary.md + displayName: 'Merge TestSummary files' + +### +### Report final results to GitHub +### + +- bash: ./jenkins/vsts-device-tests-set-status.sh "--token=$(GitHub.Token)" --device="VSTS-DDFun" + displayName: Report results to GitHub as comment / status + continueOnError: true + condition: succeededOrFailed() diff --git a/tools/devops/templates/stable-stages.yml b/tools/devops/templates/stable-stages.yml new file mode 100644 index 000000000000..041c9481e5cf --- /dev/null +++ b/tools/devops/templates/stable-stages.yml @@ -0,0 +1,95 @@ +# Template that contains the different stages to be executed in a pipeline +# +# The tamplate contains the following parameters +# +# include32b: Run tests on iOS 32b devices. +# useBetaDevices: Use beta devices instead of stable ones. +# pool: The pool/lab to use +# timeout: Timeout used in the different jobs. +# maxParallel: Max number of parallel jobs to do in a stage. +# +parameters: + +# set demands as possible parameters because we keep changing them from lab to lab :/ +- name: iOSDemand + type: string + default: 'ios' + +- name: iOS32Demand + type: string + default: 'xismoke-32' + +- name: tvOSDemand + type: string + default: 'tvos' + +- name: pool + type: string + default: 'VSEng-Xamarin-Mac-Devices' # default ddfun pool + +- name: timeout + type: number + default: 1000 + +- name: maxParallel + type: number + default: 5 + +stages: +- stage: iOSDeviceTests + displayName: 'iOS Device Tests' + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel + jobs: + - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params + parameters: + demands: ${{ parameters.iOSDemand }} + deviceLabel: 'run-ios-64-tests' + variation: "iOS" + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel}} + +- stage: tvOSDeviceTests + displayName: 'tvOS Device Tests' + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel + jobs: + - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params + parameters: + demands: ${{ parameters.tvOSDemand }} + deviceLabel: 'run-tvos-tests' + variation: "tvOS" + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel }} + +# Wait for the lab to have devices, templates do not let me do what I want with expressions :/ +#- stage: iOS32DeviceTests +# displayName: 'iOS32b Device Tests' +# dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel +# jobs: +# - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params +# parameters: +# demands: ${{ parameters.iOS32Demand }} # old QA lab capability, might change +# deviceLabel: 'run-ios-32-tests' +# variation: "iOS32b" +# pool: ${{ parameters.pool }} +# timeout: ${{ parameters.timeout }} +# maxParallel: ${{ parameters.maxParallel }} + +- stage: monitoring + displayName: 'Device test monitoring' + dependsOn: + - iOSDeviceTests + - tvOSDeviceTests +# - iOS32DeviceTests + + condition: always() # always notify of what happened. + + pool: + name: ${{ parameters.pool }} + demands: 'ios' # we should not have demands here, but the problem is that DDFun does not set all machines to be able to checout private repos :/ + + jobs: + - job: + steps: + - template: publish-results.yml # template contains the steps to get all test reports and upload them From c3d847dd020563c918bc883a38408a396f1d6e4c Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 17 Feb 2020 23:32:35 -0500 Subject: [PATCH 03/98] [Harness] Create an xml crash file when we do not have a result file. On devices that cannot reach the host via TCP we do not have a log, this means that in the if statement needs to have a case for it. The main problem is that when the device cannot connect to the host, we do not get a log OR a crash reason from the crash logs. It makes sense not to have a crash reason, because the app did not crash. In these sitations, we have to create a xml crash report (since we really do not know if we can parse the file) that will tell vsts that there was an issue. Adding the main log will let the monitoring person see the results of the test run. --- tests/xharness/AppRunner.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 3f3d7c51d2e0..b9bb76d3e73c 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -887,6 +887,11 @@ public async Task RunAsync () FailureMessage = $"Launch failure"; if (Harness.InCI) XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, "AppLaunch", FailureMessage, main_log.FullPath, XmlResultParser.Jargon.NUnitV3); + } else if (!File.Exists (listener_log.FullPath) && Harness.InCI) { + // this happens more that what we would like on devices, the main reason most of the time is that we have had netwoking problems and the + // tcp connection could not be stablished. We are going to report it as an error since we have not parsed the logs, evne when the app might have + // not crashed. + XmlResultParser.GenerateFailure (Logs, "tcp-connection", appName, Variation, "TcpConnection", "Device could not reach the host over tcp.", main_log.FullPath, XmlResultParser.Jargon.NUnitV3); } } From cb84dfdf485d68668baddba03c79ba45dbc8c0ed Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 18 Feb 2020 15:29:35 -0500 Subject: [PATCH 04/98] Fix logic for test selection using labels. --- tests/xharness/Jenkins.cs | 21 ++++++--------------- tools/devops/templates/job-matrix.yml | 2 +- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index 0e1721efac63..7227c746408f 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -226,25 +226,17 @@ bool IsIncluded (TestProject project) return false; if (project.IsBclTest) { - // logic is not that hard, lets start with the old blc, that is easy, it returns true - // if IncludeBCL || IncludeOldBCL if (!project.IsNewBclTest) return IncludeBcl || IncludeOldBCL; - // we now have to deal with the new bcls, that include the mscorlib variation, that - // is NEW && can be ignored, so lets split between all and the special case - if (project.IsMscorlib) { // not need to check if new since we check above + if (project.IsMscorlib) return IncludeMscorlib; - } else { - // simple case, is in if IncludeBCL || IncludeNew - return IncludeBcl || IncludeNewBCL; - } - + return IncludeBcl || IncludeNewBCL; } - if (IncludeMonotouch && project.IsMonotouch) - return true; + if (!IncludeMonotouch && project.IsMonotouch) + return false; - if (!IncludeXamarin && !project.IsBclTest) + if (!IncludeXamarin && !project.IsBclTest && !project.IsMonotouch) return false; if (Harness.IncludeSystemPermissionTests == false && project.Name == "introspection") @@ -648,8 +640,7 @@ Task> CreateRunDeviceTasksAsync () foreach (var task in projectTasks) { task.TimeoutMultiplier = project.TimeoutMultiplier; task.BuildOnly |= project.BuildOnly; - if (!task.Ignored && ignored) - task.Ignored = true; + task.Ignored |= ignored; } rv.AddRange (projectTasks); } diff --git a/tools/devops/templates/job-matrix.yml b/tools/devops/templates/job-matrix.yml index 15ea14d19a8d..fa67f9c3a5ac 100644 --- a/tools/devops/templates/job-matrix.yml +++ b/tools/devops/templates/job-matrix.yml @@ -62,7 +62,7 @@ jobs: TESTS: 'Monotouch' xUnitBCL: CONTEXT: '${{ parameters.variation }}-xUnit-BCL-DDFun' - TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-new-bcl-tests,skip-mscorlib-tests' # run only the new bcl tests + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-new-bcl-tests' # run only the new bcl tests TESTS: 'xUnitBCL' nUnitBCL: CONTEXT: '${{ parameters.variation }}-NUnit-BCL-DDFun' From 101a980641c1efd9d11b96d924263adaa999f62f Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 18 Feb 2020 09:15:18 -0500 Subject: [PATCH 05/98] [VSTS] Correct order of params for make. (#7927) --- tools/devops/run-tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 294cfbf86601..dc222122c58c 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -6,6 +6,6 @@ if [[ "$SYSTEM_DEBUG" == "true" ]]; then else DEBUG="" fi -make -C $DEBUG xamarin-macios/builds download -j -make -C $DEBUG xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j -make -C $DEBUG xamarin-macios/tests vsts-device-tests +make $DEBUG -C xamarin-macios/builds download -j +make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j +make $DEBUG -C xamarin-macios/tests vsts-device-tests From 804f24200e84c46dd33c3e889c9d51ec6c506366 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 00:05:20 -0500 Subject: [PATCH 06/98] Do not autoconfig xharness, we 'know' better. --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index d875998baadf..749f2935d57d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -382,7 +382,7 @@ runner: xharness/xharness.exe # installed on the system. vsts-device-tests: xharness/xharness.exe $(MAKE) -C $(TOP)/builds .stamp-mono-ios-sdk-destdir download -j - $(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --autoconf --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND) + $(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND) ifdef ENABLE_XAMARIN wrench-launch-external wrench-report-external: From 318f597c27e8b43167f85f7d70586be0fed7f2f2 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 00:05:34 -0500 Subject: [PATCH 07/98] Also add harness logs. --- tests/xharness/AppRunner.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index b9bb76d3e73c..8c9da87851ca 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -372,6 +372,7 @@ public bool EnsureCleanSimulatorState { // add our logs AND the logs of the previous task, which is the build task logs.AddRange (Directory.GetFiles (Logs.Directory)); logs.AddRange (Directory.GetFiles (BuildTask.LogDirectory)); + logs.Add (Harness.LogFile); // add the attachments and write in the new filename // add a final prefix to the file name to make sure that the VSTS test uploaded just pick // the final version, else we will upload tests more than once From 09c0d5675225f1303a6eadf4a503bff952224b0f Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 00:14:41 -0500 Subject: [PATCH 08/98] Revert "Do not autoconfig xharness, we 'know' better." This reverts commit 804f24200e84c46dd33c3e889c9d51ec6c506366. --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 749f2935d57d..d875998baadf 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -382,7 +382,7 @@ runner: xharness/xharness.exe # installed on the system. vsts-device-tests: xharness/xharness.exe $(MAKE) -C $(TOP)/builds .stamp-mono-ios-sdk-destdir download -j - $(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND) + $(Q) ulimit -n 4096 && $(SYSTEM_MONO) --debug $(CURDIR)/$< $(XHARNESS_VERBOSITY) --jenkins --autoconf --rootdir $(CURDIR) --sdkroot $(XCODE_DEVELOPER_ROOT) --use-system:true --label=skip-all-tests,run-device-tests --markdown-summary=$(CURDIR)/TestSummary.md $(TESTS_EXTRA_ARGUMENTS) $(TESTS_PERIODIC_COMMAND) ifdef ENABLE_XAMARIN wrench-launch-external wrench-report-external: From 36149a8f8e13c59ed2345d644fa9952efbc2e50b Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 00:16:42 -0500 Subject: [PATCH 09/98] Poor mans debugger. --- tests/xharness/Jenkins.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index 7227c746408f..d65b854147db 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -495,9 +495,11 @@ async Task> CreateRunSimulatorTasksAsync () continue; bool ignored = !IncludeSimulator; + Console.WriteLine (new string ('#', 50)); if (!IsIncluded (project)) ignored = true; - + Console.WriteLine ($"{project.Name} is included? {ignored}"); + Console.WriteLine (new string ('#', 50)); var ps = new List> (); if (!project.SkipiOSVariation) ps.Add (new Tuple (project, TestPlatform.iOS_Unified, ignored || !IncludeiOS64)); @@ -559,10 +561,13 @@ Task> CreateRunDeviceTasksAsync () foreach (var project in Harness.IOSTestProjects) { if (!project.IsExecutableProject) continue; - + + Console.WriteLine (new string ('#', 50)); bool ignored = !IncludeDevice; if (!IsIncluded (project)) ignored = true; + Console.WriteLine ($"{project.Name} is ignored? {ignored}"); + Console.WriteLine (new string ('#', 50)); projectTasks.Clear (); if (!project.SkipiOSVariation) { From 047aed6e92f4696cbff1c52bd4555a59afa5f753 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 09:18:07 -0500 Subject: [PATCH 10/98] Remove CWL, everything worked. --- tests/xharness/Jenkins.cs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index d65b854147db..cd6f842e8564 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -241,6 +241,7 @@ bool IsIncluded (TestProject project) if (Harness.IncludeSystemPermissionTests == false && project.Name == "introspection") return false; + return true; } @@ -495,11 +496,9 @@ async Task> CreateRunSimulatorTasksAsync () continue; bool ignored = !IncludeSimulator; - Console.WriteLine (new string ('#', 50)); if (!IsIncluded (project)) ignored = true; - Console.WriteLine ($"{project.Name} is included? {ignored}"); - Console.WriteLine (new string ('#', 50)); + var ps = new List> (); if (!project.SkipiOSVariation) ps.Add (new Tuple (project, TestPlatform.iOS_Unified, ignored || !IncludeiOS64)); @@ -561,13 +560,10 @@ Task> CreateRunDeviceTasksAsync () foreach (var project in Harness.IOSTestProjects) { if (!project.IsExecutableProject) continue; - - Console.WriteLine (new string ('#', 50)); + bool ignored = !IncludeDevice; if (!IsIncluded (project)) ignored = true; - Console.WriteLine ($"{project.Name} is ignored? {ignored}"); - Console.WriteLine (new string ('#', 50)); projectTasks.Clear (); if (!project.SkipiOSVariation) { From 0fb6d89880f3606cc2f7acf2536ae18cc45855b9 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 22:17:13 -0500 Subject: [PATCH 11/98] [Harness] Close message element correctly in NUnit3. Xml should be: ``` Foo Bar ``` But we generate: ``` Foo Bar ``` Makes the parsing of the failures impossible. --- tests/xharness/XmlResultParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xharness/XmlResultParser.cs b/tests/xharness/XmlResultParser.cs index b4ed55b703b5..fdedc8ee31ce 100644 --- a/tests/xharness/XmlResultParser.cs +++ b/tests/xharness/XmlResultParser.cs @@ -616,12 +616,12 @@ static void WriteFailure (XmlWriter writer, string message, StreamReader stderr writer.WriteStartElement ("failure"); writer.WriteStartElement ("message"); writer.WriteCData (message); + writer.WriteEndElement (); // message if (stderr != null) { writer.WriteStartElement ("stack-trace"); writer.WriteCData (stderr.ReadToEnd ()); writer.WriteEndElement (); //stack trace } - writer.WriteEndElement (); // message writer.WriteEndElement (); // failure } From 13b402fac5b9efd97958d3155b1a4e3bc7e71d61 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Feb 2020 22:48:52 -0500 Subject: [PATCH 12/98] [Harness] Provide device name in those errors where is device specific. In order to simplify the monitoring job add the device name to the following failures: * Installation * Launch * Tcp Connection All the above are most of the time due to a misconfigured device. The device name is useful information for the monitoring person to be able to reach IT and address the issue. --- tests/xharness/AppRunner.cs | 4 ++-- tests/xharness/Jenkins.cs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 827cfd3f1bf6..533cb7b6494d 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -886,12 +886,12 @@ public async Task RunAsync () // same as with a crash FailureMessage = $"Launch failure"; if (Harness.InCI) - XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, "AppLaunch", FailureMessage, main_log.FullPath, XmlResultParser.Jargon.NUnitV3); + XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, $"AppLaunch on {device_name}", $"{FailureMessage} oon {device_name}", main_log.FullPath, XmlResultParser.Jargon.NUnitV3); } else if ((!File.Exists (listener_log.FullPath) || string.IsNullOrEmpty (crash_reason)) && Harness.InCI) { // this happens more that what we would like on devices, the main reason most of the time is that we have had netwoking problems and the // tcp connection could not be stablished. We are going to report it as an error since we have not parsed the logs, evne when the app might have // not crashed. - XmlResultParser.GenerateFailure (Logs, "tcp-connection", appName, Variation, "TcpConnection", "Device could not reach the host over tcp.", main_log.FullPath, XmlResultParser.Jargon.NUnitV3); + XmlResultParser.GenerateFailure (Logs, "tcp-connection", appName, Variation, $"TcpConnection on {device_name}", $"Device {device_name} could not reach the host over tcp.", main_log.FullPath, XmlResultParser.Jargon.NUnitV3); } else if (timed_out && Harness.InCI) { XmlResultParser.GenerateFailure (Logs, "timeout", appName, Variation, "AppTimeout", $"Test run timed out after {timeout.TotalMinutes} minute(s).", main_log.FullPath, XmlResultParser.Jargon.NUnitV3); } diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index daf961e244b0..5d0353701270 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -3741,7 +3741,9 @@ protected override async Task RunTestAsync () FailureMessage = $"Install failed, exit code: {install_result.ExitCode}."; ExecutionResult = TestExecutingResult.Failed; if (Harness.InCI) - XmlResultParser.GenerateFailure (Logs, "install", runner.AppName, runner.Variation, "AppInstallation", $"Install failed, exit code: {install_result.ExitCode}", install_log.FullPath, XmlResultParser.Jargon.NUnitV3); + XmlResultParser.GenerateFailure (Logs, "install", runner.AppName, runner.Variation, + $"AppInstallation on {runner.DeviceName}", $"Install failed on {runner.DeviceName}, exit code: {install_result.ExitCode}", + install_log.FullPath, XmlResultParser.Jargon.NUnitV3); } } finally { this.install_log.Dispose (); From c4978a9a9fbac2ec2f3ba6bc205add398adb4b3c Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 21 Feb 2020 05:28:00 -0500 Subject: [PATCH 13/98] [Harness] Do not add the variation two times to failure messages. AppName should only be the app name since we are passing the varation as a parameter too. Else we end up with $"{appname} {variation} {variation}". --- tests/xharness/AppRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 827cfd3f1bf6..f7f398d58f60 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -78,7 +78,7 @@ public bool isExtension { } } - public string AppName => $"{appName} {Variation}"; + public string AppName => appName; public double TimeoutMultiplier { get; set; } = 1; From 9fd1c2d4d0f098b0170851a006e8cdf26e3c3716 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 21 Feb 2020 05:55:55 -0500 Subject: [PATCH 14/98] [Harness] Add start-time to work around a bug in the publishing tool. The publishing tool is a little fragile. In run https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=3490479&view=logs&j=67d14776-f827-5fe4-2625-2db4b5987fd1&t=fa262eec-9d97-5ba4-b4cc-a9292beecd8f I noticed that valid test runs with a failing test (launch issues) were not being uploaded. I found out that the reason is a flaw in the logic on the parser of the publishing tool. The tool assumes, that if there is no start-time, there are no test results (do remember that NUnitV3 is schemaless we don't know exactly what attrs are compulsory). The culprint is line: https://dev.azure.com/mseng/AzureDevOps/_git/AzureDevOps?path=%2FTa%2FTasks%2FPublishTestResults%2FParser%2FNUnitResultParser.cs&version=GBmaster&line=473&lineEnd=473&lineStartColumn=63&lineEndColumn=64&lineStyle=plain Basically: ```csharp if (testRunNode?.Attributes?["start-time"] != null) { // import test data } // do nothing interesting since there is no data ``` This commit fixes it by setting the start time as the current one, we dont care since it is a failure xml result. --- tests/xharness/XmlResultParser.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/xharness/XmlResultParser.cs b/tests/xharness/XmlResultParser.cs index b4ed55b703b5..f2c4cf94cbfc 100644 --- a/tests/xharness/XmlResultParser.cs +++ b/tests/xharness/XmlResultParser.cs @@ -640,7 +640,8 @@ static void GenerateNUnitV3Failure (XmlWriter writer, string title, string messa ("inconclusive", "0"), ("skipped", "0"), ("asserts", "1"), - ("date", XmlConvert.ToString (DateTime.Now, "yyyy-MM-dd")) + ("run-date", XmlConvert.ToString (DateTime.Now, "yyyy-MM-dd")), + ("start-time", DateTime.Now.ToString ("HH:mm:ss")) ); writer.WriteStartElement ("test-suite"); writer.WriteAttributeString ("type", "Assembly"); From c529e931b3353b191f9e9e2fe7b1974d03133f07 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt <51677938+whitneyschmidt@users.noreply.github.com> Date: Wed, 8 Apr 2020 14:01:44 -0400 Subject: [PATCH 15/98] [WIP][DDFun Pipeline] Test test test (#8171) * comment out most of templates/device-tests.yml * fix yaml formatting * remove condition from publish path * try modifying path to TestSummary.md * add .md * try out merge pwd * try out searching one dir up for summary .mds, cat merged summary * fix dir path * narrow down .md files grabbed * try out publishing + downloading merged file * try out some pwsh * fix yaml for testing pwsh * try out messy pwsh * try pointing at right script location * try out fuller path for pwsh script * try out fuller path for pwsh script * fix hash literal * fix pwsh bug in * try out Bearer * change pwsh to PUT, try and print to check validity * try out new evaluation of token * try print auth * try print everything * fix variables? * remove jobs from matrix * change some env var stuff * more env * try out new target_url * try GitHub.Token * add testing for Gh token * try pwd for ps1 * try changing location to pwsh path * try out ps1 from jenkins dir * print out env vars * fix target_url? * fix for string concat * fix GH url? * fix GH token? * unquote Authorization * add user agent? * Authorization -> AUthentication * try adding quotes to json * print out bash json, remove hash from pwsh json payload * fix underscore url * change put to post * try out full restmethod syntax * manually enter gh token * remove extra code that changes location * re-add real token * try using @params * tools/devops/templates/publish-results.yml * add back env vars * remove some env variables * env vars are actually needed... * try adding context? * uncomment a bunch of stuff in device-tests.yml * move comment? * try out dir path * ls some more * try out fix for run-tests.sh * try setting variable for gh_status * replace vsts device tests with dummy test * add own sh script * try setting job vars in script * try out setting var in a bash task * try out setting global var directly * fix failure * add GH status for failures, set tests to always fail: * add publish_failure.ps1 * fix dir path for script * skip iOS device test stage * re-add stages keyword * remove broken var stuff * comment out dependencie on iOS * try adding some more env vars to pwsh script * add env vars to json payload for status, run more than one test suite * switch from tvos to ios to avoid agent issues * fix broken stage * fix broken syntax * update dependencies to ios * whoops * print out commit comment json payload * fix error in publish_failure * try switching to tvos to see whether queue is shorter * whoops, try again for tvos * initial try at adding commit message in publish_failure.ps1 * fix json payload error * fix formatting and emojis * fix broken escape characters in json payload * tweak context + add AGGREGATE to status for final status, try out getting status * use api.github.com * fix combined status url * try setting status by querying GH for combined status * fix the way that we get json object status * try out accessing response like pwsh object * Get-Location in pwsh scripts * uncomment real test runs, try accessing testsummary.md for aggregate result comment * revert stuff --- jenkins/add-commit-comment.sh | 2 + jenkins/add-commit-status.sh | 2 + jenkins/publish-results-jenkins.ps1 | 74 +++++++ tools/devops/foo.sh | 4 + tools/devops/run-tests.sh | 14 +- tools/devops/templates/device-tests.yml | 24 ++- tools/devops/templates/job-matrix.yml | 25 +-- tools/devops/templates/publish-results.yml | 59 +++++- tools/devops/templates/publish_failure.ps1 | 213 +++++++++++++++++++++ tools/devops/templates/publish_results.ps1 | 115 +++++++++++ tools/devops/templates/stable-stages.yml | 27 +-- 11 files changed, 526 insertions(+), 33 deletions(-) create mode 100644 jenkins/publish-results-jenkins.ps1 create mode 100755 tools/devops/foo.sh create mode 100644 tools/devops/templates/publish_failure.ps1 create mode 100644 tools/devops/templates/publish_results.ps1 diff --git a/jenkins/add-commit-comment.sh b/jenkins/add-commit-comment.sh index e791cdcac526..b8d52dc911b8 100755 --- a/jenkins/add-commit-comment.sh +++ b/jenkins/add-commit-comment.sh @@ -87,6 +87,8 @@ if test -n "$VERBOSE"; then sed 's/^/ /' "$JSONFILE"; fi +cat $JSONFILE + if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/commits/$HASH/comments" > "$LOGFILE" 2>&1; then echo "Failed to add commit message." echo "curl output:" diff --git a/jenkins/add-commit-status.sh b/jenkins/add-commit-status.sh index a5291b7198cf..02955ab10b3a 100755 --- a/jenkins/add-commit-status.sh +++ b/jenkins/add-commit-status.sh @@ -128,6 +128,8 @@ trap cleanup EXIT printf "}\n" ) > "$JSONFILE" +cat $JSONFILE + if test -n "$VERBOSE"; then echo "JSON file:" sed 's/^/ /' "$JSONFILE"; diff --git a/jenkins/publish-results-jenkins.ps1 b/jenkins/publish-results-jenkins.ps1 new file mode 100644 index 000000000000..9ea8caf316e0 --- /dev/null +++ b/jenkins/publish-results-jenkins.ps1 @@ -0,0 +1,74 @@ +Write-Host $pwd.Path +Write-Host $PSScriptRoot +Set-Location -Path $PSScriptRoot +Write-Host $pwd.Path + +## don't need context here b/c we are combining all device tests into one post? +#$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" + +# add real device type here +# add description back in +$json_payload = @" +{ + "hash" : $Env:BUILD_REVISION, + "state" : $Env:GH_STATE, + "target-url" : $Env:TARGET_URL, + "description" : "description placeholder", + "context" : "VSTS: device tests", + "vsts-for-debugging" : $Env:VSTS_BUILD_URL + "GH-token-for-debugging" : $Env:GITHUB_TOKEN +} +"@ + + +$url = "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$BUILD_REVISION" + +Write-Host @{'Authorization' = ("token {0}" -f $GITHUB_TOKEN)} +Write-Host $json_payload + +$params = @{ + Uri = $url + Headers = @{'Authorization' = ("token {0}" -f $GITHUB_TOKEN)} + Method = 'PUT' + Body = $json_payload + ContentType = 'application/json' +} + +Write-Host $params + +$response = Invoke-RestMethod @params + +$response | ConvertTo-Json | Write-Host + +# -Uri $url +# -Method Post +# -Body $json_payload +#$response = Invoke-RestMethod -Uri $url -ContentType application/json -Method Post -Body @json_payload +<# +( + printf '{\n' + printf "\t\"state\": \"%s\",\n" "$STATE" + printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" + printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" + printf "\t\"context\": \"%s\"\n" "$CONTEXT" + printf "}\n" +) > "$JSONFILE" + +if test -n "$VERBOSE"; then + echo "JSON file:" + sed 's/^/ /' "$JSONFILE"; +fi + +if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$HASH" > "$LOGFILE" 2>&1; then + echo "Failed to add status." + echo "curl output:" + sed 's/^/ /' "$LOGFILE" + echo "Json body:" + sed 's/^/ /' "$JSONFILE" + exit 1 #> + +# printf "\t\"state\": \"%s\",\n" "$STATE" +# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" +# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" +# printf "\t\"context\": \"%s\"\n" "$CONTEXT" +# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" \ No newline at end of file diff --git a/tools/devops/foo.sh b/tools/devops/foo.sh new file mode 100755 index 000000000000..f2d31e81acd7 --- /dev/null +++ b/tools/devops/foo.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "hi" +exit 0exit 1 diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index dc222122c58c..3ba2ba63fd12 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -1,11 +1,17 @@ #!/bin/bash -ex - +ls -l +pwd +ls -l xamarin-macios/ +ls -l if [[ "$SYSTEM_DEBUG" == "true" ]]; then DEBUG="-d" else DEBUG="" fi -make $DEBUG -C xamarin-macios/builds download -j -make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j -make $DEBUG -C xamarin-macios/tests vsts-device-tests +#make $DEBUG -C xamarin-macios/builds download -j +#make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j +#make $DEBUG -C xamarin-macios/tests vsts-device-tests +echo "exit 1" > bar.sh +chmod a+x bar.sh +./bar.sh diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 5b42443a1d1d..83fa7ffd9dc1 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -90,11 +90,28 @@ steps: ### Run the device tests ### + # move this to in-line code, should be very short - bash: ./xamarin-macios/tools/devops/run-tests.sh displayName: 'Run tests' timeoutInMinutes: 600 +# if there's a failure: +# - update GH commit status +# - add commit comment +# grab env variables to add test specific text +# pwsh for PowerShell Core +# powershell is for Win only PowerShell +- pwsh: ./xamarin-macios/tools/devops/templates/publish_failure.ps1 + condition: failed() + env: + BUILD_REVISION: $(BUILD_REVISION) + GH_STATE: $(GH_STATE) + GITHUB_TOKEN: $(GitHub.Token) + CONTEXT: $(CONTEXT) + TESTS: $(TESTS) + TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) + - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' continueOnError: true @@ -126,6 +143,8 @@ steps: ### ### Push the HTML report to Azure DevOps (shows up in Summary tab as Build Artifact) ### +- bash: | + echo "$CONTEXT" >> TestSummary-$CONTEXT.md - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: HtmlReport' @@ -138,10 +157,11 @@ steps: - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: TestSummary' inputs: - targetPath: 'xamarin-macios/tests/TestSummary.md' +# targetPath: 'xamarin-macios/tests/TestSummary.md' + targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) +# condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) ### ### Cleanup after us, not having that can lead to VSMac install issues diff --git a/tools/devops/templates/job-matrix.yml b/tools/devops/templates/job-matrix.yml index 5176a64a08a9..197ed94b7de8 100644 --- a/tools/devops/templates/job-matrix.yml +++ b/tools/devops/templates/job-matrix.yml @@ -11,6 +11,7 @@ # timeout: The timeout of the jobs in minutes. # maxParallel: The amount of parallel jobs to be used + parameters: - name: variation type: string @@ -60,18 +61,18 @@ jobs: CONTEXT: '${{ parameters.variation }}-monotouch-DDFun' TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-monotouch-tests' # run all tests but the bcl ones TESTS: 'Monotouch' - xUnitBCL: - CONTEXT: '${{ parameters.variation }}-xUnit-BCL-DDFun' - TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-bcl-xunit-tests' # run only the new bcl tests - TESTS: 'xUnitBCL' - nUnitBCL: - CONTEXT: '${{ parameters.variation }}-NUnit-BCL-DDFun' - TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-bcl-nunit-tests' # run only the new bcl tests - TESTS: 'NUnitBCL' - Mscorlib: - CONTEXT: '${{ parameters.variation }}-mscorlib-DDFun' - TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-mscorlib-tests' # run only the new bcl tests - TESTS: 'mscorlib' + # xUnitBCL: + # CONTEXT: '${{ parameters.variation }}-xUnit-BCL-DDFun' + # TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-new-bcl-tests,skip-mscorlib-tests' # run only the new bcl tests + # TESTS: 'xUnitBCL' + # nUnitBCL: + # CONTEXT: '${{ parameters.variation }}-NUnit-BCL-DDFun' + # TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-old-bcl-tests' # run only the new bcl tests + # TESTS: 'NUnitBCL' + # Mscorlib: + # CONTEXT: '${{ parameters.variation }}-mscorlib-DDFun' + # TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-mscorlib-tests' # run only the new bcl tests + # TESTS: 'mscorlib' steps: - template: device-tests.yml # all the smart stuff happens in the template, just needed to set the params diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 29cf05c79dec..13931f16551b 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -22,10 +22,65 @@ steps: - bash: | set -x set -e - ls -la - cat $PIPELINE_WORKSPACE/Summaries/*.md >> ./TestSummary.md + ls -la + pwd + cat $PIPELINE_WORKSPACE/s/TestSummary-*.md >> ./TestSummary.md + cat ./TestSummary.md displayName: 'Merge TestSummary files' +### +### Publish merged TestSummary file +### +- task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Aggregate TestSummary' + inputs: +# targetPath: 'xamarin-macios/tests/TestSummary.md' + targetPath: ./TestSummary.md + artifactName: TestSummary + continueOnError: true + +### +### Report final results to GitHub +### +# - task: PowerShell@2 +# inputs: +# targetType: #filePath +# script: @json_payload = @"{"token": $TOKEN, "target": $TARGET_URL}" + +### +### Publish merged TestSummary file +### +- task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Aggregate TestSummary' + inputs: +# targetPath: 'xamarin-macios/tests/TestSummary.md' + targetPath: ./TestSummary.md + artifactName: TestSummary + continueOnError: true + + +# pwsh for PowerShell Core +# powershell is for Win PowerShell +- pwsh: ./tools/devops/templates/publish_results.ps1 + condition: succeededOrFailed() + env: + BUILD_REVISION: $(BUILD_REVISION) + GH_STATE: $(GH_STATE) + GITHUB_TOKEN: $(GitHub.Token) + CONTEXT: $(CONTEXT) + +# GITHUB_TOKEN: $(GITHUB_TOKEN) + +# VSTS_BUILD_URL='https://dev.azure.com/devdiv/ - SYSTEM_TEAMFOUNDATIONSERVERURI +# DevDiv/_build/index?buildId=3521765&view=ms.vss-test-web.test-result-details' +#VSTS_BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/index?buildId=${BUILD_BUILDID}&view=ms.vss-test-web.test-result-details" +# printf "\t\"state\": \"%s\",\n" "$STATE" +# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" +# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" +# printf "\t\"context\": \"%s\"\n" "$CONTEXT" +# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" + + ### ### Report final results to GitHub ### diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 new file mode 100644 index 000000000000..ec98c805e6b9 --- /dev/null +++ b/tools/devops/templates/publish_failure.ps1 @@ -0,0 +1,213 @@ +# When this runs, a test failure has occurred +# --state=: The status state. + +Get-Location +Set-Location Env: +Get-ChildItem + + +$target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT/_build/index?buildId=$Env:BUILD_BUILDID&view=ms.vss-test-web.test-result-details" + +## don't need context here b/c we are combining all device tests into one post? +#$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" + +# add real device type here +# add description back in +# state: only report failure, so state is always failure but we can also use AGENT_JOBSTATUS to avoid hardcoding values +$json_payload = @" +{ + "state" : "failure", + "target_url" : "$target_url", + "description" : "$Env:SYSTEM_JOBNAME", + "context" : "$Env:SYSTEM_STAGEDISPLAYNAME" +} +"@ + + +$url = "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$Env:BUILD_REVISION" + +Write-Host $json_payload +Write-Host $url + +$params = @{ + Uri = $url + Headers = @{'Authorization' = ("token {0}" -f $Env:GITHUB_TOKEN)} + Method = 'POST' + Body = $json_payload + ContentType = 'application/json' +} + +Write-Host $params + +$response = Invoke-RestMethod @params + +$response | ConvertTo-Json | Write-Host + + +$RESULT_EMOJI +If ($Env:AGENT_JOBSTATUS -eq 'Failed') +{ + $RESULT_EMOJI = ":fire: " +} Else { + $RESULT_EMOJI = ":white_check_mark: " # maybe remove this since we only add commit messages on failure? +} + +#$MESSAGE_FILE = New-Item -Path . -Name "commit-message.txt" -ItemType "file" + +$HEADER = "" +If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') +{ + $HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline\\n" +} +#Else{ +#HTML Report jenkins stuff +#} + +# SYSTEM_JOBNAME: Xamarin | Monotouch | xUnitBCL | NUnitBCL | mscorlib +# CONTEXT: tvOS | iOS | iOS32 +$DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTEXT" + +# BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" +$json_text = $HEADER + "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" + +$message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" + +$json_payload = @" +{ + "body" : "$json_text" +} +"@ + +Write-Host $json_payload + +$params = @{ + Uri = $message_url + Headers = @{'Authorization' = ("token {0}" -f $Env:GITHUB_TOKEN)} + Method = 'POST' + Body = $json_payload + ContentType = 'application/json' +} + +Write-Host $params + +$response = Invoke-RestMethod @params + +$response | ConvertTo-Json | Write-Host + + +#Where to publish payload for commit comment: "https://api.github.com/repos/xamarin/xamarin-macios/commits/$HASH/comments" + +#Example VSTS Build URL: https://dev.azure.com/devdiv/DevDiv/_build/index?buildId=3568540&view=ms.vss-test-web.test-result-details + +#"%s%s on [Azure DevOps](%s)($Env:CONTEXT)%s %s\\n\\n" "$RESULT_EMOJII" "$DESCRIPTION" "$VSTS_BUILD_URL" "$HTML_REPORT" "$RESULT_EMOJII" >> "$MESSAGE_FILE" + + +# ( +# printf '{\n' +# printf "\t\"state\": \"%s\",\n" "$STATE" +# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" +# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" +# printf "\t\"context\": \"%s\"\n" "$CONTEXT" +# printf "}\n" +# ) > "$JSONFILE" + +# if test -n "$VERBOSE"; then + + + +# # Add a GitHub commit comment + + + + +# # Query job status (always failure) and set result emoji +# GH_STATE=failure +# DESCRIPTION="Running device tests" +# RESULT_EMOJII= +# if test -n "$START"; then +# GH_STATE=pending +# DESCRIPTION="Running device tests on $DEVICE_TYPE" +# else +# case "$(echo "$AGENT_JOBSTATUS" | tr '[:upper:]' '[:lower:]')" in +# succeeded) +# GH_STATE=success +# DESCRIPTION="Device tests passed on $DEVICE_TYPE" +# RESULT_EMOJII="✅ " +# ;; +# failed | canceled | succeededwithissues | *) +# GH_STATE=error +# DESCRIPTION="Device tests completed ($AGENT_JOBSTATUS) on $DEVICE_TYPE" +# RESULT_EMOJII="🔥 " +# ;; +# esac +# fi + + +# + cat /Users/xamarinqa/azdo/_work/48/s/TestSummary.md +# + ./jenkins/add-commit-comment.sh --token=*** --file=commit-message.txt --hash=429bec0478899b5f35260efceb87d67a876ee22a +# { +# "body": "### :construction: Experimental DDFun pipeline\n\ud83d\udd25 Device tests completed (Failed) on VSTS-DDFun on [Azure DevOps](https://dev.azure.com/devdiv/DevDiv/_build/index?buildId=3568540&view=ms.vss-test-web.test-result-details)(VSTS-DDFun) \ud83d\udd25 \n\n" +# } +# Successfully added commit message to https://github.com/xamarin/xamarin-macios/commit/429bec0478899b5f35260efceb87d67a876ee22a +# + cleanup + + + +# # create a temp file to construct commit message +# # add cleanup code +# # add message header: if [[ $DEVICE_TYPE == *"DDFun"* ]]; then printf "### :construction: Experimental DDFun pipeline\\n" > "$MESSAGE_FILE" +# # add the following to the file: $DEVICE_TYPE + emoji + $DESCRIPTION + $VSTS_BUILD_URL + $HTMLREPORT + RESULT_EMOJI +# # look for testsummary.md +# # if not found +# # add "🔥 Tests failed catastrophically on $DEVICE_TYPE (no summary found)\\n" +# # if found +# # cat testsummary.md > messagefile +# if test -z "$START"; then +# # When we're done, add a GitHub comment to the commit we're testing +# MESSAGE_FILE=commit-message.txt +# cleanup () +# { +# rm -f "$MESSAGE_FILE" +# } +# trap cleanup ERR +# trap cleanup EXIT + +# HTML_REPORT="" +# if [[ $DEVICE_TYPE == *"DDFun"* ]]; then +# printf "### :construction: Experimental DDFun pipeline\\n" > "$MESSAGE_FILE" +# else +# P=$(cat tmp.p) +# HTML_REPORT=": [Html Report](http://xamarin-storage/${P}/jenkins-results/tests/index.html)" +# fi + +# printf "%s%s on [Azure DevOps](%s)($DEVICE_TYPE)%s %s\\n\\n" "$RESULT_EMOJII" "$DESCRIPTION" "$VSTS_BUILD_URL" "$HTML_REPORT" "$RESULT_EMOJII" >> "$MESSAGE_FILE" + +# FILE=$PWD/TestSummary.md +# if ! test -f "$FILE"; then +# printf "🔥 Tests failed catastrophically on $DEVICE_TYPE (no summary found)\\n" >> "$MESSAGE_FILE" +# else +# cat "$FILE" >> "$MESSAGE_FILE" +# fi + +# ./jenkins/add-commit-comment.sh --token="$TOKEN" --file="$MESSAGE_FILE" "--hash=$BUILD_REVISION" + +# # construct a json payload and publish it to GH +# # url is probably something like: +# $comment_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" + +# # need to print out JSON payload from jenkins to do comparison + + +# if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/commits/$HASH/comments" > "$LOGFILE" 2>&1; then +# echo "Failed to add commit message." +# echo "curl output:" +# sed 's/^/ /' "$LOGFILE" +# echo "Json body:" +# sed 's/^/ /' "$JSONFILE" +# exit 1 +# else +# if test -n "$VERBOSE"; then sed 's/^/ /' "$LOGFILE"; fi +# echo "Successfully added commit message to https://github.com/xamarin/xamarin-macios/commit/$HASH" +# fi + +# fi diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 new file mode 100644 index 000000000000..4e96382fd3f6 --- /dev/null +++ b/tools/devops/templates/publish_results.ps1 @@ -0,0 +1,115 @@ +# debugging code: print out all environment variables, including those passed in via yml +Get-Location +Set-Location Env: +Get-ChildItem + +# get combined status: +# success only if every status is success +# otherwise failure + +# url to query for combined status +$combined_status_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/status" + +$params = @{ + Uri = $combined_status_url + Headers = @{'Authorization' = ("token {0}" -f $Env:GITHUB_TOKEN)} + Method = 'GET' + ContentType = 'application/json' +} + +# +$response = Invoke-RestMethod @params + +Write-Host $response +$response | ConvertTo-Json | Write-Host +Write-Host "^ ConvertToJson below is ConvertFromJson `n" + +$response | ConvertTo-Json | ConvertFrom-Json | Write-Host + +#$status = $response | ConvertTo-Json | ConvertFrom-Json | Select-Object state + +Write-Host "Raw response: " + $response + +Write-Host "Response.state: " + $response.state +$state = $response.state + + +# post status to github +$target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT/_build/index?buildId=$Env:BUILD_BUILDID&view=ms.vss-test-web.test-result-details" + +## don't need context here b/c we are combining all device tests into one post? +#$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" + +# add real device type here +# add description back in +$json_payload = @" +{ + "state" : "$state", + "target_url" : "$target_url", + "description" : "description placeholder", + "context" : "VSTS: AGGREGATE device tests" +} +"@ + + +$url = "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$Env:BUILD_REVISION" + +Write-Host $json_payload +Write-Host $url + +$params = @{ + Uri = $url + Headers = @{'Authorization' = ("token {0}" -f $Env:GITHUB_TOKEN)} + Method = 'POST' + Body = $json_payload + ContentType = 'application/json' +} + +Write-Host $params + +$response = Invoke-RestMethod @params + +$response | Write-Host + + +# https://api.github.com/xamarin/xamarin-macios/commits/eea6fd1f27ba9a0ac4fa09c8e57fc87d612b6340/status + +#GET /projects/:id/repository/commits/:sha/refs + + +#https://api.github.com/repos/xamarin/xamarin-macios/statuses/$Env:BUILD_REVISION + + + +# -Uri $url +# -Method Post +# -Body $json_payload +#$response = Invoke-RestMethod -Uri $url -ContentType application/json -Method Post -Body @json_payload +<# +( + printf '{\n' + printf "\t\"state\": \"%s\",\n" "$STATE" + printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" + printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" + printf "\t\"context\": \"%s\"\n" "$CONTEXT" + printf "}\n" +) > "$JSONFILE" + +if test -n "$VERBOSE"; then + echo "JSON file:" + sed 's/^/ /' "$JSONFILE"; +fi + +if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$HASH" > "$LOGFILE" 2>&1; then + echo "Failed to add status." + echo "curl output:" + sed 's/^/ /' "$LOGFILE" + echo "Json body:" + sed 's/^/ /' "$JSONFILE" + exit 1 #> + +# printf "\t\"state\": \"%s\",\n" "$STATE" +# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" +# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" +# printf "\t\"context\": \"%s\"\n" "$CONTEXT" +# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" \ No newline at end of file diff --git a/tools/devops/templates/stable-stages.yml b/tools/devops/templates/stable-stages.yml index 041c9481e5cf..dd7484aace60 100644 --- a/tools/devops/templates/stable-stages.yml +++ b/tools/devops/templates/stable-stages.yml @@ -8,6 +8,7 @@ # timeout: Timeout used in the different jobs. # maxParallel: Max number of parallel jobs to do in a stage. # + parameters: # set demands as possible parameters because we keep changing them from lab to lab :/ @@ -36,18 +37,18 @@ parameters: default: 5 stages: -- stage: iOSDeviceTests - displayName: 'iOS Device Tests' - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel - jobs: - - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params - parameters: - demands: ${{ parameters.iOSDemand }} - deviceLabel: 'run-ios-64-tests' - variation: "iOS" - pool: ${{ parameters.pool }} - timeout: ${{ parameters.timeout }} - maxParallel: ${{ parameters.maxParallel}} +# - stage: iOSDeviceTests +# displayName: 'iOS Device Tests' +# dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel +# jobs: +# - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params +# parameters: +# demands: ${{ parameters.iOSDemand }} +# deviceLabel: 'run-ios-64-tests' +# variation: "iOS" +# pool: ${{ parameters.pool }} +# timeout: ${{ parameters.timeout }} +# maxParallel: ${{ parameters.maxParallel}} - stage: tvOSDeviceTests displayName: 'tvOS Device Tests' @@ -79,7 +80,7 @@ stages: - stage: monitoring displayName: 'Device test monitoring' dependsOn: - - iOSDeviceTests +# - iOSDeviceTests - tvOSDeviceTests # - iOS32DeviceTests From 3c997066fb9833b4a65aea2e256fd1f058ff08ed Mon Sep 17 00:00:00 2001 From: Whitney Schmidt <51677938+whitneyschmidt@users.noreply.github.com> Date: Wed, 8 Apr 2020 14:10:29 -0400 Subject: [PATCH 16/98] Delete foo.sh --- tools/devops/foo.sh | 4 ---- 1 file changed, 4 deletions(-) delete mode 100755 tools/devops/foo.sh diff --git a/tools/devops/foo.sh b/tools/devops/foo.sh deleted file mode 100755 index f2d31e81acd7..000000000000 --- a/tools/devops/foo.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "hi" -exit 0exit 1 From d5fc0de2e021d032302f0cd0bb558f197665d6b2 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt <51677938+whitneyschmidt@users.noreply.github.com> Date: Wed, 8 Apr 2020 14:14:08 -0400 Subject: [PATCH 17/98] Delete publish-results-jenkins.ps1 --- jenkins/publish-results-jenkins.ps1 | 74 ----------------------------- 1 file changed, 74 deletions(-) delete mode 100644 jenkins/publish-results-jenkins.ps1 diff --git a/jenkins/publish-results-jenkins.ps1 b/jenkins/publish-results-jenkins.ps1 deleted file mode 100644 index 9ea8caf316e0..000000000000 --- a/jenkins/publish-results-jenkins.ps1 +++ /dev/null @@ -1,74 +0,0 @@ -Write-Host $pwd.Path -Write-Host $PSScriptRoot -Set-Location -Path $PSScriptRoot -Write-Host $pwd.Path - -## don't need context here b/c we are combining all device tests into one post? -#$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" - -# add real device type here -# add description back in -$json_payload = @" -{ - "hash" : $Env:BUILD_REVISION, - "state" : $Env:GH_STATE, - "target-url" : $Env:TARGET_URL, - "description" : "description placeholder", - "context" : "VSTS: device tests", - "vsts-for-debugging" : $Env:VSTS_BUILD_URL - "GH-token-for-debugging" : $Env:GITHUB_TOKEN -} -"@ - - -$url = "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$BUILD_REVISION" - -Write-Host @{'Authorization' = ("token {0}" -f $GITHUB_TOKEN)} -Write-Host $json_payload - -$params = @{ - Uri = $url - Headers = @{'Authorization' = ("token {0}" -f $GITHUB_TOKEN)} - Method = 'PUT' - Body = $json_payload - ContentType = 'application/json' -} - -Write-Host $params - -$response = Invoke-RestMethod @params - -$response | ConvertTo-Json | Write-Host - -# -Uri $url -# -Method Post -# -Body $json_payload -#$response = Invoke-RestMethod -Uri $url -ContentType application/json -Method Post -Body @json_payload -<# -( - printf '{\n' - printf "\t\"state\": \"%s\",\n" "$STATE" - printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" - printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" - printf "\t\"context\": \"%s\"\n" "$CONTEXT" - printf "}\n" -) > "$JSONFILE" - -if test -n "$VERBOSE"; then - echo "JSON file:" - sed 's/^/ /' "$JSONFILE"; -fi - -if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$HASH" > "$LOGFILE" 2>&1; then - echo "Failed to add status." - echo "curl output:" - sed 's/^/ /' "$LOGFILE" - echo "Json body:" - sed 's/^/ /' "$JSONFILE" - exit 1 #> - -# printf "\t\"state\": \"%s\",\n" "$STATE" -# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" -# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" -# printf "\t\"context\": \"%s\"\n" "$CONTEXT" -# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" \ No newline at end of file From 57bae65cbcbe7e36cd4e1a093a4b028b480b110e Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 22 Apr 2020 19:37:54 -0400 Subject: [PATCH 18/98] try out grabbing test summary files location --- tools/devops/templates/publish_results.ps1 | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 4e96382fd3f6..78f2579c4c72 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -3,6 +3,16 @@ Get-Location Set-Location Env: Get-ChildItem + +$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIR +Write-Host $testsummary_location +Get-Content $testsummary_location +Get-Content $testsummary_location | Write-Host + +$testsummary_location = $testsummary_location + "/TestSummary.md" +Write-Host $testsummary_location +Get-Content $testsummary_location + # get combined status: # success only if every status is success # otherwise failure @@ -17,7 +27,7 @@ $params = @{ ContentType = 'application/json' } -# +# $response = Invoke-RestMethod @params Write-Host $response @@ -46,7 +56,7 @@ $json_payload = @" { "state" : "$state", "target_url" : "$target_url", - "description" : "description placeholder", + "description" : "description placeholder", "context" : "VSTS: AGGREGATE device tests" } "@ @@ -85,7 +95,7 @@ $response | Write-Host # -Method Post # -Body $json_payload #$response = Invoke-RestMethod -Uri $url -ContentType application/json -Method Post -Body @json_payload -<# +<# ( printf '{\n' printf "\t\"state\": \"%s\",\n" "$STATE" @@ -112,4 +122,4 @@ if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line to # printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" # printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" # printf "\t\"context\": \"%s\"\n" "$CONTEXT" -# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" \ No newline at end of file +# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" From 52d51fb06cf6371405bf882e622b284831bde29b Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 22 Apr 2020 20:02:14 -0400 Subject: [PATCH 19/98] change device pool --- tools/devops/device-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/device-tests.yml b/tools/devops/device-tests.yml index 38bfdc77cb1e..510ab7cfe132 100644 --- a/tools/devops/device-tests.yml +++ b/tools/devops/device-tests.yml @@ -1,4 +1,4 @@ -# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending +# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending # on the configuration to run different device tests. # # lame! cannot have the resources in the template.. :/ @@ -14,6 +14,6 @@ resources: stages: - template: templates/stable-stages.yml # all the smart stuff happens in the template, just needed to set the params parameters: - pool: 'VSEng-Xamarin-Mac-Devices' + pool: 'VSEng-Xamarin-QA' timeout: 1000 maxParallel: 5 From 469b57cf2cf24563386a19c49380aa2172391afb Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 22 Apr 2020 20:09:57 -0400 Subject: [PATCH 20/98] change capabilities for xamarin-ios device pool --- tools/devops/templates/stable-stages.yml | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tools/devops/templates/stable-stages.yml b/tools/devops/templates/stable-stages.yml index dd7484aace60..705865236dfb 100644 --- a/tools/devops/templates/stable-stages.yml +++ b/tools/devops/templates/stable-stages.yml @@ -1,5 +1,5 @@ # Template that contains the different stages to be executed in a pipeline -# +# # The tamplate contains the following parameters # # include32b: Run tests on iOS 32b devices. @@ -14,7 +14,7 @@ parameters: # set demands as possible parameters because we keep changing them from lab to lab :/ - name: iOSDemand type: string - default: 'ios' + default: 'xismoke' #ios for ddfun pool - name: iOS32Demand type: string @@ -22,10 +22,10 @@ parameters: - name: tvOSDemand type: string - default: 'tvos' + default: 'xitvos' #tvos for ddfun - name: pool - type: string + type: string default: 'VSEng-Xamarin-Mac-Devices' # default ddfun pool - name: timeout @@ -37,7 +37,7 @@ parameters: default: 5 stages: -# - stage: iOSDeviceTests +# - stage: iOSDeviceTests # displayName: 'iOS Device Tests' # dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel # jobs: @@ -46,9 +46,9 @@ stages: # demands: ${{ parameters.iOSDemand }} # deviceLabel: 'run-ios-64-tests' # variation: "iOS" -# pool: ${{ parameters.pool }} -# timeout: ${{ parameters.timeout }} -# maxParallel: ${{ parameters.maxParallel}} +# pool: ${{ parameters.pool }} +# timeout: ${{ parameters.timeout }} +# maxParallel: ${{ parameters.maxParallel}} - stage: tvOSDeviceTests displayName: 'tvOS Device Tests' @@ -59,12 +59,12 @@ stages: demands: ${{ parameters.tvOSDemand }} deviceLabel: 'run-tvos-tests' variation: "tvOS" - pool: ${{ parameters.pool }} - timeout: ${{ parameters.timeout }} - maxParallel: ${{ parameters.maxParallel }} + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel }} -# Wait for the lab to have devices, templates do not let me do what I want with expressions :/ -#- stage: iOS32DeviceTests +# Wait for the lab to have devices, templates do not let me do what I want with expressions :/ +#- stage: iOS32DeviceTests # displayName: 'iOS32b Device Tests' # dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel # jobs: @@ -73,20 +73,20 @@ stages: # demands: ${{ parameters.iOS32Demand }} # old QA lab capability, might change # deviceLabel: 'run-ios-32-tests' # variation: "iOS32b" -# pool: ${{ parameters.pool }} -# timeout: ${{ parameters.timeout }} -# maxParallel: ${{ parameters.maxParallel }} +# pool: ${{ parameters.pool }} +# timeout: ${{ parameters.timeout }} +# maxParallel: ${{ parameters.maxParallel }} - stage: monitoring displayName: 'Device test monitoring' dependsOn: # - iOSDeviceTests - tvOSDeviceTests -# - iOS32DeviceTests +# - iOS32DeviceTests condition: always() # always notify of what happened. - pool: + pool: name: ${{ parameters.pool }} demands: 'ios' # we should not have demands here, but the problem is that DDFun does not set all machines to be able to checout private repos :/ From d2732d6bbcf054a61551988afa448c44c5bff399 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 23 Apr 2020 13:54:15 -0400 Subject: [PATCH 21/98] remove buggy pwsh line from publish_results, uncomment real tests in run-tests.sh --- tools/devops/run-tests.sh | 12 ++++++------ tools/devops/templates/publish_results.ps1 | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 3ba2ba63fd12..a5941713c218 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -9,9 +9,9 @@ if [[ "$SYSTEM_DEBUG" == "true" ]]; then else DEBUG="" fi -#make $DEBUG -C xamarin-macios/builds download -j -#make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j -#make $DEBUG -C xamarin-macios/tests vsts-device-tests -echo "exit 1" > bar.sh -chmod a+x bar.sh -./bar.sh +make $DEBUG -C xamarin-macios/builds download -j +make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j +make $DEBUG -C xamarin-macios/tests vsts-device-tests +#echo "exit 1" > bar.sh +#chmod a+x bar.sh +#./bar.sh diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 78f2579c4c72..6b2686084de9 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -34,7 +34,7 @@ Write-Host $response $response | ConvertTo-Json | Write-Host Write-Host "^ ConvertToJson below is ConvertFromJson `n" -$response | ConvertTo-Json | ConvertFrom-Json | Write-Host +#$response | ConvertTo-Json | ConvertFrom-Json | Write-Host #$status = $response | ConvertTo-Json | ConvertFrom-Json | Select-Object state From ef6164ddccb46b40224cf181bb23c3269b5d0ad3 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 23 Apr 2020 17:09:03 -0400 Subject: [PATCH 22/98] Revert "change capabilities for xamarin-ios device pool" This reverts commit 469b57cf2cf24563386a19c49380aa2172391afb. --- tools/devops/templates/stable-stages.yml | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tools/devops/templates/stable-stages.yml b/tools/devops/templates/stable-stages.yml index 705865236dfb..dd7484aace60 100644 --- a/tools/devops/templates/stable-stages.yml +++ b/tools/devops/templates/stable-stages.yml @@ -1,5 +1,5 @@ # Template that contains the different stages to be executed in a pipeline -# +# # The tamplate contains the following parameters # # include32b: Run tests on iOS 32b devices. @@ -14,7 +14,7 @@ parameters: # set demands as possible parameters because we keep changing them from lab to lab :/ - name: iOSDemand type: string - default: 'xismoke' #ios for ddfun pool + default: 'ios' - name: iOS32Demand type: string @@ -22,10 +22,10 @@ parameters: - name: tvOSDemand type: string - default: 'xitvos' #tvos for ddfun + default: 'tvos' - name: pool - type: string + type: string default: 'VSEng-Xamarin-Mac-Devices' # default ddfun pool - name: timeout @@ -37,7 +37,7 @@ parameters: default: 5 stages: -# - stage: iOSDeviceTests +# - stage: iOSDeviceTests # displayName: 'iOS Device Tests' # dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel # jobs: @@ -46,9 +46,9 @@ stages: # demands: ${{ parameters.iOSDemand }} # deviceLabel: 'run-ios-64-tests' # variation: "iOS" -# pool: ${{ parameters.pool }} -# timeout: ${{ parameters.timeout }} -# maxParallel: ${{ parameters.maxParallel}} +# pool: ${{ parameters.pool }} +# timeout: ${{ parameters.timeout }} +# maxParallel: ${{ parameters.maxParallel}} - stage: tvOSDeviceTests displayName: 'tvOS Device Tests' @@ -59,12 +59,12 @@ stages: demands: ${{ parameters.tvOSDemand }} deviceLabel: 'run-tvos-tests' variation: "tvOS" - pool: ${{ parameters.pool }} - timeout: ${{ parameters.timeout }} - maxParallel: ${{ parameters.maxParallel }} + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel }} -# Wait for the lab to have devices, templates do not let me do what I want with expressions :/ -#- stage: iOS32DeviceTests +# Wait for the lab to have devices, templates do not let me do what I want with expressions :/ +#- stage: iOS32DeviceTests # displayName: 'iOS32b Device Tests' # dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel # jobs: @@ -73,20 +73,20 @@ stages: # demands: ${{ parameters.iOS32Demand }} # old QA lab capability, might change # deviceLabel: 'run-ios-32-tests' # variation: "iOS32b" -# pool: ${{ parameters.pool }} -# timeout: ${{ parameters.timeout }} -# maxParallel: ${{ parameters.maxParallel }} +# pool: ${{ parameters.pool }} +# timeout: ${{ parameters.timeout }} +# maxParallel: ${{ parameters.maxParallel }} - stage: monitoring displayName: 'Device test monitoring' dependsOn: # - iOSDeviceTests - tvOSDeviceTests -# - iOS32DeviceTests +# - iOS32DeviceTests condition: always() # always notify of what happened. - pool: + pool: name: ${{ parameters.pool }} demands: 'ios' # we should not have demands here, but the problem is that DDFun does not set all machines to be able to checout private repos :/ From 64f5c82add0c9c7fbd57fae1c7bdc8039b8db1e9 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 23 Apr 2020 17:09:23 -0400 Subject: [PATCH 23/98] Revert "change device pool" This reverts commit 52d51fb06cf6371405bf882e622b284831bde29b. --- tools/devops/device-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/device-tests.yml b/tools/devops/device-tests.yml index 510ab7cfe132..38bfdc77cb1e 100644 --- a/tools/devops/device-tests.yml +++ b/tools/devops/device-tests.yml @@ -1,4 +1,4 @@ -# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending +# xamarin-macios device pipeline, this file sets the different parameters required by stages template which will create set of stages depending # on the configuration to run different device tests. # # lame! cannot have the resources in the template.. :/ @@ -14,6 +14,6 @@ resources: stages: - template: templates/stable-stages.yml # all the smart stuff happens in the template, just needed to set the params parameters: - pool: 'VSEng-Xamarin-QA' + pool: 'VSEng-Xamarin-Mac-Devices' timeout: 1000 maxParallel: 5 From c2885a2ef673ec35340714c711509a55332a5807 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 23 Apr 2020 17:55:43 -0400 Subject: [PATCH 24/98] remove set-location from publish_results --- tools/devops/templates/publish_results.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 6b2686084de9..95d068908079 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -1,7 +1,7 @@ # debugging code: print out all environment variables, including those passed in via yml -Get-Location -Set-Location Env: -Get-ChildItem +#Get-Location +#Set-Location Env: +#Get-ChildItem $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIR From 80b84180a374e90a8029424f40ea3bce92deab18 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 24 Apr 2020 15:00:08 -0400 Subject: [PATCH 25/98] update with CORRECT env var --- tools/devops/templates/publish_results.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 95d068908079..3b41c79d6686 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -4,7 +4,7 @@ #Get-ChildItem -$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIR +$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY Write-Host $testsummary_location Get-Content $testsummary_location Get-Content $testsummary_location | Write-Host From 5b29a27861caffd7b2006f127dd7b09efbcf9f96 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 24 Apr 2020 15:08:24 -0400 Subject: [PATCH 26/98] re-add debugging code to print out env vars --- tools/devops/templates/publish_results.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 3b41c79d6686..6dcf4ced2dca 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -1,7 +1,7 @@ # debugging code: print out all environment variables, including those passed in via yml -#Get-Location -#Set-Location Env: -#Get-ChildItem +Get-Location +Set-Location Env: +Get-ChildItem $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY From 07b64325cc4832d9d65723e36ecae221972f8dbc Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 24 Apr 2020 17:53:33 -0400 Subject: [PATCH 27/98] fix accessing testsummary.md --- tools/devops/templates/publish_results.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 6dcf4ced2dca..05617ce25f7d 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -5,9 +5,7 @@ Get-ChildItem $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -Write-Host $testsummary_location -Get-Content $testsummary_location -Get-Content $testsummary_location | Write-Host +Get-ChildItem $testsummary_location $testsummary_location = $testsummary_location + "/TestSummary.md" Write-Host $testsummary_location From e73c1943d1fc8803d52fa848e81e00357b113c94 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 27 Apr 2020 15:17:25 -0400 Subject: [PATCH 28/98] first try at adding test summary to per-test failure comment, moved when test summaries get published to vsts as artifacts --- tools/devops/templates/device-tests.yml | 36 +++++++++++++++------- tools/devops/templates/publish_failure.ps1 | 12 ++++++-- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 83fa7ffd9dc1..c2b5fe3474c9 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -95,6 +95,28 @@ steps: displayName: 'Run tests' timeoutInMinutes: 600 + ### + ### If test summary does not exist, create test summary files so that we can notify + ### that no results were published at the per-test AND final results steps + ### +- bash: | + set -e + set -x + FILE="TestSummary-${CONTEXT}.md" + echo $FILE + if ! test -f $FILE; then + echo ":fire: Tests failed catastrophically (no summary found)" > $FILE + fi + displayName: "If test summary not found, create test summary" + +- task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: TestSummary' + inputs: +# targetPath: 'xamarin-macios/tests/TestSummary.md' + targetPath: TestSummary-$(CONTEXT).md + artifactName: TestSummary-$(CONTEXT) + continueOnError: true +# condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) # if there's a failure: # - update GH commit status @@ -103,7 +125,7 @@ steps: # pwsh for PowerShell Core # powershell is for Win only PowerShell - pwsh: ./xamarin-macios/tools/devops/templates/publish_failure.ps1 - condition: failed() +# condition: failed() env: BUILD_REVISION: $(BUILD_REVISION) GH_STATE: $(GH_STATE) @@ -143,8 +165,8 @@ steps: ### ### Push the HTML report to Azure DevOps (shows up in Summary tab as Build Artifact) ### -- bash: | - echo "$CONTEXT" >> TestSummary-$CONTEXT.md +#- bash: | +# echo "$CONTEXT" >> TestSummary-$CONTEXT.md - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: HtmlReport' @@ -154,14 +176,6 @@ steps: continueOnError: true condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) -- task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: TestSummary' - inputs: -# targetPath: 'xamarin-macios/tests/TestSummary.md' - targetPath: TestSummary-$(CONTEXT).md - artifactName: TestSummary-$(CONTEXT) - continueOnError: true -# condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) ### ### Cleanup after us, not having that can lead to VSMac install issues diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index ec98c805e6b9..67e9f6e66b4f 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -52,8 +52,6 @@ If ($Env:AGENT_JOBSTATUS -eq 'Failed') $RESULT_EMOJI = ":white_check_mark: " # maybe remove this since we only add commit messages on failure? } -#$MESSAGE_FILE = New-Item -Path . -Name "commit-message.txt" -ItemType "file" - $HEADER = "" If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') { @@ -70,6 +68,16 @@ $DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTE # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = $HEADER + "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" +# add contents of test summary to json_text +$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY +$testsummary_location = $testsummary_location + "/TestSummary.md" +Write-Host $testsummary_location +$test_summary = Get-Content $testsummary_location + +$json_text = $json_text + $test_summary + + + $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" $json_payload = @" From 2dfad3168a02979bcf9025de2a1c2100e2031ade Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 27 Apr 2020 15:54:25 -0400 Subject: [PATCH 29/98] change condition for yaml so that everything runs regardless of success/failure --- tools/devops/templates/device-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index c2b5fe3474c9..e791997949e0 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -99,6 +99,7 @@ steps: ### If test summary does not exist, create test summary files so that we can notify ### that no results were published at the per-test AND final results steps ### + ### TODO if provisioning fails add a different error message - bash: | set -e set -x @@ -108,6 +109,7 @@ steps: echo ":fire: Tests failed catastrophically (no summary found)" > $FILE fi displayName: "If test summary not found, create test summary" + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: TestSummary' @@ -116,7 +118,7 @@ steps: targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true -# condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) # if there's a failure: # - update GH commit status @@ -133,6 +135,7 @@ steps: CONTEXT: $(CONTEXT) TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From 55a52a5b2ef7c47c141e1f7b339ace5cf5f41d12 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 27 Apr 2020 16:42:14 -0400 Subject: [PATCH 30/98] fix *indentation* that was causing pwsh scripts not to run on failure --- tools/devops/templates/device-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index e791997949e0..38dc9891c9c3 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -135,7 +135,8 @@ steps: CONTEXT: $(CONTEXT) TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) - condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + displayName: 'add GH status & commit message' + condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From c2b3aac5c1286b129217383452f7efb23e3a62bd Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 27 Apr 2020 17:48:13 -0400 Subject: [PATCH 31/98] skip test execution, instead immediately exit w/ failure code --- tools/devops/run-tests.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index a5941713c218..3ba2ba63fd12 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -9,9 +9,9 @@ if [[ "$SYSTEM_DEBUG" == "true" ]]; then else DEBUG="" fi -make $DEBUG -C xamarin-macios/builds download -j -make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j -make $DEBUG -C xamarin-macios/tests vsts-device-tests -#echo "exit 1" > bar.sh -#chmod a+x bar.sh -#./bar.sh +#make $DEBUG -C xamarin-macios/builds download -j +#make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j +#make $DEBUG -C xamarin-macios/tests vsts-device-tests +echo "exit 1" > bar.sh +chmod a+x bar.sh +./bar.sh From b46df753796866df4788dbfd5d57998f3da62c64 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 27 Apr 2020 18:39:47 -0400 Subject: [PATCH 32/98] add step for test summary if provisioning fails, fix summary path in publish_failure --- tools/devops/templates/device-tests.yml | 13 +++++++++++++ tools/devops/templates/publish_failure.ps1 | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 38dc9891c9c3..2768a7aa705f 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -111,6 +111,19 @@ steps: displayName: "If test summary not found, create test summary" condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) +### TODO: can probably figure out way to do this in the bash script above for +### test failures +### If provisioning failed, create test summary files to indicate failure +### +- bash: | + set -e + set -x + FILE="TestSummary-${CONTEXT}.md" + echo $FILE + echo ":fire: Provisioning step failed" > $FILE + displayName: 'Add test summary for failed provisioning' + condition: eq(variables['ProvisioningProfiles'], 'failure') + - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: TestSummary' inputs: diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 67e9f6e66b4f..cb3918e2b6d8 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -70,7 +70,7 @@ $json_text = $HEADER + "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url # add contents of test summary to json_text $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -$testsummary_location = $testsummary_location + "/TestSummary.md" +$testsummary_location = $testsummary_location + "/TestSummary-$Env:CONTEXT.md" Write-Host $testsummary_location $test_summary = Get-Content $testsummary_location From ee19b9c667b2b399b625feb33cce33b2f2511b09 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 13:35:10 -0400 Subject: [PATCH 33/98] remove provisioning success condition, first try (wrong) at publishing test summary in final commit message --- tools/devops/templates/device-tests.yml | 13 +++- tools/devops/templates/publish_failure.ps1 | 7 +++ tools/devops/templates/publish_results.ps1 | 70 ++++++++++++++++++++-- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 2768a7aa705f..b102a5df125a 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -124,6 +124,13 @@ steps: displayName: 'Add test summary for failed provisioning' condition: eq(variables['ProvisioningProfiles'], 'failure') +### +### Publish test summary +### We should publish summary for the following cases: +### 1) run-tests created summary files +### 2) run-tests did not create summary files and we created them +### 3) provisioning failed +### - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: TestSummary' inputs: @@ -131,9 +138,9 @@ steps: targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + condition: and(succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) -# if there's a failure: +# if there's a test failure || test summary not generated || provisioning failed: # - update GH commit status # - add commit comment # grab env variables to add test specific text @@ -149,7 +156,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'add GH status & commit message' - condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) + condition: and(succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index cb3918e2b6d8..e726d8910cd0 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -5,6 +5,9 @@ Get-Location Set-Location Env: Get-ChildItem +### +### Construct status +### $target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT/_build/index?buildId=$Env:BUILD_BUILDID&view=ms.vss-test-web.test-result-details" @@ -44,6 +47,10 @@ $response = Invoke-RestMethod @params $response | ConvertTo-Json | Write-Host +### +### Construct commit message +### + $RESULT_EMOJI If ($Env:AGENT_JOBSTATUS -eq 'Failed') { diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 05617ce25f7d..dbe32be0f897 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -32,10 +32,6 @@ Write-Host $response $response | ConvertTo-Json | Write-Host Write-Host "^ ConvertToJson below is ConvertFromJson `n" -#$response | ConvertTo-Json | ConvertFrom-Json | Write-Host - -#$status = $response | ConvertTo-Json | ConvertFrom-Json | Select-Object state - Write-Host "Raw response: " + $response Write-Host "Response.state: " + $response.state @@ -80,6 +76,72 @@ $response = Invoke-RestMethod @params $response | Write-Host + + +### +### Construct commit message w/ aggregate test summary +### + +$RESULT_EMOJI +If ($Env:AGENT_JOBSTATUS -eq 'Failed') +{ + $RESULT_EMOJI = ":fire: " +} Else { + $RESULT_EMOJI = ":white_check_mark: " # maybe remove this since we only add commit messages on failure? +} + +$HEADER = "" +If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') +{ + $HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline\\n" +} +#Else{ +#HTML Report jenkins stuff +#} + +# SYSTEM_JOBNAME: Xamarin | Monotouch | xUnitBCL | NUnitBCL | mscorlib +# CONTEXT: tvOS | iOS | iOS32 +$DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTEXT" + +# BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" +$json_text = $HEADER + "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" + +# add contents of test summary to json_text +$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY +$testsummary_location = $testsummary_location + "/TestSummary-$Env:CONTEXT.md" +Write-Host $testsummary_location +$test_summary = Get-Content $testsummary_location + +$json_text = $json_text + $test_summary + + + +$message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" + +$json_payload = @" +{ + "body" : "$json_text" +} +"@ + +Write-Host $json_payload + +$params = @{ + Uri = $message_url + Headers = @{'Authorization' = ("token {0}" -f $Env:GITHUB_TOKEN)} + Method = 'POST' + Body = $json_payload + ContentType = 'application/json' +} + +Write-Host $params + +$response = Invoke-RestMethod @params + +$response | ConvertTo-Json | Write-Host + + + # https://api.github.com/xamarin/xamarin-macios/commits/eea6fd1f27ba9a0ac4fa09c8e57fc87d612b6340/status #GET /projects/:id/repository/commits/:sha/refs From d9494746308ab361ead239c1a3a8502bbedcf975 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 13:45:57 -0400 Subject: [PATCH 34/98] adjust final commit message for aggregate test file --- tools/devops/templates/publish_results.ps1 | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index dbe32be0f897..c133a8c24d33 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -76,39 +76,27 @@ $response = Invoke-RestMethod @params $response | Write-Host - - ### ### Construct commit message w/ aggregate test summary ### -$RESULT_EMOJI -If ($Env:AGENT_JOBSTATUS -eq 'Failed') -{ - $RESULT_EMOJI = ":fire: " -} Else { - $RESULT_EMOJI = ":white_check_mark: " # maybe remove this since we only add commit messages on failure? -} - $HEADER = "" If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') { - $HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline\\n" + $HEADER = "### :boom: :construction: TESTING Experimental DDFun pipeline\\n" } #Else{ #HTML Report jenkins stuff #} -# SYSTEM_JOBNAME: Xamarin | Monotouch | xUnitBCL | NUnitBCL | mscorlib -# CONTEXT: tvOS | iOS | iOS32 -$DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTEXT" +$DESCRIPTION="Device test aggregate results:" # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" -$json_text = $HEADER + "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" +$json_text = $HEADER + "$DESCRIPTION on [Azure DevOps]($target_url)" # add contents of test summary to json_text $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -$testsummary_location = $testsummary_location + "/TestSummary-$Env:CONTEXT.md" +$testsummary_location = $testsummary_location + "/TestSummary.md" Write-Host $testsummary_location $test_summary = Get-Content $testsummary_location From ad3620806c32fb56c0c80f2b20179ef5f943f4ed Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 13:49:08 -0400 Subject: [PATCH 35/98] fix succeeededorfailed conditions --- tools/devops/templates/device-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index b102a5df125a..068837196182 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -138,7 +138,7 @@ steps: targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: and(succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) + condition: (succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) # if there's a test failure || test summary not generated || provisioning failed: # - update GH commit status @@ -156,7 +156,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'add GH status & commit message' - condition: and(succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) + condition: (succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From 75a18d70c4a3b76b0e91dcad9d04a02f9c66c841 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 13:50:57 -0400 Subject: [PATCH 36/98] fix again... --- tools/devops/templates/device-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 068837196182..5d3bd7246187 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -138,7 +138,7 @@ steps: targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: (succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) + condition: succeededOrFailed() #, eq(variables['ProvisioningProfiles'], 'success')) # if there's a test failure || test summary not generated || provisioning failed: # - update GH commit status @@ -156,7 +156,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'add GH status & commit message' - condition: (succeededOrFailed()) #, eq(variables['ProvisioningProfiles'], 'success')) + condition: succeededOrFailed() #, eq(variables['ProvisioningProfiles'], 'success')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From 629d944c208cf686e502882efcabb90c68847d62 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 14:13:53 -0400 Subject: [PATCH 37/98] fix aggregate step --- tools/devops/templates/publish-results.yml | 24 +--------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 13931f16551b..c3ed2db28d23 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -24,7 +24,7 @@ steps: set -e ls -la pwd - cat $PIPELINE_WORKSPACE/s/TestSummary-*.md >> ./TestSummary.md + cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*.md >> ./TestSummary.md cat ./TestSummary.md displayName: 'Merge TestSummary files' @@ -39,26 +39,6 @@ steps: artifactName: TestSummary continueOnError: true -### -### Report final results to GitHub -### -# - task: PowerShell@2 -# inputs: -# targetType: #filePath -# script: @json_payload = @"{"token": $TOKEN, "target": $TARGET_URL}" - -### -### Publish merged TestSummary file -### -- task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: Aggregate TestSummary' - inputs: -# targetPath: 'xamarin-macios/tests/TestSummary.md' - targetPath: ./TestSummary.md - artifactName: TestSummary - continueOnError: true - - # pwsh for PowerShell Core # powershell is for Win PowerShell - pwsh: ./tools/devops/templates/publish_results.ps1 @@ -69,8 +49,6 @@ steps: GITHUB_TOKEN: $(GitHub.Token) CONTEXT: $(CONTEXT) -# GITHUB_TOKEN: $(GITHUB_TOKEN) - # VSTS_BUILD_URL='https://dev.azure.com/devdiv/ - SYSTEM_TEAMFOUNDATIONSERVERURI # DevDiv/_build/index?buildId=3521765&view=ms.vss-test-web.test-result-details' #VSTS_BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/index?buildId=${BUILD_BUILDID}&view=ms.vss-test-web.test-result-details" From cc2ca820a8369d59cf656b38386fb3a2bfb4b7cb Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 14:43:06 -0400 Subject: [PATCH 38/98] debug summary location --- tools/devops/templates/publish-results.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index c3ed2db28d23..93abd667ef82 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -23,6 +23,8 @@ steps: set -x set -e ls -la + ls -la $PIPELINE_WORKSPACE + ls -la $PIPELINE_WORKSPACE/Summaries pwd cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*.md >> ./TestSummary.md cat ./TestSummary.md From fb4e08b1950fb36b31b9ee3a9e74d874ce4259b1 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 15:09:56 -0400 Subject: [PATCH 39/98] debug testsummary contents --- tools/devops/templates/publish-results.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 93abd667ef82..2ebfbbe3e979 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -26,6 +26,7 @@ steps: ls -la $PIPELINE_WORKSPACE ls -la $PIPELINE_WORKSPACE/Summaries pwd + cat $PIPELINE_WORKSPACE/Summaries/TestSummary-* cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*.md >> ./TestSummary.md cat ./TestSummary.md displayName: 'Merge TestSummary files' From ead00b719d8e250e81971ef18e22c831360f95de Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 28 Apr 2020 16:09:53 -0400 Subject: [PATCH 40/98] fix dir path for aggregate --- tools/devops/templates/publish-results.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 2ebfbbe3e979..c79699e38aec 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -26,8 +26,8 @@ steps: ls -la $PIPELINE_WORKSPACE ls -la $PIPELINE_WORKSPACE/Summaries pwd - cat $PIPELINE_WORKSPACE/Summaries/TestSummary-* - cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*.md >> ./TestSummary.md + cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md + #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*.md >> ./TestSummary.md cat ./TestSummary.md displayName: 'Merge TestSummary files' From 210ff21217cac394af6312ee9a2afc3bf3868e2b Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 29 Apr 2020 17:35:53 -0400 Subject: [PATCH 41/98] add test summary file name to aggregate --- tools/devops/templates/publish-results.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index c79699e38aec..d75dd23a6669 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -26,8 +26,13 @@ steps: ls -la $PIPELINE_WORKSPACE ls -la $PIPELINE_WORKSPACE/Summaries pwd - cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md - #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*.md >> ./TestSummary.md + #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md + + for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md; + do echo $f >> ./TestSummary.md + cat $f >> ./TestSummary.md + done + cat ./TestSummary.md displayName: 'Merge TestSummary files' @@ -52,16 +57,6 @@ steps: GITHUB_TOKEN: $(GitHub.Token) CONTEXT: $(CONTEXT) -# VSTS_BUILD_URL='https://dev.azure.com/devdiv/ - SYSTEM_TEAMFOUNDATIONSERVERURI -# DevDiv/_build/index?buildId=3521765&view=ms.vss-test-web.test-result-details' -#VSTS_BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/index?buildId=${BUILD_BUILDID}&view=ms.vss-test-web.test-result-details" -# printf "\t\"state\": \"%s\",\n" "$STATE" -# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" -# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" -# printf "\t\"context\": \"%s\"\n" "$CONTEXT" -# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" - - ### ### Report final results to GitHub ### From b6a57a661ac581f01016f84dffaaa7cfa1277d91 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 30 Apr 2020 16:30:13 -0400 Subject: [PATCH 42/98] create + grab real test summary from correct location --- tools/devops/templates/device-tests.yml | 16 ++++++---------- tools/devops/templates/job-matrix.yml | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 5d3bd7246187..7a8a0b70d829 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -103,22 +103,21 @@ steps: - bash: | set -e set -x - FILE="TestSummary-${CONTEXT}.md" + FILE="xamarin-macios/tests/TestSummary.md" echo $FILE if ! test -f $FILE; then echo ":fire: Tests failed catastrophically (no summary found)" > $FILE fi - displayName: "If test summary not found, create test summary" + displayName: "If test summary does not exist, create test summary" condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) -### TODO: can probably figure out way to do this in the bash script above for -### test failures +### ### If provisioning failed, create test summary files to indicate failure ### - bash: | set -e set -x - FILE="TestSummary-${CONTEXT}.md" + FILE="xamarin-macios/tests/TestSummary.md" echo $FILE echo ":fire: Provisioning step failed" > $FILE displayName: 'Add test summary for failed provisioning' @@ -134,8 +133,8 @@ steps: - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: TestSummary' inputs: -# targetPath: 'xamarin-macios/tests/TestSummary.md' - targetPath: TestSummary-$(CONTEXT).md + targetPath: 'xamarin-macios/tests/TestSummary.md' + #targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true condition: succeededOrFailed() #, eq(variables['ProvisioningProfiles'], 'success')) @@ -189,9 +188,6 @@ steps: ### ### Push the HTML report to Azure DevOps (shows up in Summary tab as Build Artifact) ### -#- bash: | -# echo "$CONTEXT" >> TestSummary-$CONTEXT.md - - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: HtmlReport' inputs: diff --git a/tools/devops/templates/job-matrix.yml b/tools/devops/templates/job-matrix.yml index 197ed94b7de8..83d8509e1830 100644 --- a/tools/devops/templates/job-matrix.yml +++ b/tools/devops/templates/job-matrix.yml @@ -54,7 +54,7 @@ jobs: maxParallel: ${{ parameters.maxParallel }} matrix: Xamarin: - CONTEXT: '${{ parameters.variation }}' + CONTEXT: '${{ parameters.variation }}-xamarin-DDFun' TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-non-monotouch-tests' # run all tests but the bcl ones TESTS: 'Non Monotouch' Monotouch: From 6bfec0fafb8fa40bcc9081f033fc7bc6948498b1 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 30 Apr 2020 16:51:45 -0400 Subject: [PATCH 43/98] wildcard testsummary in publish-results --- tools/devops/templates/publish-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index d75dd23a6669..5538fda654b4 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -28,7 +28,7 @@ steps: pwd #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md - for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md; + for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/*; do echo $f >> ./TestSummary.md cat $f >> ./TestSummary.md done From 3c83f309c9823b75094a8dcf1a9dcb2c40ab1d09 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 30 Apr 2020 18:08:24 -0400 Subject: [PATCH 44/98] add string manipulation --- tools/devops/templates/publish-results.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 5538fda654b4..6e157ddac470 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -28,8 +28,25 @@ steps: pwd #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md - for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/*; - do echo $f >> ./TestSummary.md + for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary.md; + + prefix=$PIPELINE_WORKSPACE/Summaries/ + prefix_len = ${#prefix} + echo $prefix_len + suffix = "/TestSummary.md" + suffix_len = ${#suffix} + echo $suffix_len + + s=${f:$prefix_len} + echo $s + s_len=${#s} + diff=$((s_len - suffix_len)) + + echo $diff + s=${s:0:$diff} + echo $s + + do echo $s >> ./TestSummary.md cat $f >> ./TestSummary.md done From 2d38eebaffa2ae5e17f31f8213db842a3328722f Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 30 Apr 2020 19:39:08 -0400 Subject: [PATCH 45/98] fix syntax error for prefix --- tools/devops/templates/publish-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 6e157ddac470..975410e9a49b 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -30,7 +30,7 @@ steps: for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary.md; - prefix=$PIPELINE_WORKSPACE/Summaries/ + prefix="$PIPELINE_WORKSPACE/Summaries/" prefix_len = ${#prefix} echo $prefix_len suffix = "/TestSummary.md" From e1b50062c3b755088eedabfaba0b0871c71efff0 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 14:15:48 -0400 Subject: [PATCH 46/98] try fixing bash syntax error --- tools/devops/templates/publish-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 975410e9a49b..74cc2c6a18ae 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -30,7 +30,7 @@ steps: for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary.md; - prefix="$PIPELINE_WORKSPACE/Summaries/" + prefix="${PIPELINE_WORKSPACE}/Summaries/" prefix_len = ${#prefix} echo $prefix_len suffix = "/TestSummary.md" From 19a84211e29a23fecfd26cb4c860fdeb36bfcee7 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 14:30:29 -0400 Subject: [PATCH 47/98] fix bash for loop --- tools/devops/templates/publish-results.yml | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 74cc2c6a18ae..ef69fa30831b 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -29,25 +29,26 @@ steps: #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary.md; + do - prefix="${PIPELINE_WORKSPACE}/Summaries/" - prefix_len = ${#prefix} - echo $prefix_len - suffix = "/TestSummary.md" - suffix_len = ${#suffix} - echo $suffix_len + prefix="${PIPELINE_WORKSPACE}/Summaries/" + prefix_len = ${#prefix} + echo $prefix_len + suffix = "/TestSummary.md" + suffix_len = ${#suffix} + echo $suffix_len - s=${f:$prefix_len} - echo $s - s_len=${#s} - diff=$((s_len - suffix_len)) + s=${f:$prefix_len} + echo $s + s_len=${#s} + diff=$((s_len - suffix_len)) - echo $diff - s=${s:0:$diff} - echo $s + echo $diff + s=${s:0:$diff} + echo $s - do echo $s >> ./TestSummary.md - cat $f >> ./TestSummary.md + echo $s >> ./TestSummary.md + cat $f >> ./TestSummary.md done cat ./TestSummary.md From a7b1623e88feea5ffed605b9fc09196978f06ae7 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 14:50:31 -0400 Subject: [PATCH 48/98] fix extra spaces in bash script --- tools/devops/templates/publish-results.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index ef69fa30831b..5b7fbe07d47c 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -32,10 +32,10 @@ steps: do prefix="${PIPELINE_WORKSPACE}/Summaries/" - prefix_len = ${#prefix} + prefix_len=${#prefix} echo $prefix_len - suffix = "/TestSummary.md" - suffix_len = ${#suffix} + suffix="/TestSummary.md" + suffix_len=${#suffix} echo $suffix_len s=${f:$prefix_len} From 8bc52cbf4674ee571fa37541642abdab5deb39db Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 16:05:17 -0400 Subject: [PATCH 49/98] first try at preserving formatting in .md --- tools/devops/templates/publish-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 5b7fbe07d47c..5ab5bdb1bad1 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -48,7 +48,7 @@ steps: echo $s echo $s >> ./TestSummary.md - cat $f >> ./TestSummary.md + echo "$f" >> ./TestSummary.md done cat ./TestSummary.md From 99f8ec3c68792156303359596bfc02055fd97fa2 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 16:22:52 -0400 Subject: [PATCH 50/98] try out formatting python --- tools/devops/templates/publish-results.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 5ab5bdb1bad1..57705ee532ee 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -48,7 +48,10 @@ steps: echo $s echo $s >> ./TestSummary.md - echo "$f" >> ./TestSummary.md + #echo "$f" >> ./TestSummary.md + + python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' < "$f" >> ./TestSummary.md + done cat ./TestSummary.md From 548a9db0be0103cf9ce586e39cbb2cc5b4abde9a Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 17:05:29 -0400 Subject: [PATCH 51/98] try making not .md file --- tools/devops/templates/publish-results.yml | 4 ++-- tools/devops/templates/publish_results.ps1 | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 57705ee532ee..57ca0e0f54bc 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -47,10 +47,10 @@ steps: s=${s:0:$diff} echo $s - echo $s >> ./TestSummary.md + echo $s >> ./TestSummary #echo "$f" >> ./TestSummary.md - python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' < "$f" >> ./TestSummary.md + python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' < "$f" >> ./TestSummary done diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index c133a8c24d33..88c11981923c 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -96,12 +96,15 @@ $json_text = $HEADER + "$DESCRIPTION on [Azure DevOps]($target_url)" # add contents of test summary to json_text $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -$testsummary_location = $testsummary_location + "/TestSummary.md" +$testsummary_location = $testsummary_location + "/TestSummary" Write-Host $testsummary_location $test_summary = Get-Content $testsummary_location $json_text = $json_text + $test_summary +Write-Host "json_text + test_summary" +Write-Host $json_text + $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" From c5c63b89abf0a96035c98a844c31d7d983558971 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 17:32:45 -0400 Subject: [PATCH 52/98] try converting to json text before adding as text body to post --- tools/devops/templates/publish-results.yml | 4 ++-- tools/devops/templates/publish_results.ps1 | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 57ca0e0f54bc..17790cf7d059 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -48,9 +48,9 @@ steps: echo $s echo $s >> ./TestSummary - #echo "$f" >> ./TestSummary.md + echo "$f" >> ./TestSummary.md - python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' < "$f" >> ./TestSummary + #python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' < "$f" >> ./TestSummary done diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 88c11981923c..6eb396d29714 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -105,6 +105,10 @@ $json_text = $json_text + $test_summary Write-Host "json_text + test_summary" Write-Host $json_text +$json_text = $json_text | ConvertTo-Json +Write-Host "Convert to json" +Write-Host $json_text + $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" From 257d61eaa3af8c9b582accee8415e13111e69983 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 1 May 2020 17:47:45 -0400 Subject: [PATCH 53/98] remove quotation marks from json body text --- tools/devops/templates/publish_results.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 6eb396d29714..3076b95325aa 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -115,7 +115,7 @@ $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env $json_payload = @" { - "body" : "$json_text" + "body" : $json_text } "@ From 591c5f1e6c17b9e91bd2994d962e1782d1cb1848 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 4 May 2020 11:56:44 -0400 Subject: [PATCH 54/98] try out quotation marks --- tools/devops/templates/publish_results.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 3076b95325aa..c1a1720628a6 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -83,7 +83,7 @@ $response | Write-Host $HEADER = "" If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') { - $HEADER = "### :boom: :construction: TESTING Experimental DDFun pipeline\\n" + $HEADER = "### :boom: :construction: TESTING Experimental DDFun pipeline\n" } #Else{ #HTML Report jenkins stuff @@ -98,7 +98,7 @@ $json_text = $HEADER + "$DESCRIPTION on [Azure DevOps]($target_url)" $testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY $testsummary_location = $testsummary_location + "/TestSummary" Write-Host $testsummary_location -$test_summary = Get-Content $testsummary_location +$test_summary = Get-Content "$testsummary_location" $json_text = $json_text + $test_summary From 942eb1aa5dabb30ac573aa3e73eae5ff51a2e36e Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 5 May 2020 18:08:18 -0400 Subject: [PATCH 55/98] try out with all testsummary aggregation logic inside pwsh script --- tools/devops/templates/publish_results.ps1 | 101 +++++++-------------- 1 file changed, 35 insertions(+), 66 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index c1a1720628a6..b92eb2ce21ed 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -80,44 +80,57 @@ $response | Write-Host ### Construct commit message w/ aggregate test summary ### -$HEADER = "" If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') { - $HEADER = "### :boom: :construction: TESTING Experimental DDFun pipeline\n" + # do stuff } -#Else{ -#HTML Report jenkins stuff -#} -$DESCRIPTION="Device test aggregate results:" # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" -$json_text = $HEADER + "$DESCRIPTION on [Azure DevOps]($target_url)" +$json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" + + +# Get all test summary files +$files = Get-ChildItem "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/TestSummary-*/TestSummary.md" + +# stringbuilder for extra flavor +$msg = [System.Text.StringBuilder]::new() +$msg.AppendLine($json_text) +$msg.AppendLine() + +foreach ($file in $files) +{ + Write-Host $file + Write-Host Get-Content $file + + $msg.AppendLine("blah title from filename") + $msg.AppendLine() + + # read each line of the summary file, append it with correct \n at the end + foreach ($line in Get-Content -Path $file) + { + $msg.AppendLine($line) + } + + # new line to separate file contents + $msg.AppendLine() +} -# add contents of test summary to json_text -$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -$testsummary_location = $testsummary_location + "/TestSummary" -Write-Host $testsummary_location -$test_summary = Get-Content "$testsummary_location" -$json_text = $json_text + $test_summary -Write-Host "json_text + test_summary" -Write-Host $json_text -$json_text = $json_text | ConvertTo-Json -Write-Host "Convert to json" -Write-Host $json_text $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" -$json_payload = @" -{ - "body" : $json_text +# create pwsh object to store payload +$payload = @{ + body = $msg.ToString() } -"@ + +# convert payload to json +$json_payload = $payload | ConvertTo-json Write-Host $json_payload @@ -134,47 +147,3 @@ Write-Host $params $response = Invoke-RestMethod @params $response | ConvertTo-Json | Write-Host - - - -# https://api.github.com/xamarin/xamarin-macios/commits/eea6fd1f27ba9a0ac4fa09c8e57fc87d612b6340/status - -#GET /projects/:id/repository/commits/:sha/refs - - -#https://api.github.com/repos/xamarin/xamarin-macios/statuses/$Env:BUILD_REVISION - - - -# -Uri $url -# -Method Post -# -Body $json_payload -#$response = Invoke-RestMethod -Uri $url -ContentType application/json -Method Post -Body @json_payload -<# -( - printf '{\n' - printf "\t\"state\": \"%s\",\n" "$STATE" - printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" - printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" - printf "\t\"context\": \"%s\"\n" "$CONTEXT" - printf "}\n" -) > "$JSONFILE" - -if test -n "$VERBOSE"; then - echo "JSON file:" - sed 's/^/ /' "$JSONFILE"; -fi - -if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$HASH" > "$LOGFILE" 2>&1; then - echo "Failed to add status." - echo "curl output:" - sed 's/^/ /' "$LOGFILE" - echo "Json body:" - sed 's/^/ /' "$JSONFILE" - exit 1 #> - -# printf "\t\"state\": \"%s\",\n" "$STATE" -# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" -# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" -# printf "\t\"context\": \"%s\"\n" "$CONTEXT" -# ./jenkins/add-commit-status.sh --token="$TOKEN" --hash="$BUILD_REVISION" --state="$GH_STATE" --target-url="$VSTS_BUILD_URL" --description="$DESCRIPTION" --context="VSTS: device tests ($DEVICE_TYPE)" From 4da25bb067a53d41b95896da892f32dc7b4f376c Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 5 May 2020 18:20:29 -0400 Subject: [PATCH 56/98] fix file path for testsummaries --- tools/devops/templates/publish_results.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index b92eb2ce21ed..13530d82bf58 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -28,10 +28,6 @@ $params = @{ # $response = Invoke-RestMethod @params -Write-Host $response -$response | ConvertTo-Json | Write-Host -Write-Host "^ ConvertToJson below is ConvertFromJson `n" - Write-Host "Raw response: " + $response Write-Host "Response.state: " + $response.state @@ -91,7 +87,7 @@ $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Dev # Get all test summary files -$files = Get-ChildItem "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/TestSummary-*/TestSummary.md" +$files = Get-ChildItem "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Summaries/TestSummary-*/TestSummary.md" # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() From 7e6bb67f1ea54ed522e329c44d628f9de0d0a372 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 5 May 2020 18:41:38 -0400 Subject: [PATCH 57/98] fix file path for summaries... --- tools/devops/templates/publish_results.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 13530d82bf58..6a601114da3f 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -87,7 +87,7 @@ $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Dev # Get all test summary files -$files = Get-ChildItem "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Summaries/TestSummary-*/TestSummary.md" +$files = Get-ChildItem -Path $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Summaries/TestSummary-*/TestSummary.md # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() From 4d52b298e6e21d81993d4f4539a79d07c9cfa22b Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 5 May 2020 19:00:55 -0400 Subject: [PATCH 58/98] fix path... --- tools/devops/templates/publish_results.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 6a601114da3f..2e2b8f89f68a 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -85,9 +85,10 @@ If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" - +$dir = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY +$dir = $dir + "/Summaries/*" # Get all test summary files -$files = Get-ChildItem -Path $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Summaries/TestSummary-*/TestSummary.md +$files = Get-ChildItem -Path $dir -Include TestSummary.md # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() From ae0bdbc38caf622b19e2906b90f49a7654973be3 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 6 May 2020 12:54:34 -0400 Subject: [PATCH 59/98] try out double quotes around --- tools/devops/templates/publish_results.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 2e2b8f89f68a..a59ca91f82b0 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -86,7 +86,8 @@ If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" $dir = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -$dir = $dir + "/Summaries/*" +$dir = "$dir/Summaries/*" +Get-ChildItem $dir | Write-Host # Get all test summary files $files = Get-ChildItem -Path $dir -Include TestSummary.md From 03f85484dfcbb12f1b1a457e1f7a346cdf0d9bb3 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 6 May 2020 13:22:06 -0400 Subject: [PATCH 60/98] fix env var for dir path, add debugging print --- tools/devops/templates/publish_results.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index a59ca91f82b0..842bd88b52a7 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -85,8 +85,10 @@ If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" -$dir = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY +$dir = $Env:PIPELINE_WORKSPACE $dir = "$dir/Summaries/*" + +Write-Host $dir Get-ChildItem $dir | Write-Host # Get all test summary files $files = Get-ChildItem -Path $dir -Include TestSummary.md From cca7f2d00693979edf802b598887a0bbcf9d7219 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 6 May 2020 14:56:05 -0400 Subject: [PATCH 61/98] actually fix dir path?? --- tools/devops/templates/publish_results.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 842bd88b52a7..8c03d1b663f8 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -91,7 +91,8 @@ $dir = "$dir/Summaries/*" Write-Host $dir Get-ChildItem $dir | Write-Host # Get all test summary files -$files = Get-ChildItem -Path $dir -Include TestSummary.md +$files = Get-ChildItem $dir + # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() From 9363b88398c525f6aa05431b637eca54e5e0b5ea Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 6 May 2020 15:24:47 -0400 Subject: [PATCH 62/98] recursive call to get-childitem since we have Summaries/blah-testname/testsummary.md --- tools/devops/templates/publish_results.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 8c03d1b663f8..67a44ad58ff1 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -91,7 +91,7 @@ $dir = "$dir/Summaries/*" Write-Host $dir Get-ChildItem $dir | Write-Host # Get all test summary files -$files = Get-ChildItem $dir +$files = Get-ChildItem -Path $dir -Recurse -Include TestSummary.md # stringbuilder for extra flavor From 951dbd05fa50d54a7c4740cefab5fe6cc02a86b5 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 6 May 2020 15:49:42 -0400 Subject: [PATCH 63/98] try removing recursive call by adding more wildcards to dir path --- tools/devops/templates/publish_results.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 67a44ad58ff1..44c733875f00 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -86,12 +86,17 @@ If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" $dir = $Env:PIPELINE_WORKSPACE -$dir = "$dir/Summaries/*" +$dir = "$dir/Summaries/TestSummary-*/TestSummary.md" Write-Host $dir Get-ChildItem $dir | Write-Host + +# Test summary filepath follows the format: +# /Users/xamarinqa/azdo/_work/15/Summaries/TestSummary-DEVICE-TESTNAME/TestSummary.md +# /Users/xamarinqa/azdo/_work/15/Summaries/TestSummary-tvOS-monotouch-DDFun/TestSummary.md +# /Users/xamarinqa/azdo/_work/15/Summaries/TestSummary-tvOS-xamarin-DDFun/TestSummary.md # Get all test summary files -$files = Get-ChildItem -Path $dir -Recurse -Include TestSummary.md +$files = Get-ChildItem -Path $dir # stringbuilder for extra flavor From b642e4a51cdc2e6b193a502bb04e5708f31cc771 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 19:19:32 -0400 Subject: [PATCH 64/98] remote aggregate test summary bash task, try out publishing correctly formatted test results for per-test failure messages --- tools/devops/templates/publish-results.yml | 52 +------ tools/devops/templates/publish_failure.ps1 | 167 +++++---------------- 2 files changed, 42 insertions(+), 177 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index 17790cf7d059..d2db41e09c6f 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -19,54 +19,16 @@ steps: allowFailedBuilds: true path: $(Pipeline.Workspace)/Summaries -- bash: | - set -x - set -e - ls -la - ls -la $PIPELINE_WORKSPACE - ls -la $PIPELINE_WORKSPACE/Summaries - pwd - #cat $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary-*.md >> ./TestSummary.md - - for f in $PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary.md; - do - - prefix="${PIPELINE_WORKSPACE}/Summaries/" - prefix_len=${#prefix} - echo $prefix_len - suffix="/TestSummary.md" - suffix_len=${#suffix} - echo $suffix_len - - s=${f:$prefix_len} - echo $s - s_len=${#s} - diff=$((s_len - suffix_len)) - - echo $diff - s=${s:0:$diff} - echo $s - - echo $s >> ./TestSummary - echo "$f" >> ./TestSummary.md - - #python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' < "$f" >> ./TestSummary - - done - - cat ./TestSummary.md - displayName: 'Merge TestSummary files' - ### ### Publish merged TestSummary file ### -- task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: Aggregate TestSummary' - inputs: -# targetPath: 'xamarin-macios/tests/TestSummary.md' - targetPath: ./TestSummary.md - artifactName: TestSummary - continueOnError: true +# - task: PublishPipelineArtifact@1 +# displayName: 'Publish Artifact: Aggregate TestSummary' +# inputs: +# # targetPath: 'xamarin-macios/tests/TestSummary.md' +# targetPath: ./TestSummary.md +# artifactName: TestSummary +# continueOnError: true # pwsh for PowerShell Core # powershell is for Win PowerShell diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index e726d8910cd0..d6498e137615 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -48,7 +48,7 @@ $response | ConvertTo-Json | Write-Host ### -### Construct commit message +### Construct commit message with test summary ### $RESULT_EMOJI @@ -59,39 +59,60 @@ If ($Env:AGENT_JOBSTATUS -eq 'Failed') $RESULT_EMOJI = ":white_check_mark: " # maybe remove this since we only add commit messages on failure? } -$HEADER = "" + If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') { - $HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline\\n" + ## do stuff } #Else{ #HTML Report jenkins stuff #} + +$HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline" + + # SYSTEM_JOBNAME: Xamarin | Monotouch | xUnitBCL | NUnitBCL | mscorlib # CONTEXT: tvOS | iOS | iOS32 $DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTEXT" # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" -$json_text = $HEADER + "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" +$json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" -# add contents of test summary to json_text -$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -$testsummary_location = $testsummary_location + "/TestSummary-$Env:CONTEXT.md" -Write-Host $testsummary_location -$test_summary = Get-Content $testsummary_location +$file = $Env:PIPELINE_WORKSPACE +$file = "$file/Summaries/TestSummary-$Env:CONTEXT/TestSummary.md" -$json_text = $json_text + $test_summary +Write-Host $file +Get-ChildItem $file | Write-Host -$message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" +# stringbuilder for extra flavor +$msg = [System.Text.StringBuilder]::new() +$msg.AppendLine($HEADER) +$msg.AppendLine() +$msg.AppendLine($json_text) +$msg.AppendLine() -$json_payload = @" +# read each line of the summary file, append it with correct \n at the end +foreach ($line in Get-Content -Path $file) { - "body" : "$json_text" + $msg.AppendLine($line) } -"@ + + + +$message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" + + +# create pwsh object to store payload +$payload = @{ + body = $msg.ToString() +} + + +# convert payload to json +$json_payload = $payload | ConvertTo-json Write-Host $json_payload @@ -108,121 +129,3 @@ Write-Host $params $response = Invoke-RestMethod @params $response | ConvertTo-Json | Write-Host - - -#Where to publish payload for commit comment: "https://api.github.com/repos/xamarin/xamarin-macios/commits/$HASH/comments" - -#Example VSTS Build URL: https://dev.azure.com/devdiv/DevDiv/_build/index?buildId=3568540&view=ms.vss-test-web.test-result-details - -#"%s%s on [Azure DevOps](%s)($Env:CONTEXT)%s %s\\n\\n" "$RESULT_EMOJII" "$DESCRIPTION" "$VSTS_BUILD_URL" "$HTML_REPORT" "$RESULT_EMOJII" >> "$MESSAGE_FILE" - - -# ( -# printf '{\n' -# printf "\t\"state\": \"%s\",\n" "$STATE" -# printf "\t\"target_url\": \"%s\",\n" "$TARGET_URL" -# printf "\t\"description\": %s,\n" "$(echo -n "$DESCRIPTION" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" -# printf "\t\"context\": \"%s\"\n" "$CONTEXT" -# printf "}\n" -# ) > "$JSONFILE" - -# if test -n "$VERBOSE"; then - - - -# # Add a GitHub commit comment - - - - -# # Query job status (always failure) and set result emoji -# GH_STATE=failure -# DESCRIPTION="Running device tests" -# RESULT_EMOJII= -# if test -n "$START"; then -# GH_STATE=pending -# DESCRIPTION="Running device tests on $DEVICE_TYPE" -# else -# case "$(echo "$AGENT_JOBSTATUS" | tr '[:upper:]' '[:lower:]')" in -# succeeded) -# GH_STATE=success -# DESCRIPTION="Device tests passed on $DEVICE_TYPE" -# RESULT_EMOJII="✅ " -# ;; -# failed | canceled | succeededwithissues | *) -# GH_STATE=error -# DESCRIPTION="Device tests completed ($AGENT_JOBSTATUS) on $DEVICE_TYPE" -# RESULT_EMOJII="🔥 " -# ;; -# esac -# fi - - -# + cat /Users/xamarinqa/azdo/_work/48/s/TestSummary.md -# + ./jenkins/add-commit-comment.sh --token=*** --file=commit-message.txt --hash=429bec0478899b5f35260efceb87d67a876ee22a -# { -# "body": "### :construction: Experimental DDFun pipeline\n\ud83d\udd25 Device tests completed (Failed) on VSTS-DDFun on [Azure DevOps](https://dev.azure.com/devdiv/DevDiv/_build/index?buildId=3568540&view=ms.vss-test-web.test-result-details)(VSTS-DDFun) \ud83d\udd25 \n\n" -# } -# Successfully added commit message to https://github.com/xamarin/xamarin-macios/commit/429bec0478899b5f35260efceb87d67a876ee22a -# + cleanup - - - -# # create a temp file to construct commit message -# # add cleanup code -# # add message header: if [[ $DEVICE_TYPE == *"DDFun"* ]]; then printf "### :construction: Experimental DDFun pipeline\\n" > "$MESSAGE_FILE" -# # add the following to the file: $DEVICE_TYPE + emoji + $DESCRIPTION + $VSTS_BUILD_URL + $HTMLREPORT + RESULT_EMOJI -# # look for testsummary.md -# # if not found -# # add "🔥 Tests failed catastrophically on $DEVICE_TYPE (no summary found)\\n" -# # if found -# # cat testsummary.md > messagefile -# if test -z "$START"; then -# # When we're done, add a GitHub comment to the commit we're testing -# MESSAGE_FILE=commit-message.txt -# cleanup () -# { -# rm -f "$MESSAGE_FILE" -# } -# trap cleanup ERR -# trap cleanup EXIT - -# HTML_REPORT="" -# if [[ $DEVICE_TYPE == *"DDFun"* ]]; then -# printf "### :construction: Experimental DDFun pipeline\\n" > "$MESSAGE_FILE" -# else -# P=$(cat tmp.p) -# HTML_REPORT=": [Html Report](http://xamarin-storage/${P}/jenkins-results/tests/index.html)" -# fi - -# printf "%s%s on [Azure DevOps](%s)($DEVICE_TYPE)%s %s\\n\\n" "$RESULT_EMOJII" "$DESCRIPTION" "$VSTS_BUILD_URL" "$HTML_REPORT" "$RESULT_EMOJII" >> "$MESSAGE_FILE" - -# FILE=$PWD/TestSummary.md -# if ! test -f "$FILE"; then -# printf "🔥 Tests failed catastrophically on $DEVICE_TYPE (no summary found)\\n" >> "$MESSAGE_FILE" -# else -# cat "$FILE" >> "$MESSAGE_FILE" -# fi - -# ./jenkins/add-commit-comment.sh --token="$TOKEN" --file="$MESSAGE_FILE" "--hash=$BUILD_REVISION" - -# # construct a json payload and publish it to GH -# # url is probably something like: -# $comment_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" - -# # need to print out JSON payload from jenkins to do comparison - - -# if ! curl -f -v -H "Authorization: token $TOKEN" -H "User-Agent: command line tool" -d "@$JSONFILE" "https://api.github.com/repos/xamarin/xamarin-macios/commits/$HASH/comments" > "$LOGFILE" 2>&1; then -# echo "Failed to add commit message." -# echo "curl output:" -# sed 's/^/ /' "$LOGFILE" -# echo "Json body:" -# sed 's/^/ /' "$JSONFILE" -# exit 1 -# else -# if test -n "$VERBOSE"; then sed 's/^/ /' "$LOGFILE"; fi -# echo "Successfully added commit message to https://github.com/xamarin/xamarin-macios/commit/$HASH" -# fi - -# fi From 281f709406f4d194ce8cb8fbfab8eda4fe3037a6 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 19:38:31 -0400 Subject: [PATCH 65/98] fix leftover pwsh commands causing failure --- tools/devops/templates/publish_failure.ps1 | 2 +- tools/devops/templates/publish_results.ps1 | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index d6498e137615..afe76aeacb5a 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -83,7 +83,7 @@ $file = $Env:PIPELINE_WORKSPACE $file = "$file/Summaries/TestSummary-$Env:CONTEXT/TestSummary.md" Write-Host $file -Get-ChildItem $file | Write-Host +Get-Content $file | Write-Host diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 44c733875f00..079d890cb97e 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -3,14 +3,6 @@ Get-Location Set-Location Env: Get-ChildItem - -$testsummary_location = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -Get-ChildItem $testsummary_location - -$testsummary_location = $testsummary_location + "/TestSummary.md" -Write-Host $testsummary_location -Get-Content $testsummary_location - # get combined status: # success only if every status is success # otherwise failure From e8c3541dbe36308f9c25860c0e61a73ecd547481 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 19:54:17 -0400 Subject: [PATCH 66/98] try out more debugging messages for per-test failure --- tools/devops/templates/device-tests.yml | 2 +- tools/devops/templates/publish_failure.ps1 | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 7a8a0b70d829..101b68cff988 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -154,7 +154,7 @@ steps: CONTEXT: $(CONTEXT) TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) - displayName: 'add GH status & commit message' + displayName: 'Powershell: Add GH status & commit message' condition: succeededOrFailed() #, eq(variables['ProvisioningProfiles'], 'success')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index afe76aeacb5a..abf737a0894c 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -80,6 +80,10 @@ $DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTE $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" $file = $Env:PIPELINE_WORKSPACE +Get-ChildItem $file | Write-Host +$file = "$file/Summaries/TestSummary-$Env:CONTEXT" +Get-ChildItem $file | Write-Host + $file = "$file/Summaries/TestSummary-$Env:CONTEXT/TestSummary.md" Write-Host $file From ebde63f5ec08b181f4a6e949e4bf90f436382e11 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 20:02:34 -0400 Subject: [PATCH 67/98] try fixing per-device pwsh env var for summaries --- tools/devops/templates/publish_failure.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index abf737a0894c..eb38c7989226 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -79,7 +79,8 @@ $DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTE # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" -$file = $Env:PIPELINE_WORKSPACE +$file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY +#$Env:PIPELINE_WORKSPACE Get-ChildItem $file | Write-Host $file = "$file/Summaries/TestSummary-$Env:CONTEXT" Get-ChildItem $file | Write-Host From 11b82c0599e8e346092268534a15b8d1db2f74cc Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 20:12:41 -0400 Subject: [PATCH 68/98] actually fix file path for per-device tests? --- tools/devops/templates/publish_failure.ps1 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index eb38c7989226..f9af4b3f67bf 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -82,10 +82,7 @@ $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BU $file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY #$Env:PIPELINE_WORKSPACE Get-ChildItem $file | Write-Host -$file = "$file/Summaries/TestSummary-$Env:CONTEXT" -Get-ChildItem $file | Write-Host - -$file = "$file/Summaries/TestSummary-$Env:CONTEXT/TestSummary.md" +$file = "$file/Summaries/TestSummary-$Env:CONTEXT.md" Write-Host $file Get-Content $file | Write-Host From c43f8d4bb9fa2f8c1cff2175599fd4524b2f40d4 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 20:20:19 -0400 Subject: [PATCH 69/98] fix file path for per-test message again... --- tools/devops/templates/publish_failure.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index f9af4b3f67bf..6ea6cdf0a9cc 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -82,7 +82,7 @@ $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BU $file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY #$Env:PIPELINE_WORKSPACE Get-ChildItem $file | Write-Host -$file = "$file/Summaries/TestSummary-$Env:CONTEXT.md" +$file = "$file/TestSummary-$Env:CONTEXT.md" Write-Host $file Get-Content $file | Write-Host From cb2bae4386ba65125282c9d744bb71fc97fe7475 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 20:35:20 -0400 Subject: [PATCH 70/98] try accessing xamarin-macios rather than upload location because there seems to be a delay with access to the upload --- tools/devops/templates/publish_failure.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 6ea6cdf0a9cc..5b99b35c4dce 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -82,6 +82,12 @@ $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BU $file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY #$Env:PIPELINE_WORKSPACE Get-ChildItem $file | Write-Host + +$file = "$file/xamarin-macios" + +Write-Host "xamarin-macios children" +Get-ChildItem $file | Write-Host + $file = "$file/TestSummary-$Env:CONTEXT.md" Write-Host $file From 2ad64eb6b7a1920a22f6dd390bc7df476c372b0b Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Fri, 8 May 2020 20:45:13 -0400 Subject: [PATCH 71/98] try out xamarin-macios/tests --- tools/devops/templates/publish_failure.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 5b99b35c4dce..b5656013dbb3 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -83,9 +83,12 @@ $file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY #$Env:PIPELINE_WORKSPACE Get-ChildItem $file | Write-Host -$file = "$file/xamarin-macios" -Write-Host "xamarin-macios children" + + +$file = "$file/xamarin-macios/tests" + +Write-Host "xamarin-macios tests children" Get-ChildItem $file | Write-Host $file = "$file/TestSummary-$Env:CONTEXT.md" From 5abd35fd2e72a6edcc46762a383075b58237a6fd Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 13:22:43 -0400 Subject: [PATCH 72/98] fix file path?? --- tools/devops/templates/publish_failure.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index b5656013dbb3..35a7feba5c63 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -91,7 +91,7 @@ $file = "$file/xamarin-macios/tests" Write-Host "xamarin-macios tests children" Get-ChildItem $file | Write-Host -$file = "$file/TestSummary-$Env:CONTEXT.md" +$file = "$file/TestSummary.md" Write-Host $file Get-Content $file | Write-Host From 81e3cf73e1f4d28e21b31f58af98e508e7e7405c Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 13:55:16 -0400 Subject: [PATCH 73/98] try adding device + test info to aggregate results --- tools/devops/templates/publish_results.ps1 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 079d890cb97e..abd87734b8dc 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -96,6 +96,15 @@ $msg = [System.Text.StringBuilder]::new() $msg.AppendLine($json_text) $msg.AppendLine() + + +# Grab the test name + device info string from filepath +$prefix="$Env:PIPELINE_WORKSPACE/Summaries/TestSummary-" +$prefix_len=$prefix.length +$suffix="/TestSummary.md" +$suffix_len=$suffix.length + + foreach ($file in $files) { Write-Host $file @@ -104,6 +113,12 @@ foreach ($file in $files) $msg.AppendLine("blah title from filename") $msg.AppendLine() + + $info = $file.Substring($prefix.Length, $file.Length - $suffix.Length) + + $msg.AppendLine($info); + + # read each line of the summary file, append it with correct \n at the end foreach ($line in Get-Content -Path $file) { From 8556e2ff69c54af4aec78976344287f201fe8166 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 14:36:54 -0400 Subject: [PATCH 74/98] grab fullname then substring --- tools/devops/templates/publish_results.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index abd87734b8dc..94301e8c96a3 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -114,8 +114,7 @@ foreach ($file in $files) $msg.AppendLine() - $info = $file.Substring($prefix.Length, $file.Length - $suffix.Length) - + $info = $file.FullName.Substring($prefix.Length, $file.Length - $suffix.Length) $msg.AppendLine($info); From 1156b5f6f336c8b62451fc72f16edce46bea816f Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 15:16:39 -0400 Subject: [PATCH 75/98] fix length? --- tools/devops/templates/publish_results.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 94301e8c96a3..22c06da0fca9 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -114,7 +114,7 @@ foreach ($file in $files) $msg.AppendLine() - $info = $file.FullName.Substring($prefix.Length, $file.Length - $suffix.Length) + $info = $file.FullName.Substring($prefix.Length, $file.Length - $suffix.Length - $prefix.Length) $msg.AppendLine($info); From 1c33516c2a5c503577dcbe8e541adab4956e364c Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 16:03:11 -0400 Subject: [PATCH 76/98] fix file.fullname --- tools/devops/templates/publish_results.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 22c06da0fca9..05c19d1612e9 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -113,8 +113,10 @@ foreach ($file in $files) $msg.AppendLine("blah title from filename") $msg.AppendLine() + Write-Host $file.FullName + Write-Host $file.FullName.Length - $info = $file.FullName.Substring($prefix.Length, $file.Length - $suffix.Length - $prefix.Length) + $info = $file.FullName.Substring($prefix.Length, $file.FullName.Length - $suffix.Length - $prefix.Length) $msg.AppendLine($info); From 150dfb88fa86ffcf6b74ee97f63b1e2df0c3e5c6 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 16:42:13 -0400 Subject: [PATCH 77/98] tweak formatting --- tools/devops/templates/publish_results.ps1 | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 05c19d1612e9..4527c5956b28 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -110,20 +110,24 @@ foreach ($file in $files) Write-Host $file Write-Host Get-Content $file - $msg.AppendLine("blah title from filename") - $msg.AppendLine() - - Write-Host $file.FullName - Write-Host $file.FullName.Length $info = $file.FullName.Substring($prefix.Length, $file.FullName.Length - $suffix.Length - $prefix.Length) - $msg.AppendLine($info); + #$msg.AppendLine($info); + $first_line = 1 # read each line of the summary file, append it with correct \n at the end foreach ($line in Get-Content -Path $file) { - $msg.AppendLine($line) + if ($first_line) + { + $msg.Append($line + ": ") + $msg.AppendLine($info) + $first_line = 0 + } + else { + $msg.AppendLine($line) + } } # new line to separate file contents From 4b21d926e9e94d2eb9cbed84be90c667b136cc56 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 17:13:37 -0400 Subject: [PATCH 78/98] clean up publish_failure, add vso var to device-tests to check for test failure --- tools/devops/templates/device-tests.yml | 16 +++++++++---- tools/devops/templates/publish_results.ps1 | 26 +++------------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 101b68cff988..f882735515f1 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -91,7 +91,17 @@ steps: ### # move this to in-line code, should be very short -- bash: ./xamarin-macios/tools/devops/run-tests.sh +- bash: | + set -x + set -e + + ./xamarin-macios/tools/devops/run-tests.sh + + if [ $? -eq 0 ]; then + echo "##vso[task.setvariable variable=TestState]success" + else + echo "##vso[task.setvariable variable=TestState]failure" + fi displayName: 'Run tests' timeoutInMinutes: 600 @@ -99,7 +109,6 @@ steps: ### If test summary does not exist, create test summary files so that we can notify ### that no results were published at the per-test AND final results steps ### - ### TODO if provisioning fails add a different error message - bash: | set -e set -x @@ -134,10 +143,9 @@ steps: displayName: 'Publish Artifact: TestSummary' inputs: targetPath: 'xamarin-macios/tests/TestSummary.md' - #targetPath: TestSummary-$(CONTEXT).md artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: succeededOrFailed() #, eq(variables['ProvisioningProfiles'], 'success')) + condition: succeededOrFailed(), eq(variables['TestState'], 'failure')) # if there's a test failure || test summary not generated || provisioning failed: # - update GH commit status diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 4527c5956b28..2ffe2daadcb2 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -20,9 +20,6 @@ $params = @{ # $response = Invoke-RestMethod @params -Write-Host "Raw response: " + $response - -Write-Host "Response.state: " + $response.state $state = $response.state @@ -43,7 +40,6 @@ $json_payload = @" } "@ - $url = "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$Env:BUILD_REVISION" Write-Host $json_payload @@ -63,7 +59,6 @@ $response = Invoke-RestMethod @params $response | Write-Host - ### ### Construct commit message w/ aggregate test summary ### @@ -73,7 +68,6 @@ If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') # do stuff } - # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" @@ -90,35 +84,27 @@ Get-ChildItem $dir | Write-Host # Get all test summary files $files = Get-ChildItem -Path $dir - # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() $msg.AppendLine($json_text) $msg.AppendLine() - - # Grab the test name + device info string from filepath $prefix="$Env:PIPELINE_WORKSPACE/Summaries/TestSummary-" -$prefix_len=$prefix.length $suffix="/TestSummary.md" -$suffix_len=$suffix.length - foreach ($file in $files) { - Write-Host $file - Write-Host Get-Content $file - - $info = $file.FullName.Substring($prefix.Length, $file.FullName.Length - $suffix.Length - $prefix.Length) - #$msg.AppendLine($info); + # switch to keep track of when we are reading the first line of a summary $first_line = 1 # read each line of the summary file, append it with correct \n at the end foreach ($line in Get-Content -Path $file) { + # if reading the first line, append test + device info to the header + # $msg will look like: "Test results: tvOS-monotouch-DDFun" if ($first_line) { $msg.Append($line + ": ") @@ -134,12 +120,6 @@ foreach ($file in $files) $msg.AppendLine() } - - - - - - $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" # create pwsh object to store payload From 235c764121b81387fc0a8f48a63b974ae19c5bd7 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 17:19:30 -0400 Subject: [PATCH 79/98] fix syntax error --- tools/devops/templates/device-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index f882735515f1..aa045e454500 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -145,7 +145,7 @@ steps: targetPath: 'xamarin-macios/tests/TestSummary.md' artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: succeededOrFailed(), eq(variables['TestState'], 'failure')) + condition: and(succeededOrFailed(), eq(variables['TestState'], 'failure')) # if there's a test failure || test summary not generated || provisioning failed: # - update GH commit status From be94e204d1510d06805c7441ee980de9e2209ecc Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Mon, 11 May 2020 18:49:23 -0400 Subject: [PATCH 80/98] test with device tests passing --- tools/devops/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 3ba2ba63fd12..74fd232ed02b 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -12,6 +12,6 @@ fi #make $DEBUG -C xamarin-macios/builds download -j #make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j #make $DEBUG -C xamarin-macios/tests vsts-device-tests -echo "exit 1" > bar.sh +echo "exit 0" > bar.sh chmod a+x bar.sh ./bar.sh From 6a29febc29c35c00a7349a9ecb4c9bb52b016a47 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 14:27:08 -0400 Subject: [PATCH 81/98] fix condition on tests failure --- tools/devops/templates/device-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index aa045e454500..934ba1defa67 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -145,7 +145,7 @@ steps: targetPath: 'xamarin-macios/tests/TestSummary.md' artifactName: TestSummary-$(CONTEXT) continueOnError: true - condition: and(succeededOrFailed(), eq(variables['TestState'], 'failure')) + condition: succeededOrFailed() # if there's a test failure || test summary not generated || provisioning failed: # - update GH commit status @@ -154,7 +154,6 @@ steps: # pwsh for PowerShell Core # powershell is for Win only PowerShell - pwsh: ./xamarin-macios/tools/devops/templates/publish_failure.ps1 -# condition: failed() env: BUILD_REVISION: $(BUILD_REVISION) GH_STATE: $(GH_STATE) @@ -163,7 +162,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'Powershell: Add GH status & commit message' - condition: succeededOrFailed() #, eq(variables['ProvisioningProfiles'], 'success')) + condition: and(succeededOrFailed(), eq(variables['TestState'], 'failure')) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From 1568e7f2663189700dfebbfe853b2c65ef3061dd Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 15:35:34 -0400 Subject: [PATCH 82/98] add succeededorfailed(), update TestState to failed when no test summary file is found --- tools/devops/templates/device-tests.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 934ba1defa67..8a2d9c14949a 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -9,6 +9,8 @@ # # CONTEXT: The context that is used to set the status of the commit in github. # TESTS_EXTRA_ARGUMENTS: the extra arguments passed to the runner to decide which tests need to be ran. +# +# Condition: succeededOrFailed() - true even if a previous dependency has failed, unless the run was canceled. steps: @@ -115,7 +117,8 @@ steps: FILE="xamarin-macios/tests/TestSummary.md" echo $FILE if ! test -f $FILE; then - echo ":fire: Tests failed catastrophically (no summary found)" > $FILE + echo ":fire: Tests failed catastrophically (no summary found)" > $FILE + echo "##vso[task.setvariable variable=TestState]failure" fi displayName: "If test summary does not exist, create test summary" condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) @@ -130,7 +133,7 @@ steps: echo $FILE echo ":fire: Provisioning step failed" > $FILE displayName: 'Add test summary for failed provisioning' - condition: eq(variables['ProvisioningProfiles'], 'failure') + condition: and (succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'failure')) ### ### Publish test summary @@ -151,7 +154,7 @@ steps: # - update GH commit status # - add commit comment # grab env variables to add test specific text -# pwsh for PowerShell Core +# pwsh for PowerShell Core (Win + Mac) # powershell is for Win only PowerShell - pwsh: ./xamarin-macios/tools/devops/templates/publish_failure.ps1 env: @@ -162,7 +165,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'Powershell: Add GH status & commit message' - condition: and(succeededOrFailed(), eq(variables['TestState'], 'failure')) + condition: and(succeededOrFailed(), or (eq(variables['TestState'], 'failure'), eq(variables['ProvisioningProfiles'], 'failure'))) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From bbf6ad11f9e9c4705220f3b1df52dfdcbb771612 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 15:52:27 -0400 Subject: [PATCH 83/98] test out w/ test failures --- tools/devops/run-tests.sh | 2 +- tools/devops/templates/device-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 74fd232ed02b..3ba2ba63fd12 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -12,6 +12,6 @@ fi #make $DEBUG -C xamarin-macios/builds download -j #make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j #make $DEBUG -C xamarin-macios/tests vsts-device-tests -echo "exit 0" > bar.sh +echo "exit 1" > bar.sh chmod a+x bar.sh ./bar.sh diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 8a2d9c14949a..4a734e1e45e6 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -132,7 +132,7 @@ steps: FILE="xamarin-macios/tests/TestSummary.md" echo $FILE echo ":fire: Provisioning step failed" > $FILE - displayName: 'Add test summary for failed provisioning' + displayName: 'Provisioning failed: create test summary' condition: and (succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'failure')) ### From 86613a8812428701d725f54342606b1a15d4b680 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 16:14:21 -0400 Subject: [PATCH 84/98] give a better fake exit code --- tools/devops/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 3ba2ba63fd12..de45b6f397d9 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -12,6 +12,6 @@ fi #make $DEBUG -C xamarin-macios/builds download -j #make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j #make $DEBUG -C xamarin-macios/tests vsts-device-tests -echo "exit 1" > bar.sh +echo "exit 100" > bar.sh chmod a+x bar.sh ./bar.sh From 0512e5942cec1145a4eefab471555565d777564e Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 16:33:08 -0400 Subject: [PATCH 85/98] temporary testing change to always run publish_failure --- tools/devops/templates/device-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 4a734e1e45e6..ac3b0b3a7b72 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -165,7 +165,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'Powershell: Add GH status & commit message' - condition: and(succeededOrFailed(), or (eq(variables['TestState'], 'failure'), eq(variables['ProvisioningProfiles'], 'failure'))) + condition: always() #and(succeededOrFailed(), or (eq(variables['TestState'], 'failure'), eq(variables['ProvisioningProfiles'], 'failure'))) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' From 4207bee3484ce5a2e7ba799d745b4834092520aa Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 16:35:37 -0400 Subject: [PATCH 86/98] clean up some stuff, update status with VSTS prefix + add real description and context to aggregate status --- tools/devops/templates/publish-results.yml | 10 ++-------- tools/devops/templates/publish_failure.ps1 | 11 +---------- tools/devops/templates/publish_results.ps1 | 14 +++++++------- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/tools/devops/templates/publish-results.yml b/tools/devops/templates/publish-results.yml index d2db41e09c6f..da2821e2160b 100644 --- a/tools/devops/templates/publish-results.yml +++ b/tools/devops/templates/publish-results.yml @@ -20,19 +20,13 @@ steps: path: $(Pipeline.Workspace)/Summaries ### -### Publish merged TestSummary file +### Publish commit message and status ### -# - task: PublishPipelineArtifact@1 -# displayName: 'Publish Artifact: Aggregate TestSummary' -# inputs: -# # targetPath: 'xamarin-macios/tests/TestSummary.md' -# targetPath: ./TestSummary.md -# artifactName: TestSummary -# continueOnError: true # pwsh for PowerShell Core # powershell is for Win PowerShell - pwsh: ./tools/devops/templates/publish_results.ps1 + displayName: Publish commit message + status condition: succeededOrFailed() env: BUILD_REVISION: $(BUILD_REVISION) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 35a7feba5c63..38b7b3e9af60 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -14,14 +14,12 @@ $target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT ## don't need context here b/c we are combining all device tests into one post? #$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" -# add real device type here -# add description back in # state: only report failure, so state is always failure but we can also use AGENT_JOBSTATUS to avoid hardcoding values $json_payload = @" { "state" : "failure", "target_url" : "$target_url", - "description" : "$Env:SYSTEM_JOBNAME", + "description" : "VSTS: $Env:SYSTEM_JOBNAME", "context" : "$Env:SYSTEM_STAGEDISPLAYNAME" } "@ @@ -80,12 +78,8 @@ $DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTE $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" $file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -#$Env:PIPELINE_WORKSPACE Get-ChildItem $file | Write-Host - - - $file = "$file/xamarin-macios/tests" Write-Host "xamarin-macios tests children" @@ -111,8 +105,6 @@ foreach ($line in Get-Content -Path $file) $msg.AppendLine($line) } - - $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" @@ -121,7 +113,6 @@ $payload = @{ body = $msg.ToString() } - # convert payload to json $json_payload = $payload | ConvertTo-json diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 2ffe2daadcb2..a830fa81bb97 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -3,6 +3,11 @@ Get-Location Set-Location Env: Get-ChildItem +### +### Construct commit status with combined test results +### + + # get combined status: # success only if every status is success # otherwise failure @@ -26,17 +31,12 @@ $state = $response.state # post status to github $target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT/_build/index?buildId=$Env:BUILD_BUILDID&view=ms.vss-test-web.test-result-details" -## don't need context here b/c we are combining all device tests into one post? -#$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" - -# add real device type here -# add description back in $json_payload = @" { "state" : "$state", "target_url" : "$target_url", - "description" : "description placeholder", - "context" : "VSTS: AGGREGATE device tests" + "description" : "$state", + "context" : "VSTS: All device tests" } "@ From 855a9f461e932ab3cbdb5b1499eb72822fbd7faf Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 16:56:51 -0400 Subject: [PATCH 87/98] swap description with context since context is unique... --- tools/devops/templates/publish_failure.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 38b7b3e9af60..23bdb70d32de 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -19,8 +19,8 @@ $json_payload = @" { "state" : "failure", "target_url" : "$target_url", - "description" : "VSTS: $Env:SYSTEM_JOBNAME", - "context" : "$Env:SYSTEM_STAGEDISPLAYNAME" + "description" : "$Env:SYSTEM_STAGEDISPLAYNAME", + "context" : "VSTS: $Env:SYSTEM_JOBNAME" } "@ From e665eb6f765c4f5577591510ad00429dd566becc Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 17:14:36 -0400 Subject: [PATCH 88/98] try out not passing -e to script, ssome cleanup --- tools/devops/run-tests.sh | 4 +++- tools/devops/templates/device-tests.yml | 2 +- tools/devops/templates/publish_failure.ps1 | 6 +----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index de45b6f397d9..8d01ffc5683e 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -1,4 +1,6 @@ -#!/bin/bash -ex +#!/bin/bash -x + +# we do not want errors to fail the script, we want to print as much info as possible, so we don't pass -e to bash ls -l pwd diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index ac3b0b3a7b72..4a734e1e45e6 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -165,7 +165,7 @@ steps: TESTS: $(TESTS) TESTSARGS: $(TESTS_EXTRA_ARGUMENTS) displayName: 'Powershell: Add GH status & commit message' - condition: always() #and(succeededOrFailed(), or (eq(variables['TestState'], 'failure'), eq(variables['ProvisioningProfiles'], 'failure'))) + condition: and(succeededOrFailed(), or (eq(variables['TestState'], 'failure'), eq(variables['ProvisioningProfiles'], 'failure'))) - bash: ./xamarin-macios/tools/devops/add-summaries.sh displayName: 'Add summaries' diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 23bdb70d32de..17539179709f 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -11,10 +11,7 @@ Get-ChildItem $target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT/_build/index?buildId=$Env:BUILD_BUILDID&view=ms.vss-test-web.test-result-details" -## don't need context here b/c we are combining all device tests into one post? -#$json_payload = @"{"token": $TOKEN, "hash":$BUILD_REVISION "state": $GH_STATE, "target-url": $TARGET_URL, "description": $DESCRIPTION, "context": "VSTS: device tests $DEVICE_TYPE"}" - -# state: only report failure, so state is always failure but we can also use AGENT_JOBSTATUS to avoid hardcoding values +# state: we only report failures $json_payload = @" { "state" : "failure", @@ -24,7 +21,6 @@ $json_payload = @" } "@ - $url = "https://api.github.com/repos/xamarin/xamarin-macios/statuses/$Env:BUILD_REVISION" Write-Host $json_payload From b460224ebb46ac2d7b26975b30b55b9435678b6a Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 18:06:12 -0400 Subject: [PATCH 89/98] handle error in run-tests correctly --- tools/devops/run-tests.sh | 4 +--- tools/devops/templates/device-tests.yml | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 8d01ffc5683e..de45b6f397d9 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -1,6 +1,4 @@ -#!/bin/bash -x - -# we do not want errors to fail the script, we want to print as much info as possible, so we don't pass -e to bash +#!/bin/bash -ex ls -l pwd diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 4a734e1e45e6..7149249b57c7 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -97,13 +97,15 @@ steps: set -x set -e - ./xamarin-macios/tools/devops/run-tests.sh + EC = 0 + ./xamarin-macios/tools/devops/run-tests.sh || EC=$? - if [ $? -eq 0 ]; then + if [ $EC -eq 0 ]; then echo "##vso[task.setvariable variable=TestState]success" else echo "##vso[task.setvariable variable=TestState]failure" fi + exit $EC displayName: 'Run tests' timeoutInMinutes: 600 From cad44700c52898d46ac681810cf0ab5f072a0714 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 18:15:52 -0400 Subject: [PATCH 90/98] fix syntax --- tools/devops/templates/device-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 7149249b57c7..f502e3784be5 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -97,7 +97,7 @@ steps: set -x set -e - EC = 0 + EC=0 ./xamarin-macios/tools/devops/run-tests.sh || EC=$? if [ $EC -eq 0 ]; then From a01d5ff44084feba90e8a02971db72850df85ea9 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Tue, 12 May 2020 19:02:38 -0400 Subject: [PATCH 91/98] add bolded text, clean up a bit --- tools/devops/templates/publish_failure.ps1 | 2 +- tools/devops/templates/publish_results.ps1 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 17539179709f..b3b1b7dff643 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -68,7 +68,7 @@ $HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline" # SYSTEM_JOBNAME: Xamarin | Monotouch | xUnitBCL | NUnitBCL | mscorlib # CONTEXT: tvOS | iOS | iOS32 -$DESCRIPTION="Device test $Env:SYSTEM_JOBNAME $Env:AGENT_JOBSTATUS on $Env:CONTEXT" +$DESCRIPTION="Device test **$Env:SYSTEM_JOBNAME** $Env:AGENT_JOBSTATUS on **$Env:CONTEXT**" # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index a830fa81bb97..37cb5fcdcc1e 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -1,3 +1,4 @@ +# TODO: set up so that we only log debugging # debugging code: print out all environment variables, including those passed in via yml Get-Location Set-Location Env: From 7169485d43e2f0367e31b3b43318e7bf0ed443a7 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 13 May 2020 17:12:25 -0400 Subject: [PATCH 92/98] clean up + add try/catch to publish_results --- tools/devops/templates/publish_results.ps1 | 58 +++++++++++++--------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/tools/devops/templates/publish_results.ps1 b/tools/devops/templates/publish_results.ps1 index 37cb5fcdcc1e..890c2daba716 100644 --- a/tools/devops/templates/publish_results.ps1 +++ b/tools/devops/templates/publish_results.ps1 @@ -8,10 +8,11 @@ Get-ChildItem ### Construct commit status with combined test results ### - -# get combined status: -# success only if every status is success -# otherwise failure +### +### Get combined status: +### Success only if every status is success, otherwise failure +### If something goes wrong when we Invoke-RestMethod, status is set to error +### # url to query for combined status $combined_status_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/status" @@ -23,13 +24,18 @@ $params = @{ ContentType = 'application/json' } -# -$response = Invoke-RestMethod @params - -$state = $response.state +$state = "error" +try { + $response = Invoke-RestMethod @params + $state = $response.state +} catch { + Write-Host "Error querying for GH status: $_" +} +### +### Post commit status to GitHub +### -# post status to github $target_url = $Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI + "$Env:SYSTEM_TEAMPROJECT/_build/index?buildId=$Env:BUILD_BUILDID&view=ms.vss-test-web.test-result-details" $json_payload = @" @@ -56,9 +62,12 @@ $params = @{ Write-Host $params -$response = Invoke-RestMethod @params - -$response | Write-Host +try { + $response = Invoke-RestMethod @params + $response | Write-Host +} catch { + Write-Host "Error posting GH status: $_" +} ### ### Construct commit message w/ aggregate test summary @@ -69,11 +78,7 @@ If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') # do stuff } -# BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" -$json_text = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" - -$dir = $Env:PIPELINE_WORKSPACE -$dir = "$dir/Summaries/TestSummary-*/TestSummary.md" +$dir = "$Env:PIPELINE_WORKSPACE/Summaries/TestSummary-*/TestSummary.md" Write-Host $dir Get-ChildItem $dir | Write-Host @@ -85,20 +90,23 @@ Get-ChildItem $dir | Write-Host # Get all test summary files $files = Get-ChildItem -Path $dir +# BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" +$header = "### :boom: :construction: TESTING Experimental DDFun pipeline: Device test aggregate results: on [Azure DevOps]($target_url)" + # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() -$msg.AppendLine($json_text) +$msg.AppendLine($header) $msg.AppendLine() -# Grab the test name + device info string from filepath $prefix="$Env:PIPELINE_WORKSPACE/Summaries/TestSummary-" $suffix="/TestSummary.md" foreach ($file in $files) { + # grab the test name + device string from filepath $info = $file.FullName.Substring($prefix.Length, $file.FullName.Length - $suffix.Length - $prefix.Length) - # switch to keep track of when we are reading the first line of a summary + # switch to keep track of when we are on the first line of a summary $first_line = 1 # read each line of the summary file, append it with correct \n at the end @@ -123,7 +131,6 @@ foreach ($file in $files) $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" -# create pwsh object to store payload $payload = @{ body = $msg.ToString() } @@ -143,6 +150,9 @@ $params = @{ Write-Host $params -$response = Invoke-RestMethod @params - -$response | ConvertTo-Json | Write-Host +try { + $response = Invoke-RestMethod @params + $response | ConvertTo-Json | Write-Host +} catch { + Write-Host "Error posting GH commit message: $_" +} From 7ec48a8ab5f57cf24042681e9b26c4e8870ad913 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Wed, 13 May 2020 17:33:25 -0400 Subject: [PATCH 93/98] clean up publish_failure + add try/catch --- tools/devops/templates/device-tests.yml | 13 ++++--- tools/devops/templates/publish_failure.ps1 | 44 ++++++++-------------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index f502e3784be5..61711317ccfe 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -152,12 +152,13 @@ steps: continueOnError: true condition: succeededOrFailed() -# if there's a test failure || test summary not generated || provisioning failed: -# - update GH commit status -# - add commit comment -# grab env variables to add test specific text -# pwsh for PowerShell Core (Win + Mac) -# powershell is for Win only PowerShell +### +### If there's a test failure || test summary not generated || provisioning failed: +### - update GH commit status +### - add commit comment +### +### Note: pwsh for PowerShell Core (Win + Mac), powershell is for Win only PowerShell +### - pwsh: ./xamarin-macios/tools/devops/templates/publish_failure.ps1 env: BUILD_REVISION: $(BUILD_REVISION) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index b3b1b7dff643..85a39338eb65 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -36,36 +36,26 @@ $params = @{ Write-Host $params -$response = Invoke-RestMethod @params - -$response | ConvertTo-Json | Write-Host - +try { + $response = Invoke-RestMethod @params + $response | ConvertTo-Json | Write-Host +} catch { + Write-Host "Error posting GH status: $_" +} ### ### Construct commit message with test summary ### -$RESULT_EMOJI -If ($Env:AGENT_JOBSTATUS -eq 'Failed') -{ - $RESULT_EMOJI = ":fire: " -} Else { - $RESULT_EMOJI = ":white_check_mark: " # maybe remove this since we only add commit messages on failure? -} - +$RESULT_EMOJI = ":fire: " # we only run when there's a failure If ($Env:BUILD_DEFINITIONNAME -like '*DDFun*') { ## do stuff } -#Else{ -#HTML Report jenkins stuff -#} - $HEADER = "### :bangbang: :construction: TESTING Experimental DDFun pipeline" - # SYSTEM_JOBNAME: Xamarin | Monotouch | xUnitBCL | NUnitBCL | mscorlib # CONTEXT: tvOS | iOS | iOS32 $DESCRIPTION="Device test **$Env:SYSTEM_JOBNAME** $Env:AGENT_JOBSTATUS on **$Env:CONTEXT**" @@ -73,10 +63,9 @@ $DESCRIPTION="Device test **$Env:SYSTEM_JOBNAME** $Env:AGENT_JOBSTATUS on **$Env # BUILD_DEFINITIONNAME: Pipeline name, e.g. "iOS Device Tests [DDFun]" $json_text = "$RESULT_EMOJI $DESCRIPTION on [Azure DevOps]($target_url) ($Env:BUILD_DEFINITIONNAME) $RESULT_EMOJI" -$file = $Env:SYSTEM_DEFAULTWORKINGDIRECTORY -Get-ChildItem $file | Write-Host - -$file = "$file/xamarin-macios/tests" +# Grab test results from default working directory +# Cannot access the published pipeline artifact because it is not always available when this script runs :( +$file = "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tests" Write-Host "xamarin-macios tests children" Get-ChildItem $file | Write-Host @@ -86,8 +75,6 @@ $file = "$file/TestSummary.md" Write-Host $file Get-Content $file | Write-Host - - # stringbuilder for extra flavor $msg = [System.Text.StringBuilder]::new() $msg.AppendLine($HEADER) @@ -103,8 +90,6 @@ foreach ($line in Get-Content -Path $file) $message_url = "https://api.github.com/repos/xamarin/xamarin-macios/commits/$Env:BUILD_REVISION/comments" - -# create pwsh object to store payload $payload = @{ body = $msg.ToString() } @@ -124,6 +109,9 @@ $params = @{ Write-Host $params -$response = Invoke-RestMethod @params - -$response | ConvertTo-Json | Write-Host +try { + $response = Invoke-RestMethod @params + $response | ConvertTo-Json | Write-Host +} catch { + Write-Host "Error posting GH commit message: $_" +} From 08eea7805867e0093cf425b50820971e3ba1af15 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 14 May 2020 11:53:15 -0400 Subject: [PATCH 94/98] re-add back tests, devices --- tools/devops/run-tests.sh | 13 ++++---- tools/devops/templates/job-matrix.yml | 42 ++++++++++++------------ tools/devops/templates/stable-stages.yml | 24 +++++++------- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index de45b6f397d9..2e1ea1ecd5ac 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -3,15 +3,14 @@ ls -l pwd ls -l xamarin-macios/ -ls -l if [[ "$SYSTEM_DEBUG" == "true" ]]; then DEBUG="-d" else DEBUG="" fi -#make $DEBUG -C xamarin-macios/builds download -j -#make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j -#make $DEBUG -C xamarin-macios/tests vsts-device-tests -echo "exit 100" > bar.sh -chmod a+x bar.sh -./bar.sh +make $DEBUG -C xamarin-macios/builds download -j +make $DEBUG -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j +make $DEBUG -C xamarin-macios/tests vsts-device-tests +#echo "exit 100" > bar.sh +#chmod a+x bar.sh +#./bar.sh diff --git a/tools/devops/templates/job-matrix.yml b/tools/devops/templates/job-matrix.yml index 83d8509e1830..1f95bc156152 100644 --- a/tools/devops/templates/job-matrix.yml +++ b/tools/devops/templates/job-matrix.yml @@ -13,20 +13,20 @@ parameters: -- name: variation - type: string +- name: variation + type: string default: iOS - name: pool - type: string + type: string default: 'VSEng-Xamarin-Mac-Devices' # default ddfun pool - name: demands - type: string + type: string default: 'ios' # we default to ios devices - name: deviceLabel - type: string + type: string default: 'run-ios-64-tests' # default to iOS 64b devices - name: timeout @@ -40,14 +40,14 @@ parameters: jobs: ### -### Run all the xamarin tests, not bcl or mscorlib +### Run all the xamarin tests, not bcl or mscorlib ### -- job: DDFun +- job: DDFun displayName: 'Run ${{ parameters.variation }} Device Tests' - timeoutInMinutes: ${{ parameters.timeout }} + timeoutInMinutes: ${{ parameters.timeout }} pool: - name: ${{ parameters.pool }} + name: ${{ parameters.pool }} demands: ${{ parameters.demands }} strategy: @@ -61,18 +61,18 @@ jobs: CONTEXT: '${{ parameters.variation }}-monotouch-DDFun' TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-monotouch-tests' # run all tests but the bcl ones TESTS: 'Monotouch' - # xUnitBCL: - # CONTEXT: '${{ parameters.variation }}-xUnit-BCL-DDFun' - # TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-new-bcl-tests,skip-mscorlib-tests' # run only the new bcl tests - # TESTS: 'xUnitBCL' - # nUnitBCL: - # CONTEXT: '${{ parameters.variation }}-NUnit-BCL-DDFun' - # TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-old-bcl-tests' # run only the new bcl tests - # TESTS: 'NUnitBCL' - # Mscorlib: - # CONTEXT: '${{ parameters.variation }}-mscorlib-DDFun' - # TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-mscorlib-tests' # run only the new bcl tests - # TESTS: 'mscorlib' + xUnitBCL: + CONTEXT: '${{ parameters.variation }}-xUnit-BCL-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-new-bcl-tests,skip-mscorlib-tests' # run only the new bcl tests + TESTS: 'xUnitBCL' + nUnitBCL: + CONTEXT: '${{ parameters.variation }}-NUnit-BCL-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-old-bcl-tests' # run only the new bcl tests + TESTS: 'NUnitBCL' + Mscorlib: + CONTEXT: '${{ parameters.variation }}-mscorlib-DDFun' + TESTS_EXTRA_ARGUMENTS: '--label=${{ parameters.deviceLabel }},run-mscorlib-tests' # run only the new bcl tests + TESTS: 'mscorlib' steps: - template: device-tests.yml # all the smart stuff happens in the template, just needed to set the params diff --git a/tools/devops/templates/stable-stages.yml b/tools/devops/templates/stable-stages.yml index dd7484aace60..675eb557a8bd 100644 --- a/tools/devops/templates/stable-stages.yml +++ b/tools/devops/templates/stable-stages.yml @@ -37,18 +37,18 @@ parameters: default: 5 stages: -# - stage: iOSDeviceTests -# displayName: 'iOS Device Tests' -# dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel -# jobs: -# - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params -# parameters: -# demands: ${{ parameters.iOSDemand }} -# deviceLabel: 'run-ios-64-tests' -# variation: "iOS" -# pool: ${{ parameters.pool }} -# timeout: ${{ parameters.timeout }} -# maxParallel: ${{ parameters.maxParallel}} +- stage: iOSDeviceTests + displayName: 'iOS Device Tests' + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel + jobs: + - template: job-matrix.yml # all the smart stuff happens in the template, just needed to set the params + parameters: + demands: ${{ parameters.iOSDemand }} + deviceLabel: 'run-ios-64-tests' + variation: "iOS" + pool: ${{ parameters.pool }} + timeout: ${{ parameters.timeout }} + maxParallel: ${{ parameters.maxParallel}} - stage: tvOSDeviceTests displayName: 'tvOS Device Tests' From b5975f9f6adfa8a327c06802833747704ae69957 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 14 May 2020 15:14:06 -0400 Subject: [PATCH 95/98] set TestState to failure before we try to run tests, add partial provisioning improvements --- tools/devops/templates/device-tests.yml | 29 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 61711317ccfe..65081685480d 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -31,7 +31,14 @@ steps: rm -f ~/Library/Caches/com.xamarin.provisionator/Provisions/*p12 rm -f ~/Library/Caches/com.xamarin.provisionator/Provisions/*mobileprovision - ./maccore/tools/install-qa-provisioning-profiles.sh -v + EC=0 + ./maccore/tools/install-qa-provisioning-profiles.sh -v || EC=$? + if [ $EC -eq 0 ]; then + echo "##vso[task.setvariable variable=ProvisioningProfiles]success" + else + echo "##vso[task.setvariable variable=ProvisioningProfiles]failure" + fi + exit $EC displayName: 'Add provisioning profiles' env: LOGIN_KEYCHAIN_PASSWORD: $(OSX_KEYCHAIN_PASS) @@ -47,12 +54,13 @@ steps: if grep ignore-python3 xamarin-macios/system-dependencies.sh 2>&1 > /dev/null; then CONFIGURE_ARGS="$CONFIGURE_ARGS --ignore-python3" fi - ./xamarin-macios/system-dependencies.sh $CONFIGURE_ARGS - if [ $? -eq 0 ]; then + ./xamarin-macios/system-dependencies.sh $CONFIGURE_ARGS || EC=$? + if [ $EC -eq 0 ]; then echo "##vso[task.setvariable variable=ProvisioningProfiles]success" else echo "##vso[task.setvariable variable=ProvisioningProfiles]failure" fi + exit $EC displayName: 'Provision dependencies' timeoutInMinutes: 240 @@ -89,10 +97,19 @@ steps: displayName: 'Fix device discovery (reset launchctl)' ### -### Run the device tests +### Always set TestState to failure. If tests are skipped we do not want to have TestState == null. ### +- bash: | + set -x + set -e - # move this to in-line code, should be very short + echo "##vso[task.setvariable variable=TestState]failure" + displayName: 'Set TestState to failure before trying to run tests' + condition: succeededOrFailed() + +### +### Run the device tests +### - bash: | set -x set -e @@ -122,7 +139,7 @@ steps: echo ":fire: Tests failed catastrophically (no summary found)" > $FILE echo "##vso[task.setvariable variable=TestState]failure" fi - displayName: "If test summary does not exist, create test summary" + displayName: "Check for test summary, if does not exist create test summary" condition: and(succeededOrFailed(), eq(variables['ProvisioningProfiles'], 'success')) ### From 603930dcee2c38b628b8b0cc4fe9da3d12f67700 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 14 May 2020 15:32:21 -0400 Subject: [PATCH 96/98] make context for per-test failure _actually_ unique --- tools/devops/templates/publish_failure.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/templates/publish_failure.ps1 b/tools/devops/templates/publish_failure.ps1 index 85a39338eb65..ee5f0a3502dc 100644 --- a/tools/devops/templates/publish_failure.ps1 +++ b/tools/devops/templates/publish_failure.ps1 @@ -16,8 +16,8 @@ $json_payload = @" { "state" : "failure", "target_url" : "$target_url", - "description" : "$Env:SYSTEM_STAGEDISPLAYNAME", - "context" : "VSTS: $Env:SYSTEM_JOBNAME" + "description" : "$Env:AGENT_JOBSTATUS", + "context" : "VSTS: $Env:SYSTEM_JOBNAME $Env:SYSTEM_STAGEDISPLAYNAME" } "@ From 246b5c52b0a2d52c74e58332010182315a818f42 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 14 May 2020 15:35:02 -0400 Subject: [PATCH 97/98] fix provisioning status is always set to failure... --- tools/devops/templates/device-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 65081685480d..4d866f679262 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -54,6 +54,8 @@ steps: if grep ignore-python3 xamarin-macios/system-dependencies.sh 2>&1 > /dev/null; then CONFIGURE_ARGS="$CONFIGURE_ARGS --ignore-python3" fi + + $EC=0 ./xamarin-macios/system-dependencies.sh $CONFIGURE_ARGS || EC=$? if [ $EC -eq 0 ]; then echo "##vso[task.setvariable variable=ProvisioningProfiles]success" From 1a6c4bf2bb0635dbeb5123b457450536992c4a71 Mon Sep 17 00:00:00 2001 From: Whitney Schmidt Date: Thu, 14 May 2020 15:41:56 -0400 Subject: [PATCH 98/98] fix syntax... --- tools/devops/templates/device-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/templates/device-tests.yml b/tools/devops/templates/device-tests.yml index 4d866f679262..0d40dc30a872 100644 --- a/tools/devops/templates/device-tests.yml +++ b/tools/devops/templates/device-tests.yml @@ -55,7 +55,7 @@ steps: CONFIGURE_ARGS="$CONFIGURE_ARGS --ignore-python3" fi - $EC=0 + EC=0 ./xamarin-macios/system-dependencies.sh $CONFIGURE_ARGS || EC=$? if [ $EC -eq 0 ]; then echo "##vso[task.setvariable variable=ProvisioningProfiles]success"