From e6032a2f152d40eb682da978f87d3f6f5a27d7a3 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 24 Jun 2020 16:40:33 +0800 Subject: [PATCH 01/16] disbale byte compiling in rpm build --- scripts/release/rpm/azure-cli.spec | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index 2bb8acbe236..678fc880c15 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -1,6 +1,8 @@ # RPM spec file for Azure CLI # Definition of macros used - https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros +%global __python %{__python3} + # .el7.centos -> .el7 %if 0%{?rhel} %define dist .el%{?rhel} @@ -14,6 +16,9 @@ %define repo_path %{getenv:REPO_PATH} %define cli_lib_dir %{_libdir}/az +# Turn off automatic python bytecompilation +%undefine __brp_python_bytecompile + Summary: Azure CLI License: MIT Name: %{name} From 732808566551d84d110cf08bcf937ff2bc6ebe86 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 24 Jun 2020 18:28:27 +0800 Subject: [PATCH 02/16] modify __os_install_post --- scripts/release/rpm/azure-cli.spec | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index 678fc880c15..1dfc379c8fb 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -2,6 +2,8 @@ # Definition of macros used - https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros %global __python %{__python3} +# Turn off python byte compilation +%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') # .el7.centos -> .el7 %if 0%{?rhel} @@ -16,9 +18,6 @@ %define repo_path %{getenv:REPO_PATH} %define cli_lib_dir %{_libdir}/az -# Turn off automatic python bytecompilation -%undefine __brp_python_bytecompile - Summary: Azure CLI License: MIT Name: %{name} From 79858eceb72637c6988af3e449f30b1fbcc211d0 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Sun, 28 Jun 2020 15:01:56 +0800 Subject: [PATCH 03/16] use embeddable python for msi --- build_scripts/windows/scripts/build.cmd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index ffb19299b48..b6d24e0ebfa 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -12,7 +12,7 @@ if "%CLI_VERSION%"=="" ( set PYTHON_VERSION=3.6.6 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" -set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip" +set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-64.zip" set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip" :: Set up the output directory and temp. directories @@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR% set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch set BUILDING_DIR=%ARTIFACTS_DIR%\cli set WIX_DIR=%ARTIFACTS_DIR%\wix -set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32 +set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-64 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change set REPO_ROOT=%~dp0..\..\.. @@ -75,10 +75,10 @@ if not exist %PYTHON_DIR% ( mkdir %PYTHON_DIR% pushd %PYTHON_DIR% echo Downloading Python. - curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k - unzip -q Python366-32.zip + curl -o Python368-64.zip %PYTHON_DOWNLOAD_URL% -k + unzip -q Python368-64.zip if %errorlevel% neq 0 goto ERROR - del Python366-32.zip + del Python368-64.zip echo Python downloaded and extracted successfully. popd ) From f4bf0fd1a255ad27122ab08b13304e2f1cfcccfe Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Tue, 7 Jul 2020 18:26:22 +0800 Subject: [PATCH 04/16] add msi package test script --- build_scripts/windows/scripts/test_msi_package.py | 13 +++++++++++++ scripts/release/rpm/test_rpm_package.py | 0 2 files changed, 13 insertions(+) create mode 100644 build_scripts/windows/scripts/test_msi_package.py create mode 100644 scripts/release/rpm/test_rpm_package.py diff --git a/build_scripts/windows/scripts/test_msi_package.py b/build_scripts/windows/scripts/test_msi_package.py new file mode 100644 index 00000000000..3c2703dde1c --- /dev/null +++ b/build_scripts/windows/scripts/test_msi_package.py @@ -0,0 +1,13 @@ +import os +import sys +import subprocess + +d = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli\\command_modules' + +mod_list = [os.path.join(d, o) for o in os.listdir(d) if os.path.isdir(os.path.join(d, o))] +for mod in mod_list: + mod_name = os.path.basename(os.path.normpath(mod)) + + # command = "python -m pytest -x -v -p no:warnings --log-level WARN --junit-xml '{}\\azure_cli_tests\\{}.xml' -n auto '{}'".format(os.path.expanduser('~'), mod_name, mod) + subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', + '--junit-xml', '{}\\azure_cli_tests\\{}.xml'.format(os.path.expanduser('~'), mod_name), '-n', 'auto', mod]) diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py new file mode 100644 index 00000000000..e69de29bb2d From 92419d09976bdb4ee7027faa9cbdc961ecb8dd4c Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 8 Jul 2020 10:22:20 +0800 Subject: [PATCH 05/16] add rpm test script --- scripts/release/rpm/test_rpm_package.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index e69de29bb2d..fc9d7398ae2 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -0,0 +1,11 @@ +import os +import sys +import subprocess + +d = '/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules' +mod_list = [os.path.join(d, o) for o in os.listdir(d) if os.path.isdir(os.path.join(d, o))] +# mod_list= ['/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules/botservice'] + +for mod in mod_list: + mod_name = os.path.basename(os.path.normpath(mod)) + subprocess.call(['export PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages && python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml -n auto {}'.format(mod_name, mod)], shell=True) From 3b621afdd743e690aaba6bffb319b5d162309647 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 8 Jul 2020 13:17:28 +0800 Subject: [PATCH 06/16] fix botservice --- scripts/release/rpm/test_rpm_package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index fc9d7398ae2..e34c1b33be3 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -8,4 +8,8 @@ for mod in mod_list: mod_name = os.path.basename(os.path.normpath(mod)) + if mod_name == 'botservice': + subprocess.call(['export PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages && python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml {}'.format(mod_name, mod)], shell=True) subprocess.call(['export PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages && python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml -n auto {}'.format(mod_name, mod)], shell=True) + + From 11794381ce462dddfdb1f7cc50f1e1b34890b327 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 8 Jul 2020 21:53:58 +0800 Subject: [PATCH 07/16] update test_rpm_package.py --- .../release/rpm/setup_rpm_tests_in_docker.sh | 31 +++++++++++++++++++ scripts/release/rpm/test_rpm_package.py | 14 +++++---- 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 scripts/release/rpm/setup_rpm_tests_in_docker.sh diff --git a/scripts/release/rpm/setup_rpm_tests_in_docker.sh b/scripts/release/rpm/setup_rpm_tests_in_docker.sh new file mode 100644 index 00000000000..8fd9fcda8a8 --- /dev/null +++ b/scripts/release/rpm/setup_rpm_tests_in_docker.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This script should be run in a centos8 docker. +set -exv +yum install which -y +rpm --import https://packages.microsoft.com/keys/microsoft.asc + +sh -c 'echo -e "[azure-cli] +name=Azure CLI +baseurl=https://packages.microsoft.com/yumrepos/azure-cli +enabled=1 +gpgcheck=1 +gpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo' + +yum install azure-cli -y +ln -s /usr/bin/python3 /usr/bin/python +ln -s /usr/bin/pip3 /usr/bin/pip + +yum install git -y +git clone https://github.com/Azure/azure-cli.git +cd azure-cli/ +git checkout release +pip3 install wheel +./scripts/ci/build.sh +pip3 install pytest --prefix /usr/lib64/az +pip3 install pytest-xdist --prefix /usr/lib64/az + +find /azure-cli/artifacts/build -name "azure_cli_testsdk*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed +find /azure-cli/artifacts/build -name "azure_cli_fulltest*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed --no-deps + +python3 test_rpm_package.py \ No newline at end of file diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index e34c1b33be3..76a5d193efd 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -3,13 +3,15 @@ import subprocess d = '/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules' -mod_list = [os.path.join(d, o) for o in os.listdir(d) if os.path.isdir(os.path.join(d, o))] -# mod_list= ['/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules/botservice'] +mod_list = [os.path.join(d, o) for o in sorted(os.listdir(d)) if os.path.isdir(os.path.join(d, o)) and o != '__pycache__'] for mod in mod_list: mod_name = os.path.basename(os.path.normpath(mod)) if mod_name == 'botservice': - subprocess.call(['export PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages && python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml {}'.format(mod_name, mod)], shell=True) - subprocess.call(['export PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages && python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml -n auto {}'.format(mod_name, mod)], shell=True) - - + exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml --pyargs azure.cli.command_modules.{}'.format(mod_name, mod_name)], shell=True) + else: + exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml -n auto --pyargs azure.cli.command_modules.{}'.format(mod_name, mod_name)], shell=True) + if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected + sys.exit(exit_code) +exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/azure-cli-core.xml -n auto --pyargs azure.cli.core'], shell=True) +sys.exit(exit_code) From da68d09d107a42303169c6c216858aab89067e0a Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 8 Jul 2020 22:40:03 +0800 Subject: [PATCH 08/16] add rpm test in ci --- azure-pipelines.yml | 6 +++--- ...sts_in_docker.sh => test_rpm_in_docker.sh} | 19 +++++-------------- 2 files changed, 8 insertions(+), 17 deletions(-) rename scripts/release/rpm/{setup_rpm_tests_in_docker.sh => test_rpm_in_docker.sh} (55%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d23fba0e331..6d0dba3e270 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -639,10 +639,10 @@ jobs: echo "== Test yum package on CentOS ==" - docker pull centos:centos7 - docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/yum:/mnt/yum centos:centos7 /bin/bash -c "yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y && time az self-test && time az --version && sleep 5" + docker pull centos:centos8 + docker run --rm -e YUM_NAME=$YUM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/yum:/mnt/yum -v $(pwd):/azure-cli centos:centos8 /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - displayName: 'Bash Script' + displayName: 'Test Yum Package' - job: BuildUbuntuPackages displayName: Build Ubuntu Packages diff --git a/scripts/release/rpm/setup_rpm_tests_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh similarity index 55% rename from scripts/release/rpm/setup_rpm_tests_in_docker.sh rename to scripts/release/rpm/test_rpm_in_docker.sh index 8fd9fcda8a8..9c48bebff8c 100644 --- a/scripts/release/rpm/setup_rpm_tests_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -2,24 +2,15 @@ # This script should be run in a centos8 docker. set -exv -yum install which -y -rpm --import https://packages.microsoft.com/keys/microsoft.asc -sh -c 'echo -e "[azure-cli] -name=Azure CLI -baseurl=https://packages.microsoft.com/yumrepos/azure-cli -enabled=1 -gpgcheck=1 -gpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo' +yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y -yum install azure-cli -y ln -s /usr/bin/python3 /usr/bin/python ln -s /usr/bin/pip3 /usr/bin/pip +time az self-test +time az --version -yum install git -y -git clone https://github.com/Azure/azure-cli.git -cd azure-cli/ -git checkout release +cd /azure-cli/ pip3 install wheel ./scripts/ci/build.sh pip3 install pytest --prefix /usr/lib64/az @@ -28,4 +19,4 @@ pip3 install pytest-xdist --prefix /usr/lib64/az find /azure-cli/artifacts/build -name "azure_cli_testsdk*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed find /azure-cli/artifacts/build -name "azure_cli_fulltest*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed --no-deps -python3 test_rpm_package.py \ No newline at end of file +python3 /azure-cli/scripts/release/rpm/test_rpm_package.py From ac35f36597d83c5ec8a1352c637f887563d20c07 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 8 Jul 2020 23:08:22 +0800 Subject: [PATCH 09/16] install git in rpm test --- scripts/release/rpm/test_rpm_in_docker.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index 9c48bebff8c..9cb0f64b9b3 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -5,6 +5,8 @@ set -exv yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y +yum install git -y + ln -s /usr/bin/python3 /usr/bin/python ln -s /usr/bin/pip3 /usr/bin/pip time az self-test From 0f06480b29e318b97f820ff699222b8fd53c8a3a Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 9 Jul 2020 00:15:21 +0800 Subject: [PATCH 10/16] add license header --- build_scripts/windows/scripts/test_msi_package.py | 5 +++++ scripts/release/rpm/test_rpm_package.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/build_scripts/windows/scripts/test_msi_package.py b/build_scripts/windows/scripts/test_msi_package.py index 3c2703dde1c..b9538403398 100644 --- a/build_scripts/windows/scripts/test_msi_package.py +++ b/build_scripts/windows/scripts/test_msi_package.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import os import sys import subprocess diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 76a5d193efd..5d2543cebed 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import os import sys import subprocess From 85e13c08a3189f44f890774c3bfa57e123fa692b Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 9 Jul 2020 00:44:09 +0800 Subject: [PATCH 11/16] test network in a single thread --- scripts/release/rpm/test_rpm_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 5d2543cebed..1ee35fc2c2f 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -12,7 +12,7 @@ for mod in mod_list: mod_name = os.path.basename(os.path.normpath(mod)) - if mod_name == 'botservice': + if mod_name in ['botservice', 'network']: exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml --pyargs azure.cli.command_modules.{}'.format(mod_name, mod_name)], shell=True) else: exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml -n auto --pyargs azure.cli.command_modules.{}'.format(mod_name, mod_name)], shell=True) From b076c273ca0e93c518481f20ca45d88d253cff06 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 9 Jul 2020 11:37:29 +0800 Subject: [PATCH 12/16] modify msi test --- .gitattributes | 2 ++ .../windows/scripts/setup_msi_test.ps1 | 21 +++++++++++++++++ .../windows/scripts/test_msi_package.py | 23 ++++++++++++------- scripts/release/rpm/test_rpm_package.py | 19 ++++++++------- 4 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 build_scripts/windows/scripts/setup_msi_test.ps1 diff --git a/.gitattributes b/.gitattributes index aec323b43b9..391370e95ee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,5 @@ *.bat eol=crlf # The az script for Git Bash/Cygwin should be LF build_scripts/windows/scripts/az eol=lf +# sh scripts should be LF +*.sh eol=lf diff --git a/build_scripts/windows/scripts/setup_msi_test.ps1 b/build_scripts/windows/scripts/setup_msi_test.ps1 new file mode 100644 index 00000000000..2294dd977d2 --- /dev/null +++ b/build_scripts/windows/scripts/setup_msi_test.ps1 @@ -0,0 +1,21 @@ +# Prerequisites: +# 1. Install the MSI built with current branch +# 2. Run bash azure-cli\scripts\ci\build.sh with Git Bash first to generate artifacts under azure-cli\artifacts\build so we can use the testsdk and fulltest wheels. + +# Elevate to Admin +If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) +{ + $arguments = "& '" + $myinvocation.mycommand.definition + "'" + Start-Process powershell -Verb runAs -ArgumentList $arguments +} + +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest-xdist + +$testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_testsdk*.whl | Select-Object Name +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $testsdk.Name + +$fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $fulltest.Name + +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py \ No newline at end of file diff --git a/build_scripts/windows/scripts/test_msi_package.py b/build_scripts/windows/scripts/test_msi_package.py index b9538403398..78a4b60e3d3 100644 --- a/build_scripts/windows/scripts/test_msi_package.py +++ b/build_scripts/windows/scripts/test_msi_package.py @@ -3,16 +3,23 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +# Invoke this script in Powershell with: +# & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' test_msi_package.py + import os import sys import subprocess -d = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli\\command_modules' +root_dir = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli\\command_modules' + +mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] + +for mod_name in mod_list: + exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', + '--junit-xml', '{}\\azure_cli_test_result\\{}.xml'.format(os.path.expanduser('~'), mod_name), '-n', 'auto', '--pyargs', 'azure.cli.command_modules.{}'.format(mod_name)]) + if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected in the module + sys.exit(exit_code) -mod_list = [os.path.join(d, o) for o in os.listdir(d) if os.path.isdir(os.path.join(d, o))] -for mod in mod_list: - mod_name = os.path.basename(os.path.normpath(mod)) - - # command = "python -m pytest -x -v -p no:warnings --log-level WARN --junit-xml '{}\\azure_cli_tests\\{}.xml' -n auto '{}'".format(os.path.expanduser('~'), mod_name, mod) - subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', - '--junit-xml', '{}\\azure_cli_tests\\{}.xml'.format(os.path.expanduser('~'), mod_name), '-n', 'auto', mod]) +exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', + '--junit-xml', '{}\\azure_cli_test_result\\azure-cli-core.xml'.format(os.path.expanduser('~')), '-n', 'auto', '--pyargs', 'azure.cli.core', '--import-mode=append']) +sys.exit(exit_code) diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 1ee35fc2c2f..3ab620db9ba 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -7,16 +7,19 @@ import sys import subprocess -d = '/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules' -mod_list = [os.path.join(d, o) for o in sorted(os.listdir(d)) if os.path.isdir(os.path.join(d, o)) and o != '__pycache__'] +root_dir = '/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules' +mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] -for mod in mod_list: - mod_name = os.path.basename(os.path.normpath(mod)) +pytest_base_cmd = 'PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN' +pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd) + +for mod_name in mod_list: if mod_name in ['botservice', 'network']: - exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml --pyargs azure.cli.command_modules.{}'.format(mod_name, mod_name)], shell=True) + exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True) else: - exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/{}.xml -n auto --pyargs azure.cli.command_modules.{}'.format(mod_name, mod_name)], shell=True) - if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected + exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True) + if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected in the module sys.exit(exit_code) -exit_code = subprocess.call(['PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN --junit-xml /result/azure-cli-core.xml -n auto --pyargs azure.cli.core'], shell=True) + +exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/azure-cli-core.xml --pyargs azure.cli.core'.format(pytest_parallel_cmd)], shell=True) sys.exit(exit_code) From 1bee368f590c85c418e6883f9e917b4342c941ea Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 9 Jul 2020 13:54:44 +0800 Subject: [PATCH 13/16] fix msi test --- build_scripts/windows/scripts/setup_msi_test.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_scripts/windows/scripts/setup_msi_test.ps1 b/build_scripts/windows/scripts/setup_msi_test.ps1 index 2294dd977d2..3f63c137701 100644 --- a/build_scripts/windows/scripts/setup_msi_test.ps1 +++ b/build_scripts/windows/scripts/setup_msi_test.ps1 @@ -13,9 +13,9 @@ If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdenti & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest-xdist $testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_testsdk*.whl | Select-Object Name -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $testsdk.Name +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name) $fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $fulltest.Name +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name) & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py \ No newline at end of file From 363bd8fbe2abf173118cb11b1005064860cb3228 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 9 Jul 2020 15:14:15 +0800 Subject: [PATCH 14/16] use 32 bit python for compatibility --- build_scripts/windows/scripts/build.cmd | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index b6d24e0ebfa..f3444e46e26 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" ( echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) -set PYTHON_VERSION=3.6.6 +set PYTHON_VERSION=3.6.8 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" -set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-64.zip" +set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip" set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip" :: Set up the output directory and temp. directories @@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR% set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch set BUILDING_DIR=%ARTIFACTS_DIR%\cli set WIX_DIR=%ARTIFACTS_DIR%\wix -set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-64 +set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change set REPO_ROOT=%~dp0..\..\.. @@ -75,10 +75,10 @@ if not exist %PYTHON_DIR% ( mkdir %PYTHON_DIR% pushd %PYTHON_DIR% echo Downloading Python. - curl -o Python368-64.zip %PYTHON_DOWNLOAD_URL% -k - unzip -q Python368-64.zip + curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k + unzip -q Python368-32.zip if %errorlevel% neq 0 goto ERROR - del Python368-64.zip + del Python368-32.zip echo Python downloaded and extracted successfully. popd ) From ee39b7db7af68501e924486d6867cd711fa4e498 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 9 Jul 2020 21:15:00 +0800 Subject: [PATCH 15/16] revert msi change --- .gitattributes | 2 -- build_scripts/windows/scripts/build.cmd | 12 ++++----- .../windows/scripts/setup_msi_test.ps1 | 21 ---------------- .../windows/scripts/test_msi_package.py | 25 ------------------- 4 files changed, 6 insertions(+), 54 deletions(-) delete mode 100644 build_scripts/windows/scripts/setup_msi_test.ps1 delete mode 100644 build_scripts/windows/scripts/test_msi_package.py diff --git a/.gitattributes b/.gitattributes index 391370e95ee..aec323b43b9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,5 +13,3 @@ *.bat eol=crlf # The az script for Git Bash/Cygwin should be LF build_scripts/windows/scripts/az eol=lf -# sh scripts should be LF -*.sh eol=lf diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index f3444e46e26..ffb19299b48 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" ( echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) -set PYTHON_VERSION=3.6.8 +set PYTHON_VERSION=3.6.6 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" -set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip" +set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip" set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip" :: Set up the output directory and temp. directories @@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR% set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch set BUILDING_DIR=%ARTIFACTS_DIR%\cli set WIX_DIR=%ARTIFACTS_DIR%\wix -set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32 +set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change set REPO_ROOT=%~dp0..\..\.. @@ -75,10 +75,10 @@ if not exist %PYTHON_DIR% ( mkdir %PYTHON_DIR% pushd %PYTHON_DIR% echo Downloading Python. - curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k - unzip -q Python368-32.zip + curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k + unzip -q Python366-32.zip if %errorlevel% neq 0 goto ERROR - del Python368-32.zip + del Python366-32.zip echo Python downloaded and extracted successfully. popd ) diff --git a/build_scripts/windows/scripts/setup_msi_test.ps1 b/build_scripts/windows/scripts/setup_msi_test.ps1 deleted file mode 100644 index 3f63c137701..00000000000 --- a/build_scripts/windows/scripts/setup_msi_test.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -# Prerequisites: -# 1. Install the MSI built with current branch -# 2. Run bash azure-cli\scripts\ci\build.sh with Git Bash first to generate artifacts under azure-cli\artifacts\build so we can use the testsdk and fulltest wheels. - -# Elevate to Admin -If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) -{ - $arguments = "& '" + $myinvocation.mycommand.definition + "'" - Start-Process powershell -Verb runAs -ArgumentList $arguments -} - -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest-xdist - -$testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_testsdk*.whl | Select-Object Name -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name) - -$fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name) - -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py \ No newline at end of file diff --git a/build_scripts/windows/scripts/test_msi_package.py b/build_scripts/windows/scripts/test_msi_package.py deleted file mode 100644 index 78a4b60e3d3..00000000000 --- a/build_scripts/windows/scripts/test_msi_package.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# Invoke this script in Powershell with: -# & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' test_msi_package.py - -import os -import sys -import subprocess - -root_dir = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli\\command_modules' - -mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] - -for mod_name in mod_list: - exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', - '--junit-xml', '{}\\azure_cli_test_result\\{}.xml'.format(os.path.expanduser('~'), mod_name), '-n', 'auto', '--pyargs', 'azure.cli.command_modules.{}'.format(mod_name)]) - if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected in the module - sys.exit(exit_code) - -exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', - '--junit-xml', '{}\\azure_cli_test_result\\azure-cli-core.xml'.format(os.path.expanduser('~')), '-n', 'auto', '--pyargs', 'azure.cli.core', '--import-mode=append']) -sys.exit(exit_code) From 55bd44a1382ca676f8e90981b30c163c6d2d9fec Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Fri, 10 Jul 2020 09:47:55 +0800 Subject: [PATCH 16/16] print message when no tests found --- scripts/release/rpm/test_rpm_package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 3ab620db9ba..78eec9817b5 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -18,7 +18,9 @@ exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True) else: exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True) - if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected in the module + if exit_code == 5: + print('No tests found for {}'.format(mod_name)) + elif exit_code != 0: sys.exit(exit_code) exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/azure-cli-core.xml --pyargs azure.cli.core'.format(pytest_parallel_cmd)], shell=True)