diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d23fba0e331..6d0dba3e270 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -639,10 +639,10 @@ jobs:
 
        echo "== Test yum package on CentOS =="
 
-       docker pull centos:centos7
-       docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/yum:/mnt/yum centos:centos7 /bin/bash -c "yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y && time az self-test && time az --version && sleep 5"
+       docker pull centos:centos8
+       docker run --rm -e YUM_NAME=$YUM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/yum:/mnt/yum -v $(pwd):/azure-cli centos:centos8 /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh"
 
-    displayName: 'Bash Script'
+    displayName: 'Test Yum Package'
 
 - job: BuildUbuntuPackages
   displayName: Build Ubuntu Packages
diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec
index 2bb8acbe236..1dfc379c8fb 100644
--- a/scripts/release/rpm/azure-cli.spec
+++ b/scripts/release/rpm/azure-cli.spec
@@ -1,6 +1,10 @@
 # RPM spec file for Azure CLI
 # Definition of macros used - https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros
 
+%global __python %{__python3}
+# Turn off python byte compilation
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+
 # .el7.centos -> .el7
 %if 0%{?rhel}
   %define dist .el%{?rhel}
diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh
new file mode 100644
index 00000000000..9cb0f64b9b3
--- /dev/null
+++ b/scripts/release/rpm/test_rpm_in_docker.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# This script should be run in a centos8 docker.
+set -exv
+
+yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y
+
+yum install git -y
+
+ln -s /usr/bin/python3 /usr/bin/python
+ln -s /usr/bin/pip3 /usr/bin/pip
+time az self-test
+time az --version
+
+cd /azure-cli/
+pip3 install wheel
+./scripts/ci/build.sh
+pip3 install pytest --prefix /usr/lib64/az
+pip3 install pytest-xdist --prefix /usr/lib64/az
+
+find /azure-cli/artifacts/build -name "azure_cli_testsdk*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed
+find /azure-cli/artifacts/build -name "azure_cli_fulltest*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed --no-deps
+
+python3 /azure-cli/scripts/release/rpm/test_rpm_package.py
diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py
new file mode 100644
index 00000000000..78eec9817b5
--- /dev/null
+++ b/scripts/release/rpm/test_rpm_package.py
@@ -0,0 +1,27 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import sys
+import subprocess
+
+root_dir = '/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules'
+mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']
+
+pytest_base_cmd = 'PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN'
+pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd)
+
+for mod_name in mod_list:
+    if mod_name in ['botservice', 'network']:
+        exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True)
+    else:
+        exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True)
+    if exit_code == 5:
+        print('No tests found for {}'.format(mod_name))
+    elif exit_code != 0:
+        sys.exit(exit_code)
+
+exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/azure-cli-core.xml --pyargs azure.cli.core'.format(pytest_parallel_cmd)], shell=True)
+sys.exit(exit_code)