From 196bdf6e836f02094dd96527002800431aeb39ef Mon Sep 17 00:00:00 2001
From: Siva <quic_sivb@quicinc.com>
Date: Thu, 26 Oct 2023 14:57:09 +0530
Subject: [PATCH] [CI][ADRENO] Enhancements to Adreno specific CI utils RPC
 seem to be hanging while running bunch of tests with in single session. This
 enables running them individually. Also we don't need rerun with in same
 session that leads to hang some times. Adding PLATFORM which is needed for
 run_pytest to summarize the failures.

---
 tests/scripts/ci.py                 |  2 ++
 tests/scripts/setup-pytest-env.sh   |  4 +++-
 tests/scripts/task_python_adreno.sh | 21 ++++++++++++++++++---
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/tests/scripts/ci.py b/tests/scripts/ci.py
index 1add6bcde3eb..a2facff6d6db 100755
--- a/tests/scripts/ci.py
+++ b/tests/scripts/ci.py
@@ -189,6 +189,8 @@ def docker(
         env["SCCACHE_CACHE_SIZE"] = os.getenv("SCCACHE_CACHE_SIZE", "50G")
         env["SCCACHE_SERVER_PORT"] = os.getenv("SCCACHE_SERVER_PORT", "4226")
 
+    env["PLATFORM"] = name
+
     docker_bash = REPO_ROOT / "docker" / "bash.sh"
 
     command = [docker_bash]
diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh
index 129309f00552..7bdfbdd1df47 100755
--- a/tests/scripts/setup-pytest-env.sh
+++ b/tests/scripts/setup-pytest-env.sh
@@ -70,7 +70,9 @@ function run_pytest() {
 
     has_reruns=$(python3 -m pytest --help 2>&1 | grep 'reruns=' || true)
     if [ -n "$has_reruns" ]; then
-        extra_args+=('--reruns=3')
+        if [[ ! "${extra_args[*]}" == *"--reruns"* ]]; then
+          extra_args+=('--reruns=3')
+        fi
     fi
 
     suite_name="${test_suite_name}-${current_shard}-${ffi_type}"
diff --git a/tests/scripts/task_python_adreno.sh b/tests/scripts/task_python_adreno.sh
index 6d9c370aeca4..634d9adbd658 100755
--- a/tests/scripts/task_python_adreno.sh
+++ b/tests/scripts/task_python_adreno.sh
@@ -54,18 +54,33 @@ adb forward tcp:5002 tcp:5002
 env adb shell "cd ${TARGET_FOLDER}; killall -9 tvm_rpc-${USER}; sleep 2; LD_LIBRARY_PATH=${TARGET_FOLDER}/ ./tvm_rpc-${USER} server --host=0.0.0.0 --port=5000 --port-end=5010 --tracker=127.0.0.1:${TVM_TRACKER_PORT} --key=${RPC_DEVICE_KEY}" &
 DEVICE_PID=$!
 sleep 5 # Wait for the device connections
-trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; }" 0
+trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; cleanup; }" 0
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 # Test TVM
 make cython3
 
+# The RPC to remote Android device has issue of hang after few tests with in CI environments.
+# Lets run them individually on fresh rpc session.
 # OpenCL texture test on Adreno
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture
+TEXTURE_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder tests/python/relay/opencl_texture)
+i=0
+IFS=$'\n'
+for node_id in $TEXTURE_TESTS; do
+    echo "$node_id"
+    run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0
+    i=$((i+1))
+done
 
 # Adreno CLML test
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-openclml tests/python/contrib/test_clml
+CLML_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder tests/python/contrib/test_clml)
+i=0
+for node_id in $CLML_TESTS; do
+    echo "$node_id"
+    run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0
+    i=$((i+1))
+done
 
 kill ${TRACKER_PID}
 kill ${DEVICE_PID}