From b7d70710c354741ebb5f603135ae6abc08f439a3 Mon Sep 17 00:00:00 2001
From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
Date: Fri, 31 Jul 2026 09:30:38 +0000
Subject: [PATCH 1/5] [NVBUG-6541356][fix] align cache transceiver harness
 setup

Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
---
 .../run_cache_transceiver_test.py             | 20 ++++++++++++-
 tensorrt_llm/_torch/pyexecutor/_util.py       | 28 ++++--------------
 .../_torch/pyexecutor/kv_cache_transceiver.py | 26 ++++++++++++++++-
 .../precheck_config.py                        |  8 +++--
 .../run_precheck.py                           | 10 ++++++-
 .../test_cache_transceiver_precheck_config.py | 12 ++++++++
 .../others/test_kv_cache_transceiver.py       | 29 +++++++++++++++++--
 7 files changed, 103 insertions(+), 30 deletions(-)

diff --git a/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py b/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py
index c4ebd8c6460c..bebf82fec2e7 100644
--- a/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py
+++ b/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py
@@ -55,7 +55,10 @@
 from tensorrt_llm._torch.distributed import Distributed
 from tensorrt_llm._torch.pyexecutor.hang_detector import HangDetector
 from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2
-from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import create_kv_cache_transceiver
+from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import (
+    create_kv_cache_transceiver,
+    maybe_enable_fabric_memory_for_python_transceiver,
+)
 from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState, LlmRequestType
 from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager
 from tensorrt_llm.llmapi.llm_args import BlockReuseConfig, CacheTransceiverConfig
@@ -551,6 +554,21 @@ def reset_sock():
     zmq_sock = open_sock()
 
     cases = build_cases(cfg)
+    # The C++ fabric-memory env getter is cached on its first KV pool
+    # allocation. Enable the default before any matrix case builds a pool, even
+    # when a C++ transceiver case appears before Python+V1 in the matrix.
+    python_v1_case = next(
+        (case for case in cases if case["runtime"] == "PYTHON" and case["cache_manager"] == "V1"),
+        None,
+    )
+    if python_v1_case is not None:
+        maybe_enable_fabric_memory_for_python_transceiver(
+            CacheTransceiverConfig(
+                backend=python_v1_case["backend"],
+                transceiver_runtime="PYTHON",
+            ),
+            KVCacheManager,
+        )
     req_lens = cfg["test_matrix"]["request_lengths"]
     warmup = cfg["test_matrix"]["warmup_requests"]
     num_req = cfg["test_matrix"]["num_requests_per_length"]
diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py
index be555fa55c42..68ccd6263964 100644
--- a/tensorrt_llm/_torch/pyexecutor/_util.py
+++ b/tensorrt_llm/_torch/pyexecutor/_util.py
@@ -54,7 +54,9 @@
 from .dwdp import DwdpManager
 from .guided_decoder import GuidedDecoder
 from .kv_cache_manager_v2 import KVCacheManagerV2
-from .kv_cache_transceiver import AttentionTypeCpp, create_kv_cache_transceiver
+from .kv_cache_transceiver import (
+    AttentionTypeCpp, create_kv_cache_transceiver,
+    maybe_enable_fabric_memory_for_python_transceiver)
 from .llm_request import ExecutorResponse, LlmRequestState
 from .mamba_cache_manager import (BaseMambaCacheManager,
                                   CppMambaHybridCacheManager,
@@ -528,28 +530,8 @@ def __init__(
         self._maybe_enable_fabric_memory_for_python_transceiver()
 
     def _maybe_enable_fabric_memory_for_python_transceiver(self) -> None:
-        """Default TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 for the Python
-        transceiver on the C++ V1 KV cache manager.
-
-        The Python transceiver (KvCacheTransceiverV2) transfers KV blocks
-        directly out of the C++ pool, so the pool should be allocated with
-        fabric memory to enable MNNVL transfers. This must run before any
-        pool allocation because the C++ env getter caches the value on first
-        read. Explicit user settings are respected, and platforms without
-        fabric memory support fall back to standard allocation in C++.
-        """
-        if (self._cache_transceiver_config is None
-                or self._cache_transceiver_config.backend is None or
-                self._cache_transceiver_config.transceiver_runtime != "PYTHON"):
-            return
-        if not issubclass(self._kv_cache_manager_cls, KVCacheManager):
-            return
-        if os.environ.get("TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY") is None:
-            os.environ["TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"] = "1"
-            logger.info(
-                "Python cache transceiver with C++ KV cache manager detected; "
-                "defaulting TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 (set it "
-                "to 0 explicitly to disable)")
+        maybe_enable_fabric_memory_for_python_transceiver(
+            self._cache_transceiver_config, self._kv_cache_manager_cls)
 
     def _get_model_kv_cache_manager_cls(
         self,
diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py
index d4868acf0226..a97707c9a090 100644
--- a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py
+++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from abc import ABC, abstractmethod
-from os import getenv
+from os import environ, getenv
 from typing import Any, Dict, List, Optional
 
 import tensorrt_llm
@@ -30,10 +30,34 @@
 _DISABLE_KV_CACHE_TRANSFER_OVERLAP_ENV = "TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP"
 _DISAGG_LAYERWISE_ENV = "TRTLLM_DISAGG_LAYERWISE"
 _TRY_ZCOPY_FOR_KV_CACHE_TRANSFER_ENV = "TRTLLM_TRY_ZCOPY_FOR_KVCACHE_TRANSFER"
+_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV = "TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"
 _SUPPORTED_INFLIGHT_CANCEL_NIXL_BACKEND = "UCX"
 _disagg_inflight_cancel_enabled_cache: Optional[bool] = None
 
 
+def maybe_enable_fabric_memory_for_python_transceiver(
+        cache_transceiver_config: Optional[CacheTransceiverConfig],
+        kv_cache_manager_cls: type) -> None:
+    """Default the C++ V1 KV pool to fabric memory for the Python transceiver.
+
+    This must run before any KV pool allocation because the C++ environment
+    getter caches the value on first read. Explicit user settings are always
+    respected.
+    """
+    if (cache_transceiver_config is None
+            or cache_transceiver_config.backend is None
+            or cache_transceiver_config.transceiver_runtime != "PYTHON"):
+        return
+    if not issubclass(kv_cache_manager_cls, KVCacheManager):
+        return
+    if getenv(_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV) is None:
+        environ[_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV] = "1"
+        logger.info(
+            "Python cache transceiver with C++ KV cache manager detected; "
+            f"defaulting {_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV}=1 (set it "
+            "to 0 explicitly to disable)")
+
+
 def is_disagg_inflight_cancel_enabled() -> bool:
     """Return whether disaggregated in-flight KV transfer cancellation is enabled."""
     global _disagg_inflight_cancel_enabled_cache
diff --git a/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py b/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py
index 916f2c4678a3..d8388977ef59 100644
--- a/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py
+++ b/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py
@@ -26,6 +26,7 @@
 
 import json
 import os
+import shlex
 
 # Optional per-yaml overrides live under a `cache_transceiver_precheck:` block.
 PRECHECK_DEFAULTS = {
@@ -163,6 +164,7 @@ def precheck_prefix_lines(
         "--work-dir $testOutputDir/cache_transceiver_precheck "
         f"--benchmark-mode {benchmark_mode} --llm-src $llmSrcNode"
     )
+    model_root_env = f"LLM_MODELS_ROOT={shlex.quote(os.environ.get('LLM_MODELS_ROOT', ''))}"
     lines = [
         f"export ctPrecheckEnabled={int(enabled)}",
         # The external srun timeout must cover the driver's first-rep NIXL
@@ -172,8 +174,10 @@ def precheck_prefix_lines(
         f"{int(knobs.get('step_timeout_s', default_step_timeout_s(max_world)))}",
         "export precheckRunScript=$llmSrcNode/jenkins/scripts/perf/"
         "disaggregated/slurm_precheck_run.sh",
-        f'export pytestCommandCTXPrecheck="{ucx_tls_cmd} $CTX_WORKER_ENV_VARS {cmd} --role ctx"',
-        f'export pytestCommandGENPrecheck="{ucx_tls_cmd} $GEN_WORKER_ENV_VARS {cmd} --role gen"',
+        f'export pytestCommandCTXPrecheck="{ucx_tls_cmd} {model_root_env} '
+        f'$CTX_WORKER_ENV_VARS $PYTEST_COMMON_VARS {cmd} --role ctx"',
+        f'export pytestCommandGENPrecheck="{ucx_tls_cmd} {model_root_env} '
+        f'$GEN_WORKER_ENV_VARS $PYTEST_COMMON_VARS {cmd} --role gen"',
     ]
     if stage_name:
         # Suite name for the synthetic junit xml the gate writes on failure
diff --git a/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py b/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py
index 19c6bf70c856..5008db9fab1d 100644
--- a/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py
+++ b/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py
@@ -204,7 +204,10 @@ def load_internal_apis():
     from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING
     from tensorrt_llm._torch.pyexecutor.hang_detector import HangDetector
     from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2
-    from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import create_kv_cache_transceiver
+    from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import (
+        create_kv_cache_transceiver,
+        maybe_enable_fabric_memory_for_python_transceiver,
+    )
     from tensorrt_llm._torch.pyexecutor.llm_request import (
         LlmRequest,
         LlmRequestState,
@@ -237,6 +240,9 @@ def load_internal_apis():
         KVCacheManager=KVCacheManager,
         KVCacheManagerV2=KVCacheManagerV2,
         create_kv_cache_transceiver=create_kv_cache_transceiver,
+        maybe_enable_fabric_memory_for_python_transceiver=(
+            maybe_enable_fabric_memory_for_python_transceiver
+        ),
         LlmRequest=LlmRequest,
         LlmRequestState=LlmRequestState,
         LlmRequestType=LlmRequestType,
@@ -883,6 +889,8 @@ def setup(self, kv_shape, max_req_len):
                 "(the C++ transceiver only supports the V1 manager)"
             )
 
+        manager_cls = api.KVCacheManagerV2 if self.use_v2 else api.KVCacheManager
+        api.maybe_enable_fabric_memory_for_python_transceiver(cache_cfg, manager_cls)
         self.kvm = build_kv_cache_manager(
             kv_shape, self.plan, self.side, self.mapping, max_req_len, self.use_v2
         )
diff --git a/tests/unittest/others/test_cache_transceiver_precheck_config.py b/tests/unittest/others/test_cache_transceiver_precheck_config.py
index 0bcff41adb48..2b7c4a36fb31 100644
--- a/tests/unittest/others/test_cache_transceiver_precheck_config.py
+++ b/tests/unittest/others/test_cache_transceiver_precheck_config.py
@@ -359,6 +359,18 @@ def test_wireup_timeout_derivation():
     assert plan["wireup_timeout_s"] == 42
 
 
+def test_precheck_commands_propagate_model_root(monkeypatch):
+    monkeypatch.setenv("LLM_MODELS_ROOT", "/models with spaces")
+    lines = pcfg.precheck_prefix_lines({}, "e2e", "$config", "unset UCX_TLS &&", max_world=8)
+    commands = [line for line in lines if line.startswith("export pytestCommand")]
+
+    assert len(commands) == 2
+    for command in commands:
+        assert "LLM_MODELS_ROOT='/models with spaces'" in command
+        assert "$PYTEST_COMMON_VARS" in command
+        assert command.index("LLM_MODELS_ROOT=") < command.index("$PYTEST_COMMON_VARS")
+
+
 def _enabled_line(cfg):
     lines = pcfg.precheck_prefix_lines(cfg, "e2e", "$c", "unset &&", max_world=8)
     return next(x for x in lines if x.startswith("export ctPrecheckEnabled"))
diff --git a/tests/unittest/others/test_kv_cache_transceiver.py b/tests/unittest/others/test_kv_cache_transceiver.py
index 241a4aee0d09..d617122356ee 100644
--- a/tests/unittest/others/test_kv_cache_transceiver.py
+++ b/tests/unittest/others/test_kv_cache_transceiver.py
@@ -21,8 +21,10 @@
 import tensorrt_llm.bindings
 import tensorrt_llm.bindings.executor as trtllm
 from tensorrt_llm._torch.distributed import Distributed
-from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import \
-    create_kv_cache_transceiver
+from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2
+from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import (
+    create_kv_cache_transceiver,
+    maybe_enable_fabric_memory_for_python_transceiver)
 from tensorrt_llm._torch.pyexecutor.llm_request import (LlmRequest,
                                                         LlmRequestState)
 from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import \
@@ -42,6 +44,29 @@
 KV_TRANSFER_COMPLETION_MARGIN_S = 10.0
 
 
+@pytest.mark.parametrize(
+    "runtime,manager_cls,initial_value,expected_value",
+    [
+        ("PYTHON", KVCacheManager, None, "1"),
+        ("PYTHON", KVCacheManager, "0", "0"),
+        ("PYTHON", KVCacheManagerV2, None, None),
+        ("CPP", KVCacheManager, None, None),
+    ],
+)
+def test_maybe_enable_fabric_memory_for_python_transceiver(
+        monkeypatch, runtime, manager_cls, initial_value, expected_value):
+    env_name = "TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"
+    if initial_value is None:
+        monkeypatch.delenv(env_name, raising=False)
+    else:
+        monkeypatch.setenv(env_name, initial_value)
+    config = CacheTransceiverConfig(backend="NIXL", transceiver_runtime=runtime)
+
+    maybe_enable_fabric_memory_for_python_transceiver(config, manager_cls)
+
+    assert os.environ.get(env_name) == expected_value
+
+
 @pytest.mark.parametrize("transceiver_runtime", ["CPP", "auto"])
 def test_cpp_transceiver_rejects_mixed_mamba_manager(transceiver_runtime):
     config = CacheTransceiverConfig(backend="NIXL",

From 6c9da90a51e5cf04666573fb118063f4de5cb0c5 Mon Sep 17 00:00:00 2001
From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
Date: Mon, 3 Aug 2026 05:25:46 +0000
Subject: [PATCH 2/5] [NVBUG-6541356][fix] fix cache transceiver precheck
 environment

Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
---
 jenkins/scripts/perf/cluster_env.py           |  9 ++--
 jenkins/scripts/perf/local/submit.py          |  1 +
 jenkins/scripts/perf/submit.py                | 32 +++++++++++++++
 .../precheck_config.py                        | 18 ++++++--
 .../test_cache_transceiver_precheck_config.py | 41 +++++++++++++++----
 .../test_cache_transceiver_precheck_run.py    | 11 +++++
 tests/unittest/scripts/test_cluster_env.py    |  5 +--
 tests/unittest/scripts/test_perf_submit.py    | 34 ++++++++++++---
 8 files changed, 125 insertions(+), 26 deletions(-)

diff --git a/jenkins/scripts/perf/cluster_env.py b/jenkins/scripts/perf/cluster_env.py
index cbc342aa2d24..923975089d80 100644
--- a/jenkins/scripts/perf/cluster_env.py
+++ b/jenkins/scripts/perf/cluster_env.py
@@ -44,15 +44,12 @@
         "rocep198s0:1,rocep199s0:1,rocep205s0:1,rocep206s0:1"
         " UCX_IB_GID_INDEX=auto UCX_IB_TRAFFIC_CLASS=52 UCX_IB_SL=0",
     ),
-    # oci-aga: avoid transports that fail on this VF fabric, disable DEVX to
-    # avoid UAR allocation failures, and pin the GPU-connected rail VFs.
+    # oci-aga: use TCP over IPv4 alongside the local CUDA/shared-memory
+    # transports.
     (
         "oci-aga*",
         "*",
-        "export UCX_TLS=^tcp,rc_gda,gga UCX_IB_MLX5_DEVX=n "
-        "UCX_NET_DEVICES="
-        "rdma_vf_rail0:1,rdma_vf_rail1:1,rdma_vf_rail2:1,rdma_vf_rail3:1 "
-        "UCX_IB_TRAFFIC_CLASS=96 TRTLLM_NIXL_NUM_THREADS=1",
+        "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp UCX_TCP_AF_PRIO=inet",
     ),
     # nsc-svg: UCX picks wrong RDMA devices; pin the usable mlx5 ports.
     (
diff --git a/jenkins/scripts/perf/local/submit.py b/jenkins/scripts/perf/local/submit.py
index 8e571eb2f890..4c911c3f0e19 100755
--- a/jenkins/scripts/perf/local/submit.py
+++ b/jenkins/scripts/perf/local/submit.py
@@ -967,6 +967,7 @@ def main():
                     hardware_config.get("gpus_per_ctx_server", 0) or 0,
                     hardware_config.get("gpus_per_gen_server", 0) or 0,
                 ),
+                llm_models_root=args.llm_models_root,
             )
         )
 
diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py
index 72b176df3a95..8931bc5cc0d6 100755
--- a/jenkins/scripts/perf/submit.py
+++ b/jenkins/scripts/perf/submit.py
@@ -569,6 +569,31 @@ def _is_output_file_part(part):
     return ("", worker_pytest_command, disagg_server_pytest_command, benchmark_pytest_command)
 
 
+def _get_pytest_command_env_var(script_prefix_lines, env_name):
+    """Extract an environment assignment from the inbound pytest command."""
+    pytest_command_line = next(
+        (ln for ln in script_prefix_lines if "export pytestCommand=" in ln), None
+    )
+    if pytest_command_line is None:
+        return None
+
+    command_value = pytest_command_line.split("=", 1)[1]
+    try:
+        parts = shlex.split(command_value)
+        # A fully quoted export value is parsed as one token on the first
+        # pass. Parse that token once more to recover its command words.
+        if len(parts) == 1 and any(char.isspace() for char in parts[0]):
+            parts = shlex.split(parts[0])
+    except ValueError as error:
+        raise ValueError(f"Invalid inbound pytestCommand: {error}") from error
+
+    prefix = f"{env_name}="
+    for part in parts:
+        if part.startswith(prefix):
+            return part[len(prefix) :]
+    return None
+
+
 def get_test_output_dir(script_prefix_lines, test_case_name):
     """Build the per-test output directory from the inbound pytestCommand.
 
@@ -748,6 +773,12 @@ def main():
         ucx_tls_server_cmd = ucx_tls_cmd
 
         pytest_common_vars = ""
+        llm_models_root = _get_pytest_command_env_var(script_prefix_lines, "LLM_MODELS_ROOT")
+        if not llm_models_root:
+            raise ValueError(
+                "LLM_MODELS_ROOT is missing from the inbound pytestCommand; "
+                "the cache-transceiver precheck cannot resolve model defaults"
+            )
         script_prefix_lines.extend(
             [
                 worker_pytest_command,
@@ -802,6 +833,7 @@ def main():
                     hardware_config["gpus_per_gen_server"],
                 ),
                 stage_name=args.stage_name,
+                llm_models_root=llm_models_root,
             )
         )
         srun_args_lines.extend(
diff --git a/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py b/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py
index d8388977ef59..2dc634ca5679 100644
--- a/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py
+++ b/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py
@@ -129,7 +129,13 @@ def default_step_timeout_s(max_world):
 
 
 def precheck_prefix_lines(
-    cfg, benchmark_mode, config_path_expr, ucx_tls_cmd, max_world, stage_name=""
+    cfg,
+    benchmark_mode,
+    config_path_expr,
+    ucx_tls_cmd,
+    max_world,
+    stage_name="",
+    llm_models_root=None,
 ):
     """Launch-script export lines wiring the precheck gate.
 
@@ -164,7 +170,7 @@ def precheck_prefix_lines(
         "--work-dir $testOutputDir/cache_transceiver_precheck "
         f"--benchmark-mode {benchmark_mode} --llm-src $llmSrcNode"
     )
-    model_root_env = f"LLM_MODELS_ROOT={shlex.quote(os.environ.get('LLM_MODELS_ROOT', ''))}"
+    model_root_env = f"LLM_MODELS_ROOT={shlex.quote(llm_models_root)}" if llm_models_root else ""
     lines = [
         f"export ctPrecheckEnabled={int(enabled)}",
         # The external srun timeout must cover the driver's first-rep NIXL
@@ -174,9 +180,13 @@ def precheck_prefix_lines(
         f"{int(knobs.get('step_timeout_s', default_step_timeout_s(max_world)))}",
         "export precheckRunScript=$llmSrcNode/jenkins/scripts/perf/"
         "disaggregated/slurm_precheck_run.sh",
-        f'export pytestCommandCTXPrecheck="{ucx_tls_cmd} {model_root_env} '
+        # Quote the complete assignment as an export value. The launch script
+        # expands it into pytestCommand*Precheck, whose later eval interprets
+        # the inner shlex-quoted model path.
+        f"export ctPrecheckModelRootEnv={shlex.quote(model_root_env)}",
+        f'export pytestCommandCTXPrecheck="{ucx_tls_cmd} $ctPrecheckModelRootEnv '
         f'$CTX_WORKER_ENV_VARS $PYTEST_COMMON_VARS {cmd} --role ctx"',
-        f'export pytestCommandGENPrecheck="{ucx_tls_cmd} {model_root_env} '
+        f'export pytestCommandGENPrecheck="{ucx_tls_cmd} $ctPrecheckModelRootEnv '
         f'$GEN_WORKER_ENV_VARS $PYTEST_COMMON_VARS {cmd} --role gen"',
     ]
     if stage_name:
diff --git a/tests/unittest/others/test_cache_transceiver_precheck_config.py b/tests/unittest/others/test_cache_transceiver_precheck_config.py
index 2b7c4a36fb31..bfd8cba0abef 100644
--- a/tests/unittest/others/test_cache_transceiver_precheck_config.py
+++ b/tests/unittest/others/test_cache_transceiver_precheck_config.py
@@ -19,6 +19,8 @@
 
 import json
 import os
+import shlex
+import subprocess
 import sys
 
 import pytest
@@ -359,16 +361,41 @@ def test_wireup_timeout_derivation():
     assert plan["wireup_timeout_s"] == 42
 
 
-def test_precheck_commands_propagate_model_root(monkeypatch):
-    monkeypatch.setenv("LLM_MODELS_ROOT", "/models with spaces")
-    lines = pcfg.precheck_prefix_lines({}, "e2e", "$config", "unset UCX_TLS &&", max_world=8)
-    commands = [line for line in lines if line.startswith("export pytestCommand")]
+@pytest.mark.parametrize("model_root", ["/models with spaces", "/models/o'hare"])
+def test_precheck_commands_propagate_model_root(monkeypatch, model_root):
+    # CI provides the model root inside its inbound pytest command, not in the
+    # environment of the Python process that generates the launch script.
+    monkeypatch.delenv("LLM_MODELS_ROOT", raising=False)
+    lines = pcfg.precheck_prefix_lines(
+        {},
+        "e2e",
+        "$config",
+        "unset UCX_TLS &&",
+        max_world=8,
+        llm_models_root=model_root,
+    )
+    shell_script = "\n".join(
+        [
+            "CTX_WORKER_ENV_VARS=",
+            "GEN_WORKER_ENV_VARS=",
+            "PYTEST_COMMON_VARS=",
+            "llmSrcNode=/repo",
+            "testOutputDir=/tmp/output",
+            "config=/tmp/config.yaml",
+            *lines,
+            "printf '%s\\n' \"$pytestCommandCTXPrecheck\"",
+            "printf '%s\\n' \"$pytestCommandGENPrecheck\"",
+        ]
+    )
+
+    result = subprocess.run(
+        ["bash"], input=shell_script, capture_output=True, check=True, text=True
+    )
+    commands = result.stdout.splitlines()
 
     assert len(commands) == 2
     for command in commands:
-        assert "LLM_MODELS_ROOT='/models with spaces'" in command
-        assert "$PYTEST_COMMON_VARS" in command
-        assert command.index("LLM_MODELS_ROOT=") < command.index("$PYTEST_COMMON_VARS")
+        assert f"LLM_MODELS_ROOT={model_root}" in shlex.split(command)
 
 
 def _enabled_line(cfg):
diff --git a/tests/unittest/others/test_cache_transceiver_precheck_run.py b/tests/unittest/others/test_cache_transceiver_precheck_run.py
index ed060d1e9053..f57295d53853 100644
--- a/tests/unittest/others/test_cache_transceiver_precheck_run.py
+++ b/tests/unittest/others/test_cache_transceiver_precheck_run.py
@@ -397,6 +397,17 @@ def test_serving_resolvers(self, api):
         rt = inspect.signature(api.resolve_transceiver_runtime_auto).parameters
         assert list(rt)[:1] == ["llm_args"] and len(rt) >= 3
 
+    def test_deepseek_v4_auto_selects_kv_cache_manager_v2(self, api, tmp_path):
+        model_dir = tmp_path / "deepseek-v4"
+        model_dir.mkdir()
+        (model_dir / "config.json").write_text(
+            json.dumps({"architectures": ["DeepseekV4ForCausalLM"]})
+        )
+        side = {"use_kv_cache_manager_v2": "auto"}
+        cache_cfg = api.CacheTransceiverConfig(backend="NIXL", transceiver_runtime="PYTHON")
+
+        assert rp.resolve_model_prefs(str(model_dir), side, cache_cfg) is True
+
     def test_enum_members(self, api):
         for enum, members in (
             (api.DataType, ("FP8", "HALF", "BF16")),
diff --git a/tests/unittest/scripts/test_cluster_env.py b/tests/unittest/scripts/test_cluster_env.py
index 26905068b10f..8486854d2a78 100644
--- a/tests/unittest/scripts/test_cluster_env.py
+++ b/tests/unittest/scripts/test_cluster_env.py
@@ -83,10 +83,7 @@ def test_gpu_type_from_supported_gpus(
         ),
         (
             "oci-aga-cs-001",
-            "export UCX_TLS=^tcp,rc_gda,gga UCX_IB_MLX5_DEVX=n "
-            "UCX_NET_DEVICES="
-            "rdma_vf_rail0:1,rdma_vf_rail1:1,rdma_vf_rail2:1,rdma_vf_rail3:1 "
-            "UCX_IB_TRAFFIC_CLASS=96 TRTLLM_NIXL_NUM_THREADS=1",
+            "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp UCX_TCP_AF_PRIO=inet",
         ),
         ("aws-cmh", "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp"),
         ("aws-dfw-prod", "export UCX_TLS=^gdr_copy"),
diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py
index ed4fb8dedea7..418198763206 100644
--- a/tests/unittest/scripts/test_perf_submit.py
+++ b/tests/unittest/scripts/test_perf_submit.py
@@ -16,6 +16,7 @@
 
 import importlib.util
 import json
+import shlex
 from pathlib import Path
 from types import ModuleType
 
@@ -53,6 +54,11 @@ def submit_module(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatc
     return _load_module(request.param, monkeypatch)
 
 
+@pytest.fixture
+def ci_submit_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
+    return _load_module(SUBMIT_PATHS[0], monkeypatch)
+
+
 @pytest.fixture
 def example_submit_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
     return _load_module(EXAMPLE_SUBMIT_PATH, monkeypatch)
@@ -72,11 +78,6 @@ def test_get_benchmark_config_accepts_positive_integer(submit_module: ModuleType
     assert benchmark_config["concurrency"] == int(concurrency)
 
 
-@pytest.fixture
-def ci_submit_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
-    return _load_module(SUBMIT_PATHS[0], monkeypatch)
-
-
 @pytest.mark.parametrize(
     "concurrency",
     (True, 1.5, [], {}, "0", 0, "-1", -1, "1.5", "not-an-integer", None),
@@ -270,3 +271,26 @@ def test_ci_submit_rejects_missing_pytest_split_durations(
             script_prefix_lines,
             split_group=1,
         )
+
+
+@pytest.mark.parametrize("quote_command", [False, True])
+@pytest.mark.parametrize("model_root", ["/scratch/llm-models", "/models/o'hare with spaces"])
+def test_ci_extracts_model_root_from_inbound_pytest_command(
+    ci_submit_module, model_root, quote_command
+):
+    command = (
+        f"LLM_ROOT=/repo LLM_MODELS_ROOT={shlex.quote(model_root)} "
+        "/repo/tensorrt_llm/llmapi/trtllm-llmapi-launch pytest -q"
+    )
+    command_value = shlex.quote(command) if quote_command else command
+    prefix_lines = [f"export pytestCommand={command_value}"]
+
+    assert (
+        ci_submit_module._get_pytest_command_env_var(prefix_lines, "LLM_MODELS_ROOT") == model_root
+    )
+
+
+def test_ci_missing_model_root_is_detectable(ci_submit_module):
+    prefix_lines = ["export pytestCommand='LLM_ROOT=/repo pytest -q'"]
+
+    assert ci_submit_module._get_pytest_command_env_var(prefix_lines, "LLM_MODELS_ROOT") is None

From d920807e3ccb6fdce8a8eb26816f4579a92c14cd Mon Sep 17 00:00:00 2001
From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
Date: Mon, 3 Aug 2026 06:55:55 +0000
Subject: [PATCH 3/5] [NVBUG-6541356][test] address cache transceiver review
 feedback

Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
---
 tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py     | 6 ++++++
 tests/integration/test_lists/test-db/l0_sanity_check.yml   | 1 +
 .../others/test_cache_transceiver_precheck_config.py       | 1 +
 tests/unittest/scripts/test_perf_submit.py                 | 7 +++++++
 4 files changed, 15 insertions(+)

diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py
index a97707c9a090..65f613337fff 100644
--- a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py
+++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py
@@ -43,6 +43,12 @@ def maybe_enable_fabric_memory_for_python_transceiver(
     This must run before any KV pool allocation because the C++ environment
     getter caches the value on first read. Explicit user settings are always
     respected.
+
+    Args:
+        cache_transceiver_config: Configuration used to select the cache
+            transceiver runtime and backend.
+        kv_cache_manager_cls: KV-cache manager class to check for C++ V1 pool
+            allocation.
     """
     if (cache_transceiver_config is None
             or cache_transceiver_config.backend is None
diff --git a/tests/integration/test_lists/test-db/l0_sanity_check.yml b/tests/integration/test_lists/test-db/l0_sanity_check.yml
index db5c37afffbe..d4ae779eb363 100644
--- a/tests/integration/test_lists/test-db/l0_sanity_check.yml
+++ b/tests/integration/test_lists/test-db/l0_sanity_check.yml
@@ -29,6 +29,7 @@ l0_sanity_check:
     - llmapi/test_llm_examples.py::test_llmapi_sampling
     - llmapi/test_llm_examples.py::test_llmapi_runtime
     - examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0] ISOLATION
+    - unittest/others/test_kv_cache_transceiver.py::test_maybe_enable_fabric_memory_for_python_transceiver
     - unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[NIXL-mha-ctx_fp16_gen_fp16]
     - unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[UCX-mha-ctx_fp16_gen_fp16]
     - unittest/others/test_kv_cache_transceiver.py::test_cpp_nixl_sync_transfer_stress
diff --git a/tests/unittest/others/test_cache_transceiver_precheck_config.py b/tests/unittest/others/test_cache_transceiver_precheck_config.py
index bfd8cba0abef..1a39b801e89a 100644
--- a/tests/unittest/others/test_cache_transceiver_precheck_config.py
+++ b/tests/unittest/others/test_cache_transceiver_precheck_config.py
@@ -366,6 +366,7 @@ def test_precheck_commands_propagate_model_root(monkeypatch, model_root):
     # CI provides the model root inside its inbound pytest command, not in the
     # environment of the Python process that generates the launch script.
     monkeypatch.delenv("LLM_MODELS_ROOT", raising=False)
+    monkeypatch.delenv("TRTLLM_DISAGG_CT_PRECHECK", raising=False)
     lines = pcfg.precheck_prefix_lines(
         {},
         "e2e",
diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py
index 418198763206..21ed1d8074bc 100644
--- a/tests/unittest/scripts/test_perf_submit.py
+++ b/tests/unittest/scripts/test_perf_submit.py
@@ -294,3 +294,10 @@ def test_ci_missing_model_root_is_detectable(ci_submit_module):
     prefix_lines = ["export pytestCommand='LLM_ROOT=/repo pytest -q'"]
 
     assert ci_submit_module._get_pytest_command_env_var(prefix_lines, "LLM_MODELS_ROOT") is None
+
+
+def test_ci_rejects_invalid_pytest_command(ci_submit_module):
+    prefix_lines = ['export pytestCommand="LLM_MODELS_ROOT=/models']
+
+    with pytest.raises(ValueError, match="Invalid inbound pytestCommand"):
+        ci_submit_module._get_pytest_command_env_var(prefix_lines, "LLM_MODELS_ROOT")

From 46bef31cce5805f94975fcd6f10acc1f2a42b48f Mon Sep 17 00:00:00 2001
From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
Date: Tue, 4 Aug 2026 02:56:33 +0000
Subject: [PATCH 4/5] [NVBUG-6541356][test] assert precheck env ordering

Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
---
 .../others/test_cache_transceiver_precheck_config.py         | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/unittest/others/test_cache_transceiver_precheck_config.py b/tests/unittest/others/test_cache_transceiver_precheck_config.py
index 1a39b801e89a..0b62c502dd84 100644
--- a/tests/unittest/others/test_cache_transceiver_precheck_config.py
+++ b/tests/unittest/others/test_cache_transceiver_precheck_config.py
@@ -396,7 +396,10 @@ def test_precheck_commands_propagate_model_root(monkeypatch, model_root):
 
     assert len(commands) == 2
     for command in commands:
-        assert f"LLM_MODELS_ROOT={model_root}" in shlex.split(command)
+        tokens = shlex.split(command)
+        assignment = f"LLM_MODELS_ROOT={model_root}"
+        assert assignment in tokens
+        assert tokens.index(assignment) < tokens.index("python3")
 
 
 def _enabled_line(cfg):

From 883a7964501a1c65d26baf0a664826d765d8be32 Mon Sep 17 00:00:00 2001
From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
Date: Wed, 5 Aug 2026 06:43:44 +0000
Subject: [PATCH 5/5] unwaive b200_deepseek test

Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com>
---
 tests/integration/test_lists/waives.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt
index 55bd58d7ecf1..5887c5b7d66d 100644
--- a/tests/integration/test_lists/waives.txt
+++ b/tests/integration/test_lists/waives.txt
@@ -377,8 +377,6 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6550133)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6550133)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049)
-perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6561566)
-perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6561566)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6561566)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6561566)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6550133)