diff --git a/requirements-dev.txt b/requirements-dev.txt index 8c6766a590eb..57d64fb0436c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,7 @@ -r requirements.txt +# VisualGen LPIPS goldens were recorded with this version. Install it before +# pytest collection so already-imported Diffusers modules match the package files. +diffusers==0.38.0 boto3 einops lpips diff --git a/tests/integration/defs/examples/visual_gen/conftest.py b/tests/integration/defs/examples/visual_gen/conftest.py new file mode 100644 index 000000000000..9d65d09ef545 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/conftest.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared fixtures for VisualGen example integration tests.""" + +import pytest +from defs.trt_test_alternative import check_call + + +@pytest.fixture(scope="session") +def _visual_gen_deps(llm_venv): + """Install optional media dependencies once per VisualGen test session.""" + llm_venv.run_cmd(["-m", "pip", "install", "av"]) + check_call(["apt-get", "update", "-y"], shell=False) + check_call(["apt-get", "install", "-y", "ffmpeg"], shell=False) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip index b00403606a62..988c509f07b7 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d38286a6ee60db331a89bb3563fcc0896309b6e21228d4867ab33990c5b47ed -size 14386487 +oid sha256:78c7fde678df7b434b8f6da08d800d25bcfccb2509f73ab0c3d342eafe447100 +size 14648434 diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_fa4_fully_eager_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_fa4_fully_eager_lpips_golden_video.json new file mode 100644 index 000000000000..253c9f08e651 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_fa4_fully_eager_lpips_golden_video.json @@ -0,0 +1,25 @@ +{ + "video": "wan22_t2v_fa4_fully_eager_lpips_golden_video.mp4", + "model": "Wan2.2-T2V-A14B-Diffusers", + "source": "TensorRT-LLM VisualGen", + "prompt": "A cat sitting on a sunny windowsill watching birds outside.", + "negative_prompt": "", + "height": 480, + "width": 832, + "num_frames": 9, + "num_inference_steps": 4, + "guidance_scale": 4.0, + "seed": 42, + "frame_rate": 16.0, + "attention_backend": "FA4", + "torch_compile": false, + "torch_compile_stance": "force_eager", + "deterministic_algorithms": true, + "lpips_net": "alex", + "lpips_threshold": 0.25, + "diffusers_version": "0.38.0", + "torch_version": "2.12.0a0+0291f960b6.nv26.04.48445190", + "tensorrt_llm_version": "1.3.0rc21", + "tensorrt_llm_commit": "a0c406ff88c4a9736b5ce2f3c5eacbacdd0926d1", + "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:0783d275ffe7efc3b6093dd6c7e823cd703eae973e462c8e1e4f7ca7dad60d64" +} diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json index 35ecacdddc49..0621a4f06eec 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json @@ -11,7 +11,9 @@ "guidance_scale": 4.0, "seed": 42, "frame_rate": 16.0, + "attention_backend": "VANILLA", "torch_compile": false, + "torch_compile_stance": "force_eager", "deterministic_algorithms": true, "lpips_net": "alex", "lpips_threshold": 0.05, diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen.py b/tests/integration/defs/examples/visual_gen/test_visual_gen.py index a8ecf5e39782..baf83ac17ded 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen.py @@ -31,8 +31,6 @@ import pytest import torch import torch._inductor.config as inductor_config -from defs import conftest -from defs.common import venv_check_call from defs.trt_test_alternative import check_call from torch._inductor.async_compile import shutdown_compile_workers @@ -230,16 +228,6 @@ AESTHETIC_PREDICTOR_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "emb_reader") -@pytest.fixture(scope="session") -def _visual_gen_deps(llm_venv): - """Install av + diffusers + ffmpeg once per session (shared by all video-gen fixtures).""" - llm_venv.run_cmd(["-m", "pip", "install", "av"]) - llm_venv.run_cmd(["-m", "pip", "install", "diffusers>=0.37.0"]) - # Install ffmpeg system package required by save_video() for MP4 encoding - check_call(["apt-get", "update", "-y"], shell=False) - check_call(["apt-get", "install", "-y", "ffmpeg"], shell=False) - - @pytest.fixture(scope="session") def vbench_repo_root(llm_venv): """Clone VBench repo into workspace and install; return repo root path.""" @@ -345,8 +333,30 @@ def _precache_aesthetic_predictor(): ) from exc +def _llm_models_root(): + # Imported lazily so that re-importing this module in a torch.multiprocessing.spawn + # child (a fresh interpreter) does not run a module-level `from defs import conftest`, + # which pulls in `tensorrt_llm.bindings` -- a compiled extension absent from the source + # tree the spawned child resolves, crashing the worker before the test runs. The parent + # process already imports conftest during collection, so this deferral is free. + from defs import conftest + + return conftest.llm_models_root() + + +def _venv_check_call(*args, **kwargs): + # Deferred like _llm_models_root above: defs.common does `from tensorrt_llm import + # LLM`, which pulls in tensorrt_llm.bindings. Importing it at module load would + # crash the torch.multiprocessing.spawn child processes used by the multi-GPU LPIPS + # tests, which re-import this module before the worker fixes sys.path. Only the + # single-GPU example tests call this, and only in the parent process. + from defs.common import venv_check_call + + return venv_check_call(*args, **kwargs) + + def _lpips_model_path(*parts): - return os.path.join(conftest.llm_models_root(), *parts) + return os.path.join(_llm_models_root(), *parts) def _skip_if_missing(path, label, is_dir=False): @@ -382,7 +392,7 @@ def _golden_media_path(tmp_path, media_name, label): def _ltx2_lpips_text_encoder_path(): - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() candidates = [ os.path.join(scratch_space, LTX2_TEXT_ENCODER_SUBPATH), os.path.join(scratch_space, "gemma", LTX2_TEXT_ENCODER_SUBPATH), @@ -412,14 +422,28 @@ def _cleanup_cuda(): @contextlib.contextmanager -def _lpips_deterministic_algorithms(): - previous = torch.are_deterministic_algorithms_enabled() - os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") - torch.use_deterministic_algorithms(True) +def _lpips_deterministic_algorithms(*, fully_eager=False): + previous_deterministic = torch.are_deterministic_algorithms_enabled() + previous_warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + previous_cublas_workspace_config = os.environ.get("CUBLAS_WORKSPACE_CONFIG") + try: - yield + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + torch.use_deterministic_algorithms(True) + compiler_context = ( + torch.compiler.set_stance("force_eager") if fully_eager else contextlib.nullcontext() + ) + with compiler_context: + yield finally: - torch.use_deterministic_algorithms(previous) + torch.use_deterministic_algorithms( + previous_deterministic, + warn_only=previous_warn_only, + ) + if previous_cublas_workspace_config is None: + os.environ.pop("CUBLAS_WORKSPACE_CONFIG", None) + else: + os.environ["CUBLAS_WORKSPACE_CONFIG"] = previous_cublas_workspace_config def _save_lpips_video_mp4(video, output_path, frame_rate): @@ -624,7 +648,7 @@ def _generate_ltx2_cuda_graph_trtllm_backend_video(output_path): TorchCompileConfig, ) - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() checkpoint_path = os.path.join(scratch_space, LTX2_MODEL_CHECKPOINT_PATH) text_encoder_path = _ltx2_lpips_text_encoder_path() spatial_upsampler_path = os.path.join(scratch_space, LTX2_UPSAMPLER_SUBPATH) @@ -699,6 +723,7 @@ def _run_wan_lpips_pipeline( seed, attention_backend="VANILLA", parallel=None, + fully_eager=False, ): from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.visual_gen.args import AttentionConfig, TorchCompileConfig, VisualGenArgs @@ -712,7 +737,7 @@ def _run_wan_lpips_pipeline( ) if parallel is not None: args_kwargs["parallel_config"] = parallel - with _lpips_deterministic_algorithms(): + with _lpips_deterministic_algorithms(fully_eager=fully_eager): args = VisualGenArgs(**args_kwargs) pipeline = PipelineLoader(args).load(skip_warmup=True) try: @@ -747,7 +772,9 @@ def _generate_wan_lpips_video( guidance_scale, seed, frame_rate, + attention_backend="VANILLA", parallel=None, + fully_eager=False, ): generated_video = _run_wan_lpips_pipeline( model_path, @@ -759,7 +786,9 @@ def _generate_wan_lpips_video( num_inference_steps, guidance_scale, seed, + attention_backend=attention_backend, parallel=parallel, + fully_eager=fully_eager, ) assert generated_video is not None, "Single-GPU Wan LPIPS run produced no video" _save_lpips_video_mp4(generated_video, output_path, frame_rate=frame_rate) @@ -1128,7 +1157,7 @@ def _generate_wan_video(llm_venv, model_subpath, output_subdir): """ from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, model_subpath) if not os.path.isdir(model_path): pytest.skip( @@ -1201,7 +1230,7 @@ def _generate_ltx2_two_stage_video(llm_venv, output_subdir, linear_type="default """ from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, LTX2_MODEL_CHECKPOINT_PATH) text_encoder_path = os.path.join(scratch_space, LTX2_TEXT_ENCODER_SUBPATH) upsampler_path = os.path.join(scratch_space, LTX2_UPSAMPLER_SUBPATH) @@ -1361,7 +1390,7 @@ def _run_vbench_and_report( "custom_input", ] cmd.extend(["--dimension"] + VBENCH_DIMENSIONS) - venv_check_call(llm_venv, cmd) + _venv_check_call(llm_venv, cmd) pattern = os.path.join(output_path, "*_eval_results.json") result_files = glob.glob(pattern) @@ -1500,7 +1529,7 @@ def test_vbench_dimension_score_ltx2_two_stage_fp8( def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): """Run examples/visual_gen/quickstart_example.py end-to-end.""" - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_src = os.path.join(scratch_space, WAN_T2V_MODEL_SUBPATH) if not os.path.isdir(model_src): pytest.skip( @@ -1514,7 +1543,7 @@ def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): os.symlink(model_src, model_dst, target_is_directory=True) script_path = os.path.join(llm_root, "examples", "visual_gen", "quickstart_example.py") - venv_check_call(llm_venv, [script_path]) + _venv_check_call(llm_venv, [script_path]) output_path = os.path.join(llm_venv.get_working_directory(), "output.avi") assert os.path.isfile(output_path), f"Quickstart did not produce output.avi at {output_path}" @@ -1522,7 +1551,7 @@ def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): def test_visual_gen_api_walkthrough(_visual_gen_deps, llm_root, llm_venv): """Run examples/visual_gen/api_walkthrough.py end-to-end.""" - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_src = os.path.join(scratch_space, WAN_T2V_MODEL_SUBPATH) if not os.path.isdir(model_src): pytest.skip( @@ -1536,7 +1565,7 @@ def test_visual_gen_api_walkthrough(_visual_gen_deps, llm_root, llm_venv): os.symlink(model_src, model_dst, target_is_directory=True) script_path = os.path.join(llm_root, "examples", "visual_gen", "api_walkthrough.py") - venv_check_call(llm_venv, [script_path]) + _venv_check_call(llm_venv, [script_path]) output_path = os.path.join(llm_venv.get_working_directory(), "api_walkthrough_output.avi") assert os.path.isfile(output_path), f"API walkthrough did not produce {output_path}" @@ -1559,7 +1588,7 @@ def test_wan_t2v_example(_visual_gen_deps, llm_root, llm_venv): which runs the same script but with a no-quant YAML synthesized at runtime and additionally evaluates VBench scores. """ - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, WAN22_A14B_NVFP4_MODEL_SUBPATH) assert os.path.isdir(model_path), ( f"Model not found: {model_path} " @@ -1577,7 +1606,7 @@ def test_wan_t2v_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1613,7 +1642,7 @@ def test_flux1_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1649,7 +1678,7 @@ def test_flux2_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1688,7 +1717,7 @@ def test_ltx2_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1712,7 +1741,7 @@ def test_wan_i2v_example(_visual_gen_deps, llm_root, llm_venv): work together as documented. Uses the pre-quantized Wan 2.2 I2V A14B NVFP4 checkpoint and the default input image (cat_piano.png) bundled with the examples. """ - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, WAN22_I2V_A14B_NVFP4_MODEL_SUBPATH) if not os.path.isdir(model_path): pytest.skip( @@ -1731,7 +1760,7 @@ def test_wan_i2v_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1753,7 +1782,7 @@ def test_qwen_image_example(_visual_gen_deps, llm_root, llm_venv): ``configs/qwen-image-fp8-1gpu.yaml`` work together as documented. Uses the local Qwen-Image checkpoint and the shared FP8 blockwise dynamic-quant config. """ - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, QWEN_IMAGE_MODEL_SUBPATH) _skip_if_missing(model_path, "Qwen-Image checkpoint", is_dir=True) model_index_path = os.path.join(model_path, "model_index.json") @@ -1775,7 +1804,7 @@ def test_qwen_image_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1813,7 +1842,7 @@ def test_cosmos3_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py index a60765fe600c..30c21d673f35 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py @@ -14,7 +14,9 @@ # limitations under the License. """Multi-GPU integration tests for VisualGen LPIPS quality checks.""" +import glob import os +import sys from typing import Callable import pytest @@ -39,16 +41,21 @@ _save_lpips_video_mp4, ) -try: - from tensorrt_llm._utils import get_free_port + +def _parallel_config(**kwargs): + # Imported lazily so that mp.spawn child processes resolve tensorrt_llm only after + # _distributed_worker has prepended the installed-wheel location to sys.path. A + # module-level tensorrt_llm import would run during the child's module re-import + # (before sys.path is fixed) and resolve to the bindings-less source tree. from tensorrt_llm.visual_gen.args import ParallelConfig - MODULES_AVAILABLE = True -except ImportError: - MODULES_AVAILABLE = False + return ParallelConfig(**kwargs) + # Keep it as 0.25 as the worst case scenario at NVL72 scale WAN_MULTI_GPU_LPIPS_THRESHOLD = 0.25 +WAN22_MULTI_GPU_LPIPS_ATTENTION_BACKEND = "FA4" +WAN22_MULTI_GPU_LPIPS_GOLDEN_VIDEO = "wan22_t2v_fa4_fully_eager_lpips_golden_video.mp4" WAN22_LPIPS_MULTI_GPU_VARIANTS = [ ("ulysses4", {"ulysses_size": 4}), ("cfg2_ulysses2", {"cfg_size": 2, "ulysses_size": 2}), @@ -90,7 +97,37 @@ def cleanup_distributed(): dist.destroy_process_group() -def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs): +def _validated_tllm_site(site_dir): + """Return the realpath of ``site_dir`` after verifying it holds the installed wheel. + + The spawn workers rely on this directory to import tensorrt_llm with compiled + bindings; accepting an arbitrary path would let the import silently fall through + to the bindings-less source tree, so reject anything that does not contain the + package plus its compiled bindings extension. + """ + resolved = os.path.realpath(site_dir) if site_dir else "" + package_init = os.path.join(resolved, "tensorrt_llm", "__init__.py") + bindings = glob.glob(os.path.join(resolved, "tensorrt_llm", "bindings*.so")) + glob.glob( + os.path.join(resolved, "tensorrt_llm", "bindings", "*.so") + ) + if not (resolved and os.path.isfile(package_init) and bindings): + raise RuntimeError( + f"tllm_site={site_dir!r} does not contain an installed tensorrt_llm package " + "with compiled bindings; spawn workers would import the bindings-less " + "source tree instead of the wheel." + ) + return resolved + + +def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs, tllm_site): + # mp.spawn starts a fresh interpreter whose sys.path (set up by the integration + # `defs` harness) puts the source checkout ahead of the installed wheel, so a bare + # `import tensorrt_llm` would resolve to the bindings-less source tree and crash the + # worker. Prepend the parent's installed-package location so the child imports + # tensorrt_llm (with compiled bindings) from the wheel before any such import. + tllm_site = _validated_tllm_site(tllm_site) + sys.path[:] = [path for path in sys.path if os.path.realpath(path) != tllm_site] + sys.path.insert(0, tllm_site) try: init_distributed_worker(rank, world_size, backend, port) test_fn(rank, world_size, **kwargs) @@ -102,22 +139,32 @@ def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs): def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = True, **kwargs): - if not MODULES_AVAILABLE: + try: + import tensorrt_llm.bindings as tllm_bindings + from tensorrt_llm._utils import get_free_port + except ImportError: pytest.skip("Required modules not available") if use_cuda and torch.cuda.device_count() < world_size: pytest.skip(f"Test requires {world_size} GPUs, only {torch.cuda.device_count()} available") backend = "nccl" if use_cuda else "gloo" port = get_free_port() + # Directory containing the installed tensorrt_llm package (i.e. site-packages), + # passed to spawn workers so they prepend it to sys.path and import the wheel with + # compiled bindings instead of the source-tree package. Validated here as well so a + # bad environment fails before any worker is spawned. + tllm_site = _validated_tllm_site( + os.path.dirname(os.path.dirname(os.path.abspath(tllm_bindings.__file__))) + ) mp.spawn( _distributed_worker, - args=(world_size, backend, test_fn, port, kwargs), + args=(world_size, backend, test_fn, port, kwargs, tllm_site), nprocs=world_size, join=True, ) def _skip_if_insufficient_gpus_for_parallel(parallel): - parallel_cfg = ParallelConfig(**parallel) + parallel_cfg = _parallel_config(**parallel) required = parallel_cfg.n_workers available = torch.cuda.device_count() if available < required: @@ -128,7 +175,7 @@ def _skip_if_insufficient_gpus_for_parallel(parallel): def _wan22_lpips_distributed_worker(rank: int, world_size: int, **kwargs) -> None: parallel = kwargs["parallel"] - ParallelConfig(**parallel).validate_world_size(world_size) + _parallel_config(**parallel).validate_world_size(world_size) generated_video = _run_wan_lpips_pipeline( kwargs["model_path"], @@ -140,8 +187,9 @@ def _wan22_lpips_distributed_worker(rank: int, world_size: int, **kwargs) -> Non kwargs["num_inference_steps"], kwargs["guidance_scale"], kwargs["seed"], - attention_backend="FA4", + attention_backend=WAN22_MULTI_GPU_LPIPS_ATTENTION_BACKEND, parallel=parallel, + fully_eager=True, ) if rank == 0: @@ -160,10 +208,12 @@ def _wan22_lpips_distributed_worker(rank: int, world_size: int, **kwargs) -> Non def _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel): _skip_if_insufficient_gpus_for_parallel(parallel) - parallel_cfg = ParallelConfig(**parallel) + parallel_cfg = _parallel_config(**parallel) generated_path = tmp_path / f"wan22_t2v_generated_{variant_name}.mp4" golden_path = _golden_media_path( - tmp_path, "wan22_t2v_lpips_golden_video.mp4", "Wan 2.2 LPIPS golden video" + tmp_path, + WAN22_MULTI_GPU_LPIPS_GOLDEN_VIDEO, + "Wan 2.2 FA4 fully-eager LPIPS golden video", ) run_test_in_distributed( @@ -200,7 +250,9 @@ def _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel): WAN22_LPIPS_MULTI_GPU_VARIANTS, ids=[name for name, _ in WAN22_LPIPS_MULTI_GPU_VARIANTS], ) -def test_wan22_t2v_lpips_against_golden_multi_gpu(tmp_path, variant_name, parallel): +def test_wan22_t2v_lpips_against_golden_multi_gpu( + _visual_gen_deps, tmp_path, variant_name, parallel +): _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel) @@ -209,5 +261,5 @@ def test_wan22_t2v_lpips_against_golden_multi_gpu(tmp_path, variant_name, parall WAN22_LPIPS_TP_VARIANTS, ids=[name for name, _ in WAN22_LPIPS_TP_VARIANTS], ) -def test_wan22_t2v_lpips_against_golden_tp(tmp_path, variant_name, parallel): +def test_wan22_t2v_lpips_against_golden_tp(_visual_gen_deps, tmp_path, variant_name, parallel): _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index cef79620dc6f..7f919655bdb8 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -198,13 +198,6 @@ examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin- examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2i_lpips_against_golden SKIP (https://nvbugs/6418815) examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6437341) examples/visual_gen/test_visual_gen.py::test_ltx2_cuda_graph_trtllm_backend SKIP (https://nvbugs/6463822) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2_attn2d_2x1] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[ulysses4] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[cfg2_tp2] SKIP (https://nvbugs/6329227) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2] SKIP (https://nvbugs/6329227) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2_ulysses2] SKIP (https://nvbugs/6329227) full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) full:A100/disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6329052) full:A100X/llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp SKIP (https://nvbugs/6287561)