diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0b9abc7d..fe25613b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -70,7 +70,7 @@ jobs: include: - name: base extras: "" - mark_filter: "cpu and not slow and not style and not requires_intel and not requires_lmharness and not requires_rapidata" + mark_filter: "no_extras" - name: lmharness extras: "--extra lmharness" mark_filter: "requires_lmharness" @@ -83,6 +83,7 @@ jobs: HF_HOME: ${{ github.workspace }}/.cache/huggingface HF_DATASETS_CACHE: ${{ github.workspace }}/.cache/huggingface/datasets HUGGINGFACE_HUB_CACHE: ${{ github.workspace }}/.cache/huggingface/hub + DEFAULT_MARK_FILTER: "cpu and not slow and not style" steps: - name: Checkout code @@ -138,4 +139,4 @@ jobs: - name: Run tests with pytest-rerunfailures run: | echo "Running tests with up to 3 reruns on failure using $PYTEST_WORKERS workers..." - uv run pytest -n $PYTEST_WORKERS -m "${{ matrix.mark_filter }}" --reruns 3 --reruns-delay 10 --maxfail=1 + uv run pytest -n $PYTEST_WORKERS -m "${{ env.DEFAULT_MARK_FILTER }} and ${{ matrix.mark_filter }}" --reruns 3 --reruns-delay 10 --maxfail=1 diff --git a/tests/conftest.py b/tests/conftest.py index 6693e17c..c7bea91e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,40 +5,53 @@ # import all fixtures to make them avaliable for pytest from .fixtures import * # noqa: F403, F401 -HARDWARE_MARKS = {"cpu", "cuda", "multi_gpu"} +DEVICE_MARKS = { + "cpu": "mark test to run on CPU", + "cuda": "mark test to run only on GPU machines", + "multi_gpu": "mark test to run only on multi-GPU machines", +} +EXTRA_MARKS = { + "requires_gptq": "mark test that needs pruna[gptq]", + "requires_awq": "mark test that needs pruna[awq]", + "requires_stable_fast": "mark test that needs pruna[stable-fast]", + "requires_vllm": "mark test that needs pruna[vllm]", + "requires_intel": "mark test that needs pruna[intel]", + "requires_lmharness": "mark test that needs pruna[lmharness]", + "requires_whisper": "mark test that needs pruna[whisper]", + "requires_upscale": "mark test that needs pruna[upscale]", + "requires_rapidata": "mark test that needs pruna[rapidata]", +} def pytest_configure(config: Any) -> None: """Configure the pytest markers.""" - # Hardware marks - config.addinivalue_line("markers", "cpu: mark test to run on CPU") - config.addinivalue_line("markers", "cuda: mark test to run only on GPU machines") - config.addinivalue_line("markers", "multi_gpu: mark test to run only on multi-GPU machines") - config.addinivalue_line("markers", "high_gpu: mark test to run only on large GPUs") # e.g. H100 + # Device marks + for mark, description in DEVICE_MARKS.items(): + config.addinivalue_line("markers", f"{mark}: {description}") + config.addinivalue_line("markers", "high_gpu: mark test to run only on large GPUs") # Dependency marks for external dependencies - config.addinivalue_line("markers", "requires_gptq: mark test that needs pruna[gptq]") - config.addinivalue_line("markers", "requires_awq: mark test that needs pruna[awq]") - config.addinivalue_line("markers", "requires_stable_fast: mark test that needs pruna[stable-fast]") - config.addinivalue_line("markers", "requires_vllm: mark test that needs pruna[vllm]") - config.addinivalue_line("markers", "requires_intel: mark test that needs pruna[intel]") - config.addinivalue_line("markers", "requires_lmharness: mark test that needs pruna[lmharness]") - config.addinivalue_line("markers", "requires_whisper: mark test that needs pruna[whisper]") - config.addinivalue_line("markers", "requires_upscale: mark test that needs pruna[upscale]") - config.addinivalue_line("markers", "requires_rapidata: mark test that needs pruna[rapidata]") + for mark, description in EXTRA_MARKS.items(): + config.addinivalue_line("markers", f"{mark}: {description}") + config.addinivalue_line("markers", "no_extras: mark test that runs without optional dependency extras") # Category marks config.addinivalue_line("markers", "slow: mark test that run rather long") config.addinivalue_line("markers", "style: mark test that only check style") config.addinivalue_line("markers", "integration: mark test that is an integration test") +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(session: Any, config: Any, items: list) -> None: """Hook that is called after test collection.""" selected = [] deselected = [] for item in items: # Auto-tag unmarked tests as CPU - if not any(mark in item.keywords for mark in HARDWARE_MARKS): + if not any(mark in item.keywords for mark in DEVICE_MARKS): item.add_marker(pytest.mark.cpu) + # Auto-tag tests that do not require optional dependency extras. This + # keeps the default CI selection positive as new extras are added. + if not any(mark.name in EXTRA_MARKS for mark in item.iter_markers()): + item.add_marker(pytest.mark.no_extras) # device_parametrized generates cpu/cuda/accelerate variants for every # algorithm test, even when the algorithm's runs_on excludes that device. # The incompatible variants get collected by the CI (e.g. -m "cpu") and