diff --git a/Makefile b/Makefile index bb78d760..3943ce16 100644 --- a/Makefile +++ b/Makefile @@ -45,20 +45,36 @@ security-scans: # Run security scans (informational) uv run bandit -r src/ -ll -x "**/tests/**" || echo "Security scan completed with issues (informational)" # Test commands -test: # Run all tests +test: # Run all tests in parallel (auto-detect cores) + uv run pytest tests/ -v -n auto + +test-serial: # Run all tests serially (for debugging) uv run pytest tests/ -v -test-unit: # Run unit tests only +test-unit: # Run unit tests in parallel (auto-detect cores) + uv run pytest tests/unit/ -v -n auto -m "not integration" + +test-unit-serial: # Run unit tests serially (for debugging) uv run pytest tests/unit/ -v -m "not integration" -test-integration: # Run integration tests only +test-integration: # Run integration tests in parallel (auto-detect cores) + uv run pytest tests/integration/ -v -n auto -m integration + +test-integration-serial: # Run integration tests serially (for debugging) uv run pytest tests/integration/ -v -m integration -test-coverage: # Run tests with coverage report +test-coverage: # Run tests with coverage report (parallel by default) + uv run pytest tests/ -v -n auto -m "not serial" --cov=tetra_rp --cov-report=xml + uv run pytest tests/ -v -m "serial" --cov=tetra_rp --cov-append --cov-report=term-missing + +test-coverage-serial: # Run tests with coverage report (serial execution) uv run pytest tests/ -v --cov=tetra_rp --cov-report=term-missing -test-fast: # Run tests with fast-fail mode - uv run pytest tests/ -v -x --tb=short +test-fast: # Run tests with fast-fail mode and parallel execution + uv run pytest tests/ -v -x --tb=short -n auto + +test-workers: # Run tests with specific number of workers (e.g., WORKERS=4) + uv run pytest tests/ -v -n $(WORKERS) # Linting commands lint: # Check code with ruff @@ -78,17 +94,30 @@ typecheck: # Check types with mypy uv run mypy . # Quality gates (used in CI) -quality-check: format-check lint test-coverage # Essential quality gate for CI -quality-check-strict: format-check lint typecheck test-coverage # Strict quality gate with type checking +quality-check: format-check lint test-coverage # Essential quality gate for CI (parallel by default) +quality-check-strict: format-check lint typecheck test-coverage # Strict quality gate with type checking (parallel by default) +quality-check-serial: format-check lint test-coverage-serial # Serial quality gate for debugging # GitHub Actions specific targets -ci-quality-github: # Quality checks with GitHub Actions formatting +ci-quality-github: # Quality checks with GitHub Actions formatting (parallel by default) @echo "::group::Code formatting check" - uv run ruff format --check . + uv run ruff format --check . @echo "::endgroup::" @echo "::group::Linting" uv run ruff check . --output-format=github @echo "::endgroup::" @echo "::group::Test suite with coverage" + uv run pytest tests/ --junitxml=pytest-results.xml -v -n auto -m "not serial" --cov=tetra_rp --cov-report=xml --cov-fail-under=0 + uv run pytest tests/ --junitxml=pytest-results.xml -v -m "serial" --cov=tetra_rp --cov-append --cov-report=term-missing + @echo "::endgroup::" + +ci-quality-github-serial: # Serial quality checks for GitHub Actions (for debugging) + @echo "::group::Code formatting check" + uv run ruff format --check . + @echo "::endgroup::" + @echo "::group::Linting" + uv run ruff check . --output-format=github + @echo "::endgroup::" + @echo "::group::Test suite with coverage (serial)" uv run pytest tests/ --junitxml=pytest-results.xml -v --cov=tetra_rp --cov-report=term-missing @echo "::endgroup::" diff --git a/pyproject.toml b/pyproject.toml index 33bee984..ad2eb383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ test = [ "pytest-mock>=3.14.0", "pytest-asyncio>=1.0.0", "pytest-cov>=6.2.1", + "pytest-xdist>=3.6.1", "twine>=6.1.0", ] @@ -75,7 +76,8 @@ asyncio_default_fixture_loop_scope = "function" markers = [ "unit: Unit tests", "integration: Integration tests", - "slow: Slow tests" + "slow: Slow tests", + "serial: Tests that must run serially (not parallelized)" ] filterwarnings = [ "ignore::DeprecationWarning", diff --git a/tests/conftest.py b/tests/conftest.py index 65a7800a..478855aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,29 @@ from tetra_rp.core.utils.singleton import SingletonMixin +def pytest_configure(config): + """Configure pytest-xdist to respect the serial marker. + + Tests marked with @pytest.mark.serial will always run on the main worker, + while unmarked tests can be distributed to other workers. + """ + # This hook is called early in pytest initialization + # xdist will check for this during test distribution + + +def pytest_collection_modifyitems(config, items): + """Mark serial tests so they don't get distributed by xdist. + + This ensures that tests marked with @pytest.mark.serial run on the main + worker (worker -1 or 0) and are never distributed to other workers. + """ + for item in items: + # Check if item has the serial marker + if item.get_closest_marker("serial"): + # Add xdist marker to prevent distribution + item.add_marker(pytest.mark.xdist_group(name="serial")) + + @pytest.fixture def mock_asyncio_run_coro(): """Create a mock asyncio.run that executes coroutines.""" @@ -186,6 +209,68 @@ def sample_pod_template() -> Dict[str, Any]: } +@pytest.fixture(scope="session") +def worker_temp_dir(tmp_path_factory: pytest.TempPathFactory, worker_id: str) -> Path: + """Provide worker-specific temporary directory for file system isolation. + + Each xdist worker gets its own isolated temp directory to prevent + file system conflicts when tests write to shared paths. + + Args: + tmp_path_factory: Pytest's temporary path factory. + worker_id: Worker ID from pytest-xdist ('master' for single worker). + + Returns: + Path to worker-specific temporary directory. + """ + if worker_id == "master": + # Single worker (non-parallel) + return tmp_path_factory.mktemp("test_data") + else: + # Parallel execution - worker-specific directory + return tmp_path_factory.mktemp(f"test_data_{worker_id}") + + +@pytest.fixture(scope="session") +def worker_runpod_dir(worker_temp_dir: Path) -> Path: + """Provide worker-specific .runpod directory for state file isolation. + + Args: + worker_temp_dir: Worker-specific temporary directory. + + Returns: + Path to worker-specific .runpod directory. + """ + runpod_dir = worker_temp_dir / ".runpod" + runpod_dir.mkdir(parents=True, exist_ok=True) + return runpod_dir + + +@pytest.fixture(autouse=True) +def isolate_resource_state_file( + monkeypatch: pytest.MonkeyPatch, worker_runpod_dir: Path +) -> Path: + """Automatically isolate RESOURCE_STATE_FILE per worker. + + Patches RESOURCE_STATE_FILE and RUNPOD_FLASH_DIR to point to + worker-specific temp directory, preventing file system conflicts. + + Args: + monkeypatch: Pytest's monkeypatch fixture. + worker_runpod_dir: Worker-specific .runpod directory. + + Returns: + Path to worker-specific state file. + """ + from tetra_rp.core.resources import resource_manager + + worker_state_file = worker_runpod_dir / "resources.pkl" + monkeypatch.setattr(resource_manager, "RESOURCE_STATE_FILE", worker_state_file) + monkeypatch.setattr(resource_manager, "RUNPOD_FLASH_DIR", worker_runpod_dir) + + return worker_state_file + + @pytest.fixture(autouse=True) def reset_singletons(): """Reset singleton instances between tests. @@ -220,6 +305,17 @@ def patched_reducer_override(self, obj): # If patching fails, continue anyway - the test might still pass pass + # Clear module-level caches (worker-isolated due to process boundaries) + try: + from tetra_rp.stubs.live_serverless import _SERIALIZED_FUNCTION_CACHE + from tetra_rp.execute_class import _SERIALIZED_CLASS_CACHE + + _SERIALIZED_FUNCTION_CACHE.clear() + _SERIALIZED_CLASS_CACHE.clear() + except (ImportError, AttributeError): + # Caches may not exist in all configurations + pass + # Reset SingletonMixin instances to clear any accumulated state # This prevents old singleton instances from leaking into object graphs during pickling SingletonMixin._instances = {} @@ -242,6 +338,15 @@ def patched_reducer_override(self, obj): yield # Cleanup after test + try: + from tetra_rp.stubs.live_serverless import _SERIALIZED_FUNCTION_CACHE + from tetra_rp.execute_class import _SERIALIZED_CLASS_CACHE + + _SERIALIZED_FUNCTION_CACHE.clear() + _SERIALIZED_CLASS_CACHE.clear() + except (ImportError, AttributeError): + pass + SingletonMixin._instances = {} ResourceManager._resources = {} diff --git a/tests/integration/test_remote_concurrency.py b/tests/integration/test_remote_concurrency.py index b185a64f..dc39f188 100644 --- a/tests/integration/test_remote_concurrency.py +++ b/tests/integration/test_remote_concurrency.py @@ -28,6 +28,7 @@ from tetra_rp.protos.remote_execution import FunctionResponse +@pytest.mark.serial @pytest.mark.asyncio class TestRemoteConcurrency: """Test concurrency behavior of @remote decorated functions.""" diff --git a/tests/unit/test_concurrency_issues.py b/tests/unit/test_concurrency_issues.py index 966ea91b..449e0e76 100644 --- a/tests/unit/test_concurrency_issues.py +++ b/tests/unit/test_concurrency_issues.py @@ -85,6 +85,7 @@ async def undeploy(self) -> Dict[str, Any]: return result +@pytest.mark.serial class TestSingleton: """Test thread safety of SingletonMixin.""" @@ -139,6 +140,7 @@ def create_instance(): assert exception_count == 0 # No exceptions should occur +@pytest.mark.serial class TestResourceManagerConcurrency: """Test ResourceManager concurrency issues.""" @@ -291,6 +293,7 @@ def save_resource_2(): print(f"State loading error: {e}") +@pytest.mark.serial class TestFunctionCacheConcurrency: """Test global function cache thread safety.""" @@ -360,6 +363,7 @@ def cache_worker(worker_id: int): assert len(_SERIALIZED_FUNCTION_CACHE) > 0 +@pytest.mark.serial class TestClassCacheConcurrency: """Test class serialization cache thread safety.""" @@ -423,6 +427,7 @@ def cache_worker(worker_id: int): assert len(cache_operations) > 0 +@pytest.mark.serial class TestEndToEndConcurrency: """End-to-end tests for concurrent remote function execution.""" diff --git a/tests/unit/test_file_locking.py b/tests/unit/test_file_locking.py index c000d357..fde24d8c 100644 --- a/tests/unit/test_file_locking.py +++ b/tests/unit/test_file_locking.py @@ -78,6 +78,7 @@ def test_platform_detection_linux(self, mock_system): assert info["platform"] == "Linux" +@pytest.mark.serial class TestFileLocking: """Test cross-platform file locking functionality.""" @@ -238,6 +239,7 @@ def test_file_lock_with_write_operations(self): assert write_file.read_bytes() == b"updated data" +@pytest.mark.serial class TestPlatformSpecificLocking: """Test platform-specific locking mechanisms.""" diff --git a/tests/unit/test_load_balancer_sls_stub.py b/tests/unit/test_load_balancer_sls_stub.py index c5adcbf6..172126ab 100644 --- a/tests/unit/test_load_balancer_sls_stub.py +++ b/tests/unit/test_load_balancer_sls_stub.py @@ -264,6 +264,7 @@ def use_requests(): assert request["dependencies"] == deps +@pytest.mark.serial class TestLoadBalancerSlsStubRouting: """Test suite for routing detection between /execute and user routes.""" diff --git a/uv.lock b/uv.lock index e7d1076a..658da713 100644 --- a/uv.lock +++ b/uv.lock @@ -935,6 +935,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674 }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708 }, +] + [[package]] name = "fastapi" version = "0.121.2" @@ -2358,6 +2367,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095 }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396 }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2827,6 +2849,7 @@ test = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-mock" }, + { name = "pytest-xdist" }, { name = "twine" }, ] @@ -2853,6 +2876,7 @@ test = [ { name = "pytest-asyncio", specifier = ">=1.0.0" }, { name = "pytest-cov", specifier = ">=6.2.1" }, { name = "pytest-mock", specifier = ">=3.14.0" }, + { name = "pytest-xdist", specifier = ">=3.6.1" }, { name = "twine", specifier = ">=6.1.0" }, ]