perf(tests): make parallel test execution the default

Makefile

@@ -45,20 +45,36 @@ security-scans: # Run security scans (informational)
 	uv run bandit -r src/ -ll -x "**/tests/**" || echo "Security scan completed with issues (informational)"
 
 # Test commands
-test: # Run all tests
+test: # Run all tests in parallel (auto-detect cores)
+	uv run pytest tests/ -v -n auto
+
+test-serial: # Run all tests serially (for debugging)
 	uv run pytest tests/ -v
 
-test-unit: # Run unit tests only
+test-unit: # Run unit tests in parallel (auto-detect cores)
+	uv run pytest tests/unit/ -v -n auto -m "not integration"
+
+test-unit-serial: # Run unit tests serially (for debugging)
 	uv run pytest tests/unit/ -v -m "not integration"
 
-test-integration: # Run integration tests only
+test-integration: # Run integration tests in parallel (auto-detect cores)
+	uv run pytest tests/integration/ -v -n auto -m integration
+
+test-integration-serial: # Run integration tests serially (for debugging)
 	uv run pytest tests/integration/ -v -m integration
 
-test-coverage: # Run tests with coverage report
+test-coverage: # Run tests with coverage report (parallel by default)
+	uv run pytest tests/ -v -n auto -m "not serial" --cov=tetra_rp --cov-report=xml
+	uv run pytest tests/ -v -m "serial" --cov=tetra_rp --cov-append --cov-report=term-missing
+
+test-coverage-serial: # Run tests with coverage report (serial execution)
 	uv run pytest tests/ -v --cov=tetra_rp --cov-report=term-missing
 
-test-fast: # Run tests with fast-fail mode
-	uv run pytest tests/ -v -x --tb=short
+test-fast: # Run tests with fast-fail mode and parallel execution
+	uv run pytest tests/ -v -x --tb=short -n auto
+
+test-workers: # Run tests with specific number of workers (e.g., WORKERS=4)
+	uv run pytest tests/ -v -n $(WORKERS)
 
 # Linting commands
 lint: # Check code with ruff
@@ -78,17 +94,30 @@ typecheck: # Check types with mypy
 	uv run mypy .
 
 # Quality gates (used in CI)
-quality-check: format-check lint test-coverage # Essential quality gate for CI
-quality-check-strict: format-check lint typecheck test-coverage # Strict quality gate with type checking
+quality-check: format-check lint test-coverage # Essential quality gate for CI (parallel by default)
+quality-check-strict: format-check lint typecheck test-coverage # Strict quality gate with type checking (parallel by default)
+quality-check-serial: format-check lint test-coverage-serial # Serial quality gate for debugging
 
 # GitHub Actions specific targets
-ci-quality-github: # Quality checks with GitHub Actions formatting
+ci-quality-github: # Quality checks with GitHub Actions formatting (parallel by default)
 	@echo "::group::Code formatting check"
-	uv run ruff format --check . 
+	uv run ruff format --check .
 	@echo "::endgroup::"
 	@echo "::group::Linting"
 	uv run ruff check . --output-format=github
 	@echo "::endgroup::"
 	@echo "::group::Test suite with coverage"
+	uv run pytest tests/ --junitxml=pytest-results.xml -v -n auto -m "not serial" --cov=tetra_rp --cov-report=xml --cov-fail-under=0
+	uv run pytest tests/ --junitxml=pytest-results.xml -v -m "serial" --cov=tetra_rp --cov-append --cov-report=term-missing
+	@echo "::endgroup::"
+
+ci-quality-github-serial: # Serial quality checks for GitHub Actions (for debugging)
+	@echo "::group::Code formatting check"
+	uv run ruff format --check .
+	@echo "::endgroup::"
+	@echo "::group::Linting"
+	uv run ruff check . --output-format=github
+	@echo "::endgroup::"
+	@echo "::group::Test suite with coverage (serial)"
 	uv run pytest tests/ --junitxml=pytest-results.xml -v --cov=tetra_rp --cov-report=term-missing
 	@echo "::endgroup::"

pyproject.toml

@@ -39,6 +39,7 @@ test = [
     "pytest-mock>=3.14.0",
     "pytest-asyncio>=1.0.0",
     "pytest-cov>=6.2.1",
+    "pytest-xdist>=3.6.1",
     "twine>=6.1.0",
 ]
 
@@ -75,7 +76,8 @@ asyncio_default_fixture_loop_scope = "function"
 markers = [
     "unit: Unit tests",
     "integration: Integration tests",
-    "slow: Slow tests"
+    "slow: Slow tests",
+    "serial: Tests that must run serially (not parallelized)"
 ]
 filterwarnings = [
     "ignore::DeprecationWarning",

tests/conftest.py

@@ -21,6 +21,29 @@
 from tetra_rp.core.utils.singleton import SingletonMixin
 
 
+def pytest_configure(config):
+    """Configure pytest-xdist to respect the serial marker.
+
+    Tests marked with @pytest.mark.serial will always run on the main worker,
+    while unmarked tests can be distributed to other workers.
+    """
+    # This hook is called early in pytest initialization
+    # xdist will check for this during test distribution
+
+
+def pytest_collection_modifyitems(config, items):
+    """Mark serial tests so they don't get distributed by xdist.
+
+    This ensures that tests marked with @pytest.mark.serial run on the main
+    worker (worker -1 or 0) and are never distributed to other workers.
+    """
+    for item in items:
+        # Check if item has the serial marker
+        if item.get_closest_marker("serial"):
+            # Add xdist marker to prevent distribution
+            item.add_marker(pytest.mark.xdist_group(name="serial"))
+
+
 @pytest.fixture
 def mock_asyncio_run_coro():
     """Create a mock asyncio.run that executes coroutines."""
@@ -186,6 +209,68 @@ def sample_pod_template() -> Dict[str, Any]:
     }
 
 
+@pytest.fixture(scope="session")
+def worker_temp_dir(tmp_path_factory: pytest.TempPathFactory, worker_id: str) -> Path:
+    """Provide worker-specific temporary directory for file system isolation.
+
+    Each xdist worker gets its own isolated temp directory to prevent
+    file system conflicts when tests write to shared paths.
+
+    Args:
+        tmp_path_factory: Pytest's temporary path factory.
+        worker_id: Worker ID from pytest-xdist ('master' for single worker).
+
+    Returns:
+        Path to worker-specific temporary directory.
+    """
+    if worker_id == "master":
+        # Single worker (non-parallel)
+        return tmp_path_factory.mktemp("test_data")
+    else:
+        # Parallel execution - worker-specific directory
+        return tmp_path_factory.mktemp(f"test_data_{worker_id}")
+
+
+@pytest.fixture(scope="session")
+def worker_runpod_dir(worker_temp_dir: Path) -> Path:
+    """Provide worker-specific .runpod directory for state file isolation.
+
+    Args:
+        worker_temp_dir: Worker-specific temporary directory.
+
+    Returns:
+        Path to worker-specific .runpod directory.
+    """
+    runpod_dir = worker_temp_dir / ".runpod"
+    runpod_dir.mkdir(parents=True, exist_ok=True)
+    return runpod_dir
+
+
+@pytest.fixture(autouse=True)
+def isolate_resource_state_file(
+    monkeypatch: pytest.MonkeyPatch, worker_runpod_dir: Path
+) -> Path:
+    """Automatically isolate RESOURCE_STATE_FILE per worker.
+
+    Patches RESOURCE_STATE_FILE and RUNPOD_FLASH_DIR to point to
+    worker-specific temp directory, preventing file system conflicts.
+
+    Args:
+        monkeypatch: Pytest's monkeypatch fixture.
+        worker_runpod_dir: Worker-specific .runpod directory.
+
+    Returns:
+        Path to worker-specific state file.
+    """
+    from tetra_rp.core.resources import resource_manager
+
+    worker_state_file = worker_runpod_dir / "resources.pkl"
+    monkeypatch.setattr(resource_manager, "RESOURCE_STATE_FILE", worker_state_file)
+    monkeypatch.setattr(resource_manager, "RUNPOD_FLASH_DIR", worker_runpod_dir)
+
+    return worker_state_file
+
+
 @pytest.fixture(autouse=True)
 def reset_singletons():
     """Reset singleton instances between tests.
@@ -220,6 +305,17 @@ def patched_reducer_override(self, obj):
         # If patching fails, continue anyway - the test might still pass
         pass
 
+    # Clear module-level caches (worker-isolated due to process boundaries)
+    try:
+        from tetra_rp.stubs.live_serverless import _SERIALIZED_FUNCTION_CACHE
+        from tetra_rp.execute_class import _SERIALIZED_CLASS_CACHE
+
+        _SERIALIZED_FUNCTION_CACHE.clear()
+        _SERIALIZED_CLASS_CACHE.clear()
+    except (ImportError, AttributeError):
+        # Caches may not exist in all configurations
+        pass
+
     # Reset SingletonMixin instances to clear any accumulated state
     # This prevents old singleton instances from leaking into object graphs during pickling
     SingletonMixin._instances = {}
@@ -242,6 +338,15 @@ def patched_reducer_override(self, obj):
     yield
 
     # Cleanup after test
+    try:
+        from tetra_rp.stubs.live_serverless import _SERIALIZED_FUNCTION_CACHE
+        from tetra_rp.execute_class import _SERIALIZED_CLASS_CACHE
+
+        _SERIALIZED_FUNCTION_CACHE.clear()
+        _SERIALIZED_CLASS_CACHE.clear()
+    except (ImportError, AttributeError):
+        pass
+
     SingletonMixin._instances = {}
 
     ResourceManager._resources = {}

tests/integration/test_remote_concurrency.py

@@ -28,6 +28,7 @@
 from tetra_rp.protos.remote_execution import FunctionResponse
 
 
+@pytest.mark.serial
 @pytest.mark.asyncio
 class TestRemoteConcurrency:
     """Test concurrency behavior of @remote decorated functions."""

tests/unit/test_concurrency_issues.py

@@ -85,6 +85,7 @@ async def undeploy(self) -> Dict[str, Any]:
         return result
 
 
+@pytest.mark.serial
 class TestSingleton:
     """Test thread safety of SingletonMixin."""
 
@@ -139,6 +140,7 @@ def create_instance():
         assert exception_count == 0  # No exceptions should occur
 
 
+@pytest.mark.serial
 class TestResourceManagerConcurrency:
     """Test ResourceManager concurrency issues."""
 
@@ -291,6 +293,7 @@ def save_resource_2():
             print(f"State loading error: {e}")
 
 
+@pytest.mark.serial
 class TestFunctionCacheConcurrency:
     """Test global function cache thread safety."""
 
@@ -360,6 +363,7 @@ def cache_worker(worker_id: int):
         assert len(_SERIALIZED_FUNCTION_CACHE) > 0
 
 
+@pytest.mark.serial
 class TestClassCacheConcurrency:
     """Test class serialization cache thread safety."""
 
@@ -423,6 +427,7 @@ def cache_worker(worker_id: int):
         assert len(cache_operations) > 0
 
 
+@pytest.mark.serial
 class TestEndToEndConcurrency:
     """End-to-end tests for concurrent remote function execution."""
 

tests/unit/test_file_locking.py

@@ -78,6 +78,7 @@ def test_platform_detection_linux(self, mock_system):
         assert info["platform"] == "Linux"
 
 
+@pytest.mark.serial
 class TestFileLocking:
     """Test cross-platform file locking functionality."""
 
@@ -238,6 +239,7 @@ def test_file_lock_with_write_operations(self):
         assert write_file.read_bytes() == b"updated data"
 
 
+@pytest.mark.serial
 class TestPlatformSpecificLocking:
     """Test platform-specific locking mechanisms."""
 

tests/unit/test_load_balancer_sls_stub.py

@@ -264,6 +264,7 @@ def use_requests():
             assert request["dependencies"] == deps
 
 
+@pytest.mark.serial
 class TestLoadBalancerSlsStubRouting:
     """Test suite for routing detection between /execute and user routes."""