From 0c2c5c01bee7678bc49a3aeb0d301b7ee9bdfa13 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 21 Jul 2026 10:53:13 +0200 Subject: [PATCH] test(e2e): run mockable serve chat scenarios on the no-GPU lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chat scenarios "tool definitions accepted" and "end-to-end local model" only assert that a served model answers over HTTP (a tools-bearing request is accepted; a non-empty reply comes back) — neither checks real generation. They were @requires-gpu and so only ran on the scarce serial GPU runners, yet nothing about them needs a real GPU. Make the shared "a model is served in the background" precondition capability-aware: a real `rocm serve --managed` on a host with an AMD GPU (preserving real-serve coverage where hardware exists), and the in-process MockServer + a planted managed-service record on a no-GPU host. Drop @requires-gpu from both scenarios so they also run on the GitHub-hosted mock lane every PR, with no coverage loss — real inference stays covered by the @requires-gpu serve-*-inference scenarios. "a managed runtime is active" becomes a no-op on a no-GPU host: a managed TheRock SDK can only be installed where a GPU family selects wheels, and the mock serve needs no runtime. This only affects the two migrated scenarios; all other users stay @requires-gpu. Scope the EAI-7423 lemonade-serve xfails for these two ids to therock_family=gfx*, so they apply only on a real GPU lemonade host and the scenarios expect-pass on the mock lane (where MockServer, not a real lemonade serve, backs them) instead of XPASS-ing. serve-absent-gpu-index-rejected was evaluated for the same move but stays @requires-gpu: on a no-GPU host the GPU-required pre-flight refuses ("no usable AMD GPU") before the --gpu index is validated, so its index-specific rejection can only be observed with a real device present. Part of EAI-7484 (mock/real split for GPU serve scenarios). Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 12 +++++++---- tests/e2e-cucumber/features/chat.feature | 12 +++++++++-- .../features/model_serving.feature | 4 +++- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 9 +++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 20 ++++++++++++++++++- 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 26dc5e3c..e79bed16 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -126,9 +126,11 @@ serve_timeout_secs = 90 # On a lemonade LINUX host this needs a live lemonade serve, which reaches ready # then shuts down immediately (EAI-7423). os=linux: passes on native Windows # lemonade (run 29354305288 XPASS'd it), consistent with the other lemonade -# serve scenarios. +# serve scenarios. therock_family=gfx* scopes this to a REAL GPU host: the +# scenario now also runs on the no-GPU mock lane (backed by MockServer, not a +# real lemonade serve), where EAI-7423 does not apply and it must expect-pass. [["chat-tool-definitions-accepted"]] -when = { effective_engine = "lemonade", os = "linux" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 @@ -140,9 +142,11 @@ serve_timeout_secs = 90 # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat # path works. Only the lemonade LINUX path stays xfail: it needs a live lemonade # serve, which reaches ready then shuts down immediately (EAI-7423); native Windows -# lemonade serve works (run 29338475133 XPASS). +# lemonade serve works (run 29338475133 XPASS). therock_family=gfx* scopes this to +# a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not +# a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. [["chat-end-to-end-local-model"]] -when = { effective_engine = "lemonade", os = "linux" } +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index 486b1cf8..d0cc2609 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -41,14 +41,22 @@ Feature: Chat and endpoint detection When the user lists running services Then the served model endpoint is listed - @id:chat-tool-definitions-accepted @requires-gpu + # Runs on every lane: on a GPU host `a model is served in the background` does a + # real `rocm serve`, on the no-GPU mock lane it's backed by MockServer. The + # assertion (a tools-bearing request is accepted) is engine-agnostic, so no GPU + # is required — dropping @requires-gpu gives this per-PR mock-lane coverage. + @id:chat-tool-definitions-accepted Scenario: 5 - Chat requests that include tool definitions are accepted Given a managed runtime is active And a model is served in the background When a chat request with tool definitions is sent Then the chat response is successful - @id:chat-end-to-end-local-model @requires-gpu + # Runs on every lane (see scenario 5): real serve on a GPU host, MockServer on + # the no-GPU mock lane. Asserts only that a served model returns a non-empty + # reply, which is engine-agnostic — real generation is covered by the + # @requires-gpu serve-*-inference scenarios. + @id:chat-end-to-end-local-model Scenario: 6 - End-to-end chat through a locally served model Given a managed runtime is active And a model is served in the background diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 715e3537..27fe87f3 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -124,7 +124,9 @@ Feature: Model serving # Honest device selection: a `--gpu` index that does not exist on the host is # rejected outright, never silently remapped to another device (no device-0 - # fallback). Runs on GPU hardware. + # fallback). Runs on GPU hardware: on a no-GPU host the GPU-required pre-flight + # refuses ("no usable AMD GPU") before the index is ever validated, so the + # index-specific rejection can only be observed where a real device is present. @id:serve-absent-gpu-index-rejected @requires-gpu @requires-os:linux Scenario: 13 - Serving pinned to a GPU that does not exist is refused When the user serves a model pinned to a GPU index that does not exist diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index be28c38c..251dac2e 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -20,6 +20,15 @@ async fn setup_standard_rocm(_world: &mut E2eWorld) {} #[given("a managed runtime is active")] async fn setup_active_runtime(world: &mut E2eWorld) { + // On a no-GPU host this is a no-op: a managed TheRock SDK runtime can only be + // installed where there's a GPU family to select wheels for (see + // `runtime-install-active`, @requires-gpu). The only scenarios that reach this + // step without `@requires-gpu` are the mock-lane chat scenarios, whose serve + // is backed by MockServer and needs no runtime at all — so skip the install + // rather than attempt a multi-GiB SDK pull that can't succeed here. + if !e2e_cucumber::capability::host_capability().has_amd_gpu { + return; + } // This precondition only needs *a* runtime present — it does not assert a // clean slate — so opt into the shared runtimes tree: the first scenario to // hit an empty shared tree installs once, and every later scenario finds the diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index dfca6af5..5159f2c7 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -412,7 +412,25 @@ async fn setup_large_gpu_model(world: &mut E2eWorld) { #[given("a model is served in the background")] async fn setup_background_model(world: &mut E2eWorld) { - setup_gpu_model(world).await; + // This precondition backs behavioural chat scenarios (tool-defs accepted, + // end-to-end reply) whose When/Then talk to the served endpoint over HTTP and + // never assert on real generation — only that a served model answers. So it + // doesn't need a real GPU serve: on a host WITH an AMD GPU we still exercise + // the real `rocm serve --managed` path (extra real coverage where hardware + // exists), but on a no-GPU host we back it with the in-process MockServer + + // a planted managed-service record. This lets the scenarios drop + // `@requires-gpu` and run on the GitHub-hosted mock lane every PR, with no + // coverage loss. Real inference stays covered by the `@requires-gpu` + // serve-*-inference scenarios. + if e2e_cucumber::capability::host_capability().has_amd_gpu { + setup_gpu_model(world).await; + } else { + let mock = MockServer::start("TestModel/E2E-1B").await; + world.endpoint = Some(mock.base_url()); + world.model_name = Some("TestModel/E2E-1B".to_string()); + world.mock = Some(mock); + world.register_mock_service(); + } } #[given("the served model has been detected")]