Validate bounded-query sandbox runtime matrix

.github/workflows/test-gvisor-compat.yml

@@ -10,6 +10,40 @@ permissions:
   contents: read
 
 jobs:
+  bounded-query-isolation:
+    name: Bounded-query gVisor isolation
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Install gVisor
+        run: |
+          set -euo pipefail
+          ARCH=$(uname -m)
+          URL="https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}"
+          wget -q "${URL}/runsc" "${URL}/containerd-shim-runsc-v1"
+          chmod +x runsc containerd-shim-runsc-v1
+          sudo mv runsc containerd-shim-runsc-v1 /usr/local/bin/
+          sudo mkdir -p /etc/docker
+          printf '{"runtimes":{"runsc":{"path":"/usr/local/bin/runsc"}}}\n' |
+            sudo tee /etc/docker/daemon.json
+          sudo systemctl restart docker
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "24"
+          package-manager-cache: false
+      - name: Exercise bounded-query isolation under runsc
+        env:
+          AWF_BOUNDED_QUERY_TEST_RUNTIME: gvisor
+        run: |
+          npm ci
+          npm run build
+          npm run test:integration -- --runInBand bounded-query-isolation.test.ts
+
   install-gvisor:
     name: Install gVisor
     runs-on: ubuntu-latest

containers/bounded-query/broker/broker.js

@@ -56,11 +56,22 @@ function createBroker(params) {
   const runner = params.runner;
   const clock = params.clock || createRealClock();
   const ledger = params.ledger || createLedger(seedMap);
+  const telemetry = params.telemetry || { emit() {} };
 
   let invocationsUsed = 0;
   let tail = Promise.resolve();
   let accepting = true;
 
+  function emitQueryTelemetry(category) {
+    telemetry.emit({
+      primaryBackend: config.primaryBackend,
+      queryBackend: config.queryBackend,
+      lifecycleClass: 'query',
+      capabilityState: 'supported',
+      category,
+    });
+  }
+
   /**
    * Executes one request and reports its canonical result through
    * `respond` (called exactly once). The invocations run only through
@@ -80,6 +91,7 @@ function createBroker(params) {
     const validation = validateBoundedQueryRequest(request);
     if (!validation.valid) {
       audit.failure(invocationId, 'invalid-request', validation.errors.join('; '));
+      emitQueryTelemetry('invalid-request');
       safeRespond(CANONICAL_ERROR_JSON);
       return;
     }
@@ -89,6 +101,7 @@ function createBroker(params) {
     const seed = seedMap.get(repoKey);
     if (!seed) {
       audit.failure(invocationId, 'repo-not-allowed', privateRepo);
+      emitQueryTelemetry('repo-not-allowed');
       safeRespond(CANONICAL_ERROR_JSON);
       return;
     }
@@ -100,6 +113,7 @@ function createBroker(params) {
     const charge = queryBitsForSchema(schema);
     if (!ledger.tryDebit(repoKey, charge)) {
       audit.failure(invocationId, 'bit-budget-exhausted', `repo=${privateRepo} charge=${charge}`);
+      emitQueryTelemetry('bit-budget-exhausted');
       safeRespond(CANONICAL_ERROR_JSON);
       return;
     }
@@ -175,6 +189,7 @@ function createBroker(params) {
       // configured bucket — pathological infrastructure latency. Never emit a
       // successful result at unbucketed timing.
       audit.failure(invocationId, 'timing-bucket-overflow', failureReason ? failureReason.join(':') : undefined);
+      emitQueryTelemetry('timing-bucket-overflow');
       safeRespond(CANONICAL_ERROR_JSON);
     } else if (canonicalResult !== undefined) {
       audit.invocation({
@@ -184,9 +199,12 @@ function createBroker(params) {
         bits: charge,
         bucketMs,
       });
+      emitQueryTelemetry('success');
       safeRespond(canonicalOkJson(canonicalResult));
     } else {
-      audit.failure(invocationId, failureReason ? failureReason[0] : 'unknown', failureReason ? failureReason[1] : undefined);
+      const category = failureReason ? failureReason[0] : 'unknown';
+      audit.failure(invocationId, category, failureReason ? failureReason[1] : undefined);
+      emitQueryTelemetry(category);
       safeRespond(CANONICAL_ERROR_JSON);
     }
 
@@ -238,13 +256,15 @@ function createBroker(params) {
       // against it.
       if (invocationsUsed >= config.maxInvocations) {
         audit.failure('budget', 'invocation-count-exhausted', `max=${config.maxInvocations}`);
+        emitQueryTelemetry('invocation-count-exhausted');
         safeRespond(CANONICAL_ERROR_JSON);
         return Promise.resolve();
       }
       invocationsUsed += 1;
 
       const queued = tail.then(() => execute(request, safeRespond)).catch((error) => {
         audit.failure('queue', 'unexpected-error', error && error.message);
+        emitQueryTelemetry('unexpected-error');
         safeRespond(CANONICAL_ERROR_JSON);
       });
       tail = queued.then(

containers/bounded-query/broker/config.js

@@ -97,6 +97,10 @@ function loadConfig() {
   if (queryBackend !== 'docker' && queryBackend !== 'gvisor' && queryBackend !== 'sbx') {
     throw new Error(`Unsupported AWF_BOUNDED_QUERY_BACKEND: ${queryBackend}`);
   }
+  const primaryBackend = requireEnv('AWF_BOUNDED_QUERY_PRIMARY_BACKEND');
+  if (primaryBackend !== 'docker' && primaryBackend !== 'gvisor' && primaryBackend !== 'sbx') {
+    throw new Error(`Unsupported AWF_BOUNDED_QUERY_PRIMARY_BACKEND: ${primaryBackend}`);
+  }
 
   const tcpPortRaw = process.env.AWF_BOUNDED_QUERY_TCP_PORT;
   const tcpPort = tcpPortRaw === undefined ? undefined : parsePositiveInt('AWF_BOUNDED_QUERY_TCP_PORT');
@@ -131,6 +135,7 @@ function loadConfig() {
     // Never reuse the Docker-daemon-visible path for sbx mounts.
     sbxWorkDir,
     queryBackend,
+    primaryBackend,
     timeoutSeconds: parseTimeoutSeconds(),
     maxInvocations: parsePositiveInt('AWF_BOUNDED_QUERY_MAX_INVOCATIONS', 32),
     memoryLimit,

containers/bounded-query/broker/runtime-telemetry.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const PRIMARY_BACKENDS = new Set(['docker', 'gvisor', 'sbx']);
+const QUERY_BACKENDS = new Set(['docker', 'gvisor', 'sbx']);
+const LIFECYCLE_CLASSES = new Set(['preflight', 'startup', 'query', 'cleanup']);
+const CAPABILITY_STATES = new Set(['supported', 'unavailable', 'blocked']);
+const CATEGORY_PATTERN = /^[a-z][a-z0-9-]{0,63}$/;
+
+function assertTelemetryValue(allowed, value, field) {
+  if (!allowed.has(value)) throw new Error(`Invalid bounded-query telemetry ${field}`);
+}
+
+function buildRuntimeTelemetryRecord(event) {
+  assertTelemetryValue(PRIMARY_BACKENDS, event.primaryBackend, 'primaryBackend');
+  assertTelemetryValue(QUERY_BACKENDS, event.queryBackend, 'queryBackend');
+  assertTelemetryValue(LIFECYCLE_CLASSES, event.lifecycleClass, 'lifecycleClass');
+  assertTelemetryValue(CAPABILITY_STATES, event.capabilityState, 'capabilityState');
+  if (typeof event.category !== 'string' || !CATEGORY_PATTERN.test(event.category)) {
+    throw new Error('Invalid bounded-query telemetry category');
+  }
+  return Object.freeze({
+    primaryBackend: event.primaryBackend,
+    queryBackend: event.queryBackend,
+    lifecycleClass: event.lifecycleClass,
+    capabilityState: event.capabilityState,
+    category: event.category,
+  });
+}
+
+function createRuntimeTelemetry(auditDir) {
+  fs.mkdirSync(auditDir, { recursive: true, mode: 0o700 });
+  const telemetryPath = path.join(auditDir, 'runtime-telemetry.jsonl');
+  let fd = fs.openSync(telemetryPath, 'a', 0o600);
+  return {
+    emit(event) {
+      const record = buildRuntimeTelemetryRecord(event);
+      if (fd === undefined) return;
+      try {
+        fs.writeSync(fd, `${JSON.stringify(record)}\n`);
+      } catch {
+        process.stderr.write('[bounded-query] runtime telemetry unavailable\n');
+        try {
+          fs.closeSync(fd);
+        } catch {
+          // The generic telemetry failure above is the only safe diagnostic.
+        }
+        fd = undefined;
+      }
+    },
+  };
+}
+
+module.exports = { buildRuntimeTelemetryRecord, createRuntimeTelemetry };

containers/bounded-query/broker/server.js

@@ -9,6 +9,7 @@ const { loadConfig, loadSeedMap } = require('./config');
 const { buildRequestFromFrame, readBoundedBody } = require('./framing');
 const { CANONICAL_ERROR_JSON } = require('./protocol');
 const { createQueryRunner } = require('./query-runner');
+const { createRuntimeTelemetry } = require('./runtime-telemetry');
 
 /**
  * Bounded-query broker server.
@@ -289,15 +290,23 @@ function listenOnTcp(server, config) {
 async function main() {
   const config = loadConfig();
   const audit = createAuditLog(config.auditDir);
+  const telemetry = createRuntimeTelemetry(config.auditDir);
   const { runId, seeds } = loadSeedMap(config.seedMapPath);
   const runner = createQueryRunner(config);
 
   // Fail closed before accepting requests and reconcile containers left by a
   // prior broker process for this exact run. Queries never pull or fall back.
   await runner.assertAvailable();
   await runner.reconcileRun(runId);
+  telemetry.emit({
+    primaryBackend: config.primaryBackend,
+    queryBackend: config.queryBackend,
+    lifecycleClass: 'startup',
+    capabilityState: 'supported',
+    category: 'ready',
+  });
 
-  const broker = createBroker({ config, seedMap: seeds, runId, audit, runner });
+  const broker = createBroker({ config, seedMap: seeds, runId, audit, runner, telemetry });
   const unixServer = createServer({ broker, audit });
   const servers = [unixServer];
 
@@ -345,9 +354,23 @@ async function main() {
         new Promise((resolve) => setTimeout(resolve, SHUTDOWN_GRACE_MS)),
       ]);
       await runner.reconcileRun(runId);
+      telemetry.emit({
+        primaryBackend: config.primaryBackend,
+        queryBackend: config.queryBackend,
+        lifecycleClass: 'cleanup',
+        capabilityState: 'supported',
+        category: 'success',
+      });
       process.exit(0);
     } catch (error) {
       audit.lifecycle('shutdown-cleanup-failed', error.message);
+      telemetry.emit({
+        primaryBackend: config.primaryBackend,
+        queryBackend: config.queryBackend,
+        lifecycleClass: 'cleanup',
+        capabilityState: 'supported',
+        category: 'cleanup-failed',
+      });
       process.exit(1);
     }
   };

docs/awf-config-spec.md

@@ -1653,6 +1653,33 @@ sbx credential to the broker, and never falls back to Docker/gVisor. Enabling
 launch requires all missing controls plus a digest-pinned, Python
 standard-library-only AWF query template/bootstrap.
 
+**Independent runtime matrix.** `container.containerRuntime` selects the primary
+agent while `boundedQueries.runtime` independently selects a fresh query
+sandbox. Every accepted invocation creates one new sandbox and destroys it
+before response. The current capability matrix is:
+
+| Primary agent | Docker query | gVisor query | sbx query |
+|---|---|---|---|
+| Docker | Supported with Docker | Supported with registered `runsc` | Blocked |
+| gVisor | Supported with primary `runsc` | Supported with registered `runsc` | Blocked |
+| sbx | Supported after primary ingress probe | Supported after primary ingress and `runsc` probes | Blocked |
+
+Unavailable cells abort at preflight and never stage. A blocked sbx query is an
+expected security result, not runtime success. `"runtime": "sbx"` is both the
+explicit experimental selection and a requirement to pass every executable
+probe; it never authorizes fallback.
+
+**Runtime telemetry.** Telemetry records contain exactly `primaryBackend`,
+`queryBackend`, `lifecycleClass`, `capabilityState`, and `category`. They MUST
+NOT contain repository data or identifiers, scripts, outputs, paths, tokens,
+ingress capabilities, or daemon credentials.
+
+Promotion of sbx queries requires real-VM proof of no network/lateral access,
+all resource bounds, mount-target isolation, credential/state separation,
+canonical output behavior, and cleanup after timeout, resource failure, and
+interruption, plus a digest-pinned AWF Python-only template. Version/help
+probing alone is insufficient.
+
 The seed map the broker reads carries each repository's trusted
 `sensitivity` alongside its opaque seed id — the map is built entirely from
 AWF configuration, so a request can never choose or override its own

docs/bounded-queries.md

@@ -115,6 +115,105 @@ capability in JSON. Support remains blocked until sbx provides enforceable
 versions of all controls and AWF publishes a digest-pinned standard-library-only
 Python template/bootstrap.
 
+### Primary-agent and query runtime matrix
+
+The primary agent and each bounded query are separate sandbox decisions:
+
+- `container.containerRuntime` / `--container-runtime` selects the **primary
+  agent** runtime.
+- `boundedQueries.runtime` selects the **single-use query** runtime.
+
+The broker never reuses the primary agent sandbox. Every accepted query creates
+a new container or VM with a unique run/invocation identity and destroys it
+before returning. No combination falls back to a weaker backend.
+
+| Primary agent | Docker query | gVisor query | sbx query |
+|---|---|---|---|
+| Docker | Supported when Docker is available | Supported when `runsc` is registered | **Blocked** by mandatory sbx query probes |
+| gVisor | Supported when the primary `runsc` runtime is available | Supported when `runsc` is registered | **Blocked** by mandatory sbx query probes |
+| sbx | Supported when primary sbx and broker ingress probes pass | Supported when primary sbx, ingress, and `runsc` probes pass | **Blocked** by mandatory sbx query probes |
+
+“Supported” is capability-dependent, not an instruction to downgrade. An
+unavailable primary runtime fails at primary preflight. An unavailable query
+runtime fails at query preflight before the private root is created or any
+repository is staged. Selecting `"runtime": "sbx"` is the explicit experimental
+gate; the additional executable capability proof must also pass. With Docker
+Sandboxes `v0.37.1`, all three sbx-query cells remain blocked.
+
+Examples of independent selection:
+
+```json
+{
+  "container": { "containerRuntime": "gvisor" },
+  "boundedQueries": {
+    "enabled": true,
+    "privateRepos": [
+      { "repo": "my-org/private-service", "sensitivity": "internal" }
+    ],
+    "runtime": "docker"
+  }
+}
+```
+
+```json
+{
+  "container": { "containerRuntime": "sbx" },
+  "boundedQueries": {
+    "enabled": true,
+    "privateRepos": [
+      { "repo": "my-org/private-service", "sensitivity": "confidential" }
+    ],
+    "runtime": "gvisor"
+  }
+}
+```
+
+The second example starts only when sbx primary-agent ingress and Docker
+`runsc` query probes both pass.
+
+### Runtime telemetry
+
+AWF emits a deliberately narrow runtime telemetry record. It contains exactly:
+primary backend, query backend, lifecycle class, capability state, and
+success/failure category. It never contains repository identifiers or contents,
+scripts, raw outputs, host/container paths, tokens, ingress capabilities, or
+daemon credentials. Broker records are written to the protected
+`runtime-telemetry.jsonl` file beside the protected audit log and are never
+mounted into the agent.
+
+### Troubleshooting runtime selection
+
+| Symptom | Meaning | Action |
+|---|---|---|
+| `runsc ... not available; no fallback` | The gVisor query backend is not registered with Docker | Register `runsc`, verify it appears in `docker info --format '{{json .Runtimes}}'`, and rerun |
+| `sbx ... blocked ... mandatory query-isolation controls` | The sbx query security probe failed as designed | Read the complete missing-control list; do not substitute local policy or a weaker runtime |
+| sbx primary ingress probe fails | The primary VM cannot reach the broker through either proven ingress | Verify sbx Unix passthrough or authenticated host-loopback ingress; the agent must not start |
+| Docker host must be `unix://` | The networkless broker cannot reach a TCP daemon | Use a local Unix socket; AWF will not attach the broker to a network |
+| Matrix report says `BLOCKED` | Capability or security preflight prevented launch | Treat this as expected fail-closed status, not successful runtime execution |
+
+Run `node scripts/ci/report-bounded-query-runtime-matrix.js` after `npm run
+build` to print all nine local capability results. Use `--require
+docker/docker` (or another pair) when a smoke job must require one executable
+combination.
+
+### sbx query promotion criteria
+
+The experimental sbx query backend MUST remain blocked until all of these are
+demonstrated in real VMs, not only deterministic fakes:
+
+1. A digest-pinned AWF Python standard-library-only template/bootstrap exists.
+2. Per-VM network-none and lateral-connectivity denial are enforceable and
+   cannot be replaced by organization policy.
+3. CPU, memory, PID, aggregate disk, and per-file size limits are enforceable.
+4. Read-only seed/script mounts have explicit guest targets and expose no broker
+   state, credentials, sibling repository, or prior invocation.
+5. Timeout, OOM, PID, disk, file-size, malformed/oversized output, and
+   interruption cleanup tests all pass.
+6. Unix and authenticated sbx ingress retain byte-identical protocol behavior.
+
+Passing a version check alone, or passing only the CLI help probe, is not enough
+to promote the backend.
+
 ## Sensitivity categories
 
 Every repository carries a fixed sensitivity that sets an immutable maximum number of bits the broker may reveal about that repository across the entire AWF run. The budget is per-run only; the broker has no durable state across runs.