diff --git a/.github/workflows/test-gvisor-compat.yml b/.github/workflows/test-gvisor-compat.yml index ea25994f7..b15017907 100644 --- a/.github/workflows/test-gvisor-compat.yml +++ b/.github/workflows/test-gvisor-compat.yml @@ -10,6 +10,40 @@ permissions: contents: read jobs: + bounded-query-isolation: + name: Bounded-query gVisor isolation + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install gVisor + run: | + set -euo pipefail + ARCH=$(uname -m) + URL="https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}" + wget -q "${URL}/runsc" "${URL}/containerd-shim-runsc-v1" + chmod +x runsc containerd-shim-runsc-v1 + sudo mv runsc containerd-shim-runsc-v1 /usr/local/bin/ + sudo mkdir -p /etc/docker + printf '{"runtimes":{"runsc":{"path":"/usr/local/bin/runsc"}}}\n' | + sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + package-manager-cache: false + - name: Exercise bounded-query isolation under runsc + env: + AWF_BOUNDED_QUERY_TEST_RUNTIME: gvisor + run: | + npm ci + npm run build + npm run test:integration -- --runInBand bounded-query-isolation.test.ts + install-gvisor: name: Install gVisor runs-on: ubuntu-latest diff --git a/containers/bounded-query/broker/broker.js b/containers/bounded-query/broker/broker.js index 055a1436d..fe7d1affb 100644 --- a/containers/bounded-query/broker/broker.js +++ b/containers/bounded-query/broker/broker.js @@ -56,11 +56,22 @@ function createBroker(params) { const runner = params.runner; const clock = params.clock || createRealClock(); const ledger = params.ledger || createLedger(seedMap); + const telemetry = params.telemetry || { emit() {} }; let invocationsUsed = 0; let tail = Promise.resolve(); let accepting = true; + function emitQueryTelemetry(category) { + telemetry.emit({ + primaryBackend: config.primaryBackend, + queryBackend: config.queryBackend, + lifecycleClass: 'query', + capabilityState: 'supported', + category, + }); + } + /** * Executes one request and reports its canonical result through * `respond` (called exactly once). The invocations run only through @@ -80,6 +91,7 @@ function createBroker(params) { const validation = validateBoundedQueryRequest(request); if (!validation.valid) { audit.failure(invocationId, 'invalid-request', validation.errors.join('; ')); + emitQueryTelemetry('invalid-request'); safeRespond(CANONICAL_ERROR_JSON); return; } @@ -89,6 +101,7 @@ function createBroker(params) { const seed = seedMap.get(repoKey); if (!seed) { audit.failure(invocationId, 'repo-not-allowed', privateRepo); + emitQueryTelemetry('repo-not-allowed'); safeRespond(CANONICAL_ERROR_JSON); return; } @@ -100,6 +113,7 @@ function createBroker(params) { const charge = queryBitsForSchema(schema); if (!ledger.tryDebit(repoKey, charge)) { audit.failure(invocationId, 'bit-budget-exhausted', `repo=${privateRepo} charge=${charge}`); + emitQueryTelemetry('bit-budget-exhausted'); safeRespond(CANONICAL_ERROR_JSON); return; } @@ -175,6 +189,7 @@ function createBroker(params) { // configured bucket — pathological infrastructure latency. Never emit a // successful result at unbucketed timing. audit.failure(invocationId, 'timing-bucket-overflow', failureReason ? failureReason.join(':') : undefined); + emitQueryTelemetry('timing-bucket-overflow'); safeRespond(CANONICAL_ERROR_JSON); } else if (canonicalResult !== undefined) { audit.invocation({ @@ -184,9 +199,12 @@ function createBroker(params) { bits: charge, bucketMs, }); + emitQueryTelemetry('success'); safeRespond(canonicalOkJson(canonicalResult)); } else { - audit.failure(invocationId, failureReason ? failureReason[0] : 'unknown', failureReason ? failureReason[1] : undefined); + const category = failureReason ? failureReason[0] : 'unknown'; + audit.failure(invocationId, category, failureReason ? failureReason[1] : undefined); + emitQueryTelemetry(category); safeRespond(CANONICAL_ERROR_JSON); } @@ -238,6 +256,7 @@ function createBroker(params) { // against it. if (invocationsUsed >= config.maxInvocations) { audit.failure('budget', 'invocation-count-exhausted', `max=${config.maxInvocations}`); + emitQueryTelemetry('invocation-count-exhausted'); safeRespond(CANONICAL_ERROR_JSON); return Promise.resolve(); } @@ -245,6 +264,7 @@ function createBroker(params) { const queued = tail.then(() => execute(request, safeRespond)).catch((error) => { audit.failure('queue', 'unexpected-error', error && error.message); + emitQueryTelemetry('unexpected-error'); safeRespond(CANONICAL_ERROR_JSON); }); tail = queued.then( diff --git a/containers/bounded-query/broker/config.js b/containers/bounded-query/broker/config.js index e71d210f6..dc9a9d0ba 100644 --- a/containers/bounded-query/broker/config.js +++ b/containers/bounded-query/broker/config.js @@ -97,6 +97,10 @@ function loadConfig() { if (queryBackend !== 'docker' && queryBackend !== 'gvisor' && queryBackend !== 'sbx') { throw new Error(`Unsupported AWF_BOUNDED_QUERY_BACKEND: ${queryBackend}`); } + const primaryBackend = requireEnv('AWF_BOUNDED_QUERY_PRIMARY_BACKEND'); + if (primaryBackend !== 'docker' && primaryBackend !== 'gvisor' && primaryBackend !== 'sbx') { + throw new Error(`Unsupported AWF_BOUNDED_QUERY_PRIMARY_BACKEND: ${primaryBackend}`); + } const tcpPortRaw = process.env.AWF_BOUNDED_QUERY_TCP_PORT; const tcpPort = tcpPortRaw === undefined ? undefined : parsePositiveInt('AWF_BOUNDED_QUERY_TCP_PORT'); @@ -131,6 +135,7 @@ function loadConfig() { // Never reuse the Docker-daemon-visible path for sbx mounts. sbxWorkDir, queryBackend, + primaryBackend, timeoutSeconds: parseTimeoutSeconds(), maxInvocations: parsePositiveInt('AWF_BOUNDED_QUERY_MAX_INVOCATIONS', 32), memoryLimit, diff --git a/containers/bounded-query/broker/runtime-telemetry.js b/containers/bounded-query/broker/runtime-telemetry.js new file mode 100644 index 000000000..31be15976 --- /dev/null +++ b/containers/bounded-query/broker/runtime-telemetry.js @@ -0,0 +1,56 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const PRIMARY_BACKENDS = new Set(['docker', 'gvisor', 'sbx']); +const QUERY_BACKENDS = new Set(['docker', 'gvisor', 'sbx']); +const LIFECYCLE_CLASSES = new Set(['preflight', 'startup', 'query', 'cleanup']); +const CAPABILITY_STATES = new Set(['supported', 'unavailable', 'blocked']); +const CATEGORY_PATTERN = /^[a-z][a-z0-9-]{0,63}$/; + +function assertTelemetryValue(allowed, value, field) { + if (!allowed.has(value)) throw new Error(`Invalid bounded-query telemetry ${field}`); +} + +function buildRuntimeTelemetryRecord(event) { + assertTelemetryValue(PRIMARY_BACKENDS, event.primaryBackend, 'primaryBackend'); + assertTelemetryValue(QUERY_BACKENDS, event.queryBackend, 'queryBackend'); + assertTelemetryValue(LIFECYCLE_CLASSES, event.lifecycleClass, 'lifecycleClass'); + assertTelemetryValue(CAPABILITY_STATES, event.capabilityState, 'capabilityState'); + if (typeof event.category !== 'string' || !CATEGORY_PATTERN.test(event.category)) { + throw new Error('Invalid bounded-query telemetry category'); + } + return Object.freeze({ + primaryBackend: event.primaryBackend, + queryBackend: event.queryBackend, + lifecycleClass: event.lifecycleClass, + capabilityState: event.capabilityState, + category: event.category, + }); +} + +function createRuntimeTelemetry(auditDir) { + fs.mkdirSync(auditDir, { recursive: true, mode: 0o700 }); + const telemetryPath = path.join(auditDir, 'runtime-telemetry.jsonl'); + let fd = fs.openSync(telemetryPath, 'a', 0o600); + return { + emit(event) { + const record = buildRuntimeTelemetryRecord(event); + if (fd === undefined) return; + try { + fs.writeSync(fd, `${JSON.stringify(record)}\n`); + } catch { + process.stderr.write('[bounded-query] runtime telemetry unavailable\n'); + try { + fs.closeSync(fd); + } catch { + // The generic telemetry failure above is the only safe diagnostic. + } + fd = undefined; + } + }, + }; +} + +module.exports = { buildRuntimeTelemetryRecord, createRuntimeTelemetry }; diff --git a/containers/bounded-query/broker/server.js b/containers/bounded-query/broker/server.js index 2b78451dc..fd8176aea 100644 --- a/containers/bounded-query/broker/server.js +++ b/containers/bounded-query/broker/server.js @@ -9,6 +9,7 @@ const { loadConfig, loadSeedMap } = require('./config'); const { buildRequestFromFrame, readBoundedBody } = require('./framing'); const { CANONICAL_ERROR_JSON } = require('./protocol'); const { createQueryRunner } = require('./query-runner'); +const { createRuntimeTelemetry } = require('./runtime-telemetry'); /** * Bounded-query broker server. @@ -289,6 +290,7 @@ function listenOnTcp(server, config) { async function main() { const config = loadConfig(); const audit = createAuditLog(config.auditDir); + const telemetry = createRuntimeTelemetry(config.auditDir); const { runId, seeds } = loadSeedMap(config.seedMapPath); const runner = createQueryRunner(config); @@ -296,8 +298,15 @@ async function main() { // prior broker process for this exact run. Queries never pull or fall back. await runner.assertAvailable(); await runner.reconcileRun(runId); + telemetry.emit({ + primaryBackend: config.primaryBackend, + queryBackend: config.queryBackend, + lifecycleClass: 'startup', + capabilityState: 'supported', + category: 'ready', + }); - const broker = createBroker({ config, seedMap: seeds, runId, audit, runner }); + const broker = createBroker({ config, seedMap: seeds, runId, audit, runner, telemetry }); const unixServer = createServer({ broker, audit }); const servers = [unixServer]; @@ -345,9 +354,23 @@ async function main() { new Promise((resolve) => setTimeout(resolve, SHUTDOWN_GRACE_MS)), ]); await runner.reconcileRun(runId); + telemetry.emit({ + primaryBackend: config.primaryBackend, + queryBackend: config.queryBackend, + lifecycleClass: 'cleanup', + capabilityState: 'supported', + category: 'success', + }); process.exit(0); } catch (error) { audit.lifecycle('shutdown-cleanup-failed', error.message); + telemetry.emit({ + primaryBackend: config.primaryBackend, + queryBackend: config.queryBackend, + lifecycleClass: 'cleanup', + capabilityState: 'supported', + category: 'cleanup-failed', + }); process.exit(1); } }; diff --git a/docs/awf-config-spec.md b/docs/awf-config-spec.md index 1cab1ea19..981fa85fb 100644 --- a/docs/awf-config-spec.md +++ b/docs/awf-config-spec.md @@ -1653,6 +1653,33 @@ sbx credential to the broker, and never falls back to Docker/gVisor. Enabling launch requires all missing controls plus a digest-pinned, Python standard-library-only AWF query template/bootstrap. +**Independent runtime matrix.** `container.containerRuntime` selects the primary +agent while `boundedQueries.runtime` independently selects a fresh query +sandbox. Every accepted invocation creates one new sandbox and destroys it +before response. The current capability matrix is: + +| Primary agent | Docker query | gVisor query | sbx query | +|---|---|---|---| +| Docker | Supported with Docker | Supported with registered `runsc` | Blocked | +| gVisor | Supported with primary `runsc` | Supported with registered `runsc` | Blocked | +| sbx | Supported after primary ingress probe | Supported after primary ingress and `runsc` probes | Blocked | + +Unavailable cells abort at preflight and never stage. A blocked sbx query is an +expected security result, not runtime success. `"runtime": "sbx"` is both the +explicit experimental selection and a requirement to pass every executable +probe; it never authorizes fallback. + +**Runtime telemetry.** Telemetry records contain exactly `primaryBackend`, +`queryBackend`, `lifecycleClass`, `capabilityState`, and `category`. They MUST +NOT contain repository data or identifiers, scripts, outputs, paths, tokens, +ingress capabilities, or daemon credentials. + +Promotion of sbx queries requires real-VM proof of no network/lateral access, +all resource bounds, mount-target isolation, credential/state separation, +canonical output behavior, and cleanup after timeout, resource failure, and +interruption, plus a digest-pinned AWF Python-only template. Version/help +probing alone is insufficient. + The seed map the broker reads carries each repository's trusted `sensitivity` alongside its opaque seed id — the map is built entirely from AWF configuration, so a request can never choose or override its own diff --git a/docs/bounded-queries.md b/docs/bounded-queries.md index 9b8820dfe..30e391abe 100644 --- a/docs/bounded-queries.md +++ b/docs/bounded-queries.md @@ -115,6 +115,105 @@ capability in JSON. Support remains blocked until sbx provides enforceable versions of all controls and AWF publishes a digest-pinned standard-library-only Python template/bootstrap. +### Primary-agent and query runtime matrix + +The primary agent and each bounded query are separate sandbox decisions: + +- `container.containerRuntime` / `--container-runtime` selects the **primary + agent** runtime. +- `boundedQueries.runtime` selects the **single-use query** runtime. + +The broker never reuses the primary agent sandbox. Every accepted query creates +a new container or VM with a unique run/invocation identity and destroys it +before returning. No combination falls back to a weaker backend. + +| Primary agent | Docker query | gVisor query | sbx query | +|---|---|---|---| +| Docker | Supported when Docker is available | Supported when `runsc` is registered | **Blocked** by mandatory sbx query probes | +| gVisor | Supported when the primary `runsc` runtime is available | Supported when `runsc` is registered | **Blocked** by mandatory sbx query probes | +| sbx | Supported when primary sbx and broker ingress probes pass | Supported when primary sbx, ingress, and `runsc` probes pass | **Blocked** by mandatory sbx query probes | + +“Supported” is capability-dependent, not an instruction to downgrade. An +unavailable primary runtime fails at primary preflight. An unavailable query +runtime fails at query preflight before the private root is created or any +repository is staged. Selecting `"runtime": "sbx"` is the explicit experimental +gate; the additional executable capability proof must also pass. With Docker +Sandboxes `v0.37.1`, all three sbx-query cells remain blocked. + +Examples of independent selection: + +```json +{ + "container": { "containerRuntime": "gvisor" }, + "boundedQueries": { + "enabled": true, + "privateRepos": [ + { "repo": "my-org/private-service", "sensitivity": "internal" } + ], + "runtime": "docker" + } +} +``` + +```json +{ + "container": { "containerRuntime": "sbx" }, + "boundedQueries": { + "enabled": true, + "privateRepos": [ + { "repo": "my-org/private-service", "sensitivity": "confidential" } + ], + "runtime": "gvisor" + } +} +``` + +The second example starts only when sbx primary-agent ingress and Docker +`runsc` query probes both pass. + +### Runtime telemetry + +AWF emits a deliberately narrow runtime telemetry record. It contains exactly: +primary backend, query backend, lifecycle class, capability state, and +success/failure category. It never contains repository identifiers or contents, +scripts, raw outputs, host/container paths, tokens, ingress capabilities, or +daemon credentials. Broker records are written to the protected +`runtime-telemetry.jsonl` file beside the protected audit log and are never +mounted into the agent. + +### Troubleshooting runtime selection + +| Symptom | Meaning | Action | +|---|---|---| +| `runsc ... not available; no fallback` | The gVisor query backend is not registered with Docker | Register `runsc`, verify it appears in `docker info --format '{{json .Runtimes}}'`, and rerun | +| `sbx ... blocked ... mandatory query-isolation controls` | The sbx query security probe failed as designed | Read the complete missing-control list; do not substitute local policy or a weaker runtime | +| sbx primary ingress probe fails | The primary VM cannot reach the broker through either proven ingress | Verify sbx Unix passthrough or authenticated host-loopback ingress; the agent must not start | +| Docker host must be `unix://` | The networkless broker cannot reach a TCP daemon | Use a local Unix socket; AWF will not attach the broker to a network | +| Matrix report says `BLOCKED` | Capability or security preflight prevented launch | Treat this as expected fail-closed status, not successful runtime execution | + +Run `node scripts/ci/report-bounded-query-runtime-matrix.js` after `npm run +build` to print all nine local capability results. Use `--require +docker/docker` (or another pair) when a smoke job must require one executable +combination. + +### sbx query promotion criteria + +The experimental sbx query backend MUST remain blocked until all of these are +demonstrated in real VMs, not only deterministic fakes: + +1. A digest-pinned AWF Python standard-library-only template/bootstrap exists. +2. Per-VM network-none and lateral-connectivity denial are enforceable and + cannot be replaced by organization policy. +3. CPU, memory, PID, aggregate disk, and per-file size limits are enforceable. +4. Read-only seed/script mounts have explicit guest targets and expose no broker + state, credentials, sibling repository, or prior invocation. +5. Timeout, OOM, PID, disk, file-size, malformed/oversized output, and + interruption cleanup tests all pass. +6. Unix and authenticated sbx ingress retain byte-identical protocol behavior. + +Passing a version check alone, or passing only the CLI help probe, is not enough +to promote the backend. + ## Sensitivity categories Every repository carries a fixed sensitivity that sets an immutable maximum number of bits the broker may reveal about that repository across the entire AWF run. The budget is per-run only; the broker has no durable state across runs. diff --git a/docs/sbx-integration.md b/docs/sbx-integration.md index 802a40c89..8e41a9c52 100644 --- a/docs/sbx-integration.md +++ b/docs/sbx-integration.md @@ -96,6 +96,18 @@ this query backend before staging or Compose assembly; no sbx daemon access is passed to the broker and there is no Docker/gVisor fallback. See [Bounded Queries](bounded-queries.md#sbx-query-runtime-status). +The full 3×3 primary/query matrix is documented in +[Bounded Queries](bounded-queries.md#primary-agent-and-query-runtime-matrix). +All sbx-query cells are intentionally blocked; Docker and gVisor query +backends may run under an sbx primary agent only after its independent broker +ingress probe passes. Every query gets a new sandbox and no backend falls back. + +Promotion is gated on a digest-pinned Python-only template and real-VM proof of +network/lateral denial, PID/memory/CPU/disk/file-size enforcement, explicit +guest mount targets, credential and cross-invocation isolation, canonical +failure bytes, timing buckets, and interruption cleanup. Docker Sandboxes +`v0.37.1` cannot satisfy those controls. + ## Part 2 — How AWF uses `sbx` AWF's default backend runs the agent as a **Docker Compose service** alongside diff --git a/scripts/ci/report-bounded-query-runtime-matrix.js b/scripts/ci/report-bounded-query-runtime-matrix.js new file mode 100644 index 000000000..0a58976af --- /dev/null +++ b/scripts/ci/report-bounded-query-runtime-matrix.js @@ -0,0 +1,133 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const { spawnSync } = require('child_process'); + +const BACKENDS = ['docker', 'gvisor', 'sbx']; + +function run(command, args) { + const result = spawnSync(command, args, { + encoding: 'utf8', + timeout: 30_000, + stdio: ['ignore', 'pipe', 'pipe'], + }); + return { + ok: !result.error && result.status === 0, + stdout: result.stdout || '', + }; +} + +function collectCapabilities(commandRunner = run) { + const docker = commandRunner('docker', ['info', '--format', '{{json .Runtimes}}']); + let runtimes = {}; + if (docker.ok) { + try { + runtimes = JSON.parse(docker.stdout); + } catch { + runtimes = {}; + } + } + const gvisor = Object.prototype.hasOwnProperty.call(runtimes, 'runsc'); + // `sbx version` only proves that the binary exists. Listing is authenticated + // and non-mutating, so it also proves daemon and credential availability. + const sbxPrimary = commandRunner('sbx', ['ls']).ok; + const sbxQuery = commandRunner( + process.execPath, + ['containers/bounded-query/broker/sbx-capability-probe.js'], + ); + let sbxQuerySupported = false; + if (sbxQuery.stdout) { + try { + sbxQuerySupported = JSON.parse(sbxQuery.stdout).supported === true; + } catch { + sbxQuerySupported = false; + } + } + return { + primary: { + docker: docker.ok ? 'supported' : 'unavailable', + gvisor: gvisor ? 'supported' : 'unavailable', + sbx: sbxPrimary ? 'supported' : 'unavailable', + }, + query: { + docker: docker.ok ? 'supported' : 'unavailable', + gvisor: gvisor ? 'supported' : 'unavailable', + sbx: sbxQuerySupported ? 'supported' : 'blocked', + }, + }; +} + +function evaluate(primary, query, capabilities) { + if (capabilities.primary[primary] !== 'supported') { + return { + status: 'BLOCKED', + capability: capabilities.primary[primary], + phase: 'primary-preflight', + }; + } + if (capabilities.query[query] !== 'supported') { + return { + status: 'BLOCKED', + capability: capabilities.query[query], + phase: 'query-preflight', + }; + } + return { status: 'SUPPORTED', capability: 'supported', phase: 'ready' }; +} + +function renderMatrix(capabilities) { + const lines = [ + '## Bounded-query runtime capability matrix', + '', + '| Primary agent | Query sandbox | Result | Primary capability | Query capability | Gate |', + '|---|---|---|---|---|---|', + ]; + for (const primary of BACKENDS) { + for (const query of BACKENDS) { + const result = evaluate(primary, query, capabilities); + lines.push( + `| ${primary} | ${query} | ${result.status} | ${capabilities.primary[primary]} | ` + + `${capabilities.query[query]} | ${result.phase} |`, + ); + } + } + lines.push( + '', + '> BLOCKED is an expected fail-closed security result, not runtime success. No fallback is attempted.', + ); + return `${lines.join('\n')}\n`; +} + +function main() { + const capabilities = collectCapabilities(); + const report = renderMatrix(capabilities); + process.stdout.write(report); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, report); + } + + const requiredIndex = process.argv.indexOf('--require'); + if (requiredIndex !== -1) { + const requirement = process.argv[requiredIndex + 1] || ''; + const [primary, query] = requirement.split('/'); + if (!BACKENDS.includes(primary) || !BACKENDS.includes(query)) { + throw new Error(`Invalid --require combination: ${requirement}`); + } + const result = evaluate(primary, query, capabilities); + if (result.status !== 'SUPPORTED') { + throw new Error(`Required runtime combination ${requirement} is ${result.status} at ${result.phase}`); + } + } +} + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + } +} + +module.exports = { collectCapabilities, evaluate, renderMatrix }; diff --git a/scripts/ci/report-bounded-query-runtime-matrix.test.ts b/scripts/ci/report-bounded-query-runtime-matrix.test.ts new file mode 100644 index 000000000..62d3f09f6 --- /dev/null +++ b/scripts/ci/report-bounded-query-runtime-matrix.test.ts @@ -0,0 +1,52 @@ +import * as path from 'path'; + +/* eslint-disable @typescript-eslint/no-require-imports */ +const { collectCapabilities, evaluate, renderMatrix } = require( + path.join(__dirname, 'report-bounded-query-runtime-matrix.js'), +); +/* eslint-enable @typescript-eslint/no-require-imports */ + +describe('bounded-query runtime capability report', () => { + it('reports all nine combinations and preserves the sbx query security block', () => { + const capabilities = collectCapabilities((command: string, args: string[]) => { + if (command === 'docker') { + return { ok: true, stdout: '{"runc":{},"runsc":{}}' }; + } + if (command === 'sbx') { + expect(args).toEqual(['ls']); + return { ok: true, stdout: 'Docker Sandboxes v0.37.1' }; + } + if (args.includes('sbx-capability-probe.js')) { + return { ok: false, stdout: '{"supported":false}' }; + } + return { ok: false, stdout: '' }; + }); + const report = renderMatrix(capabilities); + const rows = report.split('\n').filter((line: string) => /^\| (docker|gvisor|sbx) /.test(line)); + expect(rows).toHaveLength(9); + expect(report).toContain('| sbx | sbx | BLOCKED | supported | blocked | query-preflight |'); + expect(report).toContain('BLOCKED is an expected fail-closed security result, not runtime success'); + }); + + it('never promotes an unavailable primary or query runtime through fallback', () => { + const capabilities = { + primary: { docker: 'supported', gvisor: 'unavailable', sbx: 'unavailable' }, + query: { docker: 'supported', gvisor: 'unavailable', sbx: 'blocked' }, + }; + expect(evaluate('gvisor', 'docker', capabilities)).toEqual({ + status: 'BLOCKED', + capability: 'unavailable', + phase: 'primary-preflight', + }); + expect(evaluate('docker', 'gvisor', capabilities)).toEqual({ + status: 'BLOCKED', + capability: 'unavailable', + phase: 'query-preflight', + }); + expect(evaluate('docker', 'sbx', capabilities)).toEqual({ + status: 'BLOCKED', + capability: 'blocked', + phase: 'query-preflight', + }); + }); +}); diff --git a/scripts/ci/smoke-bounded-queries.sh b/scripts/ci/smoke-bounded-queries.sh index d035569a2..d69b4f1dd 100755 --- a/scripts/ci/smoke-bounded-queries.sh +++ b/scripts/ci/smoke-bounded-queries.sh @@ -164,6 +164,8 @@ JSON fi echo "::endgroup::" done + + node "$workspace/scripts/ci/report-bounded-query-runtime-matrix.js" --require docker/docker } if [[ "${1:-}" == "--inside-agent" ]]; then diff --git a/src/artifact-preservation.ts b/src/artifact-preservation.ts index cd5e8dcec..72d67ff63 100644 --- a/src/artifact-preservation.ts +++ b/src/artifact-preservation.ts @@ -7,8 +7,10 @@ import { fixArtifactPermissionsForRootless } from './artifact-permissions'; import { getLocalDockerEnv } from './host-env'; import { resolveBoundedQueryPaths } from './bounded-query/paths'; -const BOUNDED_QUERY_AUDIT_CONTAINER_PATH = - 'awf-bounded-query-broker:/var/log/awf-bounded-query/bounded-query.jsonl'; +const BOUNDED_QUERY_AUDIT_FILES = [ + 'bounded-query.jsonl', + 'runtime-telemetry.jsonl', +] as const; /** * Copies the iptables audit dump from the init-signal volume to the audit directory. @@ -32,20 +34,23 @@ export function preserveIptablesAudit(workDir: string, auditDir?: string): void } if (fs.existsSync(boundedQueryRoot)) { - try { - const destination = path.join(targetAuditDir, 'bounded-query.jsonl'); - const result = execa.sync( - 'docker', - ['cp', BOUNDED_QUERY_AUDIT_CONTAINER_PATH, destination], - { env: getLocalDockerEnv(), reject: false }, - ); - if (result.exitCode === 0) { - logger.debug('Copied bounded-query broker audit to audit directory'); - } else { - logger.debug('Could not copy bounded-query audit file:', result.stderr); + for (const auditFile of BOUNDED_QUERY_AUDIT_FILES) { + try { + const source = `awf-bounded-query-broker:/var/log/awf-bounded-query/${auditFile}`; + const destination = path.join(targetAuditDir, auditFile); + const result = execa.sync( + 'docker', + ['cp', source, destination], + { env: getLocalDockerEnv(), reject: false }, + ); + if (result.exitCode === 0) { + logger.debug(`Copied bounded-query broker ${auditFile} to audit directory`); + } else { + logger.debug(`Could not copy bounded-query ${auditFile}:`, result.stderr); + } + } catch (error) { + logger.debug(`Could not copy bounded-query ${auditFile}:`, error); } - } catch (error) { - logger.debug('Could not copy bounded-query audit file:', error); } } } diff --git a/src/bounded-query/manager.test.ts b/src/bounded-query/manager.test.ts index 66341161f..9a49196cd 100644 --- a/src/bounded-query/manager.test.ts +++ b/src/bounded-query/manager.test.ts @@ -222,6 +222,46 @@ describe('prepareBoundedQueries', () => { prepareBoundedQueries(buildConfig(workDir), { env: { GH_TOKEN: 't' }, gitRunner: failing }), ).rejects.toThrow(/staging failed/); }); + + it.each(['docker', 'gvisor', 'sbx'] as const)( + 'fails query runtime %s capability preflight before directories or staging', + async (runtime) => { + const assertRuntimeAvailable = jest.fn().mockRejectedValue(new Error(`${runtime} unavailable`)); + const probeSbxUnixSocket = jest.fn(); + const config = buildConfig(workDir, { runtime }); + await expect(prepareBoundedQueries(config, { + env: { GH_TOKEN: 't' }, + gitRunner, + assertRuntimeAvailable, + probeSbxUnixSocket, + })).rejects.toThrow(`${runtime} unavailable`); + expect(assertRuntimeAvailable).toHaveBeenCalledTimes(1); + expect(probeSbxUnixSocket).not.toHaveBeenCalled(); + expect(fs.existsSync(resolveBoundedQueryPaths(workDir).root)).toBe(false); + }, + ); + + it.each([undefined, 'gvisor', 'sbx'] as const)( + 'fails primary runtime %s capability preflight before query preflight or staging', + async (containerRuntime) => { + const assertPrimaryAvailable = jest.fn().mockRejectedValue(new Error('primary unavailable')); + const assertRuntimeAvailable = jest.fn(); + const probeSbxUnixSocket = jest.fn(); + await expect(prepareBoundedQueries( + { ...buildConfig(workDir), containerRuntime }, + { + env: { GH_TOKEN: 't' }, + gitRunner, + assertPrimaryAvailable, + assertRuntimeAvailable, + probeSbxUnixSocket, + }, + )).rejects.toThrow('primary unavailable'); + expect(assertRuntimeAvailable).not.toHaveBeenCalled(); + expect(probeSbxUnixSocket).not.toHaveBeenCalled(); + expect(fs.existsSync(resolveBoundedQueryPaths(workDir).root)).toBe(false); + }, + ); }); describe('teardownBoundedQueries', () => { diff --git a/src/bounded-query/manager.ts b/src/bounded-query/manager.ts index 30d1754df..107323ba4 100644 --- a/src/bounded-query/manager.ts +++ b/src/bounded-query/manager.ts @@ -10,7 +10,11 @@ import { resolveBoundedQueryPaths, type BoundedQueryPaths, } from './paths'; -import { assertQueryRuntimeAvailable, validateBoundedQueryConfig } from './preflight'; +import { + assertPrimaryRuntimeAvailable, + assertQueryRuntimeAvailable, + validateBoundedQueryConfig, +} from './preflight'; import { writeBoundedQuerySkill } from './skill'; import { writeBoundedQueryWrapper } from './wrapper-artifact'; import { releaseSeedPermissions, resolveStagingToken, stageBoundedQuerySeeds, type GitRunner } from './staging'; @@ -19,6 +23,10 @@ import { assertBoundedQueryPrivateRootIsolated } from './mount-policy'; import { fixArtifactPermissionsForRootless } from '../artifact-permissions'; import { runtimeUsesComposeAgent } from '../container-runtime'; import { probeSbxUnixSocketMount } from '../sbx-manager'; +import { + resolveBoundedQueryPrimaryBackend, + serializeBoundedQueryRuntimeTelemetry, +} from './runtime-matrix'; /** * Bounded-query lifecycle orchestration. @@ -138,6 +146,10 @@ export interface PrepareBoundedQueriesDeps { env?: NodeJS.ProcessEnv; /** Override the sbx Unix-socket passthrough probe (tests). */ probeSbxUnixSocket?: () => Promise; + /** Override query-runtime capability preflight (tests). */ + assertRuntimeAvailable?: typeof assertQueryRuntimeAvailable; + /** Override primary-runtime capability preflight (tests). */ + assertPrimaryAvailable?: typeof assertPrimaryRuntimeAvailable; } interface SbxIngressCapabilities { @@ -184,6 +196,46 @@ export async function prepareBoundedQueries( throw new Error(`Bounded-query configuration is invalid:\n - ${errors.join('\n - ')}`); } + const primaryBackend = resolveBoundedQueryPrimaryBackend(config.containerRuntime); + const telemetryBase = { + primaryBackend, + queryBackend: boundedQueries.runtime, + lifecycleClass: 'preflight' as const, + }; + const assertRuntimeAvailable = deps.assertRuntimeAvailable ?? assertQueryRuntimeAvailable; + const assertPrimaryAvailable = deps.assertPrimaryAvailable ?? assertPrimaryRuntimeAvailable; + try { + await assertPrimaryAvailable(config.containerRuntime); + } catch (error) { + logger.info( + `Bounded-query runtime telemetry: ${serializeBoundedQueryRuntimeTelemetry({ + ...telemetryBase, + capabilityState: 'unavailable', + category: 'primary-runtime-unavailable', + })}`, + ); + throw error; + } + try { + await assertRuntimeAvailable(boundedQueries); + } catch (error) { + logger.info( + `Bounded-query runtime telemetry: ${serializeBoundedQueryRuntimeTelemetry({ + ...telemetryBase, + capabilityState: boundedQueries.runtime === 'sbx' ? 'blocked' : 'unavailable', + category: boundedQueries.runtime === 'sbx' ? 'query-security-block' : 'query-runtime-unavailable', + })}`, + ); + throw error; + } + logger.info( + `Bounded-query runtime telemetry: ${serializeBoundedQueryRuntimeTelemetry({ + ...telemetryBase, + capabilityState: 'supported', + category: 'ready', + })}`, + ); + if (runtimeUsesComposeAgent(config.containerRuntime)) { config.boundedQueryIngressTransport = 'unix'; } else { @@ -194,8 +246,6 @@ export async function prepareBoundedQueries( const paths = resolveBoundedQueryPaths(config.workDir); assertBoundedQueryPrivateRootIsolated(config, paths, env); - await assertQueryRuntimeAvailable(boundedQueries); - const token = resolveStagingToken(env); if (!token) { // Already covered by validateBoundedQueryConfig; re-checked so the token is diff --git a/src/bounded-query/mount-policy.test.ts b/src/bounded-query/mount-policy.test.ts index 98e333672..ed10b0afa 100644 --- a/src/bounded-query/mount-policy.test.ts +++ b/src/bounded-query/mount-policy.test.ts @@ -138,7 +138,7 @@ describe('bounded-query private-root mount policy', () => { fs.mkdirSync(target); fs.symlinkSync(target, alias); expect(resolvePathThroughExistingAncestor(path.join(alias, 'missing', 'leaf'))) - .toBe(path.join(target, 'missing', 'leaf')); + .toBe(path.join(fs.realpathSync.native(target), 'missing', 'leaf')); }); it('rejects relative paths before filesystem resolution', () => { diff --git a/src/bounded-query/preflight.test.ts b/src/bounded-query/preflight.test.ts index ee1d4c26d..a8bd6c24a 100644 --- a/src/bounded-query/preflight.test.ts +++ b/src/bounded-query/preflight.test.ts @@ -1,6 +1,11 @@ import type { WrapperConfig } from '../types'; import execa from 'execa'; -import { assertQueryRuntimeAvailable, preflightTestHelpers, validateBoundedQueryConfig } from './preflight'; +import { + assertPrimaryRuntimeAvailable, + assertQueryRuntimeAvailable, + preflightTestHelpers, + validateBoundedQueryConfig, +} from './preflight'; import type { BoundedQueriesConfig } from '../types'; import type { BoundedQueryRepository } from '../types/bounded-query-options'; @@ -157,10 +162,25 @@ describe('validateBoundedQueryConfig', () => { }); describe('assertQueryRuntimeAvailable', () => { - it('does not query Docker for the default runtime', async () => { - const query = jest.fn(); - await expect(assertQueryRuntimeAvailable(baseBoundedQueries, query)).resolves.toBeUndefined(); - expect(query).not.toHaveBeenCalled(); + it('requires a reachable Docker daemon for the default query runtime', async () => { + const runtimeQuery = jest.fn(); + const dockerAvailable = jest.fn().mockResolvedValue(true); + await expect( + assertQueryRuntimeAvailable(baseBoundedQueries, runtimeQuery, jest.fn(), dockerAvailable), + ).resolves.toBeUndefined(); + expect(runtimeQuery).not.toHaveBeenCalled(); + expect(dockerAvailable).toHaveBeenCalledTimes(1); + }); + + it('fails closed when the Docker query daemon is unavailable', async () => { + await expect( + assertQueryRuntimeAvailable( + baseBoundedQueries, + jest.fn(), + jest.fn(), + jest.fn().mockResolvedValue(false), + ), + ).rejects.toThrow(/Docker daemon.*not available.*never fall back/s); }); it('accepts gvisor when runsc is registered with the daemon', async () => { @@ -254,6 +274,36 @@ describe('assertQueryRuntimeAvailable', () => { }); }); + describe('assertPrimaryRuntimeAvailable', () => { + it.each([ + [undefined, 'docker'], + ['gvisor', 'gvisor'], + ['runsc', 'gvisor'], + ['sbx', 'sbx'], + ] as const)('accepts an available %s primary backend (%s)', async (runtime, _backend) => { + await expect(assertPrimaryRuntimeAvailable( + runtime, + jest.fn().mockResolvedValue(true), + jest.fn().mockResolvedValue(true), + jest.fn().mockResolvedValue(true), + )).resolves.toBeUndefined(); + }); + + it.each([ + [undefined, /Docker primary-agent runtime is unavailable/], + ['gvisor', /Primary-agent runtime "gvisor".*runsc.*never fall back/s], + ['sbx', /Primary-agent runtime "sbx" is unavailable.*never fall back/s], + ['kata', /OCI runtime "kata" is not registered.*never fall back/s], + ] as const)('fails %s before staging when its primary capability is unavailable', async (runtime, message) => { + await expect(assertPrimaryRuntimeAvailable( + runtime, + jest.fn().mockResolvedValue(false), + jest.fn().mockResolvedValue(false), + jest.fn().mockResolvedValue(false), + )).rejects.toThrow(message); + }); + }); + it('requires authenticated sbx daemon reachability and preserves only its management environment', async () => { const savedToken = process.env.SBX_AUTH_TOKEN; const savedProxy = process.env.DOCKER_SANDBOXES_PROXY; @@ -289,4 +339,15 @@ describe('assertQueryRuntimeAvailable', () => { else process.env.XDG_CONFIG_HOME = savedXdg; } }); + + it('uses authenticated sbx listing for primary availability', async () => { + mockExeca.mockResolvedValueOnce({ exitCode: 0, stdout: '[]' }); + + await expect(preflightTestHelpers.defaultSbxAvailabilityQuery()).resolves.toBe(true); + expect(mockExeca).toHaveBeenCalledWith( + 'sbx', + ['ls'], + expect.objectContaining({ reject: false }), + ); + }); }); diff --git a/src/bounded-query/preflight.ts b/src/bounded-query/preflight.ts index 2f2ab8004..3172ab798 100644 --- a/src/bounded-query/preflight.ts +++ b/src/bounded-query/preflight.ts @@ -26,6 +26,10 @@ const GVISOR_DOCKER_RUNTIME = 'runsc'; /** Detects whether the Docker daemon exposes a named OCI runtime. */ export type DockerRuntimeQuery = (runtimeName: string) => Promise; +/** Detects whether the Docker daemon required by a primary/query backend is reachable. */ +export type DockerAvailabilityQuery = () => Promise; +/** Detects whether the sbx primary-agent runtime is installed and authenticated. */ +export type SbxAvailabilityQuery = () => Promise; export interface SbxCapabilityReport { supported: boolean; @@ -51,6 +55,31 @@ const defaultDockerRuntimeQuery: DockerRuntimeQuery = async (runtimeName) => { } }; +const defaultDockerAvailabilityQuery: DockerAvailabilityQuery = async () => { + const result = await execa('docker', ['info', '--format', '{{.ServerVersion}}'], { + env: getLocalDockerEnv(), + reject: false, + timeout: 30_000, + }); + return result.exitCode === 0; +}; + +const defaultSbxAvailabilityQuery: SbxAvailabilityQuery = async () => { + try { + const managementEnv = { ...process.env }; + delete managementEnv.DOCKER_SANDBOXES_PROXY; + delete managementEnv.XDG_CONFIG_HOME; + const result = await execa('sbx', ['ls'], { + reject: false, + timeout: 10_000, + env: managementEnv, + }); + return result.exitCode === 0; + } catch { + return false; + } +}; + const SBX_AUDITED_VERSION = '0.37.1'; const SBX_REQUIRED_CREATE_FLAGS = [ '--cpus', @@ -218,6 +247,7 @@ export async function assertQueryRuntimeAvailable( boundedQueries: BoundedQueriesConfig, queryDockerRuntime: DockerRuntimeQuery = defaultDockerRuntimeQuery, querySbxCapabilities: SbxCapabilityQuery = defaultSbxCapabilityQuery, + queryDockerAvailable: DockerAvailabilityQuery = defaultDockerAvailabilityQuery, ): Promise { if (boundedQueries.runtime === 'sbx') { const report = await querySbxCapabilities(); @@ -231,7 +261,15 @@ export async function assertQueryRuntimeAvailable( return; } - if (boundedQueries.runtime !== 'gvisor') return; + if (boundedQueries.runtime === 'docker') { + if (!(await queryDockerAvailable())) { + throw new Error( + 'boundedQueries.runtime "docker" requires a reachable Docker daemon. It is not available, ' + + 'and bounded queries never fall back to another runtime.', + ); + } + return; + } if (!(await queryDockerRuntime(GVISOR_DOCKER_RUNTIME))) { throw new Error( @@ -242,12 +280,55 @@ export async function assertQueryRuntimeAvailable( } } +/** Verifies the primary-agent runtime before bounded-query repository staging. */ +export async function assertPrimaryRuntimeAvailable( + containerRuntime: string | undefined, + queryDockerRuntime: DockerRuntimeQuery = defaultDockerRuntimeQuery, + queryDockerAvailable: DockerAvailabilityQuery = defaultDockerAvailabilityQuery, + querySbxAvailable: SbxAvailabilityQuery = defaultSbxAvailabilityQuery, +): Promise { + if (containerRuntime === 'sbx') { + if (!(await querySbxAvailable())) { + throw new Error( + 'Primary-agent runtime "sbx" is unavailable. Bounded queries abort before staging and never ' + + 'fall back to a Docker or gVisor primary agent.', + ); + } + return; + } + if (containerRuntime === 'gvisor' || containerRuntime === 'runsc') { + if (!(await queryDockerRuntime(GVISOR_DOCKER_RUNTIME))) { + throw new Error( + `Primary-agent runtime "${containerRuntime}" requires the "${GVISOR_DOCKER_RUNTIME}" OCI runtime. ` + + 'It is not available, so bounded queries abort before staging and never fall back.', + ); + } + return; + } + if (containerRuntime) { + if (!(await queryDockerRuntime(containerRuntime))) { + throw new Error( + `Primary-agent OCI runtime "${containerRuntime}" is not registered with Docker. ` + + 'Bounded queries abort before staging and never fall back.', + ); + } + return; + } + if (!(await queryDockerAvailable())) { + throw new Error( + 'The Docker primary-agent runtime is unavailable. Bounded queries abort before staging and never fall back.', + ); + } +} + /** @internal Exported for focused unit tests. */ // ts-prune-ignore-next export const preflightTestHelpers = { SUPPORTED_QUERY_RUNTIMES, GVISOR_DOCKER_RUNTIME, defaultDockerRuntimeQuery, + defaultDockerAvailabilityQuery, + defaultSbxAvailabilityQuery, defaultSbxCapabilityQuery, SBX_AUDITED_VERSION, SBX_REQUIRED_CREATE_FLAGS, diff --git a/src/bounded-query/runtime-matrix.test.ts b/src/bounded-query/runtime-matrix.test.ts new file mode 100644 index 000000000..82e8cd196 --- /dev/null +++ b/src/bounded-query/runtime-matrix.test.ts @@ -0,0 +1,363 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + BOUNDED_QUERY_RUNTIME_BACKENDS, + evaluateBoundedQueryRuntimeCombination, + resolveBoundedQueryPrimaryBackend, + serializeBoundedQueryRuntimeTelemetry, + type BoundedQueryPrimaryBackend, + type BoundedQueryRuntimeCapabilities, +} from './runtime-matrix'; + +/* eslint-disable @typescript-eslint/no-require-imports */ +const brokerDir = path.join(__dirname, '..', '..', 'containers', 'bounded-query', 'broker'); +const { createBroker } = require(path.join(brokerDir, 'broker.js')); +const { createQueryRunner } = require(path.join(brokerDir, 'query-runner.js')); +const { createRuntimeTelemetry } = require(path.join(brokerDir, 'runtime-telemetry.js')); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const CANONICAL_ERROR = '{"status":"error"}'; +const CANONICAL_OK = '{"status":"ok","result":true}'; +const BOOLEAN_SCHEMA = { type: 'boolean' }; +const PRIMARY_BACKENDS = BOUNDED_QUERY_RUNTIME_BACKENDS; +const QUERY_BACKENDS = BOUNDED_QUERY_RUNTIME_BACKENDS; + +const deterministicCapabilities: BoundedQueryRuntimeCapabilities = { + primary: { + docker: 'supported', + gvisor: 'supported', + sbx: 'supported', + }, + query: { + docker: 'supported', + gvisor: 'supported', + sbx: 'blocked', + }, +}; + +const combinations = PRIMARY_BACKENDS.flatMap((primaryBackend) => + QUERY_BACKENDS.map((queryBackend) => ({ primaryBackend, queryBackend }))); +const executableCombinations = combinations.filter(({ primaryBackend, queryBackend }) => + evaluateBoundedQueryRuntimeCombination(primaryBackend, queryBackend, deterministicCapabilities).supported); +const blockedCombinations = combinations.filter(({ primaryBackend, queryBackend }) => + !evaluateBoundedQueryRuntimeCombination(primaryBackend, queryBackend, deterministicCapabilities).supported); + +interface HarnessOptions { + maxInvocations?: number; + sensitivity?: 'public' | 'internal' | 'confidential'; + output?: string; + runnerResult?: { exitCode: number; timedOut: boolean }; + processingMs?: number; +} + +async function invoke( + broker: { handle: (request: unknown, respond: (json: string) => void) => Promise }, + request: unknown, +): Promise { + let response = ''; + await broker.handle(request, (json: string) => { + response = json; + }); + return response; +} + +function createHarness( + primaryBackend: BoundedQueryPrimaryBackend, + queryBackend: 'docker' | 'gvisor', + options: HarnessOptions = {}, +) { + const outputs = new Map(); + const launches: Array> = []; + const destroyed: string[] = []; + const telemetry: Array> = []; + let now = 0; + const sleeps: number[] = []; + const config = { + primaryBackend, + queryBackend, + workDir: '/broker/private/work', + timeoutSeconds: 30, + maxInvocations: options.maxInvocations ?? 8, + }; + const workspace = { + createInvocationWorkspace: ({ + invocationId, + seedId, + script, + }: { + invocationId: string; + seedId: string; + script: string; + }) => { + expect(seedId).toBe('a'.repeat(32)); + expect(script).not.toMatch(/TOKEN|PASSWORD|docker\.sock|broker\/private/); + return { outPath: invocationId }; + }, + readQueryOutput: (outPath: string) => { + const output = outputs.get(outPath); + return output !== undefined && Buffer.byteLength(output) <= 8192 ? output : undefined; + }, + destroyInvocationWorkspace: (_workDir: string, invocationId: string) => { + destroyed.push(invocationId); + outputs.delete(invocationId); + }, + }; + const runner = { + runQueryContainer: async (params: Record) => { + launches.push(params); + now += options.processingMs ?? 0; + outputs.set(String(params.invocationId), options.output ?? 'true'); + return { + exitCode: options.runnerResult?.exitCode ?? 0, + timedOut: options.runnerResult?.timedOut ?? false, + stdout: '', + stderr: '', + }; + }, + }; + const broker = createBroker({ + config, + seedMap: new Map([ + ['octo/repo', { seedId: 'a'.repeat(32), sensitivity: options.sensitivity ?? 'internal' }], + ]), + runId: 'abcd1234', + audit: { invocation() {}, failure() {}, lifecycle() {} }, + telemetry: { emit: (event: Record) => telemetry.push(event) }, + workspace, + runner, + clock: { + nowMs: () => now, + sleep: async (ms: number) => { + sleeps.push(ms); + now += ms; + }, + }, + }); + return { broker, destroyed, launches, sleeps, telemetry }; +} + +describe('bounded-query runtime conformance matrix', () => { + it('contains every independent primary/query combination exactly once', () => { + expect(combinations).toHaveLength(9); + expect(new Set(combinations.map(({ primaryBackend, queryBackend }) => + `${primaryBackend}/${queryBackend}`)).size).toBe(9); + expect(executableCombinations).toHaveLength(6); + expect(blockedCombinations).toHaveLength(3); + }); + + it.each(blockedCombinations)( + '$primaryBackend primary + $queryBackend query fails closed at query preflight', + ({ primaryBackend, queryBackend }) => { + const result = evaluateBoundedQueryRuntimeCombination( + primaryBackend, + queryBackend, + deterministicCapabilities, + ); + expect(result).toEqual({ + primaryBackend, + queryBackend, + supported: false, + capabilityState: 'blocked', + blockedAt: 'query-preflight', + category: 'query-security-block', + }); + }, + ); + + it.each([ + ['gvisor', 'docker', 'primary-preflight', 'primary-runtime-unavailable'], + ['sbx', 'docker', 'primary-preflight', 'primary-runtime-unavailable'], + ['docker', 'gvisor', 'query-preflight', 'query-runtime-unavailable'], + ] as const)( + 'reports precise unavailable capability state for %s/%s', + (primaryBackend, queryBackend, blockedAt, category) => { + const capabilities: BoundedQueryRuntimeCapabilities = { + primary: { docker: 'supported', gvisor: 'unavailable', sbx: 'unavailable' }, + query: { docker: 'supported', gvisor: 'unavailable', sbx: 'blocked' }, + }; + expect(evaluateBoundedQueryRuntimeCombination(primaryBackend, queryBackend, capabilities)) + .toMatchObject({ supported: false, capabilityState: 'unavailable', blockedAt, category }); + }, + ); + + it.each(executableCombinations)( + '$primaryBackend primary + $queryBackend query satisfies the common behavioral contract', + async ({ primaryBackend, queryBackend }) => { + if (queryBackend === 'sbx') throw new Error('blocked sbx query combination entered executable suite'); + + const successful = createHarness(primaryBackend, queryBackend, { processingMs: 50 }); + expect(await invoke(successful.broker, { + privateRepo: 'octo/repo', + schema: BOOLEAN_SCHEMA, + script: 'finite query', + })).toBe(CANONICAL_OK); + expect(successful.launches).toHaveLength(1); + expect(successful.destroyed).toHaveLength(1); + expect(successful.sleeps).toEqual([50]); + expect(successful.telemetry).toContainEqual({ + primaryBackend, + queryBackend, + lifecycleClass: 'query', + capabilityState: 'supported', + category: 'success', + }); + expect(successful.launches[0]).not.toHaveProperty('repo'); + expect(JSON.stringify(successful.launches[0])).not.toMatch(/TOKEN|PASSWORD|docker\.sock/); + + const publicRepo = createHarness(primaryBackend, queryBackend, { + sensitivity: 'public', + maxInvocations: 2, + }); + expect(await invoke(publicRepo.broker, { + privateRepo: 'octo/repo', + schema: BOOLEAN_SCHEMA, + script: 'public query', + })).toBe(CANONICAL_OK); + expect(await invoke(publicRepo.broker, { + privateRepo: 'octo/repo', + schema: BOOLEAN_SCHEMA, + script: 'second public query', + })).toBe(CANONICAL_OK); + expect(new Set(publicRepo.launches.map((launch) => launch.invocationId)).size).toBe(2); + expect(publicRepo.destroyed).toHaveLength(2); + + const wrongRepo = createHarness(primaryBackend, queryBackend); + expect(await invoke(wrongRepo.broker, { + privateRepo: 'octo/not-configured', + schema: BOOLEAN_SCHEMA, + script: 'must not launch', + })).toBe(CANONICAL_ERROR); + expect(wrongRepo.launches).toHaveLength(0); + + const exhausted = createHarness(primaryBackend, queryBackend, { sensitivity: 'confidential' }); + const expensiveSchema = { type: 'integer', minimum: 0, maximum: 255 }; + expect(await invoke(exhausted.broker, { + privateRepo: 'octo/repo', + schema: expensiveSchema, + script: 'must not launch', + })).toBe(CANONICAL_ERROR); + expect(exhausted.launches).toHaveLength(0); + + const capped = createHarness(primaryBackend, queryBackend, { maxInvocations: 1 }); + const request = { privateRepo: 'octo/repo', schema: BOOLEAN_SCHEMA, script: 'cap query' }; + expect(await invoke(capped.broker, request)).toBe(CANONICAL_OK); + expect(await invoke(capped.broker, request)).toBe(CANONICAL_ERROR); + expect(capped.launches).toHaveLength(1); + + for (const failure of [ + { output: '{malformed', runnerResult: undefined }, + { output: 'x'.repeat(8193), runnerResult: undefined }, + { output: 'true', runnerResult: { exitCode: 137, timedOut: true } }, + { output: 'true', runnerResult: { exitCode: 137, timedOut: false } }, // OOM + { output: 'true', runnerResult: { exitCode: 152, timedOut: false } }, // file-size + { output: 'true', runnerResult: { exitCode: 1, timedOut: false } }, // PID/disk + ]) { + const failed = createHarness(primaryBackend, queryBackend, failure); + // eslint-disable-next-line no-await-in-loop + expect(await invoke(failed.broker, request)).toBe(CANONICAL_ERROR); + expect(failed.destroyed).toHaveLength(1); + } + }, + ); + + it.each(executableCombinations)( + '$primaryBackend primary + $queryBackend query derives a fresh no-network sandbox', + async ({ primaryBackend: _primaryBackend, queryBackend }) => { + if (queryBackend === 'sbx') throw new Error('blocked sbx query combination entered executable suite'); + const dockerCalls: string[][] = []; + const docker = { + runDocker: async (args: readonly string[]) => { + dockerCalls.push([...args]); + if (args[0] === 'info') { + return { exitCode: 0, timedOut: false, stdout: '{"runsc":{}}', stderr: '' }; + } + return { exitCode: 0, timedOut: false, stdout: '', stderr: '' }; + }, + }; + const runner = createQueryRunner({ + queryBackend, + hostWorkDir: '/daemon/private/work', + queryMountDir: '/query', + queryScriptPath: '/awf/query-script.py', + querySeccompPath: '/opt/awf/query-seccomp.json', + queryImage: 'ghcr.io/example/bounded-query@sha256:abc', + memoryLimit: '256m', + timeoutSeconds: 30, + queryUid: 65534, + queryGid: 65534, + }, { docker }); + await runner.assertAvailable(); + const first = runner.spec('abcd1234', '1'.repeat(16)); + const second = runner.spec('abcd1234', '2'.repeat(16)); + expect(first.containerName).not.toBe(second.containerName); + expect(first.launchArgs).toEqual(expect.arrayContaining([ + '--network', 'none', + '--read-only', + '--cap-drop', 'ALL', + '--pids-limit', '128', + ])); + expect(first.launchArgs.join(' ')).not.toMatch(/docker\.sock|broker\.sock|seed-map|GH_TOKEN/); + expect(first.launchArgs.filter((arg: string) => arg === '-v')).toHaveLength(3); + if (queryBackend === 'gvisor') expect(first.launchArgs).toEqual(expect.arrayContaining(['--runtime', 'runsc'])); + if (queryBackend === 'docker') expect(first.launchArgs).not.toContain('--runtime'); + await runner.reconcileRun('abcd1234'); + expect(dockerCalls).toContainEqual([ + 'ps', + '-aq', + '--filter', + 'label=awf.bounded-query.run=abcd1234', + ]); + }, + ); +}); + +describe('bounded-query runtime telemetry', () => { + it('serializes only the five approved fields', () => { + const serialized = serializeBoundedQueryRuntimeTelemetry({ + primaryBackend: resolveBoundedQueryPrimaryBackend('runsc'), + queryBackend: 'docker', + lifecycleClass: 'preflight', + capabilityState: 'supported', + category: 'ready', + }); + expect(JSON.parse(serialized)).toEqual({ + primaryBackend: 'gvisor', + queryBackend: 'docker', + lifecycleClass: 'preflight', + capabilityState: 'supported', + category: 'ready', + }); + }); + + it('persists exact-field records without content, paths, outputs, or credentials', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'awf-runtime-telemetry-')); + try { + const telemetry = createRuntimeTelemetry(root); + telemetry.emit({ + primaryBackend: 'sbx', + queryBackend: 'docker', + lifecycleClass: 'query', + capabilityState: 'supported', + category: 'timeout', + repo: 'must-be-ignored', + script: 'must-be-ignored', + output: 'must-be-ignored', + path: '/must-be-ignored', + token: 'must-be-ignored', + capability: 'must-be-ignored', + }); + const record = JSON.parse(fs.readFileSync(path.join(root, 'runtime-telemetry.jsonl'), 'utf8')); + expect(Object.keys(record)).toEqual([ + 'primaryBackend', + 'queryBackend', + 'lifecycleClass', + 'capabilityState', + 'category', + ]); + expect(JSON.stringify(record)).not.toContain('must-be-ignored'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/bounded-query/runtime-matrix.ts b/src/bounded-query/runtime-matrix.ts new file mode 100644 index 000000000..744538e6a --- /dev/null +++ b/src/bounded-query/runtime-matrix.ts @@ -0,0 +1,96 @@ +import type { BoundedQueryRuntime } from '../types'; + +export const BOUNDED_QUERY_RUNTIME_BACKENDS = ['docker', 'gvisor', 'sbx'] as const; + +export type BoundedQueryPrimaryBackend = (typeof BOUNDED_QUERY_RUNTIME_BACKENDS)[number]; +export type BoundedQueryCapabilityState = 'supported' | 'unavailable' | 'blocked'; + +export interface BoundedQueryRuntimeCapabilities { + primary: Readonly>; + query: Readonly>; +} + +export interface BoundedQueryRuntimeCombination { + primaryBackend: BoundedQueryPrimaryBackend; + queryBackend: BoundedQueryRuntime; + supported: boolean; + capabilityState: BoundedQueryCapabilityState; + blockedAt?: 'primary-preflight' | 'query-preflight'; + category: 'ready' | 'primary-runtime-unavailable' | 'query-runtime-unavailable' | 'query-security-block'; +} + +export interface BoundedQueryRuntimeTelemetry { + primaryBackend: BoundedQueryPrimaryBackend; + queryBackend: BoundedQueryRuntime; + lifecycleClass: 'preflight' | 'startup' | 'query' | 'cleanup'; + capabilityState: BoundedQueryCapabilityState; + category: string; +} + +/** Maps AWF's execution setting to the independent primary-agent matrix axis. */ +export function resolveBoundedQueryPrimaryBackend( + containerRuntime: string | undefined, +): BoundedQueryPrimaryBackend { + if (containerRuntime === 'gvisor' || containerRuntime === 'runsc') return 'gvisor'; + if (containerRuntime === 'sbx') return 'sbx'; + return 'docker'; +} + +/** + * Evaluates one primary/query pair without fallback. + * + * Primary availability is checked first because the primary agent cannot be + * started without it. Query availability is then checked before any repository + * staging. A blocked query capability is distinct from an unavailable binary: + * it means the runtime exists but cannot enforce AWF's mandatory controls. + */ +export function evaluateBoundedQueryRuntimeCombination( + primaryBackend: BoundedQueryPrimaryBackend, + queryBackend: BoundedQueryRuntime, + capabilities: BoundedQueryRuntimeCapabilities, +): BoundedQueryRuntimeCombination { + const primaryState = capabilities.primary[primaryBackend]; + if (primaryState !== 'supported') { + return { + primaryBackend, + queryBackend, + supported: false, + capabilityState: primaryState, + blockedAt: 'primary-preflight', + category: 'primary-runtime-unavailable', + }; + } + + const queryState = capabilities.query[queryBackend]; + if (queryState !== 'supported') { + return { + primaryBackend, + queryBackend, + supported: false, + capabilityState: queryState, + blockedAt: 'query-preflight', + category: queryState === 'blocked' ? 'query-security-block' : 'query-runtime-unavailable', + }; + } + + return { + primaryBackend, + queryBackend, + supported: true, + capabilityState: 'supported', + category: 'ready', + }; +} + +/** Serializes the intentionally narrow, path- and content-free telemetry shape. */ +export function serializeBoundedQueryRuntimeTelemetry( + event: BoundedQueryRuntimeTelemetry, +): string { + return JSON.stringify({ + primaryBackend: event.primaryBackend, + queryBackend: event.queryBackend, + lifecycleClass: event.lifecycleClass, + capabilityState: event.capabilityState, + category: event.category, + }); +} diff --git a/src/bounded-query/wrapper.test.ts b/src/bounded-query/wrapper.test.ts index 31d3a823e..2219751f3 100644 --- a/src/bounded-query/wrapper.test.ts +++ b/src/bounded-query/wrapper.test.ts @@ -208,7 +208,7 @@ describe('bounded-query wrapper', () => { for (const result of results) { expect(result).toEqual({ stdout: `${CANONICAL_ERROR}\n`, stderr: '', status: 0 }); } - }); + }, 10_000); it.each([ 'http://host.docker.internal:0/query', diff --git a/src/docker-manager-diagnostics.test.ts b/src/docker-manager-diagnostics.test.ts index 20575a38a..f9261c58c 100644 --- a/src/docker-manager-diagnostics.test.ts +++ b/src/docker-manager-diagnostics.test.ts @@ -161,7 +161,7 @@ describe('docker-manager diagnostics', () => { expect(fs.existsSync(path.join(defaultAuditDir, 'iptables-audit.txt'))).toBe(true); }); - it('should copy the bounded-query broker audit before work directory cleanup', () => { + it('should copy bounded-query audit and safe telemetry before work directory cleanup', () => { const brokerAuditDir = resolveBoundedQueryPaths(getDir()).auditDir; fs.mkdirSync(brokerAuditDir, { recursive: true }); fs.writeFileSync( @@ -183,6 +183,15 @@ describe('docker-manager diagnostics', () => { ], expect.objectContaining({ reject: false }), ); + expect(mockExecaSync).toHaveBeenCalledWith( + 'docker', + [ + 'cp', + 'awf-bounded-query-broker:/var/log/awf-bounded-query/runtime-telemetry.jsonl', + path.join(auditDir, 'runtime-telemetry.jsonl'), + ], + expect.objectContaining({ reject: false }), + ); fs.rmSync(resolveBoundedQueryPaths(getDir()).root, { recursive: true, force: true }); }); }); diff --git a/src/services/bounded-query-service.test.ts b/src/services/bounded-query-service.test.ts index 5852d232d..c1f50d0b7 100644 --- a/src/services/bounded-query-service.test.ts +++ b/src/services/bounded-query-service.test.ts @@ -110,6 +110,7 @@ describe('buildBoundedQueryService', () => { expect(environment.AWF_BOUNDED_QUERY_MEMORY).toBe('256m'); expect(environment.AWF_BOUNDED_QUERY_MAX_INVOCATIONS).toBe('9'); expect(environment.AWF_BOUNDED_QUERY_BACKEND).toBe('docker'); + expect(environment.AWF_BOUNDED_QUERY_PRIMARY_BACKEND).toBe('docker'); expect(environment.AWF_BOUNDED_QUERY_HOST_WORK_DIR).toBe(paths.workDir); }); @@ -152,6 +153,20 @@ describe('buildBoundedQueryService', () => { expect((gvisorService.environment as Record).AWF_BOUNDED_QUERY_BACKEND).toBe('gvisor'); }); + it.each([ + [undefined, 'docker'], + ['gvisor', 'gvisor'], + ['runsc', 'gvisor'], + ['sbx', 'sbx'], + ])('records primary runtime %s independently from the query backend', (containerRuntime, expected) => { + const { service: matrixService } = buildBoundedQueryService({ + config: buildConfig({ containerRuntime }, { runtime: 'docker' }), + imageConfig: imageConfig(), + }); + expect((matrixService.environment as Record).AWF_BOUNDED_QUERY_PRIMARY_BACKEND).toBe(expected); + expect((matrixService.environment as Record).AWF_BOUNDED_QUERY_BACKEND).toBe('docker'); + }); + it('uses the AWF Docker host socket when overridden, without leaking it to the agent', () => { const result = buildBoundedQueryService({ config: buildConfig({ awfDockerHost: 'unix:///run/user/1001/docker.sock' }), diff --git a/src/services/bounded-query-service.ts b/src/services/bounded-query-service.ts index e7ef56fcf..d54b77656 100644 --- a/src/services/bounded-query-service.ts +++ b/src/services/bounded-query-service.ts @@ -27,6 +27,7 @@ import { BOUNDED_QUERY_INGRESS_NETWORK, BOUNDED_QUERY_TCP_PORT, } from '../bounded-query/ingress'; +import { resolveBoundedQueryPrimaryBackend } from '../bounded-query/runtime-matrix'; /** * Compose assembly for the trusted bounded-query broker. @@ -221,6 +222,7 @@ export function buildBoundedQueryService(params: BoundedQueryServiceParams): Bou // The broker selects a fixed QueryRunner from this normalized value. // Runtime flags are never accepted from an invocation. AWF_BOUNDED_QUERY_BACKEND: boundedQueries.runtime, + AWF_BOUNDED_QUERY_PRIMARY_BACKEND: resolveBoundedQueryPrimaryBackend(config.containerRuntime), AWF_BOUNDED_QUERY_TIMEOUT: String(boundedQueries.timeout), AWF_BOUNDED_QUERY_MEMORY: boundedQueries.memoryLimit, AWF_BOUNDED_QUERY_MAX_INVOCATIONS: String(boundedQueries.maxInvocations), diff --git a/tests/integration/bounded-query-isolation.test.ts b/tests/integration/bounded-query-isolation.test.ts index 62e46ef05..0f698905a 100644 --- a/tests/integration/bounded-query-isolation.test.ts +++ b/tests/integration/bounded-query-isolation.test.ts @@ -9,6 +9,7 @@ const { buildQueryArgs } = require('../../containers/bounded-query/broker/query- describe('bounded-query Docker isolation', () => { const image = `awf-bounded-query-integration:${process.pid}`; + const queryBackend = process.env.AWF_BOUNDED_QUERY_TEST_RUNTIME === 'gvisor' ? 'gvisor' : 'docker'; let root: string; beforeAll(() => { @@ -34,7 +35,7 @@ describe('bounded-query Docker isolation', () => { }); it('executes against a writable bounded copy with no network or broker tools', () => { - const invocationId = 'integration'; + const invocationId = `integration-${process.pid}`; const invocationDir = path.join(root, invocationId); const repoDir = path.join(invocationDir, 'repo'); const outPath = path.join(invocationDir, 'out'); @@ -83,19 +84,23 @@ describe('bounded-query Docker isolation', () => { queryScriptPath: '/awf/query-script.py', querySeccompPath: path.resolve(__dirname, '../../containers/bounded-query/query-seccomp.json'), queryImage: image, - queryBackend: 'docker', + queryBackend, memoryLimit: '256m', queryUid: 65534, queryGid: 65534, }, runId: 'integration-run', invocationId, - containerName: `awf-query-integration-${process.pid}`, + runtimeName: queryBackend === 'gvisor' ? 'runsc' : undefined, }); + const containerName = args[args.indexOf('--name') + 1]; - execFileSync('docker', args, { stdio: 'pipe', timeout: 30_000 }); - - expect(fs.readFileSync(outPath, 'utf8')).toBe('{"result":"YES"}'); - expect(fs.existsSync(path.join(repoDir, 'mutation.txt'))).toBe(false); + try { + execFileSync('docker', args, { stdio: 'pipe', timeout: 30_000 }); + expect(fs.readFileSync(outPath, 'utf8')).toBe('{"result":"YES"}'); + expect(fs.existsSync(path.join(repoDir, 'mutation.txt'))).toBe(false); + } finally { + execFileSync('docker', ['rm', '--force', containerName], { stdio: 'ignore' }); + } }, 60_000); });