From 4144266e78de6ab7a5403a3e13c100ef96ff9520 Mon Sep 17 00:00:00 2001 From: Salman Muin Kayser Chishti Date: Fri, 15 May 2026 13:58:25 +0000 Subject: [PATCH] fix(docker): apply docker-host-path-prefix to all compose service mounts The agent and iptables-init containers coordinate via a shared bind-mounted init-signal directory at /tmp/awf-init. The iptables-init container writes ready/output.log there after running setup-iptables.sh, and the agent's entrypoint waits for those files before continuing. buildAgentVolumes() applies dockerHostPathPrefix to its mount sources so the agent's /tmp/awf-init bind is daemon-resolvable on split runner/Docker daemon filesystems (e.g. ARC + DinD). buildIptablesInitService() did not, so once --docker-host-path-prefix was set the two containers bound to two different daemon-side directories. The init container could complete successfully and the agent would still time out after 30s with 'No init container output log found' because its bind target stayed empty. The same gap existed in the squid, api-proxy, and cli-proxy service builders: their bind-mount sources (squid logs, SSL cert/key/db, api-proxy logs, cli-proxy logs, optional DIFC CA cert) were never run through the prefix translation, so on ARC/DinD their logs would land in daemon-local directories and optional file mounts could fail when Docker auto-creates a directory at the unstaged source path. Extract normalize/translate/applyHostPathPrefixToVolumes into a shared host-path-prefix module and call applyHostPathPrefixToVolumes() at the end of every service builder's volume list construction. agent-volumes.ts delegates to the shared helper and re-exports the helpers for backwards compatibility. doh-proxy has no bind mounts and is unchanged. Add a parameterized symmetric invariant test that walks every bind mount on every compose service and asserts the prefix is applied uniformly when set (and skipped otherwise), so any future service builder is protected against the same class of asymmetric translation bug. --- src/compose-generator.ts | 1 + src/services/agent-service.test.ts | 94 ++++++++++++++++++++++++++++++ src/services/agent-service.ts | 20 ++++++- src/services/agent-volumes.ts | 59 +++++-------------- src/services/api-proxy-service.ts | 12 ++-- src/services/cli-proxy-service.ts | 16 +++-- src/services/host-path-prefix.ts | 65 +++++++++++++++++++++ src/services/squid-service.ts | 7 ++- 8 files changed, 216 insertions(+), 58 deletions(-) create mode 100644 src/services/host-path-prefix.ts diff --git a/src/compose-generator.ts b/src/compose-generator.ts index 0b582a040..cc30de322 100644 --- a/src/compose-generator.ts +++ b/src/compose-generator.ts @@ -147,6 +147,7 @@ export function generateDockerCompose( environment, networkConfig, initSignalDir, + dockerHostPathPrefix: config.dockerHostPathPrefix, }); // ── Assemble base services ───────────────────────────────────────────────── diff --git a/src/services/agent-service.test.ts b/src/services/agent-service.test.ts index ee2ccae63..aa77a1fe4 100644 --- a/src/services/agent-service.test.ts +++ b/src/services/agent-service.test.ts @@ -132,6 +132,100 @@ describe('agent service', () => { expect(initService.restart).toBe('no'); }); + it('should mount init-signal dir without translation when dockerHostPathPrefix is unset', () => { + const result = generateDockerCompose(mockConfig, mockNetworkConfig); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const initService = result.services['iptables-init'] as any; + const volumes = initService.volumes as string[]; + + // Source path is the runner-side init-signal dir, container path is /tmp/awf-init + expect(volumes).toContain(`${mockConfig.workDir}/init-signal:/tmp/awf-init:rw`); + }); + + it('should apply dockerHostPathPrefix to the iptables-init init-signal volume', () => { + // Regression: when --docker-host-path-prefix is set (e.g. ARC + DinD), the agent + // container's init-signal mount source is prefixed via translateBindMountHostPath. + // The iptables-init container's mount source must be prefixed identically — otherwise + // the two containers bind to different daemon-side directories and the agent times + // out with "No init container output log found" because the ready file written by + // setup-iptables.sh lands in a different bind-mount target. + const configWithPrefix = { + ...mockConfig, + dockerHostPathPrefix: '/host', + }; + const result = generateDockerCompose(configWithPrefix, mockNetworkConfig); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const initService = result.services['iptables-init'] as any; + const initVolumes = initService.volumes as string[]; + const agentVolumes = result.services.agent.volumes as string[]; + + const expectedSource = `/host${mockConfig.workDir}/init-signal`; + expect(initVolumes).toContain(`${expectedSource}:/tmp/awf-init:rw`); + + // The agent must mount the SAME daemon-side source so they share the ready file. + expect(agentVolumes).toContain(`${expectedSource}:/tmp/awf-init:rw`); + }); + + it('should normalize trailing slash in dockerHostPathPrefix for iptables-init mount', () => { + const configWithPrefix = { + ...mockConfig, + dockerHostPathPrefix: '/host/', + }; + const result = generateDockerCompose(configWithPrefix, mockNetworkConfig); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const initService = result.services['iptables-init'] as any; + const initVolumes = initService.volumes as string[]; + + expect(initVolumes).toContain(`/host${mockConfig.workDir}/init-signal:/tmp/awf-init:rw`); + }); + + // Symmetric invariant: every absolute, non-kernel-virtual bind-mount source on every + // service must be prefixed when dockerHostPathPrefix is set. This catches the original + // class of bug (asymmetric translation between services that share a daemon-side dir) + // for any future service builder, not just iptables-init. + describe.each([ + { name: 'unset', prefix: undefined as string | undefined, expectPrefixed: false }, + { name: 'empty', prefix: '', expectPrefixed: false }, + { name: 'whitespace', prefix: ' ', expectPrefixed: false }, + { name: '/host', prefix: '/host', expectPrefixed: true }, + { name: '/host/ (trailing slash)', prefix: '/host/', expectPrefixed: true }, + ])('symmetric prefix translation across compose services (dockerHostPathPrefix=$name)', ({ prefix, expectPrefixed }) => { + it('every absolute, non-kernel-virtual bind-mount source is prefixed consistently', () => { + const cfg = { + ...mockConfig, + dockerHostPathPrefix: prefix, + // Exercise sibling services that also build bind mounts. + enableApiProxy: true, + difcProxyHost: 'proxy.example.com:18443', + difcProxyCaCert: '/etc/ssl/ca.crt', + }; + const result = generateDockerCompose(cfg, mockNetworkConfig); + + const allVolumes: string[] = []; + for (const [, svc] of Object.entries(result.services)) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const volumes = (svc as any).volumes as string[] | undefined; + if (Array.isArray(volumes)) allVolumes.push(...volumes); + } + + // Sanity: at least one workDir-derived mount on every service we touched + expect(allVolumes.some(v => v.includes(mockConfig.workDir))).toBe(true); + + for (const mount of allVolumes) { + const [src] = mount.split(':'); + // Skip relative sources (named volumes) and the kernel virtual / /dev/null exemptions + if (!src.startsWith('/')) continue; + if (src === '/dev/null' || src.startsWith('/dev') || src.startsWith('/sys') || src.startsWith('/proc')) continue; + + if (expectPrefixed) { + expect(src).toMatch(/^\/host(\/|$)/); + } else { + expect(src).not.toMatch(/^\/host(\/|$)/); + } + } + }); + }); + it('should apply container hardening measures', () => { const result = generateDockerCompose(mockConfig, mockNetworkConfig); const agent = result.services.agent; diff --git a/src/services/agent-service.ts b/src/services/agent-service.ts index f938656d6..013f275a3 100644 --- a/src/services/agent-service.ts +++ b/src/services/agent-service.ts @@ -15,6 +15,7 @@ import { NetworkConfig, ImageBuildConfig } from './squid-service'; // Re-export functions for backwards compatibility export { buildAgentEnvironment } from './agent-environment'; export { buildAgentVolumes } from './agent-volumes'; +import { applyHostPathPrefixToVolumes } from './host-path-prefix'; // ─── Agent Service ──────────────────────────────────────────────────────────── @@ -206,6 +207,12 @@ interface IptablesInitServiceParams { environment: Record; networkConfig: NetworkConfig; initSignalDir: string; + // When the Docker daemon resolves bind-mount sources from a different filesystem + // view than the runner (e.g. ARC + DinD), translate the init-signal mount source + // through the same prefix used for agent volumes. Without this, the agent and + // iptables-init containers land on two different daemon-side directories and the + // ready/output.log handshake silently fails ("No init container output log found"). + dockerHostPathPrefix?: string; } /** @@ -214,7 +221,16 @@ interface IptablesInitServiceParams { * without ever granting NET_ADMIN to the agent itself. */ export function buildIptablesInitService(params: IptablesInitServiceParams): any { - const { agentService, environment, networkConfig, initSignalDir } = params; + const { agentService, environment, networkConfig, initSignalDir, dockerHostPathPrefix } = params; + + // The init-signal mount must use the same source path that the agent container uses, + // otherwise the two containers bind to different daemon-side directories and the + // ready-file handshake fails. buildAgentVolumes() applies dockerHostPathPrefix to its + // mounts, so do the same here via the shared helper. + const [initSignalMount] = applyHostPathPrefixToVolumes( + [`${initSignalDir}:/tmp/awf-init:rw`], + dockerHostPathPrefix, + ); // SECURITY: iptables init container - sets up NAT rules in a separate container // that shares the agent's network namespace but NEVER gives NET_ADMIN to the agent. @@ -225,7 +241,7 @@ export function buildIptablesInitService(params: IptablesInitServiceParams): any network_mode: 'service:agent', // Only mount the init signal volume and the iptables setup script volumes: [ - `${initSignalDir}:/tmp/awf-init:rw`, + initSignalMount, ], environment: { // Pass through environment variables needed by setup-iptables.sh diff --git a/src/services/agent-volumes.ts b/src/services/agent-volumes.ts index 103297681..bcce18abe 100644 --- a/src/services/agent-volumes.ts +++ b/src/services/agent-volumes.ts @@ -4,6 +4,19 @@ import execa from 'execa'; import { SslConfig } from '../host-env'; import { logger } from '../logger'; import { WrapperConfig } from '../types'; +import { + normalizeDockerHostPathPrefix, + translateBindMountHostPath, + applyHostPathPrefixToVolumes, +} from './host-path-prefix'; + +// Re-export for backwards compatibility — call sites that previously imported +// these helpers from agent-volumes.ts continue to work. +export { + normalizeDockerHostPathPrefix, + translateBindMountHostPath, + applyHostPathPrefixToVolumes, +}; // ─── Agent Volumes ──────────────────────────────────────────────────────────── @@ -18,47 +31,6 @@ interface AgentVolumesParams { initSignalDir: string; } -function normalizeDockerHostPathPrefix(prefix: string): string { - const trimmed = prefix.trim(); - if (!trimmed) return ''; - const withLeadingSlash = trimmed.startsWith('/') ? trimmed : `/${trimmed}`; - const withoutTrailingSlash = withLeadingSlash.replace(/\/+$/, ''); - return withoutTrailingSlash || '/'; -} - -function translateBindMountHostPath(mount: string, dockerHostPathPrefix: string): string { - const parts = mount.split(':'); - if (parts.length < 2 || parts.length > 3) { - return mount; - } - - const [hostPath, containerPath, mode] = parts; - if (!hostPath.startsWith('/')) { - return mount; - } - - // Skip kernel virtual filesystems — /dev, /sys, and /proc are provided by the - // Docker daemon's own kernel, not staged runner paths. Prefixing them would look - // for non-existent directories under the runner root. - // SECURITY: /dev/null must be preserved for credential-hiding overlays. - // /proc is not bind-mounted (it's a fresh procfs via mount -t proc in entrypoint.sh - // with hidepid=2), but is included defensively to prevent accidental exposure of - // /proc/*/environ which contains auth credentials. - if (hostPath === '/dev/null' || hostPath.startsWith('/dev') || hostPath.startsWith('/sys') || hostPath.startsWith('/proc')) { - return mount; - } - - if (dockerHostPathPrefix === '/') { - return mount; - } - - const translatedHostPath = hostPath === '/' - ? dockerHostPathPrefix - : `${dockerHostPathPrefix}${hostPath}`; - - return mode ? `${translatedHostPath}:${containerPath}:${mode}` : `${translatedHostPath}:${containerPath}`; -} - const DEFAULT_DOCKER_SOCKET_PATH = '/var/run/docker.sock'; function resolveDockerSocketPath(config: WrapperConfig): string { @@ -491,10 +463,7 @@ export function buildAgentVolumes(params: AgentVolumesParams): string[] { logger.debug(`Hidden ${chrootCredentialFiles.length} credential file(s) at /host paths`); if (config.dockerHostPathPrefix) { - const dockerHostPathPrefix = normalizeDockerHostPathPrefix(config.dockerHostPathPrefix); - if (dockerHostPathPrefix) { - return agentVolumes.map(mount => translateBindMountHostPath(mount, dockerHostPathPrefix)); - } + return applyHostPathPrefixToVolumes(agentVolumes, config.dockerHostPathPrefix); } return agentVolumes; diff --git a/src/services/api-proxy-service.ts b/src/services/api-proxy-service.ts index 2703d0f14..9ab4162e8 100644 --- a/src/services/api-proxy-service.ts +++ b/src/services/api-proxy-service.ts @@ -11,6 +11,7 @@ import { WrapperConfig, API_PROXY_PORTS, API_PROXY_HEALTH_PORT } from '../types' import { pickEnvVars } from '../env-utils'; import { COPILOT_PLACEHOLDER_TOKEN } from '../constants/placeholders'; import { NetworkConfig, ImageBuildConfig } from './squid-service'; +import { applyHostPathPrefixToVolumes } from './host-path-prefix'; interface ApiProxyBuildResult { /** The api-proxy service definition to add to Docker Compose services. */ @@ -70,10 +71,13 @@ export function buildApiProxyService(params: ApiProxyServiceParams): ApiProxyBui ipv4_address: networkConfig.proxyIp, }, }, - volumes: [ - // Mount log directory for api-proxy logs - `${apiProxyLogsPath}:/var/log/api-proxy:rw`, - ], + volumes: applyHostPathPrefixToVolumes( + [ + // Mount log directory for api-proxy logs + `${apiProxyLogsPath}:/var/log/api-proxy:rw`, + ], + config.dockerHostPathPrefix, + ), environment: { // Pass API keys securely to sidecar (not visible to agent) ...(config.openaiApiKey && { OPENAI_API_KEY: config.openaiApiKey }), diff --git a/src/services/cli-proxy-service.ts b/src/services/cli-proxy-service.ts index 559f73b9d..599544121 100644 --- a/src/services/cli-proxy-service.ts +++ b/src/services/cli-proxy-service.ts @@ -4,6 +4,7 @@ import { buildRuntimeImageRef } from '../image-tag'; import { logger } from '../logger'; import { WrapperConfig, CLI_PROXY_PORT } from '../types'; import { NetworkConfig, ImageBuildConfig } from './squid-service'; +import { applyHostPathPrefixToVolumes } from './host-path-prefix'; interface CliProxyBuildResult { /** The cli-proxy service definition to add to Docker Compose services. */ @@ -52,12 +53,15 @@ export function buildCliProxyService(params: CliProxyServiceParams): CliProxyBui }, // Enable host.docker.internal resolution for connecting to host DIFC proxy extra_hosts: ['host.docker.internal:host-gateway'], - volumes: [ - // Log directory for HTTP server logs - `${cliProxyLogsPath}:/var/log/cli-proxy:rw`, - // Mount host CA cert for TLS verification - ...(config.difcProxyCaCert ? [`${config.difcProxyCaCert}:/tmp/proxy-tls/ca.crt:ro`] : []), - ], + volumes: applyHostPathPrefixToVolumes( + [ + // Log directory for HTTP server logs + `${cliProxyLogsPath}:/var/log/cli-proxy:rw`, + // Mount host CA cert for TLS verification + ...(config.difcProxyCaCert ? [`${config.difcProxyCaCert}:/tmp/proxy-tls/ca.crt:ro`] : []), + ], + config.dockerHostPathPrefix, + ), environment: { // External DIFC proxy connection info for tcp-tunnel.js AWF_DIFC_PROXY_HOST: difcProxyHost, diff --git a/src/services/host-path-prefix.ts b/src/services/host-path-prefix.ts new file mode 100644 index 000000000..4331c7efa --- /dev/null +++ b/src/services/host-path-prefix.ts @@ -0,0 +1,65 @@ +// Helpers for rewriting Docker bind-mount source paths so the daemon can +// resolve them on split runner/Docker daemon filesystems (e.g. ARC + DinD). +// +// When the runner process and the Docker daemon do not share the same root +// filesystem, bind-mount sources resolved on the runner side are not visible +// to the daemon. The user can stage the runner filesystem (or part of it) +// under a known location inside the daemon (commonly /host) and pass +// `--docker-host-path-prefix /host` so AWF rewrites every bind-mount source +// from `/foo` to `/host/foo` before handing the compose file to docker. +// +// These helpers are shared by all service builders (agent, iptables-init, +// squid, api-proxy, cli-proxy) so the rewrite is symmetric across services +// that share daemon-side directories. + +export function normalizeDockerHostPathPrefix(prefix: string): string { + const trimmed = prefix.trim(); + if (!trimmed) return ''; + const withLeadingSlash = trimmed.startsWith('/') ? trimmed : `/${trimmed}`; + const withoutTrailingSlash = withLeadingSlash.replace(/\/+$/, ''); + return withoutTrailingSlash || '/'; +} + +export function translateBindMountHostPath(mount: string, dockerHostPathPrefix: string): string { + const parts = mount.split(':'); + if (parts.length < 2 || parts.length > 3) { + return mount; + } + + const [hostPath, containerPath, mode] = parts; + if (!hostPath.startsWith('/')) { + return mount; + } + + // Skip kernel virtual filesystems — /dev, /sys, and /proc are provided by the + // Docker daemon's own kernel, not staged runner paths. Prefixing them would look + // for non-existent directories under the runner root. + // SECURITY: /dev/null must be preserved for credential-hiding overlays. + // /proc is not bind-mounted (it's a fresh procfs via mount -t proc in entrypoint.sh + // with hidepid=2), but is included defensively to prevent accidental exposure of + // /proc/*/environ which contains auth credentials. + if (hostPath === '/dev/null' || hostPath.startsWith('/dev') || hostPath.startsWith('/sys') || hostPath.startsWith('/proc')) { + return mount; + } + + if (dockerHostPathPrefix === '/') { + return mount; + } + + const translatedHostPath = hostPath === '/' + ? dockerHostPathPrefix + : `${dockerHostPathPrefix}${hostPath}`; + + return mode ? `${translatedHostPath}:${containerPath}:${mode}` : `${translatedHostPath}:${containerPath}`; +} + +// Applies dockerHostPathPrefix translation to every bind mount in the list. +// Returns the input unchanged when no prefix is set or the prefix normalises +// to an empty string. Service builders call this at the end of their volume +// list construction so the rewrite is consistent across the compose stack. +export function applyHostPathPrefixToVolumes(volumes: string[], dockerHostPathPrefix: string | undefined): string[] { + if (!dockerHostPathPrefix) return volumes; + const normalized = normalizeDockerHostPathPrefix(dockerHostPathPrefix); + if (!normalized) return volumes; + return volumes.map(mount => translateBindMountHostPath(mount, normalized)); +} diff --git a/src/services/squid-service.ts b/src/services/squid-service.ts index 36252f6ab..033f4d0d5 100644 --- a/src/services/squid-service.ts +++ b/src/services/squid-service.ts @@ -3,6 +3,7 @@ import { SslConfig, SQUID_PORT, SQUID_CONTAINER_NAME } from '../host-env'; import { parseImageTag, buildRuntimeImageRef } from '../image-tag'; import { logger } from '../logger'; import { WrapperConfig } from '../types'; +import { applyHostPathPrefixToVolumes } from './host-path-prefix'; /** Network configuration passed to service builders */ export interface NetworkConfig { @@ -56,6 +57,10 @@ export function buildSquidService(params: SquidServiceParams): any { squidVolumes.push(`${sslConfig.sslDbPath}:/var/spool/squid_ssl_db:rw`); } + // Apply --docker-host-path-prefix to all bind-mount sources so the daemon + // can resolve them on split runner/Docker daemon filesystems (e.g. ARC + DinD). + const translatedSquidVolumes = applyHostPathPrefixToVolumes(squidVolumes, config.dockerHostPathPrefix); + // Squid service configuration const squidService: any = { container_name: SQUID_CONTAINER_NAME, @@ -64,7 +69,7 @@ export function buildSquidService(params: SquidServiceParams): any { ipv4_address: networkConfig.squidIp, }, }, - volumes: squidVolumes, + volumes: translatedSquidVolumes, healthcheck: { test: ['CMD', 'nc', '-z', 'localhost', '3128'], interval: '1s',