diff --git a/docs/arc-dind.md b/docs/arc-dind.md index a40ae1a34..5b8c5082b 100644 --- a/docs/arc-dind.md +++ b/docs/arc-dind.md @@ -52,7 +52,7 @@ services: depends_on: sysroot-stage: { condition: service_completed_successfully } volumes: - - sysroot:/host:ro + - sysroot:/host:rw - /tmp/gh-aw/tool-cache:/host/tmp/gh-aw/tool-cache:ro volumes: @@ -120,6 +120,10 @@ For fine-grained control (or when not using `runner.topology`): "path": "/usr/local/bin/copilot", "targetPath": "/usr/local/bin/copilot" } + }, + "runner": { + "topology": "arc-dind", + "sysrootImage": "ghcr.io/github/gh-aw-firewall/build-tools:latest" } } ``` @@ -132,6 +136,24 @@ For fine-grained control (or when not using `runner.topology`): - `dind.stageEngineBinary`: copies an engine binary from the runner path into daemon-visible filesystem before compose startup. - `dind.stagingImage`: image used for short-lived staging containers. - `dind.workDir`: target root for DinD pre-staged directory tree (`/tmp/gh-aw` default). +- `runner.topology: "arc-dind"`: enables sysroot staging (`sysroot-stage` init service + `sysroot` volume mounted on agent at `/host:rw`). +- `runner.sysrootImage`: optional override for the sysroot image used by `runner.topology=arc-dind`. + +## Build-tools sysroot image + +When `runner.topology` is `arc-dind`, AWF starts a one-shot `sysroot-stage` service that copies +the filesystem from a build-tools image derived from the same `--image-registry` and `--image-tag` +settings as the other AWF containers (unless `runner.sysrootImage` overrides it) into a named +`sysroot` volume. The agent mounts that volume at `/host:rw`. + +This image pre-installs root-required system build dependencies (for example gcc/make/cmake, +libssl-dev/libc6-dev/libicu-dev, capsh/gosu/gh) so ARC workflow steps can stay non-root. + +## Tool cache path guidance for ARC + +If `RUNNER_TOOL_CACHE` points under `/opt` (for example `/opt/hostedtoolcache`) AWF logs a warning +in `runner.topology=arc-dind` mode because `/opt` is commonly not visible from the DinD daemon +filesystem. Prefer a shared runner/daemon path under `/tmp/gh-aw` when possible. ## Auto-detection of split filesystem setups diff --git a/src/commands/validators/network-options.test.ts b/src/commands/validators/network-options.test.ts index 723606b5d..f9ef9ef3b 100644 --- a/src/commands/validators/network-options.test.ts +++ b/src/commands/validators/network-options.test.ts @@ -60,9 +60,17 @@ function makeDefaultMocks() { } describe('validateNetworkOptions', () => { + const savedRunnerToolCache = process.env.RUNNER_TOOL_CACHE; + beforeEach(() => { jest.clearAllMocks(); makeDefaultMocks(); + delete process.env.RUNNER_TOOL_CACHE; + }); + + afterAll(() => { + if (savedRunnerToolCache === undefined) delete process.env.RUNNER_TOOL_CACHE; + else process.env.RUNNER_TOOL_CACHE = savedRunnerToolCache; }); describe('happy path', () => { @@ -261,4 +269,22 @@ describe('validateNetworkOptions', () => { expect(result.dnsOverHttps).toBe('https://1.1.1.1/dns-query'); }); }); + + describe('arc-dind RUNNER_TOOL_CACHE warnings', () => { + it('warns when RUNNER_TOOL_CACHE is under /opt in arc-dind topology', () => { + process.env.RUNNER_TOOL_CACHE = '/opt/hostedtoolcache'; + validateNetworkOptions({ runnerTopology: 'arc-dind' }); + + const warnCalls = (logger.warn as jest.Mock).mock.calls.map((c: string[]) => c[0]); + expect(warnCalls.some((m: string) => m.includes('RUNNER_TOOL_CACHE is under /opt'))).toBe(true); + }); + + it('does not warn when topology is not arc-dind', () => { + process.env.RUNNER_TOOL_CACHE = '/opt/hostedtoolcache'; + validateNetworkOptions({}); + + const warnCalls = (logger.warn as jest.Mock).mock.calls.map((c: string[]) => c[0]); + expect(warnCalls.some((m: string) => m.includes('RUNNER_TOOL_CACHE is under /opt'))).toBe(false); + }); + }); }); diff --git a/src/commands/validators/network-options.ts b/src/commands/validators/network-options.ts index 0f97e23a2..55d30430a 100644 --- a/src/commands/validators/network-options.ts +++ b/src/commands/validators/network-options.ts @@ -78,6 +78,18 @@ export function validateNetworkOptions(options: Record): Networ ); } + if (options.runnerTopology === 'arc-dind') { + const runnerToolCache = process.env.RUNNER_TOOL_CACHE?.trim(); + if (runnerToolCache === '/opt' || runnerToolCache?.startsWith('/opt/')) { + logger.warn( + '⚠️ RUNNER_TOOL_CACHE is under /opt, which is typically invisible to DinD daemons in ARC.', + ); + logger.warn( + ' Prefer a runner-visible shared path (for example under /tmp/gh-aw) for tool-cache mounts.', + ); + } + } + // --- Domain resolution -------------------------------------------------- // Resolve allowed and blocked domains (parse, merge, validate) diff --git a/src/compose-generator.test.ts b/src/compose-generator.test.ts index 3cf5ef403..ff8a54eee 100644 --- a/src/compose-generator.test.ts +++ b/src/compose-generator.test.ts @@ -354,11 +354,11 @@ describe('generateDockerCompose', () => { expect(result.volumes!.sysroot).toEqual({}); }); - it('adds sysroot:/host:ro to agent volumes', () => { + it('adds sysroot:/host:rw to agent volumes', () => { const config = { ...mockConfig, runnerTopology: 'arc-dind' as const }; const result = generateDockerCompose(config, mockNetworkConfig); - expect(result.services.agent.volumes).toContain('sysroot:/host:ro'); + expect(result.services.agent.volumes).toContain('sysroot:/host:rw'); }); it('does not retain base-system bind mounts that shadow sysroot', () => { @@ -375,16 +375,16 @@ describe('generateDockerCompose', () => { expect(volumes).not.toContain('/lib:/host/lib:ro'); expect(volumes).not.toContain('/lib64:/host/lib64:ro'); expect(volumes).not.toContain('/opt:/host/opt:ro'); - expect(volumes).not.toContain('/sys:/host/sys:ro'); - expect(volumes).not.toContain('/dev:/host/dev:ro'); + expect(volumes).toContain('/sys:/host/sys:ro'); + expect(volumes).toContain('/dev:/host/dev:ro'); expect(volumes.some(v => v.includes(':/host/usr:ro'))).toBe(false); expect(volumes.some(v => v.includes(':/host/bin:ro'))).toBe(false); expect(volumes.some(v => v.includes(':/host/sbin:ro'))).toBe(false); expect(volumes.some(v => v.includes(':/host/lib:ro'))).toBe(false); expect(volumes.some(v => v.includes(':/host/lib64:ro'))).toBe(false); expect(volumes.some(v => v.includes(':/host/opt:ro'))).toBe(false); - expect(volumes.some(v => v.includes(':/host/sys:ro'))).toBe(false); - expect(volumes.some(v => v.includes(':/host/dev:ro'))).toBe(false); + expect(volumes.filter(v => v.endsWith(':/host/sys:ro'))).toEqual(['/sys:/host/sys:ro']); + expect(volumes.filter(v => v.endsWith(':/host/dev:ro'))).toEqual(['/dev:/host/dev:ro']); }); it('does not declare sysroot volume when topology is standard', () => { diff --git a/src/compose-generator.ts b/src/compose-generator.ts index 870ef2dc9..a3a8ab864 100644 --- a/src/compose-generator.ts +++ b/src/compose-generator.ts @@ -124,8 +124,6 @@ export function generateDockerCompose( '/host/lib', '/host/lib64', '/host/opt', - '/host/sys', - '/host/dev', ]); const filteredVolumes = agentVolumes.filter(volume => { const target = volume.split(':')[1]; @@ -263,15 +261,17 @@ export function generateDockerCompose( // ── Final compose result ─────────────────────────────────────────────────── // When sysroot staging is active, declare the named volume and mount it - // on the agent at /host (replacing the per-directory system bind mounts). + // on the agent at /host (replacing the per-directory userspace bind mounts, + // while /sys and /dev remain live host mounts). const namedVolumes: Record | undefined = sysrootActive ? { sysroot: {} } : undefined; if (sysrootActive) { - // The sysroot named volume provides /host content (system binaries, libs, etc.) - // via the sysroot-stage init container instead of per-directory bind mounts. - agentVolumes.push('sysroot:/host:ro'); + // The sysroot named volume provides most /host content (system binaries, + // libs, etc.) via the sysroot-stage init container instead of per-directory + // userspace bind mounts. + agentVolumes.push('sysroot:/host:rw'); } if (networkIsolation) { diff --git a/src/config-file-validation.test.ts b/src/config-file-validation.test.ts index 55f0e5623..fc6d9e4a6 100644 --- a/src/config-file-validation.test.ts +++ b/src/config-file-validation.test.ts @@ -399,6 +399,21 @@ describe('validateAwfFileConfig', () => { expect(errors).toContain('config.container.unknown is not supported'); }); + it('rejects non-object runner', () => { + const errors = validateAwfFileConfig({ runner: 'invalid' }); + expect(errors).toContain('config.runner must be an object'); + }); + + it('rejects invalid runner field types', () => { + expect(validateAwfFileConfig({ runner: { topology: 'invalid' } })).toContain('config.runner.topology must be one of: standard, arc-dind'); + expect(validateAwfFileConfig({ runner: { sysrootImage: 123 } })).toContain('config.runner.sysrootImage must be a string'); + }); + + it('rejects unknown runner keys', () => { + const errors = validateAwfFileConfig({ runner: { unknown: true } }); + expect(errors).toContain('config.runner.unknown is not supported'); + }); + it('rejects non-object chroot', () => { const errors = validateAwfFileConfig({ chroot: 'invalid' }); expect(errors).toContain('config.chroot must be an object'); diff --git a/src/etc-mounts-branches.test.ts b/src/etc-mounts-branches.test.ts index 5cc81fbde..dfd917850 100644 --- a/src/etc-mounts-branches.test.ts +++ b/src/etc-mounts-branches.test.ts @@ -286,6 +286,14 @@ describe('system-mounts branch coverage', () => { expect(mounts).toContain('/custom/tools:/host/tmp/awf-runner-bin:ro'); }); + it('keeps live /sys and /dev mounts when sysroot mode is enabled', () => { + const mounts = buildSystemMounts('/workspace', undefined, true); + expect(mounts).toContain('/sys:/host/sys:ro'); + expect(mounts).toContain('/dev:/host/dev:ro'); + expect(mounts).not.toContain('/usr:/host/usr:ro'); + expect(mounts).not.toContain('/bin:/host/bin:ro'); + }); + it('excludes runner-bin mount when chrootBinariesSourcePath is whitespace-only', () => { const mounts = buildSystemMounts('/workspace', ' '); expect(mounts.every(m => !m.includes('awf-runner-bin'))).toBe(true); diff --git a/src/schema.test.ts b/src/schema.test.ts index 21ebda851..f9d866f10 100644 --- a/src/schema.test.ts +++ b/src/schema.test.ts @@ -244,6 +244,13 @@ describe('awf-config.schema.json', () => { expect(validate({ container: { runnerToolCachePath: 123 } })).toBe(false); }); + it('accepts runner.topology and runner.sysrootImage', () => { + expect(validate({ runner: { topology: 'arc-dind' } })).toBe(true); + expect(validate({ runner: { topology: 'invalid' } })).toBe(false); + expect(validate({ runner: { sysrootImage: 'ghcr.io/github/gh-aw-firewall/build-tools:latest' } })).toBe(true); + expect(validate({ runner: { sysrootImage: 123 } })).toBe(false); + }); + it('validates chroot.identity fields', () => { expect(validate({ chroot: { identity: { home: '/tmp/gh-aw/home', user: 'runner', uid: 1001, gid: 1001 } } })).toBe(true); expect(validate({ chroot: { identity: { uid: 1.2 } } })).toBe(false); diff --git a/src/services/agent-volumes/system-mounts.ts b/src/services/agent-volumes/system-mounts.ts index 8c3198c49..c13a6abe4 100644 --- a/src/services/agent-volumes/system-mounts.ts +++ b/src/services/agent-volumes/system-mounts.ts @@ -10,16 +10,27 @@ function normalizeChrootBinariesSourcePath(chrootBinariesSourcePath?: string): s return normalized === '/' ? undefined : normalized; } -export function buildSystemMounts(workspaceDir: string, chrootBinariesSourcePath?: string): string[] { +export function buildSystemMounts( + workspaceDir: string, + chrootBinariesSourcePath?: string, + useSysroot = false +): string[] { const mounts = [ - '/usr:/host/usr:ro', - '/bin:/host/bin:ro', - '/sbin:/host/sbin:ro', - '/lib:/host/lib:ro', - '/lib64:/host/lib64:ro', - '/opt:/host/opt:ro', - '/sys:/host/sys:ro', - '/dev:/host/dev:ro', + ...(useSysroot + ? [ + '/sys:/host/sys:ro', + '/dev:/host/dev:ro', + ] + : [ + '/usr:/host/usr:ro', + '/bin:/host/bin:ro', + '/sbin:/host/sbin:ro', + '/lib:/host/lib:ro', + '/lib64:/host/lib64:ro', + '/opt:/host/opt:ro', + '/sys:/host/sys:ro', + '/dev:/host/dev:ro', + ]), `${workspaceDir}:/host${workspaceDir}:rw`, '/tmp:/host/tmp:rw', ]; diff --git a/src/services/agent-volumes/volume-builder.ts b/src/services/agent-volumes/volume-builder.ts index 696e9105f..e74bea9be 100644 --- a/src/services/agent-volumes/volume-builder.ts +++ b/src/services/agent-volumes/volume-builder.ts @@ -38,7 +38,8 @@ export function buildAgentVolumes(params: AgentVolumesParams): string[] { logger.debug('Using selective path mounts for security'); - agentVolumes.push(...buildSystemMounts(workspaceDir, config.chrootBinariesSourcePath)); + const useSysroot = config.runnerTopology === 'arc-dind'; + agentVolumes.push(...buildSystemMounts(workspaceDir, config.chrootBinariesSourcePath, useSysroot)); agentVolumes.push(...buildHomeMounts({ config, effectiveHome, agentLogsPath, sessionStatePath })); agentVolumes.push(...buildEtcMounts(config)); agentVolumes.push(generateHostsFileMount(config)); diff --git a/src/types/platform-options.ts b/src/types/platform-options.ts index fb6ba6f1c..b0344af8a 100644 --- a/src/types/platform-options.ts +++ b/src/types/platform-options.ts @@ -1,9 +1,7 @@ /** - * GitHub platform deployment type and runner topology options. + * GitHub platform deployment type options. */ -export type RunnerTopology = 'standard' | 'arc-dind'; - export interface PlatformOptions { /** * The GitHub deployment type. Explicitly declares the environment so AWF can @@ -19,30 +17,4 @@ export interface PlatformOptions { * regardless of the resolved API target hostname. */ platformType?: 'github.com' | 'ghes' | 'ghec' | 'ghec-self-hosted'; - - /** - * Runner deployment topology. - * - * - 'standard' (default) — GitHub-hosted VM or self-hosted runner with local Docker. - * - 'arc-dind' — ARC (Actions Runner Controller) with Docker-in-Docker sidecar, - * where the runner and Docker daemon have separate filesystems. - * - * When set to 'arc-dind', AWF applies overridable defaults: - * - network.isolation = true (ARC k8s lacks NET_ADMIN) - * - dind.preStageDirs = true - * - Sysroot image activation (build-tools init container) - * - Tool cache validation (warns if under /opt) - */ - runnerTopology?: RunnerTopology; - - /** - * Container image providing system-level build tools (gcc, make, libraries) - * for the agent's chroot base on ARC/DinD. - * - * Used as an init container that copies its filesystem into a named volume - * mounted at /host. Only used when runnerTopology is 'arc-dind'. - * - * Defaults to 'ghcr.io/github/gh-aw-firewall/build-tools:'. - */ - sysrootImage?: string; } diff --git a/src/types/runner-options.ts b/src/types/runner-options.ts new file mode 100644 index 000000000..fc9970d90 --- /dev/null +++ b/src/types/runner-options.ts @@ -0,0 +1,20 @@ +/** + * Runner topology configuration options. + */ +export interface RunnerOptions { + /** + * Runner topology mode for AWF compose generation. + * + * - 'standard' (default) - GitHub-hosted VM or self-hosted runner with local Docker. + * - 'arc-dind' - ARC with Docker-in-Docker sidecar, enables sysroot staging + * for split runner/daemon filesystems. + */ + runnerTopology?: 'standard' | 'arc-dind'; + + /** + * Sysroot image used by arc-dind topology to stage build tools into /host. + * + * @default 'ghcr.io/github/gh-aw-firewall/build-tools:latest' + */ + sysrootImage?: string; +} diff --git a/src/types/wrapper-config.ts b/src/types/wrapper-config.ts index ad7ce952b..b5ef3fd06 100644 --- a/src/types/wrapper-config.ts +++ b/src/types/wrapper-config.ts @@ -14,6 +14,7 @@ import type { CliProxyOptions } from './cli-proxy-options'; import type { RateLimitOptions } from './rate-limit-options'; import type { RuntimeOptions } from './runtime-options'; import type { PlatformOptions } from './platform-options'; +import type { RunnerOptions } from './runner-options'; export type WrapperConfig = ContainerImageOptions @@ -24,4 +25,5 @@ export type WrapperConfig = & CliProxyOptions & RateLimitOptions & RuntimeOptions - & PlatformOptions; + & PlatformOptions + & RunnerOptions;