Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion docs/arc-dind.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ services:
depends_on:
sysroot-stage: { condition: service_completed_successfully }
volumes:
- sysroot:/host:ro
- sysroot:/host:rw
- /tmp/gh-aw/tool-cache:/host/tmp/gh-aw/tool-cache:ro

volumes:
Expand Down Expand Up @@ -120,6 +120,10 @@ For fine-grained control (or when not using `runner.topology`):
"path": "/usr/local/bin/copilot",
"targetPath": "/usr/local/bin/copilot"
}
},
"runner": {
"topology": "arc-dind",
"sysrootImage": "ghcr.io/github/gh-aw-firewall/build-tools:latest"
}
}
```
Expand All @@ -132,6 +136,24 @@ For fine-grained control (or when not using `runner.topology`):
- `dind.stageEngineBinary`: copies an engine binary from the runner path into daemon-visible filesystem before compose startup.
- `dind.stagingImage`: image used for short-lived staging containers.
- `dind.workDir`: target root for DinD pre-staged directory tree (`/tmp/gh-aw` default).
- `runner.topology: "arc-dind"`: enables sysroot staging (`sysroot-stage` init service + `sysroot` volume mounted on agent at `/host:rw`).
- `runner.sysrootImage`: optional override for the sysroot image used by `runner.topology=arc-dind`.

## Build-tools sysroot image

When `runner.topology` is `arc-dind`, AWF starts a one-shot `sysroot-stage` service that copies
the filesystem from a build-tools image derived from the same `--image-registry` and `--image-tag`
settings as the other AWF containers (unless `runner.sysrootImage` overrides it) into a named
`sysroot` volume. The agent mounts that volume at `/host:rw`.

This image pre-installs root-required system build dependencies (for example gcc/make/cmake,
libssl-dev/libc6-dev/libicu-dev, capsh/gosu/gh) so ARC workflow steps can stay non-root.

## Tool cache path guidance for ARC

If `RUNNER_TOOL_CACHE` points under `/opt` (for example `/opt/hostedtoolcache`) AWF logs a warning
in `runner.topology=arc-dind` mode because `/opt` is commonly not visible from the DinD daemon
filesystem. Prefer a shared runner/daemon path under `/tmp/gh-aw` when possible.

## Auto-detection of split filesystem setups

Expand Down
26 changes: 26 additions & 0 deletions src/commands/validators/network-options.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,17 @@ function makeDefaultMocks() {
}

describe('validateNetworkOptions', () => {
const savedRunnerToolCache = process.env.RUNNER_TOOL_CACHE;

beforeEach(() => {
jest.clearAllMocks();
makeDefaultMocks();
delete process.env.RUNNER_TOOL_CACHE;
});

afterAll(() => {
if (savedRunnerToolCache === undefined) delete process.env.RUNNER_TOOL_CACHE;
else process.env.RUNNER_TOOL_CACHE = savedRunnerToolCache;
});

describe('happy path', () => {
Expand Down Expand Up @@ -261,4 +269,22 @@ describe('validateNetworkOptions', () => {
expect(result.dnsOverHttps).toBe('https://1.1.1.1/dns-query');
});
});

describe('arc-dind RUNNER_TOOL_CACHE warnings', () => {
it('warns when RUNNER_TOOL_CACHE is under /opt in arc-dind topology', () => {
process.env.RUNNER_TOOL_CACHE = '/opt/hostedtoolcache';
validateNetworkOptions({ runnerTopology: 'arc-dind' });

const warnCalls = (logger.warn as jest.Mock).mock.calls.map((c: string[]) => c[0]);
expect(warnCalls.some((m: string) => m.includes('RUNNER_TOOL_CACHE is under /opt'))).toBe(true);
});

it('does not warn when topology is not arc-dind', () => {
process.env.RUNNER_TOOL_CACHE = '/opt/hostedtoolcache';
validateNetworkOptions({});

const warnCalls = (logger.warn as jest.Mock).mock.calls.map((c: string[]) => c[0]);
expect(warnCalls.some((m: string) => m.includes('RUNNER_TOOL_CACHE is under /opt'))).toBe(false);
});
});
});
12 changes: 12 additions & 0 deletions src/commands/validators/network-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,18 @@ export function validateNetworkOptions(options: Record<string, unknown>): Networ
);
}

if (options.runnerTopology === 'arc-dind') {
const runnerToolCache = process.env.RUNNER_TOOL_CACHE?.trim();
if (runnerToolCache === '/opt' || runnerToolCache?.startsWith('/opt/')) {
logger.warn(
'⚠️ RUNNER_TOOL_CACHE is under /opt, which is typically invisible to DinD daemons in ARC.',
);
logger.warn(
' Prefer a runner-visible shared path (for example under /tmp/gh-aw) for tool-cache mounts.',
);
}
}

// --- Domain resolution --------------------------------------------------

// Resolve allowed and blocked domains (parse, merge, validate)
Expand Down
12 changes: 6 additions & 6 deletions src/compose-generator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -354,11 +354,11 @@ describe('generateDockerCompose', () => {
expect(result.volumes!.sysroot).toEqual({});
});

it('adds sysroot:/host:ro to agent volumes', () => {
it('adds sysroot:/host:rw to agent volumes', () => {
const config = { ...mockConfig, runnerTopology: 'arc-dind' as const };
const result = generateDockerCompose(config, mockNetworkConfig);

expect(result.services.agent.volumes).toContain('sysroot:/host:ro');
expect(result.services.agent.volumes).toContain('sysroot:/host:rw');
});

it('does not retain base-system bind mounts that shadow sysroot', () => {
Expand All @@ -375,16 +375,16 @@ describe('generateDockerCompose', () => {
expect(volumes).not.toContain('/lib:/host/lib:ro');
expect(volumes).not.toContain('/lib64:/host/lib64:ro');
expect(volumes).not.toContain('/opt:/host/opt:ro');
expect(volumes).not.toContain('/sys:/host/sys:ro');
expect(volumes).not.toContain('/dev:/host/dev:ro');
expect(volumes).toContain('/sys:/host/sys:ro');
expect(volumes).toContain('/dev:/host/dev:ro');
expect(volumes.some(v => v.includes(':/host/usr:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/bin:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/sbin:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/lib:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/lib64:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/opt:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/sys:ro'))).toBe(false);
expect(volumes.some(v => v.includes(':/host/dev:ro'))).toBe(false);
expect(volumes.filter(v => v.endsWith(':/host/sys:ro'))).toEqual(['/sys:/host/sys:ro']);
expect(volumes.filter(v => v.endsWith(':/host/dev:ro'))).toEqual(['/dev:/host/dev:ro']);
});

it('does not declare sysroot volume when topology is standard', () => {
Expand Down
12 changes: 6 additions & 6 deletions src/compose-generator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,6 @@ export function generateDockerCompose(
'/host/lib',
'/host/lib64',
'/host/opt',
'/host/sys',
'/host/dev',
]);
const filteredVolumes = agentVolumes.filter(volume => {
const target = volume.split(':')[1];
Expand Down Expand Up @@ -263,15 +261,17 @@ export function generateDockerCompose(
// ── Final compose result ───────────────────────────────────────────────────

// When sysroot staging is active, declare the named volume and mount it
// on the agent at /host (replacing the per-directory system bind mounts).
// on the agent at /host (replacing the per-directory userspace bind mounts,
// while /sys and /dev remain live host mounts).
const namedVolumes: Record<string, any> | undefined = sysrootActive
? { sysroot: {} }
: undefined;

if (sysrootActive) {
// The sysroot named volume provides /host content (system binaries, libs, etc.)
// via the sysroot-stage init container instead of per-directory bind mounts.
agentVolumes.push('sysroot:/host:ro');
// The sysroot named volume provides most /host content (system binaries,
// libs, etc.) via the sysroot-stage init container instead of per-directory
// userspace bind mounts.
agentVolumes.push('sysroot:/host:rw');
}

if (networkIsolation) {
Expand Down
15 changes: 15 additions & 0 deletions src/config-file-validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,21 @@ describe('validateAwfFileConfig', () => {
expect(errors).toContain('config.container.unknown is not supported');
});

it('rejects non-object runner', () => {
const errors = validateAwfFileConfig({ runner: 'invalid' });
expect(errors).toContain('config.runner must be an object');
});

it('rejects invalid runner field types', () => {
expect(validateAwfFileConfig({ runner: { topology: 'invalid' } })).toContain('config.runner.topology must be one of: standard, arc-dind');
expect(validateAwfFileConfig({ runner: { sysrootImage: 123 } })).toContain('config.runner.sysrootImage must be a string');
});

it('rejects unknown runner keys', () => {
const errors = validateAwfFileConfig({ runner: { unknown: true } });
expect(errors).toContain('config.runner.unknown is not supported');
});

it('rejects non-object chroot', () => {
const errors = validateAwfFileConfig({ chroot: 'invalid' });
expect(errors).toContain('config.chroot must be an object');
Expand Down
8 changes: 8 additions & 0 deletions src/etc-mounts-branches.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,14 @@ describe('system-mounts branch coverage', () => {
expect(mounts).toContain('/custom/tools:/host/tmp/awf-runner-bin:ro');
});

it('keeps live /sys and /dev mounts when sysroot mode is enabled', () => {
const mounts = buildSystemMounts('/workspace', undefined, true);
expect(mounts).toContain('/sys:/host/sys:ro');
expect(mounts).toContain('/dev:/host/dev:ro');
expect(mounts).not.toContain('/usr:/host/usr:ro');
expect(mounts).not.toContain('/bin:/host/bin:ro');
});

it('excludes runner-bin mount when chrootBinariesSourcePath is whitespace-only', () => {
const mounts = buildSystemMounts('/workspace', ' ');
expect(mounts.every(m => !m.includes('awf-runner-bin'))).toBe(true);
Expand Down
7 changes: 7 additions & 0 deletions src/schema.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,13 @@ describe('awf-config.schema.json', () => {
expect(validate({ container: { runnerToolCachePath: 123 } })).toBe(false);
});

it('accepts runner.topology and runner.sysrootImage', () => {
expect(validate({ runner: { topology: 'arc-dind' } })).toBe(true);
expect(validate({ runner: { topology: 'invalid' } })).toBe(false);
expect(validate({ runner: { sysrootImage: 'ghcr.io/github/gh-aw-firewall/build-tools:latest' } })).toBe(true);
expect(validate({ runner: { sysrootImage: 123 } })).toBe(false);
});

it('validates chroot.identity fields', () => {
expect(validate({ chroot: { identity: { home: '/tmp/gh-aw/home', user: 'runner', uid: 1001, gid: 1001 } } })).toBe(true);
expect(validate({ chroot: { identity: { uid: 1.2 } } })).toBe(false);
Expand Down
29 changes: 20 additions & 9 deletions src/services/agent-volumes/system-mounts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,27 @@ function normalizeChrootBinariesSourcePath(chrootBinariesSourcePath?: string): s
return normalized === '/' ? undefined : normalized;
}

export function buildSystemMounts(workspaceDir: string, chrootBinariesSourcePath?: string): string[] {
export function buildSystemMounts(
workspaceDir: string,
chrootBinariesSourcePath?: string,
useSysroot = false
): string[] {
const mounts = [
'/usr:/host/usr:ro',
'/bin:/host/bin:ro',
'/sbin:/host/sbin:ro',
'/lib:/host/lib:ro',
'/lib64:/host/lib64:ro',
'/opt:/host/opt:ro',
'/sys:/host/sys:ro',
'/dev:/host/dev:ro',
...(useSysroot
? [
'/sys:/host/sys:ro',
'/dev:/host/dev:ro',
]
: [
'/usr:/host/usr:ro',
'/bin:/host/bin:ro',
'/sbin:/host/sbin:ro',
'/lib:/host/lib:ro',
'/lib64:/host/lib64:ro',
'/opt:/host/opt:ro',
'/sys:/host/sys:ro',
'/dev:/host/dev:ro',
]),
`${workspaceDir}:/host${workspaceDir}:rw`,
'/tmp:/host/tmp:rw',
];
Comment on lines 18 to 36

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch -- sysroot mode now keeps /sys and /dev bind mounts (live kernel VFS) while only skipping the userspace dirs (/usr, /bin, /sbin, /lib, /lib64, /opt) that the sysroot volume provides.

Expand Down
3 changes: 2 additions & 1 deletion src/services/agent-volumes/volume-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ export function buildAgentVolumes(params: AgentVolumesParams): string[] {

logger.debug('Using selective path mounts for security');

agentVolumes.push(...buildSystemMounts(workspaceDir, config.chrootBinariesSourcePath));
const useSysroot = config.runnerTopology === 'arc-dind';
agentVolumes.push(...buildSystemMounts(workspaceDir, config.chrootBinariesSourcePath, useSysroot));
agentVolumes.push(...buildHomeMounts({ config, effectiveHome, agentLogsPath, sessionStatePath }));
agentVolumes.push(...buildEtcMounts(config));
agentVolumes.push(generateHostsFileMount(config));
Expand Down
30 changes: 1 addition & 29 deletions src/types/platform-options.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
/**
* GitHub platform deployment type and runner topology options.
* GitHub platform deployment type options.
*/

export type RunnerTopology = 'standard' | 'arc-dind';

export interface PlatformOptions {
/**
* The GitHub deployment type. Explicitly declares the environment so AWF can
Expand All @@ -19,30 +17,4 @@ export interface PlatformOptions {
* regardless of the resolved API target hostname.
*/
platformType?: 'github.com' | 'ghes' | 'ghec' | 'ghec-self-hosted';

/**
* Runner deployment topology.
*
* - 'standard' (default) — GitHub-hosted VM or self-hosted runner with local Docker.
* - 'arc-dind' — ARC (Actions Runner Controller) with Docker-in-Docker sidecar,
* where the runner and Docker daemon have separate filesystems.
*
* When set to 'arc-dind', AWF applies overridable defaults:
* - network.isolation = true (ARC k8s lacks NET_ADMIN)
* - dind.preStageDirs = true
* - Sysroot image activation (build-tools init container)
* - Tool cache validation (warns if under /opt)
*/
runnerTopology?: RunnerTopology;

/**
* Container image providing system-level build tools (gcc, make, libraries)
* for the agent's chroot base on ARC/DinD.
*
* Used as an init container that copies its filesystem into a named volume
* mounted at /host. Only used when runnerTopology is 'arc-dind'.
*
* Defaults to 'ghcr.io/github/gh-aw-firewall/build-tools:<imageTag>'.
*/
sysrootImage?: string;
}
20 changes: 20 additions & 0 deletions src/types/runner-options.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/**
* Runner topology configuration options.
*/
export interface RunnerOptions {
/**
* Runner topology mode for AWF compose generation.
*
* - 'standard' (default) - GitHub-hosted VM or self-hosted runner with local Docker.
* - 'arc-dind' - ARC with Docker-in-Docker sidecar, enables sysroot staging
* for split runner/daemon filesystems.
*/
runnerTopology?: 'standard' | 'arc-dind';

/**
* Sysroot image used by arc-dind topology to stage build tools into /host.
*
* @default 'ghcr.io/github/gh-aw-firewall/build-tools:latest'
*/
sysrootImage?: string;
}
4 changes: 3 additions & 1 deletion src/types/wrapper-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type { CliProxyOptions } from './cli-proxy-options';
import type { RateLimitOptions } from './rate-limit-options';
import type { RuntimeOptions } from './runtime-options';
import type { PlatformOptions } from './platform-options';
import type { RunnerOptions } from './runner-options';

export type WrapperConfig =
ContainerImageOptions
Expand All @@ -24,4 +25,5 @@ export type WrapperConfig =
& CliProxyOptions
& RateLimitOptions
& RuntimeOptions
& PlatformOptions;
& PlatformOptions
& RunnerOptions;
Loading