Skip to content

Commit 680b5fa

Browse files
committed
firecracker: make docker -p work in guest kernels without raw table support
Docker 28 configures direct-access filtering via iptables `-t raw` for published ports. In Firecracker guest VMs this can fail with: iptables ... can't initialize iptables table `raw': Table does not exist because our current guest kernel configs all have `CONFIG_IP_NF_RAW` disabled. Changes: - Set `DOCKER_INSECURE_NO_IPTABLES_RAW=1` unconditionally when starting dockerd in goinit. - Keep an inline note explaining why this is safe for our VM sandboxing model. - Update docker-in-firecracker coverage to include `docker run -p ...` in `TestFirecrackerRunWithDockerOverUDS`. - Update guest API hash expectation. - Add TODOs at the top of all guest kernel config files to enable `CONFIG_NF_TABLES` and remove the env-var fallback. Repro (before/after): run the same `bb execute` command with and without `--action_env=DOCKER_INSECURE_NO_IPTABLES_RAW=1`; `docker run -p ...` fails without it and succeeds with it.
1 parent dd70c6f commit 680b5fa

File tree

5 files changed

+9
-5
lines changed

5 files changed

+9
-5
lines changed

enterprise/server/cmd/goinit/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,8 +192,9 @@ func startDockerd(ctx context.Context) error {
192192
if *enableDockerdTCP {
193193
args = append(args, "--host=unix:///var/run/docker.sock", "--host=tcp://0.0.0.0:2375", "--tls=false")
194194
}
195-
196195
cmd := exec.CommandContext(ctx, "dockerd", args...)
196+
// Note: despite the big scary INSECURE env var name, dockerd is completely sandboxed inside a VM, so it's secure for our usage. Once we upgrade our guest kernels to support nf tables, we can remove this.
197+
cmd.Env = append(os.Environ(), "DOCKER_INSECURE_NO_IPTABLES_RAW=1")
197198
// TODO(https://github.com/buildbuddy-io/buildbuddy-internal/issues/3306):
198199
// enable logging by default
199200
if *enableLogging {

enterprise/server/remote_execution/containers/firecracker/firecracker_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ func TestGuestAPIVersion(t *testing.T) {
150150
// Note that if you go with option 1, ALL VM snapshots will be invalidated
151151
// which will negatively affect customer experience. Be careful!
152152
const (
153-
expectedHash = "25c5043d9c3d465c5fe2e974da0e532fe113365182aa8fe59c0c1e028064562b"
153+
expectedHash = "d6e20637585cf821192d1b13d34b87316307ae286b4c31d27307052a3d7df45c"
154154
expectedVersion = "17"
155155
)
156156
assert.Equal(t, expectedHash, firecracker.GuestAPIHash)
@@ -2472,7 +2472,7 @@ func TestFirecrackerRunWithDockerOverUDS(t *testing.T) {
24722472
docker pull ` + busyboxImage + ` &>/dev/null
24732473
24742474
# Try running a few commands
2475-
docker run --rm ` + busyboxImage + ` echo Hello
2475+
docker run --rm -p 127.0.0.1:18080:80 ` + busyboxImage + ` echo Hello
24762476
docker run --rm ` + busyboxImage + ` echo world
24772477
24782478
# Check what storage driver docker is using

enterprise/vmsupport/kernel/microvm-kernel-aarch64-v5.10.config

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# TODO: Enable CONFIG_NF_TABLES and remove the DOCKER_INSECURE_NO_IPTABLES_RAW fallback in enterprise/server/cmd/goinit/main.go.
12
CONFIG_CC_VERSION_TEXT="gcc10-gcc (GCC) 10.5.0 20230707 (Red Hat 10.5.0-1)"
23
CONFIG_CC_IS_GCC=y
34
CONFIG_GCC_VERSION=100500
@@ -3113,4 +3114,4 @@ CONFIG_CC_HAS_SANCOV_TRACE_PC=y
31133114
# CONFIG_RUNTIME_TESTING_MENU is not set
31143115
# CONFIG_MEMTEST is not set
31153116
# end of Kernel Testing and Coverage
3116-
# end of Kernel hacking
3117+
# end of Kernel hacking

enterprise/vmsupport/kernel/microvm-kernel-x86_64-v5.15.config

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# TODO: Enable CONFIG_NF_TABLES and remove the DOCKER_INSECURE_NO_IPTABLES_RAW fallback in enterprise/server/cmd/goinit/main.go.
12
#
23
# Config copied from https://github.com/firecracker-microvm/firecracker/blob/main/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config
34
# Linux/x86 5.10.0 Kernel Configuration

enterprise/vmsupport/kernel/microvm-kernel-x86_64-v6.1.config

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# TODO: Enable CONFIG_NF_TABLES and remove the DOCKER_INSECURE_NO_IPTABLES_RAW fallback in enterprise/server/cmd/goinit/main.go.
12
# Config copied from https://github.com/firecracker-microvm/firecracker/blob/main/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config
23
# BuildBuddy-specific modifications:
34
# - Set CONFIG_PCI=y (see https://github.com/firecracker-microvm/firecracker/issues/4881)
@@ -3238,4 +3239,4 @@ CONFIG_ARCH_USE_MEMTEST=y
32383239
# Rust hacking
32393240
#
32403241
# end of Rust hacking
3241-
# end of Kernel hacking
3242+
# end of Kernel hacking

0 commit comments

Comments
 (0)