Revert "Enable ipv6, nftables, NAT, and nf_raw for Firecracker amd64 guests" (#11554)

Reverts #11542

Author: bduffany

deps.bzl

@@ -216,14 +216,14 @@ def install_static_dependencies(workspace_name = "buildbuddy"):
     )
     http_file(
         name = "org_kernel_git_linux_kernel-vmlinux",
-        sha256 = "7e972d01a27e7f3c3e01997e2b202a03e9eef279cd183fbdd2638a0901812052",
-        urls = ["https://storage.googleapis.com/buildbuddy-tools/binaries/linux/vmlinux-x86_64-v5.15-7e972d01a27e7f3c3e01997e2b202a03e9eef279cd183fbdd2638a0901812052"],
+        sha256 = "ea6432b7a097873dd6ffb32481de57fa9c3bb2d2c68a2568e39a1d3decdd003d",
+        urls = ["https://storage.googleapis.com/buildbuddy-tools/binaries/linux/vmlinux-x86_64-v5.15-ea6432b7a097873dd6ffb32481de57fa9c3bb2d2c68a2568e39a1d3decdd003d"],
         executable = True,
     )
     http_file(
         name = "org_kernel_git_linux_kernel-vmlinux-6.1",
-        sha256 = "04c7910a6ee6666f4ac713ce2a42ffaa49e7fd2d2d9268a123a38e3e84c5a8bd",
-        urls = ["https://storage.googleapis.com/buildbuddy-tools/binaries/linux/vmlinux-x86_64-v6.1-04c7910a6ee6666f4ac713ce2a42ffaa49e7fd2d2d9268a123a38e3e84c5a8bd"],
+        sha256 = "221765c1c163d7f4687c0fba573c47a17ada6cbe4063c16e6205fabc7066fd15",
+        urls = ["https://storage.googleapis.com/buildbuddy-tools/binaries/linux/vmlinux-x86_64-v6.1-221765c1c163d7f4687c0fba573c47a17ada6cbe4063c16e6205fabc7066fd15"],
         executable = True,
     )
     http_file(

enterprise/server/cmd/goinit/main.go

@@ -12,7 +12,6 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"syscall"
@@ -194,11 +193,8 @@ func startDockerd(ctx context.Context) error {
 		args = append(args, "--host=unix:///var/run/docker.sock", "--host=tcp://0.0.0.0:2375", "--tls=false")
 	}
 	cmd := exec.CommandContext(ctx, "dockerd", args...)
-	// TODO(bduffany): update arm64 image and remove this check for arm64 as well.
-	if runtime.GOARCH != "amd64" {
-		// Note: despite the big scary INSECURE env var name, dockerd is completely sandboxed inside a VM, so it's secure for our usage. Once we upgrade our guest kernels to support nf tables, we can remove this.
-		cmd.Env = append(os.Environ(), "DOCKER_INSECURE_NO_IPTABLES_RAW=1")
-	}
+	// Note: despite the big scary INSECURE env var name, dockerd is completely sandboxed inside a VM, so it's secure for our usage. Once we upgrade our guest kernels to support nf tables, we can remove this.
+	cmd.Env = append(os.Environ(), "DOCKER_INSECURE_NO_IPTABLES_RAW=1")
 	// TODO(https://github.com/buildbuddy-io/buildbuddy-internal/issues/3306):
 	// enable logging by default
 	if *enableLogging {

enterprise/server/remote_execution/containers/firecracker/firecracker.go

@@ -1620,10 +1620,7 @@ func getBootArgs(vmConfig *fcpb.VMConfiguration) string {
 		"i8042.nopnp",
 		"i8042.dumbkbd",
 		"tsc=reliable",
-	}
-	if runtime.GOARCH != "amd64" {
-		// TODO(bduffany): update arm64 image and enable ipv6 on arm64 as well.
-		kernelArgs = append(kernelArgs, "ipv6.disable=1")
+		"ipv6.disable=1",
 	}
 	if networkingEnabled(vmConfig.NetworkMode) {
 		kernelArgs = append(kernelArgs, machineIPBootArgs)

enterprise/server/remote_execution/containers/firecracker/firecracker_test.go

@@ -13,7 +13,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -79,7 +78,6 @@ const (
 	ubuntuImage                 = "mirror.gcr.io/library/ubuntu:20.04"
 	imageWithDockerInstalled    = "gcr.io/flame-public/executor-docker-default:enterprise-v1.6.0"
 	imageWithDockerV28Installed = platform.Ubuntu24_04Image
-	dockerDindImage             = "gcr.io/flame-public/test-docker-dind@sha256:68f6d9ab84623d1116c5432a3b924a07ee09960e6129ca1cb03ef14010588cb4"
 
 	// Minimum memory needed for a firecracker VM. This may need to be increased
 	// if the size of initrd.cpio increases.
@@ -153,7 +151,7 @@ func TestGuestAPIVersion(t *testing.T) {
 	// Note that if you go with option 1, ALL VM snapshots will be invalidated
 	// which will negatively affect customer experience. Be careful!
 	const (
-		expectedHash    = "6856b316f80dc7e3ed9939148aee3e59640e1c749ec9d344e3a3c70a70422155"
+		expectedHash    = "4a0b9e65e9db406124d1bb745e3e9e0c7ce3d2d01f282bc73d0addab8a2f0a39"
 		expectedVersion = "18"
 	)
 	assert.Equal(t, expectedHash, firecracker.GuestAPIHash)
@@ -2509,70 +2507,6 @@ func TestFirecrackerRunNOPWithZeroDisk(t *testing.T) {
 	assert.Equal(t, "/workspace\n", string(res.Stdout))
 }
 
-func TestFirecrackerRunWithIPv6Enabled(t *testing.T) {
-	if runtime.GOARCH != "amd64" {
-		// TODO(bduffany): update arm64 image and enable this test
-		t.Skipf("test is not yet supported on arm64")
-	}
-
-	ctx := context.Background()
-	env := getTestEnv(ctx, t, envOpts{})
-	rootDir := testfs.MakeTempDir(t)
-	workDir := testfs.MakeDirAll(t, rootDir, "work")
-	cmd := &repb.Command{
-		Arguments: []string{"sh", "-c", `
-			set -e
-
-			# IPv4 should be available with external networking.
-			if [ ! -r /proc/sys/net/ipv4/ip_forward ]; then
-				echo "missing /proc/sys/net/ipv4/ip_forward" >&2
-				exit 1
-			fi
-			if ! grep -Eq '^[[:space:]]*eth0:' /proc/net/dev; then
-				echo "expected eth0 device; got:" >&2
-				cat /proc/net/dev >&2
-				exit 1
-			fi
-
-			# IPv6 should also be enabled.
-			if grep -q 'ipv6.disable=1' /proc/cmdline; then
-				echo "kernel cmdline has ipv6.disable=1: $(cat /proc/cmdline)" >&2
-				exit 1
-			fi
-			if [ "$(cat /proc/sys/net/ipv6/conf/all/disable_ipv6)" != "0" ] || [ "$(cat /proc/sys/net/ipv6/conf/default/disable_ipv6)" != "0" ]; then
-				echo "IPv6 disable flags: all=$(cat /proc/sys/net/ipv6/conf/all/disable_ipv6) default=$(cat /proc/sys/net/ipv6/conf/default/disable_ipv6)" >&2
-				exit 1
-			fi
-			if ! grep -q . /proc/net/if_inet6; then
-				echo "expected non-empty /proc/net/if_inet6" >&2
-				cat /proc/net/if_inet6 >&2 || true
-				exit 1
-			fi
-			echo ipv4_ipv6_enabled
-		`},
-	}
-	opts := firecracker.ContainerOpts{
-		ContainerImage:         busyboxImage,
-		ActionWorkingDirectory: workDir,
-		VMConfiguration: &fcpb.VMConfiguration{
-			NumCpus:           1,
-			MemSizeMb:         2500,
-			NetworkMode:       fcpb.NetworkMode_NETWORK_MODE_EXTERNAL,
-			ScratchDiskSizeMb: 100,
-		},
-		ExecutorConfig: getExecutorConfig(t),
-	}
-	c, err := firecracker.NewContainer(ctx, env, &repb.ExecutionTask{}, opts)
-	require.NoError(t, err)
-
-	// Run will handle the full lifecycle: no need to call Remove() here.
-	res := c.Run(ctx, cmd, opts.ActionWorkingDirectory, oci.Credentials{})
-	require.NoError(t, res.Error)
-	assert.Equal(t, 0, res.ExitCode)
-	assert.Equal(t, "", string(res.Stderr))
-	assert.Equal(t, "ipv4_ipv6_enabled\n", string(res.Stdout))
-}
-
 func testFirecrackerRunWithDockerOverUDS(t *testing.T, containerImage string) {
 	if *skipDockerTests {
 		t.Skip()
@@ -2583,17 +2517,15 @@ func testFirecrackerRunWithDockerOverUDS(t *testing.T, containerImage string) {
 	rootDir := testfs.MakeTempDir(t)
 	workDir := testfs.MakeDirAll(t, rootDir, "work")
 	cmd := &repb.Command{
-		Arguments: []string{"sh", "-c", `
+		Arguments: []string{"bash", "-c", `
 			set -e
 
 			# Discard pull output to make the output deterministic
-			docker pull ` + busyboxImage + ` >/dev/null 2>&1
-
-			# Test basic command
-			docker run --rm ` + busyboxImage + ` echo Hello
+			docker pull ` + busyboxImage + ` &>/dev/null
 
-			# Test port publishing
-			docker run --rm -p 127.0.0.1:18080:80 ` + busyboxImage + ` echo world
+			# Try running a few commands
+			docker run --rm -p 127.0.0.1:18080:80 ` + busyboxImage + ` echo Hello
+			docker run --rm ` + busyboxImage + ` echo world
 
 			# Check what storage driver docker is using
 			docker info 2>/dev/null | grep 'Storage Driver'
@@ -2624,15 +2556,11 @@ func testFirecrackerRunWithDockerOverUDS(t *testing.T, containerImage string) {
 	}
 
 	assert.Equal(t, 0, res.ExitCode)
-	stdout := string(res.Stdout)
+	expectedStorageDriver := "vfs"
 	if snaputil.IsChunkedSnapshotSharingEnabled() {
-		// Docker may report the native overlay-backed fast path as either the
-		// legacy graphdriver name ("overlay2") or the newer containerd
-		// snapshotter name ("overlayfs"), depending on daemon configuration.
-		assert.Regexp(t, `^Hello\nworld\n Storage Driver: (overlay2|overlayfs)\n$`, stdout, "stdout should contain docker output with a native overlay storage driver")
-	} else {
-		assert.Equal(t, "Hello\nworld\n Storage Driver: vfs\n", stdout, "stdout should contain docker output")
+		expectedStorageDriver = "overlay2"
 	}
+	assert.Equal(t, "Hello\nworld\n Storage Driver: "+expectedStorageDriver+"\n", string(res.Stdout), "stdout should contain pwd output")
 	assert.Equal(t, "", string(res.Stderr), "stderr should be empty")
 }
 
@@ -2641,24 +2569,9 @@ func TestFirecrackerRunWithDockerOverUDS(t *testing.T) {
 }
 
 func TestFirecrackerRunWithDockerV28OverUDS(t *testing.T) {
-	// docker v28 requires nf_raw in order to bind ports, so this tests that the
-	// 'raw' table is properly set up in the guest.
 	testFirecrackerRunWithDockerOverUDS(t, imageWithDockerV28Installed)
 }
 
-func TestFirecrackerRunWithDockerDindOverUDS(t *testing.T) {
-	if runtime.GOARCH != "amd64" {
-		// TODO(bduffany): make this work on arm64
-		t.Skipf("test is not yet supported on arm64")
-	}
-
-	// docker:dind has docker but doesn't have iptables-legacy, so this tests
-	// that we've properly set up the newer nftables-based iptables in the
-	// guest. It also tests that we've set up NAT correctly which is also needed
-	// to make this image work.
-	testFirecrackerRunWithDockerOverUDS(t, dockerDindImage)
-}
-
 func TestFirecrackerRunWithDockerOverTCP(t *testing.T) {
 	if *skipDockerTests {
 		t.Skip()

enterprise/vmsupport/kernel/microvm-kernel-x86_64-v5.15.config

@@ -1,12 +1,13 @@
+# TODO: For our current iptables-legacy path, enable CONFIG_IP_NF_RAW (and CONFIG_IP6_NF_RAW if needed), or fully migrate to nftables (CONFIG_NF_TABLES + non-legacy iptables), then remove the DOCKER_INSECURE_NO_IPTABLES_RAW fallback in enterprise/server/cmd/goinit/main.go.
+#
 # Config copied from https://github.com/firecracker-microvm/firecracker/blob/main/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config
 # Linux/x86 5.10.0 Kernel Configuration
 #
 # Buildbuddy modifications:
 #
 # To support running docker in the VM:
 # * Enabled CONFIG_NETFILTER and related options
-# * Enabled CONFIG_NF_TABLES plus IPv4 / IPv6 raw table support
-# * Enabled CONFIG_IPV6 and related IPv6 netfilter options
+# * Enabled CONFIG_IPV6 and related options
 #
 # To support running OCI in the VM:
 # * Enabled CONFIG_SECCOMP
@@ -1015,13 +1016,7 @@ CONFIG_NF_NAT_PROTO_SCTP=y
 # CONFIG_NF_NAT_TFTP is not set
 CONFIG_NF_NAT_REDIRECT=y
 CONFIG_NETFILTER_SYNPROXY=y
-CONFIG_NF_TABLES=y
-CONFIG_NF_TABLES_INET=y
-CONFIG_NFT_CT=y
-CONFIG_NFT_MASQ=y
-CONFIG_NFT_REDIR=y
-CONFIG_NFT_NAT=y
-CONFIG_NFT_COMPAT=y
+# CONFIG_NF_TABLES is not set
 CONFIG_NETFILTER_XTABLES=y
 
 CONFIG_NETFILTER_XT_NAT=y
@@ -1080,7 +1075,6 @@ CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
 # IP: Netfilter Configuration
 #
 CONFIG_NF_DEFRAG_IPV4=y
-CONFIG_NF_TABLES_IPV4=y
 CONFIG_NF_CONNTRACK_IPV4=y
 # CONFIG_NF_SOCKET_IPV4 is not set
 # CONFIG_NF_DUP_IPV4 is not set
@@ -1107,40 +1101,9 @@ CONFIG_IP_NF_MANGLE=y
 # CONFIG_IP_NF_TARGET_CLUSTERIP is not set
 # CONFIG_IP_NF_TARGET_ECN is not set
 # CONFIG_IP_NF_TARGET_TTL is not set
-CONFIG_IP_NF_RAW=y
+# CONFIG_IP_NF_RAW is not set
 # CONFIG_IP_NF_SECURITY is not set
 
-#
-# IPv6: Netfilter Configuration
-#
-# CONFIG_NF_SOCKET_IPV6 is not set
-# CONFIG_NF_DUP_IPV6 is not set
-CONFIG_NF_TABLES_IPV6=y
-CONFIG_NF_REJECT_IPV6=y
-CONFIG_NF_LOG_IPV6=y
-CONFIG_IP6_NF_IPTABLES=y
-# CONFIG_IP6_NF_MATCH_AH is not set
-# CONFIG_IP6_NF_MATCH_EUI64 is not set
-# CONFIG_IP6_NF_MATCH_FRAG is not set
-# CONFIG_IP6_NF_MATCH_OPTS is not set
-# CONFIG_IP6_NF_MATCH_HL is not set
-# CONFIG_IP6_NF_MATCH_IPV6HEADER is not set
-# CONFIG_IP6_NF_MATCH_MH is not set
-# CONFIG_IP6_NF_MATCH_RPFILTER is not set
-# CONFIG_IP6_NF_MATCH_RT is not set
-# CONFIG_IP6_NF_TARGET_HL is not set
-CONFIG_IP6_NF_FILTER=y
-CONFIG_IP6_NF_TARGET_REJECT=y
-CONFIG_IP6_NF_TARGET_SYNPROXY=y
-CONFIG_IP6_NF_MANGLE=y
-CONFIG_IP6_NF_RAW=y
-# CONFIG_IP6_NF_SECURITY is not set
-CONFIG_IP6_NF_NAT=y
-CONFIG_IP6_NF_TARGET_MASQUERADE=y
-# end of IPv6: Netfilter Configuration
-
-CONFIG_NF_DEFRAG_IPV6=y
-
 ###### END BUILDBUDDY ADDED ######
 
 #

enterprise/vmsupport/kernel/microvm-kernel-x86_64-v6.1.config

@@ -1,9 +1,9 @@
+# TODO: For our current iptables-legacy path, enable CONFIG_IP_NF_RAW (and CONFIG_IP6_NF_RAW if needed), or fully migrate to nftables (CONFIG_NF_TABLES + non-legacy iptables), then remove the DOCKER_INSECURE_NO_IPTABLES_RAW fallback in enterprise/server/cmd/goinit/main.go.
 # Config copied from https://github.com/firecracker-microvm/firecracker/blob/main/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config
 # BuildBuddy-specific modifications:
 # - Set CONFIG_PCI=y (see https://github.com/firecracker-microvm/firecracker/issues/4881)
 # - Set CONFIG_FUSE_FS=y for FUSE support
 # - Set CONFIG_TUN=y for networking
-# - Enable CONFIG_NF_TABLES plus IPv4 / IPv6 raw table support for Docker networking
 
 #
 # Automatically generated file; DO NOT EDIT.
@@ -1094,13 +1094,7 @@ CONFIG_NF_NAT=y
 CONFIG_NF_NAT_REDIRECT=y
 CONFIG_NF_NAT_MASQUERADE=y
 CONFIG_NETFILTER_SYNPROXY=y
-CONFIG_NF_TABLES=y
-CONFIG_NF_TABLES_INET=y
-CONFIG_NFT_CT=y
-CONFIG_NFT_MASQ=y
-CONFIG_NFT_REDIR=y
-CONFIG_NFT_NAT=y
-CONFIG_NFT_COMPAT=y
+# CONFIG_NF_TABLES is not set
 CONFIG_NETFILTER_XTABLES=y
 CONFIG_NETFILTER_XTABLES_COMPAT=y
 
@@ -1194,7 +1188,6 @@ CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
 # IP: Netfilter Configuration
 #
 CONFIG_NF_DEFRAG_IPV4=y
-CONFIG_NF_TABLES_IPV4=y
 # CONFIG_NF_SOCKET_IPV4 is not set
 # CONFIG_NF_TPROXY_IPV4 is not set
 # CONFIG_NF_DUP_IPV4 is not set
@@ -1217,7 +1210,7 @@ CONFIG_IP_NF_MANGLE=y
 # CONFIG_IP_NF_TARGET_CLUSTERIP is not set
 # CONFIG_IP_NF_TARGET_ECN is not set
 # CONFIG_IP_NF_TARGET_TTL is not set
-CONFIG_IP_NF_RAW=y
+# CONFIG_IP_NF_RAW is not set
 # CONFIG_IP_NF_SECURITY is not set
 # CONFIG_IP_NF_ARPTABLES is not set
 # end of IP: Netfilter Configuration
@@ -1228,7 +1221,6 @@ CONFIG_IP_NF_RAW=y
 # CONFIG_NF_SOCKET_IPV6 is not set
 # CONFIG_NF_TPROXY_IPV6 is not set
 # CONFIG_NF_DUP_IPV6 is not set
-CONFIG_NF_TABLES_IPV6=y
 CONFIG_NF_REJECT_IPV6=y
 CONFIG_NF_LOG_IPV6=y
 CONFIG_IP6_NF_IPTABLES=y
@@ -1247,7 +1239,7 @@ CONFIG_IP6_NF_FILTER=y
 CONFIG_IP6_NF_TARGET_REJECT=y
 CONFIG_IP6_NF_TARGET_SYNPROXY=y
 CONFIG_IP6_NF_MANGLE=y
-CONFIG_IP6_NF_RAW=y
+# CONFIG_IP6_NF_RAW is not set
 # CONFIG_IP6_NF_SECURITY is not set
 CONFIG_IP6_NF_NAT=y
 CONFIG_IP6_NF_TARGET_MASQUERADE=y