cli/grpc: fall back to the legacy endpoint on native transport failure

An intermediary between the client and the daemon may not relay HTTP/2
even though the daemon's API version advertises native gRPC support —
Docker Desktop's API proxy does not, at the time of writing. Probe
native connections with a health-check RPC (any RPC-level outcome,
including Unimplemented, proves the transport) and fall back to the
legacy upgrade endpoint on transport-level failure.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

Author: ndeloof

cli/grpc/grpc.go

@@ -11,10 +11,12 @@
 // [Connect] hides those transport details: it returns a [grpc.ClientConn]
 // reaching the daemon, whatever the transport of the current endpoint (unix
 // or npipe socket, tcp with or without TLS, or a connection helper such as
-// ssh://), falling back to the legacy upgrade endpoint for older daemons.
-// Note that the legacy endpoint only exposes the daemon's built-in gRPC
-// services: services published by daemon extensions require a daemon serving
-// gRPC natively.
+// ssh://), falling back to the legacy upgrade endpoint for older daemons —
+// or when an intermediary between the client and the daemon does not relay
+// HTTP/2 even though the daemon's API version advertises native support
+// (Docker Desktop's API proxy, at the time of writing). Note that the legacy
+// endpoint only exposes the daemon's built-in gRPC services: services
+// published by daemon extensions require a daemon serving gRPC natively.
 package grpc
 
 import (
@@ -29,8 +31,11 @@ import (
 	"github.com/moby/moby/client"
 	"github.com/moby/moby/client/pkg/versions"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	healthpb "google.golang.org/grpc/health/grpc_health_v1"
+	"google.golang.org/grpc/status"
 )
 
 // nativeAPIVersion is the minimum API version where the daemon serves gRPC
@@ -87,9 +92,9 @@ func WithDialOptions(opts ...grpc.DialOption) Opt {
 }
 
 // Connect returns a gRPC client connection to the daemon of dockerCLI's
-// current endpoint. ctx is only used to query the daemon's API version; the
-// returned connection is lazy: it is not bound to ctx, and is only
-// established when the first RPC is made.
+// current endpoint. ctx bounds the daemon API version query and the probing
+// of native connections; the returned connection is not bound to it, and
+// behaves lazily afterwards.
 func Connect(ctx context.Context, dockerCLI DockerCLI, opts ...Opt) (*grpc.ClientConn, error) {
 	return Dial(ctx, dockerCLI.Client(), dockerCLI.DockerEndpoint(), opts...)
 }
@@ -105,11 +110,34 @@ func Dial(ctx context.Context, apiClient APIClient, ep docker.Endpoint, opts ...
 		return nil, fmt.Errorf("establishing gRPC connection to the daemon: %w", err)
 	}
 	if ping.APIVersion != "" && !versions.LessThan(ping.APIVersion, nativeAPIVersion) {
-		return dialNative(apiClient, ep, &cfg)
+		conn, err := dialNative(apiClient, ep, &cfg)
+		if err != nil {
+			return nil, err
+		}
+		if probeTransport(ctx, conn) == nil {
+			return conn, nil
+		}
+		// The daemon's API version advertises native gRPC support but the
+		// transport doesn't get through: an intermediary between the client
+		// and the daemon (Docker Desktop's API proxy, at the time of writing)
+		// may not relay HTTP/2. Fall back to the legacy upgrade endpoint.
+		_ = conn.Close()
 	}
 	return dialLegacy(apiClient, &cfg)
 }
 
+// probeTransport verifies conn actually reaches an HTTP/2 server. Any
+// RPC-level outcome — including Unimplemented from a daemon not serving the
+// health service — proves the transport; only transport-level failures
+// (codes.Unavailable) are reported.
+func probeTransport(ctx context.Context, conn *grpc.ClientConn) error {
+	_, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{})
+	if status.Code(err) == codes.Unavailable {
+		return err
+	}
+	return nil
+}
+
 // dialNative connects to a daemon serving gRPC natively on its API endpoint:
 // ALPN-negotiated HTTP/2 for TLS endpoints, cleartext HTTP/2 (h2c) over the
 // API client's own dialer otherwise. The client dialer covers unix and npipe

cli/grpc/grpc_test.go

@@ -102,6 +102,59 @@ func TestDialNative(t *testing.T) {
 	assert.Check(t, !apiClient.hijackCalled.Load())
 }
 
+// startHTTP1Server answers any connection with a plain HTTP/1.1 response,
+// standing in for an intermediary that does not relay HTTP/2 (e.g. Docker
+// Desktop's API proxy).
+func startHTTP1Server(t *testing.T) string {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	assert.NilError(t, err)
+	go func() {
+		for {
+			conn, err := l.Accept()
+			if err != nil {
+				return
+			}
+			go func(c net.Conn) {
+				buf := make([]byte, 1024)
+				_, _ = c.Read(buf)
+				_, _ = c.Write([]byte("HTTP/1.1 505 HTTP Version Not Supported\r\nContent-Length: 0\r\n\r\n"))
+				_ = c.Close()
+			}(conn)
+		}
+	}()
+	t.Cleanup(func() { _ = l.Close() })
+	return l.Addr().String()
+}
+
+func TestDialNativeFallsBackToLegacy(t *testing.T) {
+	// The daemon advertises native support but an HTTP/1.1-only intermediary
+	// sits on the endpoint: the probe fails at transport level and the
+	// connection falls back to the legacy upgrade endpoint.
+	http1Addr := startHTTP1Server(t)
+	grpcAddr := startServer(t)
+	apiClient := &fakeAPIClient{
+		apiVersion: "1.53",
+		dialer: func(context.Context) (net.Conn, error) {
+			return net.Dial("tcp", http1Addr)
+		},
+		hijack: func(context.Context, string, string, map[string][]string) (net.Conn, error) {
+			return net.Dial("tcp", grpcAddr)
+		},
+	}
+
+	conn, err := Dial(t.Context(), apiClient, docker.Endpoint{
+		EndpointMeta: docker.EndpointMeta{Host: "unix:///var/run/docker.sock"},
+	})
+	assert.NilError(t, err)
+	defer conn.Close()
+
+	checkHealth(t, conn)
+	assert.Check(t, apiClient.dialerCalled.Load())
+	assert.Check(t, apiClient.hijackCalled.Load())
+	assert.Check(t, is.Equal(apiClient.hijackURL, "/grpc"))
+}
+
 func TestDialLegacy(t *testing.T) {
 	for _, apiVersion := range []string{"", "1.52"} {
 		t.Run("api-version-"+apiVersion, func(t *testing.T) {