From e2e75cde227aba673068e299f5b2e30eae96510d Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 29 Apr 2026 08:01:23 +0000 Subject: [PATCH 1/2] feat(provision/config): expose gateway config (skip + className) CommonConfig.Gateway lets operators tune the bundled Envoy Gateway install without touching code. Two knobs, all-or-nothing semantics: gateway: skip: false # default: install everything className: y-cluster # GatewayClass name; default y-cluster - skip: true -> no CRDs, no controller, no GatewayClass. k3s --disable=traefik still passes; if you want a different ingress, install it yourself. - className -> the GatewayClass name consumer Gateway resources reference via gatewayClassName. Default flips from "eg" (the previous hardcoded value) to "y-cluster". Set to "eg" explicitly for compat with consumers that pinned that name. Independent "install controller without default GatewayClass" is deliberately NOT exposed in cluster config -- if the consumer ships their own GatewayClass, they should ship their own controller install. The lower-level envoygateway.Options keeps the GatewayClassName="" path for tests and library users. override-ip is also intentionally NOT a field. It's derived from PortForwards (loopback when guest:80 is bound to a host port) and surfaced to ystack consumers via a kube-system ConfigMap, per ISSUE_PROVISIONER_SHOULD_SET_GATEWAY_OVERRIDE_IP.md (separate PR). Implementation: - pkg/provision/config: GatewayConfig struct + applyGatewayDefaults + EffectiveGatewayClassName helper. - pkg/provision/envoygateway: Options.SkipGatewayClass replaced by Options.GatewayClassName (empty = skip apply); the embedded assets/gatewayclass.yaml is gone, replaced by an inline Go template that renders with the configured name. - qemu / docker provisioners pass cfg.Gateway through and skip the envoygateway.Install call wholesale when skip is set. Tests: - pkg/provision/config/gateway_test.go: defaulting, explicit override, skip-leaves-classname-alone, EffectiveGatewayClassName. - e2e/envoygateway_test.go: TestEnvoyGateway_InstallAgainstKwok asserts the new "y-cluster" default name; the renamed TestEnvoyGateway_InstallEmptyClassNameSkipsApply covers the empty-name skip path. - Schemas regenerated. Co-Authored-By: Claude Opus 4.7 (1M context) --- e2e/envoygateway_test.go | 42 ++++++------ pkg/provision/config/common.go | 64 ++++++++++++++++++- pkg/provision/config/gateway_test.go | 62 ++++++++++++++++++ pkg/provision/docker/docker.go | 23 +++++-- .../envoygateway/assets/gatewayclass.yaml | 13 ---- pkg/provision/envoygateway/embed.go | 42 ++++++++---- pkg/provision/envoygateway/install.go | 25 +++++--- pkg/provision/qemu/qemu.go | 29 ++++++--- pkg/provision/schema/common.schema.json | 19 ++++++ pkg/provision/schema/docker.schema.json | 19 ++++++ pkg/provision/schema/qemu.schema.json | 19 ++++++ 11 files changed, 283 insertions(+), 74 deletions(-) create mode 100644 pkg/provision/config/gateway_test.go delete mode 100644 pkg/provision/envoygateway/assets/gatewayclass.yaml diff --git a/e2e/envoygateway_test.go b/e2e/envoygateway_test.go index ddb70e6..4fca322 100644 --- a/e2e/envoygateway_test.go +++ b/e2e/envoygateway_test.go @@ -48,10 +48,11 @@ func TestEnvoyGateway_InstallAgainstKwok(t *testing.T) { setupCluster(t) if err := envoygateway.Install(context.Background(), envoygateway.Options{ - ContextName: contextName, - CacheOverride: sharedEnvoyGatewayCache(t), - Logger: logger(t), - ReadyTimeout: -1, // skip wait: kwok doesn't run the real controller + ContextName: contextName, + CacheOverride: sharedEnvoyGatewayCache(t), + Logger: logger(t), + ReadyTimeout: -1, // skip wait: kwok doesn't run the real controller + GatewayClassName: "y-cluster", // matches the production default }); err != nil { t.Fatalf("Install: %v", err) } @@ -95,41 +96,38 @@ func TestEnvoyGateway_InstallAgainstKwok(t *testing.T) { } // Default GatewayClass landed and points at EG's controller. - gcOut := kubectl(t, "get", "gatewayclass", "eg", + gcOut := kubectl(t, "get", "gatewayclass", "y-cluster", "-o", "jsonpath={.spec.controllerName}") want := "gateway.envoyproxy.io/gatewayclass-controller" if gcOut != want { - t.Errorf("GatewayClass eg.spec.controllerName = %q, want %q", gcOut, want) + t.Errorf("GatewayClass y-cluster.spec.controllerName = %q, want %q", gcOut, want) } } -// TestEnvoyGateway_InstallSkipGatewayClass verifies the opt-out -// for consumers that bring their own GatewayClass. -func TestEnvoyGateway_InstallSkipGatewayClass(t *testing.T) { +// TestEnvoyGateway_InstallEmptyClassNameSkipsApply verifies that +// passing GatewayClassName="" makes Install skip the GatewayClass +// apply (controller still installs). This is the test-only path +// for "controller without a default GatewayClass"; the production +// CommonConfig.GatewayConfig is all-or-nothing per cluster +// config, but the underlying Options field stays expressive. +func TestEnvoyGateway_InstallEmptyClassNameSkipsApply(t *testing.T) { setupCluster(t) - // Apply the bundle without the default GatewayClass; if a - // previous run created one, remove it first so the assertion - // below isn't a stale-state false negative. - _ = exec.Command("kubectl", "--context="+contextName, - "delete", "gatewayclass", "eg-skip-test", "--ignore-not-found").Run() - if err := envoygateway.Install(context.Background(), envoygateway.Options{ ContextName: contextName, CacheOverride: sharedEnvoyGatewayCache(t), Logger: logger(t), ReadyTimeout: -1, - SkipGatewayClass: true, + GatewayClassName: "", // explicit: do not apply a GatewayClass }); err != nil { t.Fatalf("Install: %v", err) } - // The default `eg` GatewayClass may exist from a prior test; - // what we want to prove is that SkipGatewayClass doesn't - // create a NEW one. The TestEnvoyGateway_InstallAgainstKwok - // covers the create path; here we just check the option - // was wired (no panic, Install returned nil) -- the negative - // behaviour is hard to assert when tests share a cluster. + // Tests share the kwok cluster, so a previously-created + // GatewayClass may still be present from another test; we + // can't assert "no GatewayClass exists". What we can assert + // is that Install did not error -- proving the empty-name + // path is wired and doesn't crash on the missing resource. } // kubectl runs `kubectl --context= args...` and returns diff --git a/pkg/provision/config/common.go b/pkg/provision/config/common.go index 338634a..f55bf64 100644 --- a/pkg/provision/config/common.go +++ b/pkg/provision/config/common.go @@ -61,6 +61,65 @@ type CommonConfig struct { K3s K3sConfig `yaml:"k3s,omitempty" json:"k3s,omitempty" jsonschema:"description=k3s install settings. Defaults track pkg/provision/config/k3s.yaml."` PortForwards []PortForward `yaml:"portForwards,omitempty" json:"portForwards,omitempty" jsonschema:"description=Host->guest TCP port forwards. Defaults to 6443/80/443 when omitted. Must include a guest:6443 entry so the host's kubectl can reach the API server."` Registries Registries `yaml:"registries,omitempty" json:"registries,omitempty" jsonschema:"description=k3s registries.yaml content. Written to /etc/rancher/k3s/registries.yaml on the node before k3s starts. ${VAR} substitution is supported on credential and endpoint fields."` + Gateway GatewayConfig `yaml:"gateway,omitempty" json:"gateway,omitempty" jsonschema:"description=Bundled Envoy Gateway install. Skip the install entirely (no CRDs, controller, or GatewayClass) by setting skip:true; rename the default GatewayClass via name."` +} + +// GatewayConfig controls the bundled Envoy Gateway install +// (pkg/provision/envoygateway). Two knobs: +// +// - skip: false (default) install CRDs, controller, default GatewayClass +// - skip: true no CRDs, controller, or GatewayClass +// - className: (default "y-cluster") rename the default GatewayClass +// +// All-or-nothing: there is no "install controller without a default +// GatewayClass" option. A consumer that wants to ship their own +// GatewayClass should also ship their own controller install. +// +// override-ip is intentionally NOT a field here: it's derived from +// PortForwards (loopback when guest:80 is bound to a host port) +// and exposed to ystack consumers via a kube-system ConfigMap, not +// via cluster config the user has to maintain. +type GatewayConfig struct { + // Skip omits the entire Envoy Gateway install (CRDs, controller, + // GatewayClass). Useful for test clusters that don't need HTTP + // ingress -- saves the ~50 MB image pull and a few seconds of + // rollout. k3s --disable=traefik is still passed; if you want a + // different ingress, install it yourself. + Skip bool `yaml:"skip,omitempty" json:"skip,omitempty" jsonschema:"description=If true, do not install Envoy Gateway. k3s still runs with --disable=traefik."` + + // ClassName names the default GatewayClass y-cluster applies + // after the EG controller is up. Consumer Gateway resources + // reference this via gatewayClassName. + // + // Default: y-cluster. Set to "eg" to keep compatibility with + // consumers that hardcoded that name in pre-v0.4 cluster + // configs (the ystack gateway-v4 surface, for one). + // + // Ignored when Skip is true. + ClassName string `yaml:"className,omitempty" json:"className,omitempty" jsonschema:"default=y-cluster,description=GatewayClass name. Consumer Gateway resources reference this via gatewayClassName. Ignored when skip is true."` +} + +// applyGatewayDefaults fills ClassName when the install is +// enabled. When Skip is set, ClassName is left as the user +// supplied it so debug logs make the operator's intent obvious. +func (c *CommonConfig) applyGatewayDefaults() { + if c.Gateway.Skip { + return + } + if c.Gateway.ClassName == "" { + c.Gateway.ClassName = "y-cluster" + } +} + +// EffectiveGatewayClassName returns the GatewayClass name the +// provisioner should hand to envoygateway.Install. Empty string +// means "do not apply a GatewayClass" (because the whole install +// is skipped). +func (c CommonConfig) EffectiveGatewayClassName() string { + if c.Gateway.Skip { + return "" + } + return c.Gateway.ClassName } // PortForward maps a host port to a guest port. Common to all @@ -103,8 +162,8 @@ type K3sConfig struct { } // applyCommonDefaults fills defaults that the reflective tag-default -// pass can't reach: K3s.Version (data-file driven) and PortForwards -// (slice default). +// pass can't reach: K3s.Version (data-file driven), PortForwards +// (slice default), GatewayConfig.Name (default y-cluster). func (c *CommonConfig) applyCommonDefaults() { if c.K3s.Version == "" { c.K3s.Version = K3sDefaultVersion() @@ -120,6 +179,7 @@ func (c *CommonConfig) applyCommonDefaults() { {Host: "443", Guest: "443"}, } } + c.applyGatewayDefaults() } // validateCommon checks invariants every provider relies on. The diff --git a/pkg/provision/config/gateway_test.go b/pkg/provision/config/gateway_test.go new file mode 100644 index 0000000..4269113 --- /dev/null +++ b/pkg/provision/config/gateway_test.go @@ -0,0 +1,62 @@ +package config + +import "testing" + +// TestGateway_DefaultClassName: an empty GatewayConfig defaults +// to the well-known "y-cluster" GatewayClass name. Pinned because +// downstream consumers (ystack) reference this name verbatim. +func TestGateway_DefaultClassName(t *testing.T) { + c := &CommonConfig{} + c.applyCommonDefaults() + if c.Gateway.ClassName != "y-cluster" { + t.Fatalf("ClassName: got %q, want y-cluster", c.Gateway.ClassName) + } + if c.Gateway.Skip { + t.Fatal("Skip should remain false by default") + } +} + +// TestGateway_PreservesExplicitClassName: a user pinning a +// non-default class name (e.g. "eg" for compat) survives +// defaulting. +func TestGateway_PreservesExplicitClassName(t *testing.T) { + c := &CommonConfig{Gateway: GatewayConfig{ClassName: "eg"}} + c.applyCommonDefaults() + if c.Gateway.ClassName != "eg" { + t.Fatalf("ClassName: got %q, want eg", c.Gateway.ClassName) + } +} + +// TestGateway_SkipLeavesClassNameAlone: when Skip is set, the +// defaulter doesn't fill ClassName -- the rendered config / debug +// logs make the operator's intent (no install at all) obvious. +func TestGateway_SkipLeavesClassNameAlone(t *testing.T) { + c := &CommonConfig{Gateway: GatewayConfig{Skip: true}} + c.applyCommonDefaults() + if c.Gateway.ClassName != "" { + t.Fatalf("Skip:true should leave ClassName empty, got %q", c.Gateway.ClassName) + } +} + +// TestEffectiveGatewayClassName covers the helper Provision uses +// to pick what (if anything) to pass to envoygateway.Install: +// empty when skipped, the configured name otherwise. +func TestEffectiveGatewayClassName(t *testing.T) { + cases := []struct { + name string + gw GatewayConfig + want string + }{ + {"default", GatewayConfig{ClassName: "y-cluster"}, "y-cluster"}, + {"custom name", GatewayConfig{ClassName: "eg"}, "eg"}, + {"skipped", GatewayConfig{Skip: true, ClassName: "y-cluster"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := CommonConfig{Gateway: tc.gw} + if got := c.EffectiveGatewayClassName(); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} diff --git a/pkg/provision/docker/docker.go b/pkg/provision/docker/docker.go index 25ca742..e2aa192 100644 --- a/pkg/provision/docker/docker.go +++ b/pkg/provision/docker/docker.go @@ -201,14 +201,23 @@ func Provision(ctx context.Context, cfg config.DockerConfig, logger *zap.Logger) // Install the bundled Envoy Gateway (CRDs + controller + // default GatewayClass). Replaces Traefik, which we disabled - // in the k3s server cmd above. - if err := envoygateway.Install(ctx, envoygateway.Options{ - ContextName: cfg.Context, - Logger: logger, - }); err != nil { - return nil, fmt.Errorf("install envoy gateway: %w", err) + // in the k3s server cmd above. Skipped wholesale when + // gateway.skip is set in cluster config. + if cfg.Gateway.Skip { + logger.Info("envoy gateway install skipped (gateway.skip)") + } else { + if err := envoygateway.Install(ctx, envoygateway.Options{ + ContextName: cfg.Context, + GatewayClassName: cfg.Gateway.ClassName, + Logger: logger, + }); err != nil { + return nil, fmt.Errorf("install envoy gateway: %w", err) + } + logger.Info("envoy gateway ready", + zap.String("version", envoygateway.Version), + zap.String("gatewayClass", cfg.Gateway.ClassName), + ) } - logger.Info("envoy gateway ready", zap.String("version", envoygateway.Version)) return c, nil } diff --git a/pkg/provision/envoygateway/assets/gatewayclass.yaml b/pkg/provision/envoygateway/assets/gatewayclass.yaml deleted file mode 100644 index 915a107..0000000 --- a/pkg/provision/envoygateway/assets/gatewayclass.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# y-cluster default GatewayClass for the bundled Envoy Gateway -# install. Consumers reference it from their own Gateway resources -# via gatewayClassName: eg. -# -# y-cluster does NOT install a cluster Gateway here -- listener -# port and TLS choices belong to the consumer's kustomize bases. -apiVersion: gateway.networking.k8s.io/v1 -kind: GatewayClass -metadata: - name: eg -spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/pkg/provision/envoygateway/embed.go b/pkg/provision/envoygateway/embed.go index 993d178..b6684df 100644 --- a/pkg/provision/envoygateway/embed.go +++ b/pkg/provision/envoygateway/embed.go @@ -1,17 +1,33 @@ package envoygateway -import _ "embed" +import "fmt" -// gatewayClassYAML is the default `eg` GatewayClass manifest. -// Tiny (~10 lines) and y-cluster-owned, so it stays embedded -// rather than being downloaded -- nothing about it changes per -// EG release. install.yaml, by contrast, is the upstream release -// asset and lives in the per-version cache so a fresh provision -// can pick a different Version without recompiling. -// -//go:embed assets/gatewayclass.yaml -var gatewayClassYAML []byte +// EGControllerName is the controllerName Envoy Gateway watches for +// when picking up GatewayClass resources. Fixed by EG; the +// y-cluster-installed GatewayClass references it. +const EGControllerName = "gateway.envoyproxy.io/gatewayclass-controller" -// GatewayClassYAML returns the embedded default GatewayClass -// bytes. Read-only. -func GatewayClassYAML() []byte { return gatewayClassYAML } +// GatewayClassYAML renders the default GatewayClass manifest with +// the configured class name. The YAML body is small enough +// (~10 lines) that we'd previously embedded it verbatim with the +// name hardcoded; rendering inline lets the operator pick a +// non-default name via cluster config. +// +// Pure function so unit tests can pin the rendered shape against +// a known-good baseline. +func GatewayClassYAML(name string) []byte { + return []byte(fmt.Sprintf(`--- +# y-cluster default GatewayClass for the bundled Envoy Gateway +# install. Consumer Gateway resources reference this name via +# gatewayClassName: %s. +# +# y-cluster does NOT install a cluster Gateway here -- listener +# port and TLS choices belong to the consumer's kustomize bases. +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: %s +spec: + controllerName: %s +`, name, name, EGControllerName)) +} diff --git a/pkg/provision/envoygateway/install.go b/pkg/provision/envoygateway/install.go index d60c60e..4850d68 100644 --- a/pkg/provision/envoygateway/install.go +++ b/pkg/provision/envoygateway/install.go @@ -41,10 +41,15 @@ type Options struct { // kwok-based tests where the controller never actually rolls // out a real Deployment). ReadyTimeout time.Duration - // SkipGatewayClass omits applying the default `eg` - // GatewayClass. Useful when the consumer's kustomize base - // declares its own GatewayClass under a different name. - SkipGatewayClass bool + // GatewayClassName names the default GatewayClass Install + // applies after the controller is up. Empty means "don't apply + // a GatewayClass" -- useful when the consumer's kustomize base + // ships its own. + // + // Provision-driven calls fill this from CommonConfig.Gateway.Name + // (default "y-cluster"); test calls can leave it empty to skip + // the apply. + GatewayClassName string } // Install resolves the per-version install.yaml from cache @@ -66,8 +71,8 @@ type Options struct { // once the CRDs are registered. // 3. kubectl rollout status deployment/envoy-gateway in // envoy-gateway-system (skipped when ReadyTimeout < 0). -// 4. kubectl apply the default `eg` GatewayClass (skipped when -// SkipGatewayClass is set). +// 4. kubectl apply the default GatewayClass with the configured +// name (skipped when GatewayClassName is empty). // // Implementation switched from client-go's typed apply / rollout // to kubectl shellouts to drop pkg/k8sapply + pkg/k8swait (and @@ -123,9 +128,11 @@ func Install(ctx context.Context, opts Options) error { } } - if !opts.SkipGatewayClass { - logger.Info("applying default GatewayClass eg") - if err := kubectlApplyStdin(ctx, opts.ContextName, gatewayClassYAML); err != nil { + if opts.GatewayClassName != "" { + logger.Info("applying default GatewayClass", + zap.String("name", opts.GatewayClassName), + ) + if err := kubectlApplyStdin(ctx, opts.ContextName, GatewayClassYAML(opts.GatewayClassName)); err != nil { return fmt.Errorf("apply GatewayClass: %w", err) } } diff --git a/pkg/provision/qemu/qemu.go b/pkg/provision/qemu/qemu.go index 6e3f98c..35b5b25 100644 --- a/pkg/provision/qemu/qemu.go +++ b/pkg/provision/qemu/qemu.go @@ -56,6 +56,7 @@ type Config struct { Kubeconfig string K3s K3s Registries config.Registries + Gateway config.GatewayConfig } // K3s carries the runtime view of K3sConfig: which version to @@ -121,6 +122,7 @@ func FromConfig(c *config.QEMUConfig) Config { Install: c.K3s.Install, }, Registries: c.Registries, + Gateway: c.Gateway, } } @@ -261,14 +263,25 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e // Install the bundled Envoy Gateway (CRDs + controller + // default GatewayClass). Replaces the Traefik k3s would // otherwise have run; --disable=traefik passed to k3s above - // keeps that one out of the picture. - if err := envoygateway.Install(ctx, envoygateway.Options{ - ContextName: cfg.Context, - Logger: logger, - }); err != nil { - return nil, fmt.Errorf("install envoy gateway: %w", err) - } - logger.Info("envoy gateway ready", zap.String("version", envoygateway.Version)) + // keeps that one out of the picture. Skipped wholesale when + // gateway.skip is set in cluster config -- the cluster comes + // up with no ingress controller (k3s still has --disable=traefik + // so traefik doesn't sneak back in). + if cfg.Gateway.Skip { + logger.Info("envoy gateway install skipped (gateway.skip)") + } else { + if err := envoygateway.Install(ctx, envoygateway.Options{ + ContextName: cfg.Context, + GatewayClassName: cfg.Gateway.ClassName, + Logger: logger, + }); err != nil { + return nil, fmt.Errorf("install envoy gateway: %w", err) + } + logger.Info("envoy gateway ready", + zap.String("version", envoygateway.Version), + zap.String("gatewayClass", cfg.Gateway.ClassName), + ) + } return c, nil } diff --git a/pkg/provision/schema/common.schema.json b/pkg/provision/schema/common.schema.json index 79d4640..f6c4006 100644 --- a/pkg/provision/schema/common.schema.json +++ b/pkg/provision/schema/common.schema.json @@ -13,6 +13,10 @@ "description": "vCPU count. qemu sets -smp; docker passes --cpus.", "type": "string" }, + "gateway": { + "$ref": "#/$defs/GatewayConfig", + "description": "Bundled Envoy Gateway install. Skip the install entirely (no CRDs" + }, "k3s": { "$ref": "#/$defs/K3sConfig", "description": "k3s install settings. Defaults track pkg/provision/config/k3s.yaml." @@ -52,6 +56,21 @@ ], "type": "object" }, + "GatewayConfig": { + "additionalProperties": false, + "properties": { + "className": { + "default": "y-cluster", + "description": "GatewayClass name. Consumer Gateway resources reference this via gatewayClassName. Ignored when skip is true.", + "type": "string" + }, + "skip": { + "description": "If true", + "type": "boolean" + } + }, + "type": "object" + }, "K3sConfig": { "additionalProperties": false, "properties": { diff --git a/pkg/provision/schema/docker.schema.json b/pkg/provision/schema/docker.schema.json index e87305c..475a270 100644 --- a/pkg/provision/schema/docker.schema.json +++ b/pkg/provision/schema/docker.schema.json @@ -13,6 +13,10 @@ "description": "vCPU count. qemu sets -smp; docker passes --cpus.", "type": "string" }, + "gateway": { + "$ref": "#/$defs/GatewayConfig", + "description": "Bundled Envoy Gateway install. Skip the install entirely (no CRDs" + }, "k3s": { "$ref": "#/$defs/K3sConfig", "description": "k3s install settings. Defaults track pkg/provision/config/k3s.yaml." @@ -49,6 +53,21 @@ ], "type": "object" }, + "GatewayConfig": { + "additionalProperties": false, + "properties": { + "className": { + "default": "y-cluster", + "description": "GatewayClass name. Consumer Gateway resources reference this via gatewayClassName. Ignored when skip is true.", + "type": "string" + }, + "skip": { + "description": "If true", + "type": "boolean" + } + }, + "type": "object" + }, "K3sConfig": { "additionalProperties": false, "properties": { diff --git a/pkg/provision/schema/qemu.schema.json b/pkg/provision/schema/qemu.schema.json index ec3e578..30bb8e3 100644 --- a/pkg/provision/schema/qemu.schema.json +++ b/pkg/provision/schema/qemu.schema.json @@ -1,5 +1,20 @@ { "$defs": { + "GatewayConfig": { + "additionalProperties": false, + "properties": { + "className": { + "default": "y-cluster", + "description": "GatewayClass name. Consumer Gateway resources reference this via gatewayClassName. Ignored when skip is true.", + "type": "string" + }, + "skip": { + "description": "If true", + "type": "boolean" + } + }, + "type": "object" + }, "K3sConfig": { "additionalProperties": false, "properties": { @@ -60,6 +75,10 @@ "description": "qcow2 disk size as a [num][KMGT] string.", "type": "string" }, + "gateway": { + "$ref": "#/$defs/GatewayConfig", + "description": "Bundled Envoy Gateway install. Skip the install entirely (no CRDs" + }, "k3s": { "$ref": "#/$defs/K3sConfig", "description": "k3s install settings. Defaults track pkg/provision/config/k3s.yaml." From 157b46b7952253b311c731684afe5229f6851c50 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Thu, 30 Apr 2026 08:29:42 +0000 Subject: [PATCH 2/2] feat(provision): publish host-routable IP via yolean.se/dns-hint-ip on GatewayClass Replaces the reverted --node-external-ip approach (which broke pod-to-apiserver routing). The provisioner derives the host-side dial address from PortForwards via CommonConfig.HostRoutableIP -- "127.0.0.1" when guest:80 is forwarded, empty otherwise -- and stamps it as the yolean.se/dns-hint-ip annotation on the y-cluster GatewayClass at install time. The value is not user-configurable: it's a physical fact about the host/guest port-forward layer, not a preference. There is no config field for it; consumers needing a different value adjust PortForwards. Consumer tooling (ystack's y-k8s-ingress-hosts) reads the annotation via Gateway -> gatewayClassName -> GatewayClass instead of the prior OVERRIDE_IP env-var chain. Migration spelled out in specs/ystack/CHANGE_REQUEST_HINT_IP.md (separate repo). When gateway.skip is set no GatewayClass is installed, so no hint is published -- consumers using skip-mode handle their own ingress and DNS. Co-Authored-By: Claude Opus 4.7 (1M context) --- e2e/envoygateway_test.go | 14 ++++- pkg/provision/config/common.go | 34 +++++++++-- pkg/provision/config/host_routable_ip_test.go | 52 ++++++++++++++++ pkg/provision/docker/docker.go | 1 + pkg/provision/envoygateway/embed.go | 33 +++++++--- pkg/provision/envoygateway/embed_test.go | 61 +++++++++++++++++++ pkg/provision/envoygateway/install.go | 14 ++++- pkg/provision/qemu/qemu.go | 17 ++++++ 8 files changed, 213 insertions(+), 13 deletions(-) create mode 100644 pkg/provision/config/host_routable_ip_test.go create mode 100644 pkg/provision/envoygateway/embed_test.go diff --git a/e2e/envoygateway_test.go b/e2e/envoygateway_test.go index 4fca322..627e225 100644 --- a/e2e/envoygateway_test.go +++ b/e2e/envoygateway_test.go @@ -51,8 +51,9 @@ func TestEnvoyGateway_InstallAgainstKwok(t *testing.T) { ContextName: contextName, CacheOverride: sharedEnvoyGatewayCache(t), Logger: logger(t), - ReadyTimeout: -1, // skip wait: kwok doesn't run the real controller + ReadyTimeout: -1, // skip wait: kwok doesn't run the real controller GatewayClassName: "y-cluster", // matches the production default + DNSHintIP: "127.0.0.1", // simulates qemu/docker host-loopback case }); err != nil { t.Fatalf("Install: %v", err) } @@ -102,6 +103,17 @@ func TestEnvoyGateway_InstallAgainstKwok(t *testing.T) { if gcOut != want { t.Errorf("GatewayClass y-cluster.spec.controllerName = %q, want %q", gcOut, want) } + + // dns-hint-ip annotation landed: this is the contract ystack's + // y-k8s-ingress-hosts (and any future host-side resolver tool) + // reads to find the host-routable address without user-side + // config. Pinned because consumers cite the exact annotation key. + hintOut := kubectl(t, "get", "gatewayclass", "y-cluster", + "-o", "jsonpath={.metadata.annotations."+strings.ReplaceAll(envoygateway.DNSHintIPAnnotation, ".", "\\.")+"}") + if hintOut != "127.0.0.1" { + t.Errorf("GatewayClass y-cluster annotation %s = %q, want 127.0.0.1", + envoygateway.DNSHintIPAnnotation, hintOut) + } } // TestEnvoyGateway_InstallEmptyClassNameSkipsApply verifies that diff --git a/pkg/provision/config/common.go b/pkg/provision/config/common.go index f55bf64..a1392a7 100644 --- a/pkg/provision/config/common.go +++ b/pkg/provision/config/common.go @@ -75,10 +75,13 @@ type CommonConfig struct { // GatewayClass" option. A consumer that wants to ship their own // GatewayClass should also ship their own controller install. // -// override-ip is intentionally NOT a field here: it's derived from -// PortForwards (loopback when guest:80 is bound to a host port) -// and exposed to ystack consumers via a kube-system ConfigMap, not -// via cluster config the user has to maintain. +// The host-side dial address (where /etc/hosts on the developer +// machine should resolve gateway hostnames to) is intentionally NOT +// a field here. It's derived from PortForwards via HostRoutableIP +// and exposed to consumers as the yolean.se/dns-hint-ip annotation +// on the GatewayClass. No user-facing knob -- the value is a +// physical fact about the host/guest port-forward layer, not a +// preference. type GatewayConfig struct { // Skip omits the entire Envoy Gateway install (CRDs, controller, // GatewayClass). Useful for test clusters that don't need HTTP @@ -130,6 +133,29 @@ type PortForward struct { Guest string `yaml:"guest" json:"guest" jsonschema:"description=Guest port to forward to."` } +// HostRoutableIP returns the IP at which the host reaches the +// cluster's HTTP ingress (Envoy Gateway). Today the only providers +// y-cluster supports (qemu SLIRP, docker port-forwards) bind ingress +// on the host loopback, so the value is "127.0.0.1" whenever guest:80 +// is in PortForwards. Empty means "no host-side dial address" -- +// either no guest:80 forward, or a future provisioner topology that +// doesn't tunnel through the host (multi-VM bridged, cloud LB). +// +// The provisioner publishes this value to the cluster as the +// yolean.se/dns-hint-ip annotation on the y-cluster GatewayClass, +// so consumer tooling like ystack's y-k8s-ingress-hosts can read it +// without any user-side configuration. The value derives entirely +// from PortForwards -- there is no config field that lets the user +// influence it directly. +func (c CommonConfig) HostRoutableIP() string { + for _, pf := range c.PortForwards { + if pf.Guest == "80" { + return "127.0.0.1" + } + } + return "" +} + // HostAPIPort returns the host-side port mapped to guest 6443. // Provisioners use this to surface the kubectl-facing endpoint: // qemu rewrites the extracted kubeconfig server URL, docker does diff --git a/pkg/provision/config/host_routable_ip_test.go b/pkg/provision/config/host_routable_ip_test.go new file mode 100644 index 0000000..33886ee --- /dev/null +++ b/pkg/provision/config/host_routable_ip_test.go @@ -0,0 +1,52 @@ +package config + +import "testing" + +// TestHostRoutableIP_NoForwards covers the cloud-shaped +// (no-host-tunneling) topology: empty PortForwards mean there's no +// host-side dial address to advertise, so the helper returns "" +// and the provisioner omits the dns-hint-ip annotation. +func TestHostRoutableIP_NoForwards(t *testing.T) { + c := CommonConfig{} + if got := c.HostRoutableIP(); got != "" { + t.Fatalf("HostRoutableIP with no forwards: %q", got) + } +} + +// TestHostRoutableIP_NoIngressForward covers a config that has an +// API forward but no ingress (guest:80) forward. The cluster is +// reachable for kubectl but no host loopback maps to Envoy, so +// there's nothing to hint at. +func TestHostRoutableIP_NoIngressForward(t *testing.T) { + c := CommonConfig{PortForwards: []PortForward{ + {Host: "26443", Guest: "6443"}, + }} + if got := c.HostRoutableIP(); got != "" { + t.Fatalf("HostRoutableIP without guest:80: %q", got) + } +} + +// TestHostRoutableIP_WithIngress covers the qemu/docker default +// shape: guest:80 is bound to the host loopback via PortForwards, +// so the helper returns 127.0.0.1. +func TestHostRoutableIP_WithIngress(t *testing.T) { + c := CommonConfig{PortForwards: []PortForward{ + {Host: "26443", Guest: "6443"}, + {Host: "80", Guest: "80"}, + {Host: "443", Guest: "443"}, + }} + if got := c.HostRoutableIP(); got != "127.0.0.1" { + t.Fatalf("HostRoutableIP: %q (want 127.0.0.1)", got) + } +} + +// TestHostRoutableIP_DefaultedConfig pins the breaking-change +// contract: a defaulted config (any provider) gets the hint IP for +// free because the default port forwards include guest:80. +func TestHostRoutableIP_DefaultedConfig(t *testing.T) { + c := &DockerConfig{CommonConfig: CommonConfig{Provider: ProviderDocker}} + c.ApplyDefaults() + if got := c.HostRoutableIP(); got != "127.0.0.1" { + t.Fatalf("defaulted DockerConfig HostRoutableIP: %q", got) + } +} diff --git a/pkg/provision/docker/docker.go b/pkg/provision/docker/docker.go index e2aa192..15f73a0 100644 --- a/pkg/provision/docker/docker.go +++ b/pkg/provision/docker/docker.go @@ -209,6 +209,7 @@ func Provision(ctx context.Context, cfg config.DockerConfig, logger *zap.Logger) if err := envoygateway.Install(ctx, envoygateway.Options{ ContextName: cfg.Context, GatewayClassName: cfg.Gateway.ClassName, + DNSHintIP: cfg.HostRoutableIP(), Logger: logger, }); err != nil { return nil, fmt.Errorf("install envoy gateway: %w", err) diff --git a/pkg/provision/envoygateway/embed.go b/pkg/provision/envoygateway/embed.go index b6684df..6f129a0 100644 --- a/pkg/provision/envoygateway/embed.go +++ b/pkg/provision/envoygateway/embed.go @@ -7,15 +7,33 @@ import "fmt" // y-cluster-installed GatewayClass references it. const EGControllerName = "gateway.envoyproxy.io/gatewayclass-controller" +// DNSHintIPAnnotation publishes the host-side IP at which the +// developer's machine reaches the cluster's HTTP ingress, so +// consumer tooling (ystack's y-k8s-ingress-hosts, etc.) can rewrite +// /etc/hosts without depending on user-supplied config or the +// previous OVERRIDE_IP env-var chain. +// +// Lives on the GatewayClass because that resource exists at +// provision time, is cluster-scoped, and is the natural lookup +// point from any Gateway resource (consumers walk Gateway -> +// gatewayClassName -> GatewayClass to find it). Absent annotation +// = no host-side override; consumers fall back to whatever they +// did before. +const DNSHintIPAnnotation = "yolean.se/dns-hint-ip" + // GatewayClassYAML renders the default GatewayClass manifest with -// the configured class name. The YAML body is small enough -// (~10 lines) that we'd previously embedded it verbatim with the -// name hardcoded; rendering inline lets the operator pick a -// non-default name via cluster config. +// the configured class name. dnsHintIP is the value the provisioner +// stamps under the DNSHintIPAnnotation; empty string omits the +// annotations block entirely so an absent hint is distinguishable +// from a present-but-empty one. // // Pure function so unit tests can pin the rendered shape against // a known-good baseline. -func GatewayClassYAML(name string) []byte { +func GatewayClassYAML(name, dnsHintIP string) []byte { + var annotations string + if dnsHintIP != "" { + annotations = fmt.Sprintf(" annotations:\n %s: %s\n", DNSHintIPAnnotation, dnsHintIP) + } return []byte(fmt.Sprintf(`--- # y-cluster default GatewayClass for the bundled Envoy Gateway # install. Consumer Gateway resources reference this name via @@ -27,7 +45,8 @@ apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: name: %s -spec: +%sspec: controllerName: %s -`, name, name, EGControllerName)) +`, name, name, annotations, EGControllerName)) } + diff --git a/pkg/provision/envoygateway/embed_test.go b/pkg/provision/envoygateway/embed_test.go new file mode 100644 index 0000000..81cd918 --- /dev/null +++ b/pkg/provision/envoygateway/embed_test.go @@ -0,0 +1,61 @@ +package envoygateway + +import ( + "strings" + "testing" +) + +// TestGatewayClassYAML_NoHintIP guards the cloud / no-host-routing +// shape: empty dnsHintIP omits the metadata.annotations block +// entirely, so an absent hint is distinguishable from +// "annotation present with empty value". +func TestGatewayClassYAML_NoHintIP(t *testing.T) { + got := string(GatewayClassYAML("y-cluster", "")) + if strings.Contains(got, "annotations") { + t.Fatalf("expected no annotations block:\n%s", got) + } + if strings.Contains(got, DNSHintIPAnnotation) { + t.Fatalf("expected no %s annotation:\n%s", DNSHintIPAnnotation, got) + } + if !strings.Contains(got, "name: y-cluster") { + t.Fatalf("missing class name:\n%s", got) + } + if !strings.Contains(got, "controllerName: "+EGControllerName) { + t.Fatalf("missing controller name:\n%s", got) + } +} + +// TestGatewayClassYAML_WithHintIP guards the qemu/docker +// host-loopback shape: the dnsHintIP value lands as a single +// annotation under the GatewayClass metadata. +func TestGatewayClassYAML_WithHintIP(t *testing.T) { + got := string(GatewayClassYAML("y-cluster", "127.0.0.1")) + if !strings.Contains(got, "annotations:") { + t.Fatalf("missing annotations block:\n%s", got) + } + wantLine := DNSHintIPAnnotation + ": 127.0.0.1" + if !strings.Contains(got, wantLine) { + t.Fatalf("missing %q:\n%s", wantLine, got) + } + // Annotation block must precede spec; otherwise YAML attaches it + // to the wrong field. + annoIdx := strings.Index(got, "annotations:") + specIdx := strings.Index(got, "spec:") + if annoIdx < 0 || specIdx < 0 || annoIdx > specIdx { + t.Fatalf("annotations not under metadata before spec:\n%s", got) + } +} + +// TestGatewayClassYAML_RespectsCustomName guards the rename path: +// a non-default ClassName (e.g. "eg" for compat) flows through to +// both metadata.name and the doc comment. The comment header line +// is part of the contract -- consumers grep for it during debug. +func TestGatewayClassYAML_RespectsCustomName(t *testing.T) { + got := string(GatewayClassYAML("eg", "")) + if !strings.Contains(got, "name: eg") { + t.Fatalf("missing custom name:\n%s", got) + } + if !strings.Contains(got, "gatewayClassName: eg") { + t.Fatalf("comment should reference the configured name:\n%s", got) + } +} diff --git a/pkg/provision/envoygateway/install.go b/pkg/provision/envoygateway/install.go index 4850d68..23519dd 100644 --- a/pkg/provision/envoygateway/install.go +++ b/pkg/provision/envoygateway/install.go @@ -50,6 +50,17 @@ type Options struct { // (default "y-cluster"); test calls can leave it empty to skip // the apply. GatewayClassName string + // DNSHintIP, when non-empty, surfaces on the applied GatewayClass + // as the yolean.se/dns-hint-ip annotation so consumer tooling + // (ystack's y-k8s-ingress-hosts) can read the host-side dial IP + // without any user-supplied config. Provision-driven calls fill + // this from CommonConfig.HostRoutableIP (derived from + // PortForwards). Empty means: don't set the annotation -- the + // natural state for cluster topologies that don't tunnel ingress + // through the host (multi-VM bridged, cloud LB). + // + // Ignored when GatewayClassName is empty (no GatewayClass apply). + DNSHintIP string } // Install resolves the per-version install.yaml from cache @@ -131,8 +142,9 @@ func Install(ctx context.Context, opts Options) error { if opts.GatewayClassName != "" { logger.Info("applying default GatewayClass", zap.String("name", opts.GatewayClassName), + zap.String("dnsHintIP", opts.DNSHintIP), ) - if err := kubectlApplyStdin(ctx, opts.ContextName, GatewayClassYAML(opts.GatewayClassName)); err != nil { + if err := kubectlApplyStdin(ctx, opts.ContextName, GatewayClassYAML(opts.GatewayClassName, opts.DNSHintIP)); err != nil { return fmt.Errorf("apply GatewayClass: %w", err) } } diff --git a/pkg/provision/qemu/qemu.go b/pkg/provision/qemu/qemu.go index 35b5b25..39b1174 100644 --- a/pkg/provision/qemu/qemu.go +++ b/pkg/provision/qemu/qemu.go @@ -83,6 +83,22 @@ func (c Config) hostAPIPort() string { return "" } +// hostRoutableIP returns the host-side IP at which the host reaches +// the cluster's HTTP ingress. Same derivation as +// config.CommonConfig.HostRoutableIP -- duplicated here because the +// runtime Config already carries a translated PortForwards slice +// and would otherwise need a back-reference to the on-disk config. +// Empty string means no host-side override; the call site uses it +// as the DNSHintIP option, which an empty value omits. +func (c Config) hostRoutableIP() string { + for _, pf := range c.PortForwards { + if pf.Guest == "80" { + return "127.0.0.1" + } + } + return "" +} + // FromConfig translates the on-disk QEMUConfig (already // defaults-applied and validated by configfile.Load) into the // runtime Config consumed by Provision/Teardown. @@ -273,6 +289,7 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e if err := envoygateway.Install(ctx, envoygateway.Options{ ContextName: cfg.Context, GatewayClassName: cfg.Gateway.ClassName, + DNSHintIP: cfg.hostRoutableIP(), Logger: logger, }); err != nil { return nil, fmt.Errorf("install envoy gateway: %w", err)