Reduce min keepalive interval to 1s

cmd/main.go

@@ -151,7 +151,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	authenticator, prefix, err := config.LoadConfiguration(
+	authenticator, prefix, option, err := config.LoadConfiguration(
 		context.Background(),
 		mgr.GetAPIReader(),
 		mgr.GetScheme(),
@@ -211,14 +211,16 @@ func main() {
 			ResourceKey:  "jumpstarter-kind",
 			NameKey:      "jumpstarter-name",
 		}),
+		ServerOption: option,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create service", "service", "Controller")
 		os.Exit(1)
 	}
 
 	if err = (&service.RouterService{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		ServerOption: option,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create service", "service", "Router")
 		os.Exit(1)

deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/cms/controller-cm.yaml

@@ -10,4 +10,10 @@ metadata:
     deployment.timestamp: {{ .Values.global.timestamp | quote }}
     {{ end }}
 data:
+  # backwards compatibility
+  # TODO: remove in 0.7.0
+  {{ if .Values.authenticationConfig }}
   authentication: {{- .Values.authenticationConfig | toYaml | indent 1 }}
+  {{ end }}
+  config: |
+{{ .Values.config | toYaml | indent 4 }}

deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml

@@ -9,6 +9,8 @@ metadata:
     {{ if .Values.global.timestamp }}
     deployment.timestamp: {{ .Values.global.timestamp | quote }}
     {{ end }}
+  annotations:
+    configmap-sha256: {{ include (print $.Template.BasePath "/cms/controller-cm.yaml") . | sha256sum }}
 spec:
   selector:
     matchLabels:

deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-route.yaml

@@ -5,6 +5,9 @@ metadata:
   labels:
     external-exposed: "true"
     shard: external
+  annotations:
+    haproxy.router.openshift.io/timeout: 2d
+    haproxy.router.openshift.io/timeout-tunnel: 2d
   name: jumpstarter-controller-route
   namespace: {{ default .Release.Namespace .Values.namespace }}
 spec:
@@ -28,4 +31,4 @@ spec:
     name: jumpstarter-grpc
     weight: 100
   wildcardPolicy: None
-{{ end }}
+{{ end }}

deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/router-route.yaml

@@ -5,6 +5,9 @@ metadata:
   labels:
     external-exposed: "true"
     shard: external
+  annotations:
+    haproxy.router.openshift.io/timeout: 2d
+    haproxy.router.openshift.io/timeout-tunnel: 2d
   name: jumpstarter-router-route
   namespace: {{ default .Release.Namespace .Values.namespace }}
 spec:
@@ -33,4 +36,4 @@ spec:
     name: jumpstarter-router-grpc
     weight: 100
   wildcardPolicy: None
-{{ end }}
+{{ end }}

deploy/helm/jumpstarter/values.yaml

@@ -30,7 +30,12 @@ global:
 ## @param jumpstarter-controller.imagePullPolicy Image pull policy for the controller.
 
 ## @param jumpstarter-controller.namespace Namespace where the controller will be deployed, defaults to global.namespace.
-## @param jumpstarter-controller.authenticationConfig Configuration for OIDC authentication, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-authentication-configuration for documentation
+
+## @param jumpstarter-controller.config.grpc.keepalive.minTime. The minimum amount of time a client should wait before sending a keepalive ping.
+## @param jumpstarter-controller.config.grpc.keepalive.permitWithoutStream. Whether to allow keepalive pings even when there are no active streams(RPCs).
+
+## @param jumpstarter-controller.config.authentication.internal.prefix. Prefix to add to the subject claim of the tokens issued by the builtin authenticator.
+## @param jumpstarter-controller.config.authentication.jwt. External OIDC authentication, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-authentication-configuration for documentation
 
 ## @section Ingress And Route parameters
 ## @descriptionStart This section contains parameters for the Ingress and Route configurations.
@@ -66,33 +71,40 @@ jumpstarter-controller:
 
     namespace: ""
 
-    authenticationConfig: |
-      apiVersion: jumpstarter.dev/v1alpha1
-      kind: AuthenticationConfiguration
-      # jwt:
-      # - issuer:
-      #     url: https://10.239.206.8:5556/dex
-      #     audiences:
-      #     - jumpstarter
-      #     audienceMatchPolicy: MatchAny
-      #     certificateAuthority: |
-      #       -----BEGIN CERTIFICATE-----
-      #       MIIB/DCCAYKgAwIBAgIIcpC2uS+SjEIwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV
-      #       bWluaWNhIHJvb3QgY2EgNzI5MGI2MCAXDTI1MDIwMzE5MzMyNVoYDzIxMjUwMjAz
-      #       MTkzMzI1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA3MjkwYjYwdjAQBgcq
-      #       hkjOPQIBBgUrgQQAIgNiAAQzezKJ4My35HPeoJvvzTjhS2uJMBYrYfrs5csxZjiy
-      #       q8ORrHM539XhWlA6sVZODhzcF2KL4mC9xKz/yIrsws+LKsIWNHGGmIPEKFYnHBGw
-      #       VBGeARvhpzZP/9frJXAN/8ejgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW
-      #       MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud
-      #       DgQWBBSZRBCUuP3ta2xsfjnWIjvgvz4fojAfBgNVHSMEGDAWgBSZRBCUuP3ta2xs
-      #       fjnWIjvgvz4fojAKBggqhkjOPQQDAwNoADBlAjADql5Ks5wh181iUa1ZBnx4XOVe
-      #       l0l7I+mwlwJSPmkZHxruWZTx7gQU4tfDCr+UuzUCMQC2aDXRb17cphipK4gzbExv
-      #       EDLExjhHAqMPrKDmT0jHIi7Bbos38/1tyZ/IoKjLnv0=
-      #       -----END CERTIFICATE-----
-      #   claimMappings:
-      #     username:
-      #       claim: "sub"
-      #       prefix: ""
+    config:
+      grpc:
+        keepalive:
+          # Safety: potentially makes server vulnerable to DDoS
+          # https://grpc.io/docs/guides/keepalive/#how-configuring-keepalive-affects-a-call
+          minTime: 3s
+          permitWithoutStream: true
+      authentication:
+        internal:
+          prefix: "internal:"
+        # jwt:
+        # - issuer:
+        #     url: https://10.239.206.8:5556/dex
+        #     audiences:
+        #     - jumpstarter
+        #     audienceMatchPolicy: MatchAny
+        #     certificateAuthority: |
+        #       -----BEGIN CERTIFICATE-----
+        #       MIIB/DCCAYKgAwIBAgIIcpC2uS+SjEIwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV
+        #       bWluaWNhIHJvb3QgY2EgNzI5MGI2MCAXDTI1MDIwMzE5MzMyNVoYDzIxMjUwMjAz
+        #       MTkzMzI1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA3MjkwYjYwdjAQBgcq
+        #       hkjOPQIBBgUrgQQAIgNiAAQzezKJ4My35HPeoJvvzTjhS2uJMBYrYfrs5csxZjiy
+        #       q8ORrHM539XhWlA6sVZODhzcF2KL4mC9xKz/yIrsws+LKsIWNHGGmIPEKFYnHBGw
+        #       VBGeARvhpzZP/9frJXAN/8ejgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW
+        #       MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud
+        #       DgQWBBSZRBCUuP3ta2xsfjnWIjvgvz4fojAfBgNVHSMEGDAWgBSZRBCUuP3ta2xs
+        #       fjnWIjvgvz4fojAKBggqhkjOPQQDAwNoADBlAjADql5Ks5wh181iUa1ZBnx4XOVe
+        #       l0l7I+mwlwJSPmkZHxruWZTx7gQU4tfDCr+UuzUCMQC2aDXRb17cphipK4gzbExv
+        #       EDLExjhHAqMPrKDmT0jHIi7Bbos38/1tyZ/IoKjLnv0=
+        #       -----END CERTIFICATE-----
+        #   claimMappings:
+        #     username:
+        #       claim: "sub"
+        #       prefix: ""
 
     grpc:
       hostname: ""

internal/config/config.go

@@ -3,10 +3,14 @@ package config
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/jumpstarter-dev/jumpstarter-controller/internal/oidc"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/keepalive"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -18,27 +22,59 @@ func LoadConfiguration(
 	key client.ObjectKey,
 	signer *oidc.Signer,
 	certificateAuthority string,
-) (authenticator.Token, string, error) {
+) (authenticator.Token, string, grpc.ServerOption, error) {
 	var configmap corev1.ConfigMap
 	if err := client.Get(ctx, key, &configmap); err != nil {
-		return nil, "", err
+		return nil, "", nil, err
 	}
 
 	rawAuthenticationConfiguration, ok := configmap.Data["authentication"]
+	if ok {
+		// backwards compatibility
+		// TODO: remove in 0.7.0
+		authenticator, prefix, err := oidc.LoadAuthenticationConfiguration(
+			ctx,
+			scheme,
+			[]byte(rawAuthenticationConfiguration),
+			signer,
+			certificateAuthority,
+		)
+		if err != nil {
+			return nil, "", nil, err
+		}
+
+		return authenticator, prefix, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime:             1 * time.Second,
+			PermitWithoutStream: true,
+		}), nil
+	}
+
+	rawConfig, ok := configmap.Data["config"]
 	if !ok {
-		return nil, "", fmt.Errorf("LoadConfiguration: missing authentication section")
+		return nil, "", nil, fmt.Errorf("LoadConfiguration: missing config section")
 	}
 
-	authenticator, prefix, err := oidc.LoadAuthenticationConfiguration(
+	var config Config
+	err := yaml.UnmarshalStrict([]byte(rawConfig), &config)
+	if err != nil {
+		return nil, "", nil, err
+	}
+
+	authenticator, prefix, err := LoadAuthenticationConfiguration(
 		ctx,
 		scheme,
-		[]byte(rawAuthenticationConfiguration),
+		config.Authentication,
 		signer,
 		certificateAuthority,
 	)
 	if err != nil {
-		return nil, "", err
+		return nil, "", nil, err
+	}
+
+	serverOptions, err := LoadGrpcConfiguration(config.Grpc)
+	if err != nil {
+		return nil, "", nil, err
 	}
 
-	return authenticator, prefix, nil
+	return authenticator, prefix, serverOptions, nil
 }

internal/config/grpc.go

@@ -0,0 +1,20 @@
+package config
+
+import (
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/keepalive"
+)
+
+func LoadGrpcConfiguration(config Grpc) (grpc.ServerOption, error) {
+	minTime, err := time.ParseDuration(config.Keepalive.MinTime)
+	if err != nil {
+		return nil, err
+	}
+
+	return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+		MinTime:             minTime,
+		PermitWithoutStream: config.Keepalive.PermitWithoutStream,
+	}), nil
+}

internal/config/oidc.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"context"
+
+	"github.com/jumpstarter-dev/jumpstarter-controller/internal/oidc"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	koidc "k8s.io/apiserver/plugin/pkg/authenticator/token/oidc"
+)
+
+func LoadAuthenticationConfiguration(
+	ctx context.Context,
+	scheme *runtime.Scheme,
+	config Authentication,
+	signer *oidc.Signer,
+	certificateAuthority string,
+) (authenticator.Token, string, error) {
+	if config.Internal.Prefix == "" {
+		config.Internal.Prefix = "internal:"
+	}
+
+	config.JWT = append(config.JWT, apiserverv1beta1.JWTAuthenticator{
+		Issuer: apiserverv1beta1.Issuer{
+			URL:                  signer.Issuer(),
+			CertificateAuthority: certificateAuthority,
+			Audiences:            []string{signer.Audience()},
+		},
+		ClaimMappings: apiserverv1beta1.ClaimMappings{
+			Username: apiserverv1beta1.PrefixedClaimOrExpression{
+				Claim:  "sub",
+				Prefix: &config.Internal.Prefix,
+			},
+		},
+	})
+
+	authn, err := newJWTAuthenticator(
+		ctx,
+		scheme,
+		config,
+	)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return authn, config.Internal.Prefix, nil
+}
+
+// Reference: https://github.com/kubernetes/kubernetes/blob/v1.32.1/pkg/kubeapiserver/authenticator/config.go#L244
+func newJWTAuthenticator(
+	ctx context.Context,
+	scheme *runtime.Scheme,
+	config Authentication,
+) (authenticator.Token, error) {
+	var jwtAuthenticators []authenticator.Token
+	for _, jwtAuthenticator := range config.JWT {
+		var oidcCAContent koidc.CAContentProvider
+		if len(jwtAuthenticator.Issuer.CertificateAuthority) > 0 {
+			var oidcCAError error
+			oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent(
+				"oidc-authenticator",
+				[]byte(jwtAuthenticator.Issuer.CertificateAuthority),
+			)
+			if oidcCAError != nil {
+				return nil, oidcCAError
+			}
+		}
+		var jwtAuthenticatorUnversioned apiserver.JWTAuthenticator
+		if err := scheme.Convert(&jwtAuthenticator, &jwtAuthenticatorUnversioned, nil); err != nil {
+			return nil, err
+		}
+		oidcAuth, err := koidc.New(ctx, koidc.Options{
+			JWTAuthenticator:     jwtAuthenticatorUnversioned,
+			CAContentProvider:    oidcCAContent,
+			SupportedSigningAlgs: koidc.AllValidSigningAlgorithms(),
+		})
+		if err != nil {
+			return nil, err
+		}
+		jwtAuthenticators = append(jwtAuthenticators, oidcAuth)
+	}
+	return tokenunion.NewFailOnError(jwtAuthenticators...), nil
+}

internal/config/types.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1"
+)
+
+type Config struct {
+	Authentication Authentication `json:"authentication"`
+	Grpc           Grpc           `json:"grpc"`
+}
+
+type Authentication struct {
+	Internal Internal                            `json:"internal"`
+	JWT      []apiserverv1beta1.JWTAuthenticator `json:"jwt"`
+}
+
+type Internal struct {
+	Prefix string `json:"prefix"`
+}
+
+type Grpc struct {
+	Keepalive Keepalive `json:"keepalive"`
+}
+
+type Keepalive struct {
+	MinTime             string `json:"minTime"`
+	PermitWithoutStream bool   `json:"permitWithoutStream"`
+}