diff --git a/cmd/main.go b/cmd/main.go index d05f2920..d7514979 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -151,7 +151,7 @@ func main() { os.Exit(1) } - authenticator, prefix, err := config.LoadConfiguration( + authenticator, prefix, option, err := config.LoadConfiguration( context.Background(), mgr.GetAPIReader(), mgr.GetScheme(), @@ -211,14 +211,16 @@ func main() { ResourceKey: "jumpstarter-kind", NameKey: "jumpstarter-name", }), + ServerOption: option, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create service", "service", "Controller") os.Exit(1) } if err = (&service.RouterService{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ServerOption: option, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create service", "service", "Router") os.Exit(1) diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/cms/controller-cm.yaml b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/cms/controller-cm.yaml index 620f09f0..ccf834ee 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/cms/controller-cm.yaml +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/cms/controller-cm.yaml @@ -10,4 +10,10 @@ metadata: deployment.timestamp: {{ .Values.global.timestamp | quote }} {{ end }} data: + # backwards compatibility + # TODO: remove in 0.7.0 + {{ if .Values.authenticationConfig }} authentication: {{- .Values.authenticationConfig | toYaml | indent 1 }} + {{ end }} + config: | +{{ .Values.config | toYaml | indent 4 }} diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml index 6fb8d932..0549d9fc 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml @@ -9,6 +9,8 @@ metadata: {{ if .Values.global.timestamp }} deployment.timestamp: {{ .Values.global.timestamp | quote }} {{ end }} + annotations: + configmap-sha256: {{ include (print $.Template.BasePath "/cms/controller-cm.yaml") . | sha256sum }} spec: selector: matchLabels: diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-route.yaml b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-route.yaml index 3d040679..f7a781f0 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-route.yaml +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-route.yaml @@ -5,6 +5,9 @@ metadata: labels: external-exposed: "true" shard: external + annotations: + haproxy.router.openshift.io/timeout: 2d + haproxy.router.openshift.io/timeout-tunnel: 2d name: jumpstarter-controller-route namespace: {{ default .Release.Namespace .Values.namespace }} spec: @@ -28,4 +31,4 @@ spec: name: jumpstarter-grpc weight: 100 wildcardPolicy: None -{{ end }} \ No newline at end of file +{{ end }} diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/router-route.yaml b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/router-route.yaml index 9220726c..e0659fbe 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/router-route.yaml +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/router-route.yaml @@ -5,6 +5,9 @@ metadata: labels: external-exposed: "true" shard: external + annotations: + haproxy.router.openshift.io/timeout: 2d + haproxy.router.openshift.io/timeout-tunnel: 2d name: jumpstarter-router-route namespace: {{ default .Release.Namespace .Values.namespace }} spec: @@ -33,4 +36,4 @@ spec: name: jumpstarter-router-grpc weight: 100 wildcardPolicy: None -{{ end }} \ No newline at end of file +{{ end }} diff --git a/deploy/helm/jumpstarter/values.yaml b/deploy/helm/jumpstarter/values.yaml index 6b90c60c..9d80f712 100644 --- a/deploy/helm/jumpstarter/values.yaml +++ b/deploy/helm/jumpstarter/values.yaml @@ -30,7 +30,12 @@ global: ## @param jumpstarter-controller.imagePullPolicy Image pull policy for the controller. ## @param jumpstarter-controller.namespace Namespace where the controller will be deployed, defaults to global.namespace. -## @param jumpstarter-controller.authenticationConfig Configuration for OIDC authentication, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-authentication-configuration for documentation + +## @param jumpstarter-controller.config.grpc.keepalive.minTime. The minimum amount of time a client should wait before sending a keepalive ping. +## @param jumpstarter-controller.config.grpc.keepalive.permitWithoutStream. Whether to allow keepalive pings even when there are no active streams(RPCs). + +## @param jumpstarter-controller.config.authentication.internal.prefix. Prefix to add to the subject claim of the tokens issued by the builtin authenticator. +## @param jumpstarter-controller.config.authentication.jwt. External OIDC authentication, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-authentication-configuration for documentation ## @section Ingress And Route parameters ## @descriptionStart This section contains parameters for the Ingress and Route configurations. @@ -66,33 +71,40 @@ jumpstarter-controller: namespace: "" - authenticationConfig: | - apiVersion: jumpstarter.dev/v1alpha1 - kind: AuthenticationConfiguration - # jwt: - # - issuer: - # url: https://10.239.206.8:5556/dex - # audiences: - # - jumpstarter - # audienceMatchPolicy: MatchAny - # certificateAuthority: | - # -----BEGIN CERTIFICATE----- - # MIIB/DCCAYKgAwIBAgIIcpC2uS+SjEIwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV - # bWluaWNhIHJvb3QgY2EgNzI5MGI2MCAXDTI1MDIwMzE5MzMyNVoYDzIxMjUwMjAz - # MTkzMzI1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA3MjkwYjYwdjAQBgcq - # hkjOPQIBBgUrgQQAIgNiAAQzezKJ4My35HPeoJvvzTjhS2uJMBYrYfrs5csxZjiy - # q8ORrHM539XhWlA6sVZODhzcF2KL4mC9xKz/yIrsws+LKsIWNHGGmIPEKFYnHBGw - # VBGeARvhpzZP/9frJXAN/8ejgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW - # MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud - # DgQWBBSZRBCUuP3ta2xsfjnWIjvgvz4fojAfBgNVHSMEGDAWgBSZRBCUuP3ta2xs - # fjnWIjvgvz4fojAKBggqhkjOPQQDAwNoADBlAjADql5Ks5wh181iUa1ZBnx4XOVe - # l0l7I+mwlwJSPmkZHxruWZTx7gQU4tfDCr+UuzUCMQC2aDXRb17cphipK4gzbExv - # EDLExjhHAqMPrKDmT0jHIi7Bbos38/1tyZ/IoKjLnv0= - # -----END CERTIFICATE----- - # claimMappings: - # username: - # claim: "sub" - # prefix: "" + config: + grpc: + keepalive: + # Safety: potentially makes server vulnerable to DDoS + # https://grpc.io/docs/guides/keepalive/#how-configuring-keepalive-affects-a-call + minTime: 3s + permitWithoutStream: true + authentication: + internal: + prefix: "internal:" + # jwt: + # - issuer: + # url: https://10.239.206.8:5556/dex + # audiences: + # - jumpstarter + # audienceMatchPolicy: MatchAny + # certificateAuthority: | + # -----BEGIN CERTIFICATE----- + # MIIB/DCCAYKgAwIBAgIIcpC2uS+SjEIwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV + # bWluaWNhIHJvb3QgY2EgNzI5MGI2MCAXDTI1MDIwMzE5MzMyNVoYDzIxMjUwMjAz + # MTkzMzI1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA3MjkwYjYwdjAQBgcq + # hkjOPQIBBgUrgQQAIgNiAAQzezKJ4My35HPeoJvvzTjhS2uJMBYrYfrs5csxZjiy + # q8ORrHM539XhWlA6sVZODhzcF2KL4mC9xKz/yIrsws+LKsIWNHGGmIPEKFYnHBGw + # VBGeARvhpzZP/9frJXAN/8ejgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW + # MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud + # DgQWBBSZRBCUuP3ta2xsfjnWIjvgvz4fojAfBgNVHSMEGDAWgBSZRBCUuP3ta2xs + # fjnWIjvgvz4fojAKBggqhkjOPQQDAwNoADBlAjADql5Ks5wh181iUa1ZBnx4XOVe + # l0l7I+mwlwJSPmkZHxruWZTx7gQU4tfDCr+UuzUCMQC2aDXRb17cphipK4gzbExv + # EDLExjhHAqMPrKDmT0jHIi7Bbos38/1tyZ/IoKjLnv0= + # -----END CERTIFICATE----- + # claimMappings: + # username: + # claim: "sub" + # prefix: "" grpc: hostname: "" diff --git a/internal/config/config.go b/internal/config/config.go index c3d6b315..e538469d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,10 +3,14 @@ package config import ( "context" "fmt" + "time" "github.com/jumpstarter-dev/jumpstarter-controller/internal/oidc" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/apiserver/pkg/authentication/authenticator" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -18,27 +22,59 @@ func LoadConfiguration( key client.ObjectKey, signer *oidc.Signer, certificateAuthority string, -) (authenticator.Token, string, error) { +) (authenticator.Token, string, grpc.ServerOption, error) { var configmap corev1.ConfigMap if err := client.Get(ctx, key, &configmap); err != nil { - return nil, "", err + return nil, "", nil, err } rawAuthenticationConfiguration, ok := configmap.Data["authentication"] + if ok { + // backwards compatibility + // TODO: remove in 0.7.0 + authenticator, prefix, err := oidc.LoadAuthenticationConfiguration( + ctx, + scheme, + []byte(rawAuthenticationConfiguration), + signer, + certificateAuthority, + ) + if err != nil { + return nil, "", nil, err + } + + return authenticator, prefix, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 1 * time.Second, + PermitWithoutStream: true, + }), nil + } + + rawConfig, ok := configmap.Data["config"] if !ok { - return nil, "", fmt.Errorf("LoadConfiguration: missing authentication section") + return nil, "", nil, fmt.Errorf("LoadConfiguration: missing config section") } - authenticator, prefix, err := oidc.LoadAuthenticationConfiguration( + var config Config + err := yaml.UnmarshalStrict([]byte(rawConfig), &config) + if err != nil { + return nil, "", nil, err + } + + authenticator, prefix, err := LoadAuthenticationConfiguration( ctx, scheme, - []byte(rawAuthenticationConfiguration), + config.Authentication, signer, certificateAuthority, ) if err != nil { - return nil, "", err + return nil, "", nil, err + } + + serverOptions, err := LoadGrpcConfiguration(config.Grpc) + if err != nil { + return nil, "", nil, err } - return authenticator, prefix, nil + return authenticator, prefix, serverOptions, nil } diff --git a/internal/config/grpc.go b/internal/config/grpc.go new file mode 100644 index 00000000..dcb40097 --- /dev/null +++ b/internal/config/grpc.go @@ -0,0 +1,20 @@ +package config + +import ( + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +func LoadGrpcConfiguration(config Grpc) (grpc.ServerOption, error) { + minTime, err := time.ParseDuration(config.Keepalive.MinTime) + if err != nil { + return nil, err + } + + return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: minTime, + PermitWithoutStream: config.Keepalive.PermitWithoutStream, + }), nil +} diff --git a/internal/config/oidc.go b/internal/config/oidc.go new file mode 100644 index 00000000..47e37f76 --- /dev/null +++ b/internal/config/oidc.go @@ -0,0 +1,87 @@ +package config + +import ( + "context" + + "github.com/jumpstarter-dev/jumpstarter-controller/internal/oidc" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/apis/apiserver" + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" + "k8s.io/apiserver/pkg/authentication/authenticator" + tokenunion "k8s.io/apiserver/pkg/authentication/token/union" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + koidc "k8s.io/apiserver/plugin/pkg/authenticator/token/oidc" +) + +func LoadAuthenticationConfiguration( + ctx context.Context, + scheme *runtime.Scheme, + config Authentication, + signer *oidc.Signer, + certificateAuthority string, +) (authenticator.Token, string, error) { + if config.Internal.Prefix == "" { + config.Internal.Prefix = "internal:" + } + + config.JWT = append(config.JWT, apiserverv1beta1.JWTAuthenticator{ + Issuer: apiserverv1beta1.Issuer{ + URL: signer.Issuer(), + CertificateAuthority: certificateAuthority, + Audiences: []string{signer.Audience()}, + }, + ClaimMappings: apiserverv1beta1.ClaimMappings{ + Username: apiserverv1beta1.PrefixedClaimOrExpression{ + Claim: "sub", + Prefix: &config.Internal.Prefix, + }, + }, + }) + + authn, err := newJWTAuthenticator( + ctx, + scheme, + config, + ) + if err != nil { + return nil, "", err + } + + return authn, config.Internal.Prefix, nil +} + +// Reference: https://github.com/kubernetes/kubernetes/blob/v1.32.1/pkg/kubeapiserver/authenticator/config.go#L244 +func newJWTAuthenticator( + ctx context.Context, + scheme *runtime.Scheme, + config Authentication, +) (authenticator.Token, error) { + var jwtAuthenticators []authenticator.Token + for _, jwtAuthenticator := range config.JWT { + var oidcCAContent koidc.CAContentProvider + if len(jwtAuthenticator.Issuer.CertificateAuthority) > 0 { + var oidcCAError error + oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent( + "oidc-authenticator", + []byte(jwtAuthenticator.Issuer.CertificateAuthority), + ) + if oidcCAError != nil { + return nil, oidcCAError + } + } + var jwtAuthenticatorUnversioned apiserver.JWTAuthenticator + if err := scheme.Convert(&jwtAuthenticator, &jwtAuthenticatorUnversioned, nil); err != nil { + return nil, err + } + oidcAuth, err := koidc.New(ctx, koidc.Options{ + JWTAuthenticator: jwtAuthenticatorUnversioned, + CAContentProvider: oidcCAContent, + SupportedSigningAlgs: koidc.AllValidSigningAlgorithms(), + }) + if err != nil { + return nil, err + } + jwtAuthenticators = append(jwtAuthenticators, oidcAuth) + } + return tokenunion.NewFailOnError(jwtAuthenticators...), nil +} diff --git a/internal/config/types.go b/internal/config/types.go new file mode 100644 index 00000000..28726dfe --- /dev/null +++ b/internal/config/types.go @@ -0,0 +1,28 @@ +package config + +import ( + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" +) + +type Config struct { + Authentication Authentication `json:"authentication"` + Grpc Grpc `json:"grpc"` +} + +type Authentication struct { + Internal Internal `json:"internal"` + JWT []apiserverv1beta1.JWTAuthenticator `json:"jwt"` +} + +type Internal struct { + Prefix string `json:"prefix"` +} + +type Grpc struct { + Keepalive Keepalive `json:"keepalive"` +} + +type Keepalive struct { + MinTime string `json:"minTime"` + PermitWithoutStream bool `json:"permitWithoutStream"` +} diff --git a/internal/service/controller_service.go b/internal/service/controller_service.go index 06a591e9..5c4497ac 100644 --- a/internal/service/controller_service.go +++ b/internal/service/controller_service.go @@ -69,6 +69,7 @@ type ControllerService struct { Authn authentication.ContextAuthenticator Authz authorizer.Authorizer Attr authorization.ContextAttributesGetter + ServerOption grpc.ServerOption listenQueues sync.Map } @@ -661,7 +662,7 @@ func (s *ControllerService) Start(ctx context.Context) error { } server := grpc.NewServer( - KeepaliveEnforcementPolicy(), + s.ServerOption, grpc.UnaryInterceptor(func( gctx context.Context, req any, diff --git a/internal/service/grpc.go b/internal/service/grpc.go deleted file mode 100644 index a56a27a0..00000000 --- a/internal/service/grpc.go +++ /dev/null @@ -1,15 +0,0 @@ -package service - -import ( - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" -) - -func KeepaliveEnforcementPolicy() grpc.ServerOption { - return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ - MinTime: 30 * time.Second, - PermitWithoutStream: true, - }) -} diff --git a/internal/service/router_service.go b/internal/service/router_service.go index 344b4374..067581a0 100644 --- a/internal/service/router_service.go +++ b/internal/service/router_service.go @@ -40,8 +40,9 @@ import ( type RouterService struct { pb.UnimplementedRouterServiceServer client.Client - Scheme *runtime.Scheme - pending sync.Map + Scheme *runtime.Scheme + ServerOption grpc.ServerOption + pending sync.Map } type streamContext struct { @@ -124,7 +125,7 @@ func (s *RouterService) Start(ctx context.Context) error { server := grpc.NewServer( grpc.Creds(credentials.NewServerTLSFromCert(cert)), - KeepaliveEnforcementPolicy(), + s.ServerOption, ) pb.RegisterRouterServiceServer(server, s)