diff --git a/deploy/charts/venafi-kubernetes-agent/README.md b/deploy/charts/venafi-kubernetes-agent/README.md index a59d18f6..5238e421 100644 --- a/deploy/charts/venafi-kubernetes-agent/README.md +++ b/deploy/charts/venafi-kubernetes-agent/README.md @@ -146,11 +146,12 @@ You should see the following events for your service account: | authentication.secretKey | string | `"privatekey.pem"` | Key name in the referenced secret | | authentication.secretName | string | `"agent-credentials"` | Name of the secret containing the private key | | command | list | `[]` | Specify the command to run overriding default binary. | -| config | object | `{"clientId":"","clusterDescription":"","clusterName":"","configmap":{"key":null,"name":null},"period":"0h1m0s","server":"https://api.venafi.cloud/"}` | Configuration section for the Venafi Kubernetes Agent itself | +| config | object | `{"clientId":"","clusterDescription":"","clusterName":"","configmap":{"key":null,"name":null},"ignoredSecretTypes":["kubernetes.io/service-account-token","kubernetes.io/dockercfg","kubernetes.io/dockerconfigjson","kubernetes.io/basic-auth","kubernetes.io/ssh-auth","bootstrap.kubernetes.io/token","helm.sh/release.v1"],"period":"0h1m0s","server":"https://api.venafi.cloud/"}` | Configuration section for the Venafi Kubernetes Agent itself | | config.clientId | string | `""` | The client-id returned from the Venafi Control Plane | | config.clusterDescription | string | `""` | Description for the cluster resource if it needs to be created in Venafi Control Plane | | config.clusterName | string | `""` | Name for the cluster resource if it needs to be created in Venafi Control Plane | | config.configmap | object | `{"key":null,"name":null}` | Specify ConfigMap details to load config from an existing resource. This should be blank by default unless you have you own config. | +| config.ignoredSecretTypes | list | `["kubernetes.io/service-account-token","kubernetes.io/dockercfg","kubernetes.io/dockerconfigjson","kubernetes.io/basic-auth","kubernetes.io/ssh-auth","bootstrap.kubernetes.io/token","helm.sh/release.v1"]` | Reduce the memory usage of the agent and reduce the load on the Kubernetes API server by omitting various common Secret types when listing Secrets. These Secret types will be added to a "type!=" field selector in the agent config. * https://docs.venafi.cloud/vaas/k8s-components/t-cfg-tlspk-agent/#configuration * https://kubernetes.io/docs/concepts/configuration/secret/#secret-types * https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/#list-of-supported-fields | | config.period | string | `"0h1m0s"` | Send data back to the platform every minute unless changed | | config.server | string | `"https://api.venafi.cloud/"` | Overrides the server if using a proxy in your environment For the EU variant use: https://api.venafi.eu/ | | extraArgs | list | `[]` | Specify additional arguments to pass to the agent binary. For example `["--strict", "--oneshot"]` | @@ -176,7 +177,7 @@ You should see the following events for your service account: | podSecurityContext | object | `{}` | Optional Pod (all containers) `SecurityContext` options, see https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod. | | replicaCount | int | `1` | default replicas, do not scale up | | resources | object | `{"limits":{"memory":"500Mi"},"requests":{"cpu":"200m","memory":"200Mi"}}` | Set resource requests and limits for the pod. Read [Venafi Kubernetes components deployment best practices](https://docs.venafi.cloud/vaas/k8s-components/c-k8s-components-best-practice/#scaling) to learn how to choose suitable CPU and memory resource requests and limits. | -| securityContext | object | `{"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"runAsUser":1000}` | Add Container specific SecurityContext settings to the container. Takes precedence over `podSecurityContext` when set. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-capabilities-for-a-container | +| securityContext | object | `{"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | Add Container specific SecurityContext settings to the container. Takes precedence over `podSecurityContext` when set. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-capabilities-for-a-container | | serviceAccount.annotations | object | `{}` | Annotations YAML to add to the service account | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | | serviceAccount.name | string | `""` | The name of the service account to use. If blank and `serviceAccount.create` is true, a name is generated using the fullname template of the release. | diff --git a/deploy/charts/venafi-kubernetes-agent/templates/configmap.yaml b/deploy/charts/venafi-kubernetes-agent/templates/configmap.yaml index 59ce52fc..3002e721 100644 --- a/deploy/charts/venafi-kubernetes-agent/templates/configmap.yaml +++ b/deploy/charts/venafi-kubernetes-agent/templates/configmap.yaml @@ -89,6 +89,12 @@ data: resource-type: version: v1 resource: secrets + {{- with .Values.config.ignoredSecretTypes }} + field-selectors: + {{- range . }} + - type!={{ . }} + {{- end }} + {{- end }} - kind: "k8s-dynamic" name: "k8s/certificates" config: @@ -202,5 +208,3 @@ data: version: v1 resource: issuers {{- end }} - - diff --git a/deploy/charts/venafi-kubernetes-agent/values.yaml b/deploy/charts/venafi-kubernetes-agent/values.yaml index 8502a92d..b87fb89e 100644 --- a/deploy/charts/venafi-kubernetes-agent/values.yaml +++ b/deploy/charts/venafi-kubernetes-agent/values.yaml @@ -186,6 +186,22 @@ config: # -- Description for the cluster resource if it needs to be created in Venafi Control Plane clusterDescription: "" + # -- Reduce the memory usage of the agent and reduce the load on the Kubernetes + # API server by omitting various common Secret types when listing Secrets. + # These Secret types will be added to a "type!=" field selector in the + # agent config. + # * https://docs.venafi.cloud/vaas/k8s-components/t-cfg-tlspk-agent/#configuration + # * https://kubernetes.io/docs/concepts/configuration/secret/#secret-types + # * https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/#list-of-supported-fields + ignoredSecretTypes: + - kubernetes.io/service-account-token + - kubernetes.io/dockercfg + - kubernetes.io/dockerconfigjson + - kubernetes.io/basic-auth + - kubernetes.io/ssh-auth + - bootstrap.kubernetes.io/token + - helm.sh/release.v1 + # -- Specify ConfigMap details to load config from an existing resource. # This should be blank by default unless you have you own config. configmap: diff --git a/docs/datagatherers/k8s-dynamic.md b/docs/datagatherers/k8s-dynamic.md index 6230642e..2ccd5574 100644 --- a/docs/datagatherers/k8s-dynamic.md +++ b/docs/datagatherers/k8s-dynamic.md @@ -64,7 +64,7 @@ resource referenced in the `kind` for that datagatherer. There is an example `ClusterRole` and `ClusterRoleBinding` which can be found in [`./deployment/kubernetes/base/00-rbac.yaml`](./deployment/kubernetes/base/00-rbac.yaml). -# Secrets +## Secrets Secrets can be gathered using the following config: @@ -79,4 +79,30 @@ Secrets can be gathered using the following config: Before Secrets are sent to the Preflight backend, they are redacted so no secret data is transmitted. See [`fieldfilter.go`](./../../pkg/datagatherer/k8s/fieldfilter.go) to see the details of which fields are filteres and which ones are redacted. -> **All resource other than Kubernetes Secrets are sent in full, so make sure that you don't store secret information on arbitrary resources.** \ No newline at end of file +> **All resource other than Kubernetes Secrets are sent in full, so make sure that you don't store secret information on arbitrary resources.** + + +## Field Selectors + +You can use [field selectors](https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/#list-of-supported-fields) +to include or exclude certain resources. +For example, you can reduce the memory usage of the agent and reduce the load on the Kubernetes +API server by omitting various common [Secret types](https://kubernetes.io/docs/concepts/configuration/secret/#secret-types) +when listing Secrets. + +```yaml +- kind: "k8s-dynamic" + name: "k8s/secrets" + config: + resource-type: + version: v1 + resource: secrets + field-selectors: + - type!=kubernetes.io/service-account-token + - type!=kubernetes.io/dockercfg + - type!=kubernetes.io/dockerconfigjson + - type!=kubernetes.io/basic-auth + - type!=kubernetes.io/ssh-auth, + - type!=bootstrap.kubernetes.io/token + - type!=helm.sh/release.v1 +``` diff --git a/examples/one-shot-secret.yaml b/examples/one-shot-secret.yaml new file mode 100644 index 00000000..08f6d8d9 --- /dev/null +++ b/examples/one-shot-secret.yaml @@ -0,0 +1,29 @@ +# one-shot-secret.yaml +# +# An example configuration file which can be used for local testing. +# It gathers only secrets and it does not attempt to upload to Venafi. +# For example: +# +# builds/preflight agent \ +# --agent-config-file examples/one-shot-secret.yaml \ +# --one-shot \ +# --output-path output.json +# +organization_id: "my-organization" +cluster_id: "my_cluster" +period: 1m +data-gatherers: +- kind: "k8s-dynamic" + name: "k8s/secrets" + config: + resource-type: + version: v1 + resource: secrets + field-selectors: + - type!=kubernetes.io/service-account-token + - type!=kubernetes.io/dockercfg + - type!=kubernetes.io/dockerconfigjson + - type!=kubernetes.io/basic-auth + - type!=kubernetes.io/ssh-auth, + - type!=bootstrap.kubernetes.io/token + - type!=helm.sh/release.v1 diff --git a/pkg/datagatherer/k8s/dynamic.go b/pkg/datagatherer/k8s/dynamic.go index c9007017..6f224cbd 100644 --- a/pkg/datagatherer/k8s/dynamic.go +++ b/pkg/datagatherer/k8s/dynamic.go @@ -39,6 +39,8 @@ type ConfigDynamic struct { ExcludeNamespaces []string `yaml:"exclude-namespaces"` // IncludeNamespaces is a list of namespaces to include. IncludeNamespaces []string `yaml:"include-namespaces"` + // FieldSelectors is a list of field selectors to use when listing this resource + FieldSelectors []string `yaml:"field-selectors"` } // UnmarshalYAML unmarshals the ConfigDynamic resolving GroupVersionResource. @@ -52,6 +54,7 @@ func (c *ConfigDynamic) UnmarshalYAML(unmarshal func(interface{}) error) error { } `yaml:"resource-type"` ExcludeNamespaces []string `yaml:"exclude-namespaces"` IncludeNamespaces []string `yaml:"include-namespaces"` + FieldSelectors []string `yaml:"field-selectors"` }{} err := unmarshal(&aux) if err != nil { @@ -64,6 +67,7 @@ func (c *ConfigDynamic) UnmarshalYAML(unmarshal func(interface{}) error) error { c.GroupVersionResource.Resource = aux.ResourceType.Resource c.ExcludeNamespaces = aux.ExcludeNamespaces c.IncludeNamespaces = aux.IncludeNamespaces + c.FieldSelectors = aux.FieldSelectors return nil } @@ -79,6 +83,16 @@ func (c *ConfigDynamic) validate() error { errors = append(errors, "invalid configuration: GroupVersionResource.Resource cannot be empty") } + for i, selectorString := range c.FieldSelectors { + if selectorString == "" { + errors = append(errors, fmt.Sprintf("invalid field selector %d: must not be empty", i)) + } + _, err := fields.ParseSelector(selectorString) + if err != nil { + errors = append(errors, fmt.Sprintf("invalid field selector %d: %s", i, err)) + } + } + if len(errors) > 0 { return fmt.Errorf(strings.Join(errors, ", ")) } @@ -150,7 +164,15 @@ func (c *ConfigDynamic) newDataGathererWithClient(ctx context.Context, cl dynami return nil, err } // init shared informer for selected namespaces - fieldSelector := generateFieldSelector(c.ExcludeNamespaces) + fieldSelector := generateExcludedNamespacesFieldSelector(c.ExcludeNamespaces) + + // Add any custom field selectors to the excluded namespaces selector + // The selectors have already been validated, so it is safe to use + // ParseSelectorOrDie here. + for _, selectorString := range c.FieldSelectors { + fieldSelector = fields.AndSelectors(fieldSelector, fields.ParseSelectorOrDie(selectorString)) + } + // init cache to store gathered resources dgCache := cache.New(5*time.Minute, 30*time.Second) @@ -159,7 +181,7 @@ func (c *ConfigDynamic) newDataGathererWithClient(ctx context.Context, cl dynami cl: cl, k8sClientSet: clientset, groupVersionResource: c.GroupVersionResource, - fieldSelector: fieldSelector, + fieldSelector: fieldSelector.String(), namespaces: c.IncludeNamespaces, cache: dgCache, } @@ -177,7 +199,7 @@ func (c *ConfigDynamic) newDataGathererWithClient(ctx context.Context, cl dynami 60*time.Second, informers.WithNamespace(metav1.NamespaceAll), informers.WithTweakListOptions(func(options *metav1.ListOptions) { - options.FieldSelector = fieldSelector + options.FieldSelector = fieldSelector.String() })) newDataGatherer.nativeSharedInformer = factory informer := informerFunc(factory) @@ -200,7 +222,7 @@ func (c *ConfigDynamic) newDataGathererWithClient(ctx context.Context, cl dynami cl, 60*time.Second, metav1.NamespaceAll, - func(options *metav1.ListOptions) { options.FieldSelector = fieldSelector }, + func(options *metav1.ListOptions) { options.FieldSelector = fieldSelector.String() }, ) resourceInformer := factory.ForResource(c.GroupVersionResource) informer := resourceInformer.Informer() @@ -420,17 +442,17 @@ func namespaceResourceInterface(iface dynamic.NamespaceableResourceInterface, na return iface.Namespace(namespace) } -// generateFieldSelector creates a field selector string from a list of -// namespaces to exclude. -func generateFieldSelector(excludeNamespaces []string) string { - fieldSelector := fields.Nothing() +// generateExcludedNamespacesFieldSelector creates a field selector string from +// a list of namespaces to exclude. +func generateExcludedNamespacesFieldSelector(excludeNamespaces []string) fields.Selector { + var selectors []fields.Selector for _, excludeNamespace := range excludeNamespaces { if excludeNamespace == "" { continue } - fieldSelector = fields.AndSelectors(fields.OneTermNotEqualSelector("metadata.namespace", excludeNamespace), fieldSelector) + selectors = append(selectors, fields.OneTermNotEqualSelector("metadata.namespace", excludeNamespace)) } - return fieldSelector.String() + return fields.AndSelectors(selectors...) } func isIncludedNamespace(namespace string, namespaces []string) bool { diff --git a/pkg/datagatherer/k8s/dynamic_test.go b/pkg/datagatherer/k8s/dynamic_test.go index 150304ff..21bdb715 100644 --- a/pkg/datagatherer/k8s/dynamic_test.go +++ b/pkg/datagatherer/k8s/dynamic_test.go @@ -105,8 +105,12 @@ func sortGatheredResources(list []*api.GatheredResource) { func TestNewDataGathererWithClientAndDynamicInformer(t *testing.T) { ctx := context.Background() config := ConfigDynamic{ - IncludeNamespaces: []string{"a"}, + ExcludeNamespaces: []string{"kube-system"}, GroupVersionResource: schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"}, + FieldSelectors: []string{ + "type!=kubernetes.io/service-account-token", + "type!=kubernetes.io/dockercfg", + }, } cl := fake.NewSimpleDynamicClient(runtime.NewScheme()) dg, err := config.newDataGathererWithClient(ctx, cl, nil) @@ -121,7 +125,8 @@ func TestNewDataGathererWithClientAndDynamicInformer(t *testing.T) { groupVersionResource: config.GroupVersionResource, // it's important that the namespaces are set as the IncludeNamespaces // during initialization - namespaces: config.IncludeNamespaces, + namespaces: config.IncludeNamespaces, + fieldSelector: "metadata.namespace!=kube-system,type!=kubernetes.io/service-account-token,type!=kubernetes.io/dockercfg", } gatherer := dg.(*DataGathererDynamic) @@ -150,6 +155,9 @@ func TestNewDataGathererWithClientAndDynamicInformer(t *testing.T) { if gatherer.nativeSharedInformer != nil { t.Errorf("unexpected nativeSharedInformer value: %v. should be nil", gatherer.nativeSharedInformer) } + if !reflect.DeepEqual(gatherer.fieldSelector, expected.fieldSelector) { + t.Errorf("expected %v, got %v", expected.fieldSelector, gatherer.fieldSelector) + } } func TestNewDataGathererWithClientAndSharedIndexInformer(t *testing.T) { @@ -216,6 +224,8 @@ exclude-namespaces: # from the config file include-namespaces: - default +field-selectors: +- type!=kubernetes.io/service-account-token ` expectedGVR := schema.GroupVersionResource{ @@ -231,6 +241,10 @@ include-namespaces: expectedIncludeNamespaces := []string{"default"} + expectedFieldSelectors := []string{ + "type!=kubernetes.io/service-account-token", + } + cfg := ConfigDynamic{} err := yaml.Unmarshal([]byte(textCfg), &cfg) if err != nil { @@ -251,6 +265,9 @@ include-namespaces: if got, want := cfg.IncludeNamespaces, expectedIncludeNamespaces; !reflect.DeepEqual(got, want) { t.Errorf("IncludeNamespaces does not match: got=%+v want=%+v", got, want) } + if got, want := cfg.FieldSelectors, expectedFieldSelectors; !reflect.DeepEqual(got, want) { + t.Errorf("FieldSelectors does not match: got=%+v want=%+v", got, want) + } } func TestConfigDynamicValidate(t *testing.T) { @@ -275,17 +292,42 @@ func TestConfigDynamicValidate(t *testing.T) { }, ExpectedError: "cannot set excluded and included namespaces", }, + { + Config: ConfigDynamic{ + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "secrets", + }, + FieldSelectors: []string{""}, + }, + ExpectedError: "invalid field selector 0: must not be empty", + }, + { + Config: ConfigDynamic{ + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "secrets", + }, + FieldSelectors: []string{"foo"}, + }, + ExpectedError: "invalid field selector 0: invalid selector: 'foo'; can't understand 'foo'", + }, } for _, test := range tests { err := test.Config.validate() - if !strings.Contains(err.Error(), test.ExpectedError) { + if err == nil && test.ExpectedError != "" { + t.Errorf("expected error: %q, got: nil", test.ExpectedError) + } + if err != nil && !strings.Contains(err.Error(), test.ExpectedError) { t.Errorf("expected %s, got %s", test.ExpectedError, err.Error()) } } } -func TestGenerateFieldSelector(t *testing.T) { +func TestGenerateExcludedNamespacesFieldSelector(t *testing.T) { tests := []struct { ExcludeNamespaces []string ExpectedFieldSelector string @@ -300,19 +342,19 @@ func TestGenerateFieldSelector(t *testing.T) { ExcludeNamespaces: []string{ "kube-system", }, - ExpectedFieldSelector: "metadata.namespace!=kube-system,", + ExpectedFieldSelector: "metadata.namespace!=kube-system", }, { ExcludeNamespaces: []string{ "kube-system", "my-namespace", }, - ExpectedFieldSelector: "metadata.namespace!=my-namespace,metadata.namespace!=kube-system,", + ExpectedFieldSelector: "metadata.namespace!=kube-system,metadata.namespace!=my-namespace", }, } for _, test := range tests { - fieldSelector := generateFieldSelector(test.ExcludeNamespaces) + fieldSelector := generateExcludedNamespacesFieldSelector(test.ExcludeNamespaces).String() if fieldSelector != test.ExpectedFieldSelector { t.Errorf("ExpectedFieldSelector does not match: got=%+v want=%+v", fieldSelector, test.ExpectedFieldSelector) }