From e1e243e447989cfccd040a888a63817e4491001e Mon Sep 17 00:00:00 2001 From: Michael Vittrup Larsen Date: Tue, 13 Jun 2023 07:29:12 +0200 Subject: [PATCH 1/3] Add ServiceMonitor and test deployments --- Makefile.local | 19 +- .../templates/metrics-service.yaml | 6 +- .../templates/servicemonitor.yaml | 21 +++ .../values.schema.json | 176 ++++++++++++------ charts/bifrost-gateway-controller/values.yaml | 13 +- test-data/kube-prometheus-stack-values.yaml | 27 +++ 6 files changed, 197 insertions(+), 65 deletions(-) create mode 100644 charts/bifrost-gateway-controller/templates/servicemonitor.yaml create mode 100644 test-data/kube-prometheus-stack-values.yaml diff --git a/Makefile.local b/Makefile.local index d9a1d37a..f7cbae65 100644 --- a/Makefile.local +++ b/Makefile.local @@ -106,6 +106,16 @@ deploy-multitool: deploy-external-dns: helm upgrade -i --repo https://kubernetes-sigs.github.io/external-dns external-dns external-dns --version 1.12.2 --values test-data/external-dns-values.yaml +################# +.PHONY: deploy-prometheus-operator-crds +deploy-prometheus-operator-crds: + helm upgrade -i --repo https://prometheus-community.github.io/helm-charts prometheus-operator-crds prometheus-operator-crds --version 4.0.2 + +################# +.PHONY: deploy-prometheus +deploy-prometheus: + helm upgrade -i --repo https://prometheus-community.github.io/helm-charts kube-prometheus-stack kube-prometheus-stack --version 46.8.0 -n default --values test-data/kube-prometheus-stack-values.yaml + ################# .PHONY: deploy-metrics-server deploy-metrics-server: @@ -114,7 +124,6 @@ deploy-metrics-server: ################# .PHONY: deploy-kube-state-metrics deploy-kube-state-metrics: - helm upgrade -i --repo https://prometheus-community.github.io/helm-charts prometheus-operator-crds prometheus-operator-crds --version 4.0.2 helm upgrade -i --repo https://prometheus-community.github.io/helm-charts kube-state-metrics kube-state-metrics --version 5.7.0 -n kube-system --values test-data/kube-state-metrics-values.yaml ################# @@ -210,12 +219,16 @@ BIFROST_VERSION ?= 0.1.6 deploy-controller-helm: helm upgrade -i bifrost-gateway-controller-helm oci://ghcr.io/tv2-oss/bifrost-gateway-controller-helm --version ${BIFROST_VERSION} --values charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml -n bifrost-gateway-controller-system --create-namespace +.PHONY: deploy-controller-local-helm +deploy-controller-local-helm: + helm upgrade -i bifrost-gateway-controller-helm charts/bifrost-gateway-controller --values charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml -n bifrost-gateway-controller-system --create-namespace --set prometheus.monitor.enabled=true + .PHONY: deploy-controller-aws-helm deploy-controller-aws-helm: helm upgrade -i bifrost-gateway-controller-helm oci://ghcr.io/tv2-oss/bifrost-gateway-controller-helm --version ${BIFROST_VERSION} --values charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml -n bifrost-gateway-controller-system --create-namespace -.PHONY: undeploy-controller -undeploy-controller: +.PHONY: undeploy-controller-helm +undeploy-controller-helm: helm uninstall -n bifrost-gateway-controller-system bifrost-gateway-controller-helm ################# diff --git a/charts/bifrost-gateway-controller/templates/metrics-service.yaml b/charts/bifrost-gateway-controller/templates/metrics-service.yaml index fff56316..3bcf5804 100644 --- a/charts/bifrost-gateway-controller/templates/metrics-service.yaml +++ b/charts/bifrost-gateway-controller/templates/metrics-service.yaml @@ -6,9 +6,11 @@ metadata: control-plane: manager {{- include "gateway-controller.labels" . | nindent 4 }} spec: - type: {{ .Values.metricsService.type }} + type: {{ .Values.prometheus.service.type }} selector: control-plane: manager {{- include "gateway-controller.selectorLabels" . | nindent 4 }} ports: - {{- .Values.metricsService.ports | toYaml | nindent 2 -}} + - name: http + port: {{ .Values.prometheus.service.port }} + protocol: TCP diff --git a/charts/bifrost-gateway-controller/templates/servicemonitor.yaml b/charts/bifrost-gateway-controller/templates/servicemonitor.yaml new file mode 100644 index 00000000..7bff447c --- /dev/null +++ b/charts/bifrost-gateway-controller/templates/servicemonitor.yaml @@ -0,0 +1,21 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "gateway-controller.fullname" . }}-manager-metrics + labels: + control-plane: manager + {{- include "gateway-controller.labels" . | nindent 4 }} +spec: + jobLabel: {{ .Release.Name }} + selector: + matchLabels: + control-plane: manager + {{- include "gateway-controller.selectorLabels" . | nindent 4 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: http + path: /metrics +{{- end }} diff --git a/charts/bifrost-gateway-controller/values.schema.json b/charts/bifrost-gateway-controller/values.schema.json index d7671d82..737aab2b 100644 --- a/charts/bifrost-gateway-controller/values.schema.json +++ b/charts/bifrost-gateway-controller/values.schema.json @@ -1,60 +1,65 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "required": [ - "controllerManager", - "serviceAccount", - "metricsService" - ], + "$schema": "http://json-schema.org/schema#", + "type": "object", "properties": { "controllerManager": { - "required": [ - "manager", - "annotations", - "replicas", - "deploymentStrategy" - ], + "type": "object", "properties": { + "annotations": { + "type": "object" + }, + "deploymentStrategy": { + "type": "object", + "properties": { + "type": { + "type": "string" + } + } + }, "manager": { - "required": [ - "image", - "resources", - "livenessProbe", - "readinessProbe" - ], + "type": "object", "properties": { "image": { - "required": [ - "repository", - "name", - "tag", - "pullPolicy" - ], + "type": "object", "properties": { - "repository": { + "name": { "type": "string" }, - "name": { + "pullPolicy": { "type": "string" }, - "tag": { + "repository": { "type": "string" }, - "pullPolicy": { + "tag": { "type": "string" } } }, - "resources": { - "type": "object" - }, "livenessProbe": { - "type": "object" + "type": "object", + "properties": { + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + }, + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + } + } }, "logging": { - "required": [ - "format", - "level" - ], + "type": "object", "properties": { "format": { "type": "string" @@ -64,9 +69,6 @@ } } }, - "readinessProbe": { - "type": "object" - }, "rbac": { "type": "object", "properties": { @@ -74,36 +76,100 @@ "type": "array" } } + }, + "readinessProbe": { + "type": "object", + "properties": { + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + }, + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + } + } } } }, - "annotationer": { + "podAnnotations": { "type": "object" }, "replicas": { "type": "integer" + } + } + }, + "prometheus": { + "type": "object", + "properties": { + "monitor": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } }, - "deploymentStrategy": { + "service": { "type": "object", - "required": [ - "type" - ] + "properties": { + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } + } } } }, "serviceAccount": { "type": "object", - "required": [ - "create", - "annotations" - ] - }, - "metricsService": { - "type": "object", - "required": [ - "ports", - "type" - ] + "properties": { + "annotations": { + "type": "object" + }, + "create": { + "type": "boolean" + } + } } } } diff --git a/charts/bifrost-gateway-controller/values.yaml b/charts/bifrost-gateway-controller/values.yaml index 327547eb..6baef08f 100644 --- a/charts/bifrost-gateway-controller/values.yaml +++ b/charts/bifrost-gateway-controller/values.yaml @@ -72,9 +72,12 @@ serviceAccount: # Annotations to add to the service account annotations: {} -metricsService: - ports: - - name: http +# -- Prometheus metrics +prometheus: + # -- Metrics service specification + service: port: 8080 - protocol: TCP - type: ClusterIP + type: ClusterIP + # -- Prometheus-operator ServiceMonitor metrics endpoint specification + monitor: + enabled: false diff --git a/test-data/kube-prometheus-stack-values.yaml b/test-data/kube-prometheus-stack-values.yaml new file mode 100644 index 00000000..2cbfa662 --- /dev/null +++ b/test-data/kube-prometheus-stack-values.yaml @@ -0,0 +1,27 @@ +prometheus: + prometheusSpec: + serviceMonitorSelectorNilUsesHelmValues: false +alertmanager: + enabled: false +grafana: + enabled: false +kubernetesServiceMonitors: + enabled: false +kubeApiServer: + enabled: false +kubelet: + enabled: false +kubeControllerManager: + enabled: false +coreDns: + enabled: false +kubeEtcd: + enabled: false +kubeScheduler: + enabled: false +kubeProxy: + enabled: false +kubeStateMetrics: + enabled: false +nodeExporter: + enabled: false From 648166579d8df32e69b5bdb342b9a9f29eca4e77 Mon Sep 17 00:00:00 2001 From: Michael Vittrup Larsen Date: Tue, 13 Jun 2023 07:31:44 +0200 Subject: [PATCH 2/3] Update chart CHANGELOG with PR info --- charts/bifrost-gateway-controller/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/bifrost-gateway-controller/CHANGELOG.md b/charts/bifrost-gateway-controller/CHANGELOG.md index 75288012..3df73b54 100644 --- a/charts/bifrost-gateway-controller/CHANGELOG.md +++ b/charts/bifrost-gateway-controller/CHANGELOG.md @@ -3,6 +3,7 @@ ## [UNRELEASED] - Example text, add your PR info according to example below below this line. Do not bump chart version in Chart.yaml unless a chart release will be made following your PR. +- Add ServiceMonitor CRD to enable metrics endpoint discovery and configuration ([#202](https://github.com/tv2-oss/bifrost-gateway-controller/pull/202)) [@michaelvl](https://github.com/michaelvl) ## [0.1.7] From 2c0b74a8446b42c8421a4ffc3c24b51cdc3b3977 Mon Sep 17 00:00:00 2001 From: Michael Vittrup Larsen Date: Tue, 13 Jun 2023 07:32:36 +0200 Subject: [PATCH 3/3] Update helm docs --- charts/bifrost-gateway-controller/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/bifrost-gateway-controller/README.md b/charts/bifrost-gateway-controller/README.md index fb410c3a..cdd372d2 100644 --- a/charts/bifrost-gateway-controller/README.md +++ b/charts/bifrost-gateway-controller/README.md @@ -31,10 +31,9 @@ Gateway API driven management of network infrastructure across Kubernetes and cl | controllerManager.manager.resources.requests.memory | string | `"64Mi"` | | | controllerManager.podAnnotations | object | `{}` | | | controllerManager.replicas | int | `1` | | -| metricsService.ports[0].name | string | `"http"` | | -| metricsService.ports[0].port | int | `8080` | | -| metricsService.ports[0].protocol | string | `"TCP"` | | -| metricsService.type | string | `"ClusterIP"` | | +| prometheus | object | `{"monitor":{"enabled":false},"service":{"port":8080,"type":"ClusterIP"}}` | Prometheus metrics | +| prometheus.monitor | object | `{"enabled":false}` | Prometheus-operator ServiceMonitor metrics endpoint specification | +| prometheus.service | object | `{"port":8080,"type":"ClusterIP"}` | Metrics service specification | | serviceAccount.annotations | object | `{}` | | | serviceAccount.create | bool | `true` | |