WIP Add a Prometheus metrics+alerting stack tweakable per hosting scenario

converge-generic/kustomization.yaml

@@ -5,3 +5,5 @@ bases:
 #- ../registry/generic,kafka  # Note that kafka isn't a base (as it has its own namespace) and must be applied separately
 - ../registry/generic,minio
 # included in generic,minio: - ../registry/node-update-hosts   # Required unless nodes' container runtime can look up cluster DNS for registry services
+
+- ../monitoring/instance-now

monitoring/README.md

@@ -0,0 +1,46 @@
+# Monitoring stack
+
+We've observed... that when developers start to factor in
+the existence of metrics collection ("scrape") and alerting rules
+-- both generic such as not-enough-replicas or job-failed and service specific --
+it becomes a handy solution to many difficult coding problems.
+Instead of error handling by assumption we expose a metric on some case
+and make sure the alert for it works.
+Helps immensely to leave it to humans to interpret the condition,
+instead of implementing an automated solution up front.
+
+At Yolean we've run [kube-prometheus](https://github.com/coreos/kube-prometheus) for many years
+but, while it is a good collection of tools + rules + dashboards,
+we feel it is time to start to opt-in to components individuallly.
+
+Requirements:
+ * Lean enough to be always-on in development clusters.
+ * Expose alerts that a dev environment can poll for,
+   to make monitoring + alerting rules a first class impl tool.
+ * Comes with basic sanity checks on the stack.
+
+Tools:
+ * Prometheus, the de-facto standard with k8s
+ * Grafana, the de-facto standard with k8s
+ * Alertmanager, the de-facto standard with Prometheus
+ * [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) both for `kubectl top` and Prometheus
+ * [node exporter](https://github.com/prometheus/node_exporter)
+ * [Prometheus Operatior](https://github.com/coreos/prometheus-operator/) appears to be rock solid and the `ServiceMonitor` resources really delivers capabilities that service discovery base don annotations can't, such as scraping multiple containers and having custom intervals.
+ * [kubernetes-mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin)
+ * [kube-prometheus' alerting rules](https://github.com/coreos/kube-prometheus/commits/master/manifests/prometheus-rules.yaml) with stuff like clock skew
+ * https://github.com/coreos/kube-prometheus/commits/master/manifests/grafana-dashboardDefinitions.yaml also heavily duplicating kubernetes-mixin
+ * https://github.com/brancz/kubernetes-grafana "exists because the Grafana stack in kube-prometheus has gotten close to unmaintainable"
+
+Dashboards:
+ * Maybe https://github.com/cloudworkz/kube-eagle
+
+Web UIs, as a complement:
+ * https://srcco.de/posts/kubernetes-web-uis-in-2019.html
+ * Can we have basic top + node health cheaply and independent of the stack?
+
+Prometheus instances:
+ * `now` denotes the instance(s) you'll turn to for observing current state of the cluster, for example when troubleshooting.
+   Metrics are of the operational type, volumes are high and retention very low. This is also the instance to use for non-business alerts.
+
+Specific to k3s:
+ * https://github.com/rancher/k3s/issues/425 but let's focus on amd64

monitoring/alertmanager/alertmanager.yaml

@@ -0,0 +1,12 @@
+global:
+  resolve_timeout: 5m
+route:
+  group_by: ['job']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 12h
+  receiver: 'webhook'
+receivers:
+- name: 'webhook'
+  webhook_configs:
+  - url: 'http://alertmanagerwh:30500/'

monitoring/alertmanager/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+- main-alertmanager.yaml
+generatorOptions:
+ disableNameSuffixHash: true
+secretGenerator:
+- name: alertmanager-main
+  files:
+  - alertmanager.yaml

monitoring/alertmanager/main-alertmanager.yaml

@@ -0,0 +1,9 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  name: main
+spec:
+  version: 0.19.0
+  baseImage: prom/alertmanager
+  sha: 7dbf4949a317a056d11ed8f379826b04d0665fad5b9334e1d69b23e946056cd3
+  replicas: 1

monitoring/instance-now/kustomization.yaml

@@ -0,0 +1,12 @@
+namespace: ystack-monitoring
+bases:
+- ../rbac-base/
+- ../alertmanager/
+resources:
+- now-prometheus.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+secretGenerator:
+- name: prometheus-now-additional
+  files:
+  - now-additional.yaml

monitoring/instance-now/now-additional.yaml

@@ -0,0 +1,37 @@
+
+# https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
+# https://github.com/prometheus/prometheus/blob/85a3c974b760642bbce00d795db231326a09edb9/documentation/examples/prometheus-kubernetes.yml
+# (before https://github.com/prometheus/prometheus/commit/03a9e7f72e072c6d29f422425d8acd91a957836b#diff-416ffe38e4360ad5f27d8a034963c260)
+# * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
+# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the pod's declared ports (default is a port-free target if none are declared).
+- job_name: kubernetes-pods
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  - action: keep
+    regex: true
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+  - action: replace
+    regex: (.+)
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_path
+    target_label: __metrics_path__
+  - action: replace
+    regex: ([^:]+)(?::\d+)?;(\d+)
+    replacement: $1:$2
+    source_labels:
+    - __address__
+    - __meta_kubernetes_pod_annotation_prometheus_io_port
+    target_label: __address__
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_namespace
+    target_label: kubernetes_namespace
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_pod_name
+    target_label: kubernetes_pod_name

monitoring/instance-now/now-prometheus.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: now
+  labels:
+    ystack: now
+spec:
+  version: 2.13.1
+  baseImage: prom/prometheus
+  sha: 0a8caa2e9f19907608915db6e62a67383fe44b9876a467b297ee6f64e51dd58a
+  replicas: 1
+  serviceAccountName: prometheus
+  serviceMonitorSelector:
+    matchLabels:
+      ystack: now
+  ruleSelector:
+    matchLabels:
+      ystack: now
+  retention: 8h
+  additionalScrapeConfigs:
+    name: prometheus-now-additional
+    key: now-additional.yaml
+  alerting:
+    alertmanagers:
+    - namespace: ystack-monitoring
+      name: alertmanager-operated # works as long as there's only one alertmanager resource in the namespace
+      port: web

monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alertmanager-main
+  labels:
+    proxy: alertmanager-main
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      proxy: alertmanager-main
+  template:
+    metadata:
+      labels:
+        proxy: alertmanager-main
+      annotations:
+        prometheus.io/path: /stats/prometheus
+        prometheus.io/port: "9901"
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+      - name: envoy
+        image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f
+        ports:
+        - name: proxy
+          containerPort: 80
+        - name: admin
+          containerPort: 9901
+        command:
+        - /usr/local/bin/envoy
+        - -l info
+        args:
+        - --config-yaml
+        - |
+          admin:
+            access_log_path: /tmp/admin_access.log
+            address:
+              socket_address:
+                protocol: TCP
+                address: 0.0.0.0
+                port_value: 9901
+          static_resources:
+            listeners:
+            - name: listener_0
+              address:
+                socket_address:
+                  protocol: TCP
+                  address: 0.0.0.0
+                  port_value: 80
+              filter_chains:
+              - filters:
+                - name: envoy.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                    stat_prefix: ingress_http
+                    route_config:
+                      name: local_route
+                      virtual_hosts:
+                      - name: local_service
+                        domains: ["*"]
+                        routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: ystack_monitoring_alertmanager_main
+                    http_filters:
+                    - name: envoy.router
+            clusters:
+            - name: ystack_monitoring_alertmanager_main
+              connect_timeout: 0.25s
+              type: LOGICAL_DNS
+              dns_lookup_family: V4_ONLY
+              lb_policy: ROUND_ROBIN
+              load_assignment:
+                cluster_name: ystack_monitoring_alertmanager_main
+                endpoints:
+                - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          # there's only the "main" instance
+                          address: alertmanager-operated.ystack-monitoring
+                          port_value: 9093

monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alertmanager-main
+spec:
+  selector:
+    proxy: alertmanager-main
+  ports:
+  - protocol: TCP
+    port: 80

monitoring/proxy-for-kubfwd/kustomization.yaml

@@ -0,0 +1,6 @@
+namespace: ystack
+resources:
+- alertmanager-main-proxy-service.yaml
+- alertmanager-main-proxy-deployment.yaml
+- prometheus-now-proxy-service.yaml
+- prometheus-now-proxy-deployment.yaml

monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-now
+  labels:
+    proxy: prometheus-now
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      proxy: prometheus-now
+  template:
+    metadata:
+      labels:
+        proxy: prometheus-now
+      annotations:
+        prometheus.io/path: /stats/prometheus
+        prometheus.io/port: "9901"
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+      - name: envoy
+        image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f
+        ports:
+        - name: proxy
+          containerPort: 80
+        - name: admin
+          containerPort: 9901
+        command:
+        - /usr/local/bin/envoy
+        - -l info
+        args:
+        - --config-yaml
+        - |
+          admin:
+            access_log_path: /tmp/admin_access.log
+            address:
+              socket_address:
+                protocol: TCP
+                address: 0.0.0.0
+                port_value: 9901
+          static_resources:
+            listeners:
+            - name: listener_0
+              address:
+                socket_address:
+                  protocol: TCP
+                  address: 0.0.0.0
+                  port_value: 80
+              filter_chains:
+              - filters:
+                - name: envoy.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                    stat_prefix: ingress_http
+                    route_config:
+                      name: local_route
+                      virtual_hosts:
+                      - name: local_service
+                        domains: ["*"]
+                        routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: ystack_monitoring_prometheus_now
+                    http_filters:
+                    - name: envoy.router
+            clusters:
+            - name: ystack_monitoring_prometheus_now
+              connect_timeout: 0.25s
+              type: LOGICAL_DNS
+              dns_lookup_family: V4_ONLY
+              lb_policy: ROUND_ROBIN
+              load_assignment:
+                cluster_name: ystack_monitoring_prometheus_now
+                endpoints:
+                - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          # there's only the "now" instance
+                          address: prometheus-operated.ystack-monitoring
+                          port_value: 9090

monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-now
+spec:
+  selector:
+    proxy: prometheus-now
+  ports:
+  - protocol: TCP
+    port: 80

monitoring/rbac-base/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- prometheus-cluster-role-binding.yaml
+- prometheus-cluster-role.yaml
+- prometheus-service-account.yaml

monitoring/rbac-base/prometheus-cluster-role-binding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus

monitoring/rbac-base/prometheus-cluster-role.yaml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources:
+  - configmaps
+  verbs: ["get"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]