diff --git a/converge-generic/kustomization.yaml b/converge-generic/kustomization.yaml index 176fbad1..37cfce56 100644 --- a/converge-generic/kustomization.yaml +++ b/converge-generic/kustomization.yaml @@ -5,3 +5,5 @@ bases: #- ../registry/generic,kafka # Note that kafka isn't a base (as it has its own namespace) and must be applied separately - ../registry/generic,minio # included in generic,minio: - ../registry/node-update-hosts # Required unless nodes' container runtime can look up cluster DNS for registry services + +- ../monitoring/instance-now diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 00000000..087c462d --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,46 @@ +# Monitoring stack + +We've observed... that when developers start to factor in +the existence of metrics collection ("scrape") and alerting rules +-- both generic such as not-enough-replicas or job-failed and service specific -- +it becomes a handy solution to many difficult coding problems. +Instead of error handling by assumption we expose a metric on some case +and make sure the alert for it works. +Helps immensely to leave it to humans to interpret the condition, +instead of implementing an automated solution up front. + +At Yolean we've run [kube-prometheus](https://github.com/coreos/kube-prometheus) for many years +but, while it is a good collection of tools + rules + dashboards, +we feel it is time to start to opt-in to components individuallly. + +Requirements: + * Lean enough to be always-on in development clusters. + * Expose alerts that a dev environment can poll for, + to make monitoring + alerting rules a first class impl tool. + * Comes with basic sanity checks on the stack. + +Tools: + * Prometheus, the de-facto standard with k8s + * Grafana, the de-facto standard with k8s + * Alertmanager, the de-facto standard with Prometheus + * [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) both for `kubectl top` and Prometheus + * [node exporter](https://github.com/prometheus/node_exporter) + * [Prometheus Operatior](https://github.com/coreos/prometheus-operator/) appears to be rock solid and the `ServiceMonitor` resources really delivers capabilities that service discovery base don annotations can't, such as scraping multiple containers and having custom intervals. + * [kubernetes-mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin) + * [kube-prometheus' alerting rules](https://github.com/coreos/kube-prometheus/commits/master/manifests/prometheus-rules.yaml) with stuff like clock skew + * https://github.com/coreos/kube-prometheus/commits/master/manifests/grafana-dashboardDefinitions.yaml also heavily duplicating kubernetes-mixin + * https://github.com/brancz/kubernetes-grafana "exists because the Grafana stack in kube-prometheus has gotten close to unmaintainable" + +Dashboards: + * Maybe https://github.com/cloudworkz/kube-eagle + +Web UIs, as a complement: + * https://srcco.de/posts/kubernetes-web-uis-in-2019.html + * Can we have basic top + node health cheaply and independent of the stack? + +Prometheus instances: + * `now` denotes the instance(s) you'll turn to for observing current state of the cluster, for example when troubleshooting. + Metrics are of the operational type, volumes are high and retention very low. This is also the instance to use for non-business alerts. + +Specific to k3s: + * https://github.com/rancher/k3s/issues/425 but let's focus on amd64 diff --git a/monitoring/alertmanager/alertmanager.yaml b/monitoring/alertmanager/alertmanager.yaml new file mode 100644 index 00000000..f08a2106 --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yaml @@ -0,0 +1,12 @@ +global: + resolve_timeout: 5m +route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'webhook' +receivers: +- name: 'webhook' + webhook_configs: + - url: 'http://alertmanagerwh:30500/' diff --git a/monitoring/alertmanager/kustomization.yaml b/monitoring/alertmanager/kustomization.yaml new file mode 100644 index 00000000..538e9d10 --- /dev/null +++ b/monitoring/alertmanager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- main-alertmanager.yaml +generatorOptions: + disableNameSuffixHash: true +secretGenerator: +- name: alertmanager-main + files: + - alertmanager.yaml diff --git a/monitoring/alertmanager/main-alertmanager.yaml b/monitoring/alertmanager/main-alertmanager.yaml new file mode 100644 index 00000000..90001cb1 --- /dev/null +++ b/monitoring/alertmanager/main-alertmanager.yaml @@ -0,0 +1,9 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: main +spec: + version: 0.19.0 + baseImage: prom/alertmanager + sha: 7dbf4949a317a056d11ed8f379826b04d0665fad5b9334e1d69b23e946056cd3 + replicas: 1 diff --git a/monitoring/instance-now/kustomization.yaml b/monitoring/instance-now/kustomization.yaml new file mode 100644 index 00000000..f6ea2353 --- /dev/null +++ b/monitoring/instance-now/kustomization.yaml @@ -0,0 +1,12 @@ +namespace: ystack-monitoring +bases: +- ../rbac-base/ +- ../alertmanager/ +resources: +- now-prometheus.yaml +generatorOptions: + disableNameSuffixHash: true +secretGenerator: +- name: prometheus-now-additional + files: + - now-additional.yaml diff --git a/monitoring/instance-now/now-additional.yaml b/monitoring/instance-now/now-additional.yaml new file mode 100644 index 00000000..f8e13d6f --- /dev/null +++ b/monitoring/instance-now/now-additional.yaml @@ -0,0 +1,37 @@ + +# https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod +# https://github.com/prometheus/prometheus/blob/85a3c974b760642bbce00d795db231326a09edb9/documentation/examples/prometheus-kubernetes.yml +# (before https://github.com/prometheus/prometheus/commit/03a9e7f72e072c6d29f422425d8acd91a957836b#diff-416ffe38e4360ad5f27d8a034963c260) +# * `prometheus.io/scrape`: Only scrape pods that have a value of `true` +# * `prometheus.io/path`: If the metrics path is not `/metrics` override this. +# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the pod's declared ports (default is a port-free target if none are declared). +- job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name diff --git a/monitoring/instance-now/now-prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml new file mode 100644 index 00000000..9f8f8318 --- /dev/null +++ b/monitoring/instance-now/now-prometheus.yaml @@ -0,0 +1,27 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: now + labels: + ystack: now +spec: + version: 2.13.1 + baseImage: prom/prometheus + sha: 0a8caa2e9f19907608915db6e62a67383fe44b9876a467b297ee6f64e51dd58a + replicas: 1 + serviceAccountName: prometheus + serviceMonitorSelector: + matchLabels: + ystack: now + ruleSelector: + matchLabels: + ystack: now + retention: 8h + additionalScrapeConfigs: + name: prometheus-now-additional + key: now-additional.yaml + alerting: + alertmanagers: + - namespace: ystack-monitoring + name: alertmanager-operated # works as long as there's only one alertmanager resource in the namespace + port: web diff --git a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml new file mode 100644 index 00000000..48af3a60 --- /dev/null +++ b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager-main + labels: + proxy: alertmanager-main +spec: + replicas: 1 + selector: + matchLabels: + proxy: alertmanager-main + template: + metadata: + labels: + proxy: alertmanager-main + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "9901" + prometheus.io/scrape: "true" + spec: + containers: + - name: envoy + image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f + ports: + - name: proxy + containerPort: 80 + - name: admin + containerPort: 9901 + command: + - /usr/local/bin/envoy + - -l info + args: + - --config-yaml + - | + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9901 + static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: ystack_monitoring_alertmanager_main + http_filters: + - name: envoy.router + clusters: + - name: ystack_monitoring_alertmanager_main + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: ystack_monitoring_alertmanager_main + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + # there's only the "main" instance + address: alertmanager-operated.ystack-monitoring + port_value: 9093 diff --git a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml new file mode 100644 index 00000000..dc757438 --- /dev/null +++ b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: alertmanager-main +spec: + selector: + proxy: alertmanager-main + ports: + - protocol: TCP + port: 80 diff --git a/monitoring/proxy-for-kubfwd/kustomization.yaml b/monitoring/proxy-for-kubfwd/kustomization.yaml new file mode 100644 index 00000000..c23d43fb --- /dev/null +++ b/monitoring/proxy-for-kubfwd/kustomization.yaml @@ -0,0 +1,6 @@ +namespace: ystack +resources: +- alertmanager-main-proxy-service.yaml +- alertmanager-main-proxy-deployment.yaml +- prometheus-now-proxy-service.yaml +- prometheus-now-proxy-deployment.yaml diff --git a/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml new file mode 100644 index 00000000..c6488764 --- /dev/null +++ b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-now + labels: + proxy: prometheus-now +spec: + replicas: 1 + selector: + matchLabels: + proxy: prometheus-now + template: + metadata: + labels: + proxy: prometheus-now + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "9901" + prometheus.io/scrape: "true" + spec: + containers: + - name: envoy + image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f + ports: + - name: proxy + containerPort: 80 + - name: admin + containerPort: 9901 + command: + - /usr/local/bin/envoy + - -l info + args: + - --config-yaml + - | + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9901 + static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: ystack_monitoring_prometheus_now + http_filters: + - name: envoy.router + clusters: + - name: ystack_monitoring_prometheus_now + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: ystack_monitoring_prometheus_now + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + # there's only the "now" instance + address: prometheus-operated.ystack-monitoring + port_value: 9090 diff --git a/monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml new file mode 100644 index 00000000..be84326e --- /dev/null +++ b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-now +spec: + selector: + proxy: prometheus-now + ports: + - protocol: TCP + port: 80 diff --git a/monitoring/rbac-base/kustomization.yaml b/monitoring/rbac-base/kustomization.yaml new file mode 100644 index 00000000..eaa99e33 --- /dev/null +++ b/monitoring/rbac-base/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- prometheus-cluster-role-binding.yaml +- prometheus-cluster-role.yaml +- prometheus-service-account.yaml diff --git a/monitoring/rbac-base/prometheus-cluster-role-binding.yaml b/monitoring/rbac-base/prometheus-cluster-role-binding.yaml new file mode 100644 index 00000000..2fdd1ed6 --- /dev/null +++ b/monitoring/rbac-base/prometheus-cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus diff --git a/monitoring/rbac-base/prometheus-cluster-role.yaml b/monitoring/rbac-base/prometheus-cluster-role.yaml new file mode 100644 index 00000000..a85422ec --- /dev/null +++ b/monitoring/rbac-base/prometheus-cluster-role.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/monitoring/rbac-base/prometheus-service-account.yaml b/monitoring/rbac-base/prometheus-service-account.yaml new file mode 100644 index 00000000..f3fb283c --- /dev/null +++ b/monitoring/rbac-base/prometheus-service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus diff --git a/monitoring/test-alert/example-alert-always.yaml b/monitoring/test-alert/example-alert-always.yaml new file mode 100644 index 00000000..e82b593b --- /dev/null +++ b/monitoring/test-alert/example-alert-always.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + creationTimestamp: null + labels: + ystack: now + name: test-alert-always +spec: + groups: + - name: ./example.rules + rules: + - alert: ExampleAlert + expr: vector(1) + labels: + severity: none diff --git a/monitoring/test-alert/kustomization.yaml b/monitoring/test-alert/kustomization.yaml new file mode 100644 index 00000000..0253ae94 --- /dev/null +++ b/monitoring/test-alert/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: ystack-monitoring +resources: +- example-alert-always.yaml diff --git a/node-ip/daemonset.yaml b/node-ip/daemonset.yaml new file mode 100644 index 00000000..333852f3 --- /dev/null +++ b/node-ip/daemonset.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ip + labels: + app: node-ip +spec: + selector: + matchLabels: + app: node-ip + template: + metadata: + labels: + app: node-ip + spec: + hostNetwork: true + containers: + - name: pause + image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610 diff --git a/node-ip/kustomization.yaml b/node-ip/kustomization.yaml new file mode 100644 index 00000000..03a53ccf --- /dev/null +++ b/node-ip/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- service-headless.yaml +- daemonset.yaml diff --git a/node-ip/service-headless.yaml b/node-ip/service-headless.yaml new file mode 100644 index 00000000..9fcdb58a --- /dev/null +++ b/node-ip/service-headless.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + name: node-ip +spec: + clusterIP: None + selector: + app: node-ip