From 740e7a2a78b82131f3b61f5f8278dea5da0ef3a3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 1 Oct 2019 05:15:03 +0200
Subject: [PATCH 01/17] Current prometheus-operator /example/rbac/prometheus

---
 monitoring/README.md                          | 38 +++++++++++++++++++
 monitoring/kustomization.yaml                 |  8 ++++
 .../prometheus-cluster-role-binding.yaml      | 12 ++++++
 monitoring/prometheus-cluster-role.yaml       | 18 +++++++++
 monitoring/prometheus-service-account.yaml    |  4 ++
 monitoring/prometheus.yaml                    | 17 +++++++++
 6 files changed, 97 insertions(+)
 create mode 100644 monitoring/README.md
 create mode 100644 monitoring/kustomization.yaml
 create mode 100644 monitoring/prometheus-cluster-role-binding.yaml
 create mode 100644 monitoring/prometheus-cluster-role.yaml
 create mode 100644 monitoring/prometheus-service-account.yaml
 create mode 100644 monitoring/prometheus.yaml

diff --git a/monitoring/README.md b/monitoring/README.md
new file mode 100644
index 00000000..812da0df
--- /dev/null
+++ b/monitoring/README.md
@@ -0,0 +1,38 @@
+# Monitoring stack
+
+We've observed... that when developers start to factor in
+the existence of metrics collection ("scrape") and alerting rules
+-- both generic such as not-enough-replicas or job-failed and service specific --
+it becomes a handy solution to many difficult coding problems.
+Instead of error handling by assumption we expose a metric on some case
+and make sure the alert for it works.
+Helps immensely to leave it to humans to interpret the condition,
+instead of implementing an automated solution up front.
+
+At Yolean we've run [kube-prometheus](https://github.com/coreos/kube-prometheus) for many years
+but, while it is a good collection of tools + rules + dashboards,
+we feel it is time to start to opt-in to components individuallly.
+
+Requirements:
+ * Lean enough to be always-on in development clusters.
+ * Expose alerts that a dev environment can poll for,
+   to make monitoring + alerting rules a first class impl tool.
+ * Comes with basic sanity checks on the stack.
+
+Tools:
+ * Prometheus, the de-facto standard with k8s
+ * Grafana, the de-facto standard with k8s
+ * Alertmanager, the de-facto standard with Prometheus
+ * [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) both for `kubectl top` and Prometheus
+ * [node exporter](https://github.com/prometheus/node_exporter)
+ * [Prometheus Operatior](https://github.com/coreos/prometheus-operator/) appears to be rock solid and the `ServiceMonitor` resources really delivers capabilities that service discovery base don annotations can't, such as scraping multiple containers and having custom intervals.
+
+Dashboards:
+ * Maybe https://github.com/cloudworkz/kube-eagle
+
+Web UIs, as a complement:
+ * https://srcco.de/posts/kubernetes-web-uis-in-2019.html
+ * Can we have basic top + node health cheaply and independent of the stack?
+
+Prometheus instances:
+
diff --git a/monitoring/kustomization.yaml b/monitoring/kustomization.yaml
new file mode 100644
index 00000000..8b9dbb82
--- /dev/null
+++ b/monitoring/kustomization.yaml
@@ -0,0 +1,8 @@
+namespace: thanos
+bases:
+- github.com/coreos/prometheus-operator/?ref=ab1f8da8d9195339266742dcef535df5b51522e1
+resources:
+- prometheus-cluster-role-binding.yaml
+- prometheus-cluster-role.yaml
+- prometheus-service-account.yaml
+- prometheus.yaml
diff --git a/monitoring/prometheus-cluster-role-binding.yaml b/monitoring/prometheus-cluster-role-binding.yaml
new file mode 100644
index 00000000..e255bcea
--- /dev/null
+++ b/monitoring/prometheus-cluster-role-binding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus
+  namespace: default
diff --git a/monitoring/prometheus-cluster-role.yaml b/monitoring/prometheus-cluster-role.yaml
new file mode 100644
index 00000000..a85422ec
--- /dev/null
+++ b/monitoring/prometheus-cluster-role.yaml
@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources:
+  - configmaps
+  verbs: ["get"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
diff --git a/monitoring/prometheus-service-account.yaml b/monitoring/prometheus-service-account.yaml
new file mode 100644
index 00000000..f3fb283c
--- /dev/null
+++ b/monitoring/prometheus-service-account.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
diff --git a/monitoring/prometheus.yaml b/monitoring/prometheus.yaml
new file mode 100644
index 00000000..2adaef13
--- /dev/null
+++ b/monitoring/prometheus.yaml
@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: prometheus
+  labels:
+    prometheus: prometheus
+spec:
+  replicas: 2
+  serviceAccountName: prometheus
+  serviceMonitorSelector:
+    matchLabels:
+      team: frontend
+  alerting:
+    alertmanagers:
+    - namespace: default
+      name: alertmanager
+      port: web

From 86a01b97ee8f504e7c8f70215bea8dbdad3e5374 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 1 Oct 2019 05:40:08 +0200
Subject: [PATCH 02/17] Tracks nodes' IPs as endpoints, kubectl get endpoints
 node-ip

---
 monitoring/README.md          |  3 ++-
 monitoring/kustomization.yaml |  2 +-
 monitoring/prometheus.yaml    | 16 +++++++---------
 node-ip/daemonset.yaml        | 19 +++++++++++++++++++
 node-ip/kustomization.yaml    |  3 +++
 node-ip/service-headless.yaml |  8 ++++++++
 6 files changed, 40 insertions(+), 11 deletions(-)
 create mode 100644 node-ip/daemonset.yaml
 create mode 100644 node-ip/kustomization.yaml
 create mode 100644 node-ip/service-headless.yaml

diff --git a/monitoring/README.md b/monitoring/README.md
index 812da0df..f8640bdf 100644
--- a/monitoring/README.md
+++ b/monitoring/README.md
@@ -35,4 +35,5 @@ Web UIs, as a complement:
  * Can we have basic top + node health cheaply and independent of the stack?
 
 Prometheus instances:
-
+ * `now` denotes the instance(s) you'll turn to for observing current state of the cluster, for example when troubleshooting.
+   Metrics are of the operational type, volumes are high and retention very low. This is also the instance to use for non-business alerts.
diff --git a/monitoring/kustomization.yaml b/monitoring/kustomization.yaml
index 8b9dbb82..9b861ffe 100644
--- a/monitoring/kustomization.yaml
+++ b/monitoring/kustomization.yaml
@@ -1,4 +1,4 @@
-namespace: thanos
+namespace: ystack
 bases:
 - github.com/coreos/prometheus-operator/?ref=ab1f8da8d9195339266742dcef535df5b51522e1
 resources:
diff --git a/monitoring/prometheus.yaml b/monitoring/prometheus.yaml
index 2adaef13..19ff31e1 100644
--- a/monitoring/prometheus.yaml
+++ b/monitoring/prometheus.yaml
@@ -1,17 +1,15 @@
 apiVersion: monitoring.coreos.com/v1
 kind: Prometheus
 metadata:
-  name: prometheus
+  name: now
   labels:
-    prometheus: prometheus
+    ystack-view: now
 spec:
-  replicas: 2
+  version: 2.13.0-rc.0
+  baseImage: prom/prometheus
+  sha: cf639ff2b67634c03c33fc9b7923a9702da1bb031b899688125ca8718b0ce949
+  replicas: 1
   serviceAccountName: prometheus
   serviceMonitorSelector:
     matchLabels:
-      team: frontend
-  alerting:
-    alertmanagers:
-    - namespace: default
-      name: alertmanager
-      port: web
+      ystack-view: now
diff --git a/node-ip/daemonset.yaml b/node-ip/daemonset.yaml
new file mode 100644
index 00000000..333852f3
--- /dev/null
+++ b/node-ip/daemonset.yaml
@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-ip
+  labels:
+    app: node-ip
+spec:
+  selector:
+    matchLabels:
+      app: node-ip
+  template:
+    metadata:
+      labels:
+        app: node-ip
+    spec:
+      hostNetwork: true
+      containers:
+      - name: pause
+        image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610
diff --git a/node-ip/kustomization.yaml b/node-ip/kustomization.yaml
new file mode 100644
index 00000000..03a53ccf
--- /dev/null
+++ b/node-ip/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- service-headless.yaml
+- daemonset.yaml
diff --git a/node-ip/service-headless.yaml b/node-ip/service-headless.yaml
new file mode 100644
index 00000000..9fcdb58a
--- /dev/null
+++ b/node-ip/service-headless.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-ip
+spec:
+  clusterIP: None
+  selector:
+    app: node-ip

From 6151ae269105ef8448ecf2445ac9299c0c5424f2 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 1 Oct 2019 05:57:29 +0200
Subject: [PATCH 03/17] Exposes containerd for scraping, but actual use is
 blocked by

https://github.com/containerd/containerd/issues/3412

k3s has upgraded to 1.3.0 in master since 834f7df7ecb8eb8be27ebb300f2bcb23f24397b2
and ofc we don't know when GKE's _CONTAINERD nodes will upgrade.
---
 bin/y-cluster-provision-k3s-multipass         | 53 +++++++++++++++++++
 monitoring/containerd/kustomization.yaml      |  4 ++
 .../containerd/nodes-containerd-service.yaml  | 10 ++++
 3 files changed, 67 insertions(+)
 create mode 100644 monitoring/containerd/kustomization.yaml
 create mode 100644 monitoring/containerd/nodes-containerd-service.yaml

diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass
index 800a7960..48244d39 100755
--- a/bin/y-cluster-provision-k3s-multipass
+++ b/bin/y-cluster-provision-k3s-multipass
@@ -46,6 +46,59 @@ INSTALLER_REVISION=18bd921cddee1e95cc03467a1b9636ddacd9d670
 export INSTALL_K3S_VERSION=v1.0.0
 curl -sfL https://github.com/rancher/k3s/raw/$INSTALLER_REVISION/install.sh | sh -
 
+K3S_DATA_DIR=/var/lib/rancher/k3s
+
+mkdir -p     $K3S_DATA_DIR/agent/etc/containerd
+cat <<EOF >> $K3S_DATA_DIR/agent/etc/containerd/config.toml.tmpl
+[metrics]
+address = "0.0.0.0:1338"
+
+# https://github.com/rancher/k3s/blob/v0.10.2/pkg/agent/templates/templates.go
+[plugins.opt]
+  path = "{{ .NodeConfig.Containerd.Opt }}"
+[plugins.cri]
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+{{- if .IsRunningInUserNS }}
+  disable_cgroup = true
+  disable_apparmor = true
+  restrict_oom_score_adj = true
+{{end}}
+{{- if .NodeConfig.AgentConfig.PauseImage }}
+  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{end}}
+{{- if not .NodeConfig.NoFlannel }}
+[plugins.cri.cni]
+  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
+{{end}}
+[plugins.cri.containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+{{ if .PrivateRegistryConfig }}
+{{ if .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors]{{end}}
+{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors."{{$k}}"]
+  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
+{{end}}
+{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins.cri.registry.configs."{{$k}}".auth]
+  {{ if $v.Auth.Username }}username = "{{ $v.Auth.Username }}"{{end}}
+  {{ if $v.Auth.Password }}password = "{{ $v.Auth.Password }}"{{end}}
+  {{ if $v.Auth.Auth }}auth = "{{ $v.Auth.Auth }}"{{end}}
+  {{ if $v.Auth.IdentityToken }}identity_token = "{{ $v.Auth.IdentityToken }}"{{end}}
+{{end}}
+{{ if $v.TLS }}
+[plugins.cri.registry.configs."{{$k}}".tls]
+  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
+  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
+  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
+{{end}}
+{{end}}
+{{end}}
+EOF
+
 service k3s start
 k3s crictl info
 
diff --git a/monitoring/containerd/kustomization.yaml b/monitoring/containerd/kustomization.yaml
new file mode 100644
index 00000000..88dad886
--- /dev/null
+++ b/monitoring/containerd/kustomization.yaml
@@ -0,0 +1,4 @@
+bases:
+- ../../node-ip
+resources:
+- nodes-containerd-service.yaml
diff --git a/monitoring/containerd/nodes-containerd-service.yaml b/monitoring/containerd/nodes-containerd-service.yaml
new file mode 100644
index 00000000..6d51fe7f
--- /dev/null
+++ b/monitoring/containerd/nodes-containerd-service.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: containerd
+spec:
+  selector:
+    app: node-ip
+  ports:
+  - port: 1338
+    name: metrics

From 1ef547444614aa70fbd9b60e7dbfe4193ff600cc Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 27 Oct 2019 16:35:17 +0100
Subject: [PATCH 04/17] Prometheus Operator takes 2m+20M resources when idle,

and is quite impractical with Kustomize due to:

 - Large repo that takes a long time to fetch
 - Namespace "default" is hard coded in bundle.yaml

... so let's install it by default.
---
 monitoring/kustomization.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/monitoring/kustomization.yaml b/monitoring/kustomization.yaml
index 9b861ffe..ed7d10af 100644
--- a/monitoring/kustomization.yaml
+++ b/monitoring/kustomization.yaml
@@ -1,6 +1,4 @@
 namespace: ystack
-bases:
-- github.com/coreos/prometheus-operator/?ref=ab1f8da8d9195339266742dcef535df5b51522e1
 resources:
 - prometheus-cluster-role-binding.yaml
 - prometheus-cluster-role.yaml

From 4593a463ff03b612d25ddc4445f2f3638117dfe8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 27 Oct 2019 17:03:38 +0100
Subject: [PATCH 05/17] Starts a prometheus instance accessible over kubefwd:

http://prometheus-now-0.prometheus-operated.ystack.svc.cluster.local:9090/

prometheus-now-0.prometheus-operated.ystack.svc.cluster.local:9090/graph
---
 converge-generic/kustomization.yaml                          | 2 ++
 monitoring/instance-now/kustomization.yaml                   | 5 +++++
 .../{prometheus.yaml => instance-now/now-prometheus.yaml}    | 0
 monitoring/{ => rbac-base}/kustomization.yaml                | 2 --
 .../{ => rbac-base}/prometheus-cluster-role-binding.yaml     | 1 -
 monitoring/{ => rbac-base}/prometheus-cluster-role.yaml      | 0
 monitoring/{ => rbac-base}/prometheus-service-account.yaml   | 0
 7 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 monitoring/instance-now/kustomization.yaml
 rename monitoring/{prometheus.yaml => instance-now/now-prometheus.yaml} (100%)
 rename monitoring/{ => rbac-base}/kustomization.yaml (76%)
 rename monitoring/{ => rbac-base}/prometheus-cluster-role-binding.yaml (91%)
 rename monitoring/{ => rbac-base}/prometheus-cluster-role.yaml (100%)
 rename monitoring/{ => rbac-base}/prometheus-service-account.yaml (100%)

diff --git a/converge-generic/kustomization.yaml b/converge-generic/kustomization.yaml
index 176fbad1..37cfce56 100644
--- a/converge-generic/kustomization.yaml
+++ b/converge-generic/kustomization.yaml
@@ -5,3 +5,5 @@ bases:
 #- ../registry/generic,kafka  # Note that kafka isn't a base (as it has its own namespace) and must be applied separately
 - ../registry/generic,minio
 # included in generic,minio: - ../registry/node-update-hosts   # Required unless nodes' container runtime can look up cluster DNS for registry services
+
+- ../monitoring/instance-now
diff --git a/monitoring/instance-now/kustomization.yaml b/monitoring/instance-now/kustomization.yaml
new file mode 100644
index 00000000..ac38381b
--- /dev/null
+++ b/monitoring/instance-now/kustomization.yaml
@@ -0,0 +1,5 @@
+namespace: ystack
+bases:
+- ../rbac-base/
+resources:
+- now-prometheus.yaml
diff --git a/monitoring/prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml
similarity index 100%
rename from monitoring/prometheus.yaml
rename to monitoring/instance-now/now-prometheus.yaml
diff --git a/monitoring/kustomization.yaml b/monitoring/rbac-base/kustomization.yaml
similarity index 76%
rename from monitoring/kustomization.yaml
rename to monitoring/rbac-base/kustomization.yaml
index ed7d10af..eaa99e33 100644
--- a/monitoring/kustomization.yaml
+++ b/monitoring/rbac-base/kustomization.yaml
@@ -1,6 +1,4 @@
-namespace: ystack
 resources:
 - prometheus-cluster-role-binding.yaml
 - prometheus-cluster-role.yaml
 - prometheus-service-account.yaml
-- prometheus.yaml
diff --git a/monitoring/prometheus-cluster-role-binding.yaml b/monitoring/rbac-base/prometheus-cluster-role-binding.yaml
similarity index 91%
rename from monitoring/prometheus-cluster-role-binding.yaml
rename to monitoring/rbac-base/prometheus-cluster-role-binding.yaml
index e255bcea..2fdd1ed6 100644
--- a/monitoring/prometheus-cluster-role-binding.yaml
+++ b/monitoring/rbac-base/prometheus-cluster-role-binding.yaml
@@ -9,4 +9,3 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: prometheus
-  namespace: default
diff --git a/monitoring/prometheus-cluster-role.yaml b/monitoring/rbac-base/prometheus-cluster-role.yaml
similarity index 100%
rename from monitoring/prometheus-cluster-role.yaml
rename to monitoring/rbac-base/prometheus-cluster-role.yaml
diff --git a/monitoring/prometheus-service-account.yaml b/monitoring/rbac-base/prometheus-service-account.yaml
similarity index 100%
rename from monitoring/prometheus-service-account.yaml
rename to monitoring/rbac-base/prometheus-service-account.yaml

From 69691760251449eef6da9fb6cdf214c4dd2a6842 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 27 Oct 2019 17:04:57 +0100
Subject: [PATCH 06/17] Current Prometheus release, a shorter label name for
 rules, now-ish retention

---
 monitoring/instance-now/now-prometheus.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/monitoring/instance-now/now-prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml
index 19ff31e1..faa1baef 100644
--- a/monitoring/instance-now/now-prometheus.yaml
+++ b/monitoring/instance-now/now-prometheus.yaml
@@ -3,13 +3,14 @@ kind: Prometheus
 metadata:
   name: now
   labels:
-    ystack-view: now
+    ystack: now
 spec:
-  version: 2.13.0-rc.0
+  version: 2.13.1
   baseImage: prom/prometheus
-  sha: cf639ff2b67634c03c33fc9b7923a9702da1bb031b899688125ca8718b0ce949
+  sha: 0a8caa2e9f19907608915db6e62a67383fe44b9876a467b297ee6f64e51dd58a
   replicas: 1
   serviceAccountName: prometheus
   serviceMonitorSelector:
     matchLabels:
-      ystack-view: now
+      ystack: now
+  retention: 8h

From c41c3daa8b1c46081e32a00f6ddb647953aafebc Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 27 Oct 2019 19:24:09 +0100
Subject: [PATCH 07/17] Let pods with a single container enable monitoring
 using conventional annotations

... despite such annotations being inflexible.
The rule with ServiceMonitor having to be in the same namespace as the
prometheus instance is quite impractical, in particular with Kustomize.
---
 monitoring/instance-now/kustomization.yaml  |  6 ++++
 monitoring/instance-now/now-additional.yaml | 37 +++++++++++++++++++++
 monitoring/instance-now/now-prometheus.yaml |  3 ++
 3 files changed, 46 insertions(+)
 create mode 100644 monitoring/instance-now/now-additional.yaml

diff --git a/monitoring/instance-now/kustomization.yaml b/monitoring/instance-now/kustomization.yaml
index ac38381b..5d411efd 100644
--- a/monitoring/instance-now/kustomization.yaml
+++ b/monitoring/instance-now/kustomization.yaml
@@ -3,3 +3,9 @@ bases:
 - ../rbac-base/
 resources:
 - now-prometheus.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+secretGenerator:
+- name: prometheus-now-additional
+  files:
+  - now-additional.yaml
diff --git a/monitoring/instance-now/now-additional.yaml b/monitoring/instance-now/now-additional.yaml
new file mode 100644
index 00000000..f8e13d6f
--- /dev/null
+++ b/monitoring/instance-now/now-additional.yaml
@@ -0,0 +1,37 @@
+
+# https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
+# https://github.com/prometheus/prometheus/blob/85a3c974b760642bbce00d795db231326a09edb9/documentation/examples/prometheus-kubernetes.yml
+# (before https://github.com/prometheus/prometheus/commit/03a9e7f72e072c6d29f422425d8acd91a957836b#diff-416ffe38e4360ad5f27d8a034963c260)
+# * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
+# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the pod's declared ports (default is a port-free target if none are declared).
+- job_name: kubernetes-pods
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  - action: keep
+    regex: true
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+  - action: replace
+    regex: (.+)
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_path
+    target_label: __metrics_path__
+  - action: replace
+    regex: ([^:]+)(?::\d+)?;(\d+)
+    replacement: $1:$2
+    source_labels:
+    - __address__
+    - __meta_kubernetes_pod_annotation_prometheus_io_port
+    target_label: __address__
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_namespace
+    target_label: kubernetes_namespace
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_pod_name
+    target_label: kubernetes_pod_name
diff --git a/monitoring/instance-now/now-prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml
index faa1baef..6cb41111 100644
--- a/monitoring/instance-now/now-prometheus.yaml
+++ b/monitoring/instance-now/now-prometheus.yaml
@@ -14,3 +14,6 @@ spec:
     matchLabels:
       ystack: now
   retention: 8h
+  additionalScrapeConfigs:
+    name: prometheus-now-additional
+    key: now-additional.yaml

From d215276e6bae5e228d312db97192f8e153695f60 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 18 Nov 2019 14:34:11 +0100
Subject: [PATCH 08/17] Based on operator repo's example/user-guides/alerting

breaks kubefwd, but reachable with:
kubectl -n ystack port-forward svc/prometheus-operated 9090
kubectl -n ystack port-forward svc/alertmanager-operated 9093

... and the alert doesn't work
---
 monitoring/alertmanager/alertmanager.yaml       | 12 ++++++++++++
 monitoring/alertmanager/kustomization.yaml      |  8 ++++++++
 monitoring/alertmanager/main-alertmanager.yaml  |  9 +++++++++
 monitoring/instance-now/kustomization.yaml      |  1 +
 monitoring/instance-now/now-prometheus.yaml     |  8 ++++++++
 monitoring/test-alert/example-alert-always.yaml | 13 +++++++++++++
 monitoring/test-alert/kustomization.yaml        |  2 ++
 7 files changed, 53 insertions(+)
 create mode 100644 monitoring/alertmanager/alertmanager.yaml
 create mode 100644 monitoring/alertmanager/kustomization.yaml
 create mode 100644 monitoring/alertmanager/main-alertmanager.yaml
 create mode 100644 monitoring/test-alert/example-alert-always.yaml
 create mode 100644 monitoring/test-alert/kustomization.yaml

diff --git a/monitoring/alertmanager/alertmanager.yaml b/monitoring/alertmanager/alertmanager.yaml
new file mode 100644
index 00000000..f08a2106
--- /dev/null
+++ b/monitoring/alertmanager/alertmanager.yaml
@@ -0,0 +1,12 @@
+global:
+  resolve_timeout: 5m
+route:
+  group_by: ['job']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 12h
+  receiver: 'webhook'
+receivers:
+- name: 'webhook'
+  webhook_configs:
+  - url: 'http://alertmanagerwh:30500/'
diff --git a/monitoring/alertmanager/kustomization.yaml b/monitoring/alertmanager/kustomization.yaml
new file mode 100644
index 00000000..538e9d10
--- /dev/null
+++ b/monitoring/alertmanager/kustomization.yaml
@@ -0,0 +1,8 @@
+resources:
+- main-alertmanager.yaml
+generatorOptions:
+ disableNameSuffixHash: true
+secretGenerator:
+- name: alertmanager-main
+  files:
+  - alertmanager.yaml
diff --git a/monitoring/alertmanager/main-alertmanager.yaml b/monitoring/alertmanager/main-alertmanager.yaml
new file mode 100644
index 00000000..90001cb1
--- /dev/null
+++ b/monitoring/alertmanager/main-alertmanager.yaml
@@ -0,0 +1,9 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  name: main
+spec:
+  version: 0.19.0
+  baseImage: prom/alertmanager
+  sha: 7dbf4949a317a056d11ed8f379826b04d0665fad5b9334e1d69b23e946056cd3
+  replicas: 1
diff --git a/monitoring/instance-now/kustomization.yaml b/monitoring/instance-now/kustomization.yaml
index 5d411efd..27deb6ca 100644
--- a/monitoring/instance-now/kustomization.yaml
+++ b/monitoring/instance-now/kustomization.yaml
@@ -1,6 +1,7 @@
 namespace: ystack
 bases:
 - ../rbac-base/
+- ../alertmanager/
 resources:
 - now-prometheus.yaml
 generatorOptions:
diff --git a/monitoring/instance-now/now-prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml
index 6cb41111..851f8172 100644
--- a/monitoring/instance-now/now-prometheus.yaml
+++ b/monitoring/instance-now/now-prometheus.yaml
@@ -13,7 +13,15 @@ spec:
   serviceMonitorSelector:
     matchLabels:
       ystack: now
+  ruleSelector:
+    matchLabels:
+      ystack: now
   retention: 8h
   additionalScrapeConfigs:
     name: prometheus-now-additional
     key: now-additional.yaml
+  alerting:
+    alertmanagers:
+    - namespace: ystack
+      name: alertmanager-main
+      port: web
diff --git a/monitoring/test-alert/example-alert-always.yaml b/monitoring/test-alert/example-alert-always.yaml
new file mode 100644
index 00000000..ff1bbbf5
--- /dev/null
+++ b/monitoring/test-alert/example-alert-always.yaml
@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  creationTimestamp: null
+  labels:
+    ystack: now
+  name: test-alert-always
+spec:
+  groups:
+  - name: ./example.rules
+    rules:
+    - alert: ExampleAlert
+      expr: vector(1)
diff --git a/monitoring/test-alert/kustomization.yaml b/monitoring/test-alert/kustomization.yaml
new file mode 100644
index 00000000..665d5fae
--- /dev/null
+++ b/monitoring/test-alert/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+- example-alert-always.yaml

From 8c9346f71395bf79c32d7f44bb60dac7ff38f85e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 18 Nov 2019 14:55:23 +0100
Subject: [PATCH 09/17] Revert "Exposes containerd for scraping" because of
 frequest

provision failures at k3s crictl info with the message
failed to connect: failed to connect, make sure you are running as root and the runtime has been started: context deadline exceeded

This reverts commit 09712775dc3b2a3f6bb677ec689aaac1b62bc230.
---
 bin/y-cluster-provision-k3s-multipass         | 53 -------------------
 monitoring/containerd/kustomization.yaml      |  4 --
 .../containerd/nodes-containerd-service.yaml  | 10 ----
 3 files changed, 67 deletions(-)
 delete mode 100644 monitoring/containerd/kustomization.yaml
 delete mode 100644 monitoring/containerd/nodes-containerd-service.yaml

diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass
index 48244d39..800a7960 100755
--- a/bin/y-cluster-provision-k3s-multipass
+++ b/bin/y-cluster-provision-k3s-multipass
@@ -46,59 +46,6 @@ INSTALLER_REVISION=18bd921cddee1e95cc03467a1b9636ddacd9d670
 export INSTALL_K3S_VERSION=v1.0.0
 curl -sfL https://github.com/rancher/k3s/raw/$INSTALLER_REVISION/install.sh | sh -
 
-K3S_DATA_DIR=/var/lib/rancher/k3s
-
-mkdir -p     $K3S_DATA_DIR/agent/etc/containerd
-cat <<EOF >> $K3S_DATA_DIR/agent/etc/containerd/config.toml.tmpl
-[metrics]
-address = "0.0.0.0:1338"
-
-# https://github.com/rancher/k3s/blob/v0.10.2/pkg/agent/templates/templates.go
-[plugins.opt]
-  path = "{{ .NodeConfig.Containerd.Opt }}"
-[plugins.cri]
-  stream_server_address = "127.0.0.1"
-  stream_server_port = "10010"
-{{- if .IsRunningInUserNS }}
-  disable_cgroup = true
-  disable_apparmor = true
-  restrict_oom_score_adj = true
-{{end}}
-{{- if .NodeConfig.AgentConfig.PauseImage }}
-  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-{{end}}
-{{- if not .NodeConfig.NoFlannel }}
-[plugins.cri.cni]
-  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-{{end}}
-[plugins.cri.containerd.runtimes.runc]
-  runtime_type = "io.containerd.runc.v2"
-{{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins.cri.registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins.cri.registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Configs }}
-{{ if $v.Auth }}
-[plugins.cri.registry.configs."{{$k}}".auth]
-  {{ if $v.Auth.Username }}username = "{{ $v.Auth.Username }}"{{end}}
-  {{ if $v.Auth.Password }}password = "{{ $v.Auth.Password }}"{{end}}
-  {{ if $v.Auth.Auth }}auth = "{{ $v.Auth.Auth }}"{{end}}
-  {{ if $v.Auth.IdentityToken }}identity_token = "{{ $v.Auth.IdentityToken }}"{{end}}
-{{end}}
-{{ if $v.TLS }}
-[plugins.cri.registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-{{end}}
-{{end}}
-{{end}}
-EOF
-
 service k3s start
 k3s crictl info
 
diff --git a/monitoring/containerd/kustomization.yaml b/monitoring/containerd/kustomization.yaml
deleted file mode 100644
index 88dad886..00000000
--- a/monitoring/containerd/kustomization.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-bases:
-- ../../node-ip
-resources:
-- nodes-containerd-service.yaml
diff --git a/monitoring/containerd/nodes-containerd-service.yaml b/monitoring/containerd/nodes-containerd-service.yaml
deleted file mode 100644
index 6d51fe7f..00000000
--- a/monitoring/containerd/nodes-containerd-service.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: containerd
-spec:
-  selector:
-    app: node-ip
-  ports:
-  - port: 1338
-    name: metrics

From dc12e42d1406a870fe400911c31ad570ee1807a7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 18 Nov 2019 15:00:47 +0100
Subject: [PATCH 10/17] Moves monitoring to a separate namespace

because kubefwd found so much to forward, and failed to actually forward alertmanager.
We do need local access to prometheus but kubefwd was pretty much my only reason
to keep monotiring resources in the ystack namespaces. It's cleaner this way.
---
 monitoring/instance-now/kustomization.yaml  | 2 +-
 monitoring/instance-now/now-prometheus.yaml | 2 +-
 monitoring/test-alert/kustomization.yaml    | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/monitoring/instance-now/kustomization.yaml b/monitoring/instance-now/kustomization.yaml
index 27deb6ca..f6ea2353 100644
--- a/monitoring/instance-now/kustomization.yaml
+++ b/monitoring/instance-now/kustomization.yaml
@@ -1,4 +1,4 @@
-namespace: ystack
+namespace: ystack-monitoring
 bases:
 - ../rbac-base/
 - ../alertmanager/
diff --git a/monitoring/instance-now/now-prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml
index 851f8172..12c4a3d0 100644
--- a/monitoring/instance-now/now-prometheus.yaml
+++ b/monitoring/instance-now/now-prometheus.yaml
@@ -22,6 +22,6 @@ spec:
     key: now-additional.yaml
   alerting:
     alertmanagers:
-    - namespace: ystack
+    - namespace: ystack-monitoring
       name: alertmanager-main
       port: web
diff --git a/monitoring/test-alert/kustomization.yaml b/monitoring/test-alert/kustomization.yaml
index 665d5fae..0253ae94 100644
--- a/monitoring/test-alert/kustomization.yaml
+++ b/monitoring/test-alert/kustomization.yaml
@@ -1,2 +1,3 @@
+namespace: ystack-monitoring
 resources:
 - example-alert-always.yaml

From cf3a6adf9360a297ffc3947cfff1b135d043b682 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 18 Nov 2019 15:29:51 +0100
Subject: [PATCH 11/17] Now we get one alertmanager under Endpoints in
 Prometheus' Status

For some reason I failed to get service discovery with a plain alertmanager-main service.

To troubleshoot I did:
kubectl -n ystack-monitoring get secret prometheus-now -o=jsonpath='{.data.prometheus\.yaml\.gz}' | base64 -D - | gunzip -
kubectl -n ystack-monitoring get endpoints alertmanager-operated
---
 monitoring/instance-now/now-prometheus.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monitoring/instance-now/now-prometheus.yaml b/monitoring/instance-now/now-prometheus.yaml
index 12c4a3d0..9f8f8318 100644
--- a/monitoring/instance-now/now-prometheus.yaml
+++ b/monitoring/instance-now/now-prometheus.yaml
@@ -23,5 +23,5 @@ spec:
   alerting:
     alertmanagers:
     - namespace: ystack-monitoring
-      name: alertmanager-main
+      name: alertmanager-operated # works as long as there's only one alertmanager resource in the namespace
       port: web

From b3aedbff39ac8c6f902b54bef8cd893c34088d54 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 18 Nov 2019 15:32:03 +0100
Subject: [PATCH 12/17] Lets Alertnamanger filter out this one when paging

---
 monitoring/test-alert/example-alert-always.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/monitoring/test-alert/example-alert-always.yaml b/monitoring/test-alert/example-alert-always.yaml
index ff1bbbf5..e82b593b 100644
--- a/monitoring/test-alert/example-alert-always.yaml
+++ b/monitoring/test-alert/example-alert-always.yaml
@@ -11,3 +11,5 @@ spec:
     rules:
     - alert: ExampleAlert
       expr: vector(1)
+      labels:
+        severity: none

From 349983bf72733d72d5dea80811c8cdbfaa6d5574 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 19 Nov 2019 04:09:45 +0100
Subject: [PATCH 13/17] Only a bit more research

---
 monitoring/README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/monitoring/README.md b/monitoring/README.md
index f8640bdf..087c462d 100644
--- a/monitoring/README.md
+++ b/monitoring/README.md
@@ -26,6 +26,10 @@ Tools:
  * [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) both for `kubectl top` and Prometheus
  * [node exporter](https://github.com/prometheus/node_exporter)
  * [Prometheus Operatior](https://github.com/coreos/prometheus-operator/) appears to be rock solid and the `ServiceMonitor` resources really delivers capabilities that service discovery base don annotations can't, such as scraping multiple containers and having custom intervals.
+ * [kubernetes-mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin)
+ * [kube-prometheus' alerting rules](https://github.com/coreos/kube-prometheus/commits/master/manifests/prometheus-rules.yaml) with stuff like clock skew
+ * https://github.com/coreos/kube-prometheus/commits/master/manifests/grafana-dashboardDefinitions.yaml also heavily duplicating kubernetes-mixin
+ * https://github.com/brancz/kubernetes-grafana "exists because the Grafana stack in kube-prometheus has gotten close to unmaintainable"
 
 Dashboards:
  * Maybe https://github.com/cloudworkz/kube-eagle
@@ -37,3 +41,6 @@ Web UIs, as a complement:
 Prometheus instances:
  * `now` denotes the instance(s) you'll turn to for observing current state of the cluster, for example when troubleshooting.
    Metrics are of the operational type, volumes are high and retention very low. This is also the instance to use for non-business alerts.
+
+Specific to k3s:
+ * https://github.com/rancher/k3s/issues/425 but let's focus on amd64

From c55c4966c9e77d756fc70aa3498dc075ccb2888b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 19 Nov 2019 05:11:56 +0100
Subject: [PATCH 14/17] Starts an envoy config from example

https://github.com/envoyproxy/envoy/blob/master/configs/google_com_proxy.v2.yaml
---
 .../alertmanager-main-proxy-deployment.yaml   | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml

diff --git a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
new file mode 100644
index 00000000..d8b02967
--- /dev/null
+++ b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alertmanager-main
+  labels:
+    proxy: alertmanager-main
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      proxy: alertmanager-main
+  template:
+    metadata:
+      labels:
+        proxy: alertmanager-main
+    spec:
+      containers:
+      - name: envoy
+        image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f
+        ports:
+        - containerPort: 80
+        command:
+        - /usr/local/bin/envoy
+        - -l info
+        args:
+        - --config-yaml
+        - |
+          admin:
+            access_log_path: /tmp/admin_access.log
+            address:
+              socket_address:
+                protocol: TCP
+                address: 127.0.0.1
+                port_value: 9901
+          static_resources:
+            listeners:
+            - name: listener_0
+              address:
+                socket_address:
+                  protocol: TCP
+                  address: 0.0.0.0
+                  port_value: 10000
+              filter_chains:
+              - filters:
+                - name: envoy.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                    stat_prefix: ingress_http
+                    route_config:
+                      name: local_route
+                      virtual_hosts:
+                      - name: local_service
+                        domains: ["*"]
+                        routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            host_rewrite: www.google.com
+                            cluster: service_google
+                    http_filters:
+                    - name: envoy.router
+            clusters:
+            - name: service_google
+              connect_timeout: 0.25s
+              type: LOGICAL_DNS
+              # Comment out the following line to test on v6 networks
+              dns_lookup_family: V4_ONLY
+              lb_policy: ROUND_ROBIN
+              load_assignment:
+                cluster_name: service_google
+                endpoints:
+                - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: www.google.com
+                          port_value: 443
+              transport_socket:
+                name: envoy.transport_sockets.tls
+                typed_config:
+                  "@type": type.googleapis.com/envoy.api.v2.auth.UpstreamTlsContext
+                  sni: www.google.com

From 39c38ba9a01c78392f0e861cbb5dc2dc4ecd5136 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 19 Nov 2019 05:18:19 +0100
Subject: [PATCH 15/17] Working proxy

---
 .../alertmanager-main-proxy-service.yaml               | 10 ++++++++++
 monitoring/proxy-for-kubfwd/kustomization.yaml         |  4 ++++
 2 files changed, 14 insertions(+)
 create mode 100644 monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml
 create mode 100644 monitoring/proxy-for-kubfwd/kustomization.yaml

diff --git a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml
new file mode 100644
index 00000000..dc757438
--- /dev/null
+++ b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-service.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alertmanager-main
+spec:
+  selector:
+    proxy: alertmanager-main
+  ports:
+  - protocol: TCP
+    port: 80
diff --git a/monitoring/proxy-for-kubfwd/kustomization.yaml b/monitoring/proxy-for-kubfwd/kustomization.yaml
new file mode 100644
index 00000000..f80eb534
--- /dev/null
+++ b/monitoring/proxy-for-kubfwd/kustomization.yaml
@@ -0,0 +1,4 @@
+namespace: ystack
+resources:
+- alertmanager-main-proxy-service.yaml
+- alertmanager-main-proxy-deployment.yaml

From 4b7421dcb30d540d163532995ef4c6644999fa87 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 19 Nov 2019 05:37:47 +0100
Subject: [PATCH 16/17] Monitoring stack is now conveniently kubefwded

---
 .../alertmanager-main-proxy-deployment.yaml   | 31 +++----
 .../proxy-for-kubfwd/kustomization.yaml       |  2 +
 .../prometheus-now-proxy-deployment.yaml      | 83 +++++++++++++++++++
 .../prometheus-now-proxy-service.yaml         | 10 +++
 4 files changed, 111 insertions(+), 15 deletions(-)
 create mode 100644 monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml
 create mode 100644 monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml

diff --git a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
index d8b02967..17317acb 100644
--- a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
+++ b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
@@ -13,12 +13,19 @@ spec:
     metadata:
       labels:
         proxy: alertmanager-main
+      annotations:
+        prometheus.io/path: /stats/prometheus?usedonly
+        prometheus.io/port: "9901"
+        prometheus.io/scrape: "true"
     spec:
       containers:
       - name: envoy
         image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f
         ports:
-        - containerPort: 80
+        - name: proxy
+          containerPort: 80
+        - name: admin
+          containerPort: 9901
         command:
         - /usr/local/bin/envoy
         - -l info
@@ -30,7 +37,7 @@ spec:
             address:
               socket_address:
                 protocol: TCP
-                address: 127.0.0.1
+                address: 0.0.0.0
                 port_value: 9901
           static_resources:
             listeners:
@@ -39,7 +46,7 @@ spec:
                 socket_address:
                   protocol: TCP
                   address: 0.0.0.0
-                  port_value: 10000
+                  port_value: 80
               filter_chains:
               - filters:
                 - name: envoy.http_connection_manager
@@ -55,28 +62,22 @@ spec:
                         - match:
                             prefix: "/"
                           route:
-                            host_rewrite: www.google.com
-                            cluster: service_google
+                            cluster: ystack_monitoring_alertmanager_main
                     http_filters:
                     - name: envoy.router
             clusters:
-            - name: service_google
+            - name: ystack_monitoring_alertmanager_main
               connect_timeout: 0.25s
               type: LOGICAL_DNS
-              # Comment out the following line to test on v6 networks
               dns_lookup_family: V4_ONLY
               lb_policy: ROUND_ROBIN
               load_assignment:
-                cluster_name: service_google
+                cluster_name: ystack_monitoring_alertmanager_main
                 endpoints:
                 - lb_endpoints:
                   - endpoint:
                       address:
                         socket_address:
-                          address: www.google.com
-                          port_value: 443
-              transport_socket:
-                name: envoy.transport_sockets.tls
-                typed_config:
-                  "@type": type.googleapis.com/envoy.api.v2.auth.UpstreamTlsContext
-                  sni: www.google.com
+                          # there's only the "main" instance
+                          address: alertmanager-operated.ystack-monitoring
+                          port_value: 9093
diff --git a/monitoring/proxy-for-kubfwd/kustomization.yaml b/monitoring/proxy-for-kubfwd/kustomization.yaml
index f80eb534..c23d43fb 100644
--- a/monitoring/proxy-for-kubfwd/kustomization.yaml
+++ b/monitoring/proxy-for-kubfwd/kustomization.yaml
@@ -2,3 +2,5 @@ namespace: ystack
 resources:
 - alertmanager-main-proxy-service.yaml
 - alertmanager-main-proxy-deployment.yaml
+- prometheus-now-proxy-service.yaml
+- prometheus-now-proxy-deployment.yaml
diff --git a/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml
new file mode 100644
index 00000000..16dc71a1
--- /dev/null
+++ b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml
@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-now
+  labels:
+    proxy: prometheus-now
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      proxy: prometheus-now
+  template:
+    metadata:
+      labels:
+        proxy: prometheus-now
+      annotations:
+        prometheus.io/path: /stats/prometheus?usedonly
+        prometheus.io/port: "9901"
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+      - name: envoy
+        image: envoyproxy/envoy:v1.12.1@sha256:c485247f63de16331fbe56aec8e9d8a83da76ca0fa8dc872691a09b8d99a314f
+        ports:
+        - name: proxy
+          containerPort: 80
+        - name: admin
+          containerPort: 9901
+        command:
+        - /usr/local/bin/envoy
+        - -l info
+        args:
+        - --config-yaml
+        - |
+          admin:
+            access_log_path: /tmp/admin_access.log
+            address:
+              socket_address:
+                protocol: TCP
+                address: 0.0.0.0
+                port_value: 9901
+          static_resources:
+            listeners:
+            - name: listener_0
+              address:
+                socket_address:
+                  protocol: TCP
+                  address: 0.0.0.0
+                  port_value: 80
+              filter_chains:
+              - filters:
+                - name: envoy.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                    stat_prefix: ingress_http
+                    route_config:
+                      name: local_route
+                      virtual_hosts:
+                      - name: local_service
+                        domains: ["*"]
+                        routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: ystack_monitoring_prometheus_now
+                    http_filters:
+                    - name: envoy.router
+            clusters:
+            - name: ystack_monitoring_prometheus_now
+              connect_timeout: 0.25s
+              type: LOGICAL_DNS
+              dns_lookup_family: V4_ONLY
+              lb_policy: ROUND_ROBIN
+              load_assignment:
+                cluster_name: ystack_monitoring_prometheus_now
+                endpoints:
+                - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          # there's only the "now" instance
+                          address: prometheus-operated.ystack-monitoring
+                          port_value: 9090
diff --git a/monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml
new file mode 100644
index 00000000..be84326e
--- /dev/null
+++ b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-service.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-now
+spec:
+  selector:
+    proxy: prometheus-now
+  ports:
+  - protocol: TCP
+    port: 80

From e0d5ae59cbd2c50738150892dd8eea7b912c8325 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 19 Nov 2019 05:39:33 +0100
Subject: [PATCH 17/17] ?usedonly resulted in 404 for prometheus, but worked
 with curl

---
 .../proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml    | 2 +-
 .../proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
index 17317acb..48af3a60 100644
--- a/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
+++ b/monitoring/proxy-for-kubfwd/alertmanager-main-proxy-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         proxy: alertmanager-main
       annotations:
-        prometheus.io/path: /stats/prometheus?usedonly
+        prometheus.io/path: /stats/prometheus
         prometheus.io/port: "9901"
         prometheus.io/scrape: "true"
     spec:
diff --git a/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml
index 16dc71a1..c6488764 100644
--- a/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml
+++ b/monitoring/proxy-for-kubfwd/prometheus-now-proxy-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         proxy: prometheus-now
       annotations:
-        prometheus.io/path: /stats/prometheus?usedonly
+        prometheus.io/path: /stats/prometheus
         prometheus.io/port: "9901"
         prometheus.io/scrape: "true"
     spec: