feat: update Open Telemetry configuration to support optional remote write pushing of metrics and log

charts/hedera-network/config-files/otel-collector-config.yaml

@@ -18,9 +18,19 @@ exporters:
   prometheus:
     endpoint: "0.0.0.0:8889"
     const_labels:
-      label1: "{{ .otelDefaults.nameOverride }}-label1"
+      source: p-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="p-otel-collector"}
+  {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}
+  prometheusremotewrite:
+    endpoint: "{{ .otelDefaults.prometheusRemoteWrite.endpoint }}"
+    tls:
+      insecure: true
+    external_labels:
+      source: prw-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="prw-otel-collector"}
+  {{- end }}
+#  logging:
+#    verbosity: detailed
+#    sampling_initial: 5
 
-  logging:
 
 #  jaeger:
 #    endpoint: jaeger-all-in-one:14250
@@ -34,11 +44,14 @@ exporters:
 
 processors:
   batch:
-
 extensions:
   health_check:
 
 service:
+#  telemetry:
+#    logs:
+#      level: "debug"
+#      development: true
   extensions: [health_check]
   pipelines:
     #traces:
@@ -48,4 +61,9 @@ service:
     metrics:
       receivers: [otlp, hostmetrics]
       processors: [batch]
-      exporters: [logging, prometheus]
+      exporters: 
+         - prometheus
+         {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}
+         - prometheusremotewrite
+         {{- end }}
+#      exporters: [logging, prometheus{{ if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}, prometheusremotewrite{{ end }}]

charts/hedera-network/templates/services/network-node-svc.yaml

@@ -27,4 +27,8 @@ spec:
     protocol: TCP
     port: 8888
     targetPort: 8888
+  - name: prometheus
+    protocol: TCP
+    port: 8889
+    targetPort: 8889
 {{- end }}

charts/hedera-network/templates/sidecars/_otel-collector.tpl

@@ -17,6 +17,9 @@
     - name: otel-otlp
       containerPort: 4317
       protocol: TCP
+    - name: prometheus
+      containerPort: 8889
+      protocol: TCP
   {{- with default $defaults.livenessProbe $otel.livenessProbe }}
   livenessProbe:
     {{- toYaml . | nindent 4 }}
@@ -27,7 +30,7 @@
   {{- end }}
   volumeMounts:
     - name: otel-collector-volume
-      mountPath: /etc/otel-collector-config.yaml
+      mountPath: /etc/otelcol-contrib/config.yaml
       subPath: config.yaml #key in the configmap
       readOnly: true
   {{- with default $defaults.resources $otel.resources }}

charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml

@@ -12,4 +12,6 @@ spec:
   endpoints:
     - port: otel-metrics
       interval: 5s
+    - port: prometheus
+      interval: 5s
 {{- end }}

charts/hedera-network/values.yaml

@@ -157,7 +157,7 @@ defaults:
       resources: {}
     otelCollector:
       enable: "true"
-      nameOverride: "otel-collector"
+      nameOverride: ""
       image:
         registry: "docker.io"
         repository: "otel/opentelemetry-collector-contrib"
@@ -172,6 +172,9 @@ defaults:
           path: /
           port: otel-health
       resources: {}
+      prometheusRemoteWrite:
+        enable: "false"
+        endpoint: "http://prometheus.default.svc:9090/api/v1/write"
 
 # This configures the minio tenant subchart
 # Reference for configuration: https://github.com/minio/operator/blob/master/helm/tenant/values.yaml

dev/scripts/telemetry.sh

@@ -81,7 +81,7 @@ function deploy-prometheus-example-app() {
 	echo "PROMETHEUS_EXAMPLE_APP_YAML: ${PROMETHEUS_EXAMPLE_APP_YAML}"
   echo "-----------------------------------------------------------------------------------------------------"
 	kubectl create -f "${PROMETHEUS_EXAMPLE_APP_YAML}"
-	kubectl wait --for=condition=Ready pods -l  app=prometheus-example-app -n default --timeout 60
+	kubectl wait --for=condition=Ready pods -l  app=prometheus-example-app -n default --timeout 60s
 }
 
 function destroy-prometheus-example-app() {

dev/telemetry/prometheus/prometheus.yaml

@@ -11,6 +11,7 @@ spec:
     requests:
       memory: 400Mi
   enableAdminAPI: false
+  enableRemoteWriteReceiver: true
 ---
 apiVersion: v1
 kind: Service