Add: Project + Instrumentation

Author: AnkitJodhani

.gitignore

@@ -1,2 +1,3 @@
 **/*.pptx
 
+**/**/node_modules

day-2/custom_kube_prometheus_stack.yml

@@ -0,0 +1,19 @@
+alertmanager:
+  alertmanagerSpec:
+    # Selects Alertmanager configuration based on these labels. Ensure that the Alertmanager configuration has matching labels.
+    # ✅ Solves error: Misconfigured Alertmanager selectors can lead to missing alert configurations.
+    # ✅ Solves error: Alertmanager wasn't able to findout the applied CRD (kind: Alertmanagerconfig)
+    alertmanagerConfigSelector:
+      matchLabels:
+        release: monitoring
+
+    # Sets the number of Alertmanager replicas to 3 for high availability.
+    # ✅ Solves error: Single replica can cause alerting issues during pod failures.
+    # ✅ Solves error: Alertmanager Cluster Status is Disabled (GitHub issue)
+    replicas: 2
+
+    # Sets the strategy for matching Alertmanager configurations. 'None' means no specific matching strategy.
+    # ✅ Solves error: Incorrect matcher strategy can lead to unhandled alert configurations.
+    # ✅ Solves error: Get rid of namespace matchers when creating AlertManagerConfig (GitHub issue)
+    alertmanagerConfigMatcherStrategy:
+      type: None

day-2/readme.md

@@ -91,9 +91,11 @@ helm repo update
 kubectl create ns monitoring
 ```
 ```bash
-helm install monitoring \
-  --namespace monitoring \
-  prometheus-community/kube-prometheus-stack
+cd day-2
+
+helm install monitoring prometheus-community/kube-prometheus-stack \
+-n monitoring \
+-f custom_kube_prometheus_stack.yml
 ```
 
 ### ✅ Step 4: Verify the Installation

day-4/alerts-alertmanager-servicemonitor-manifest/alertmangerconfig.yml

@@ -0,0 +1,34 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: main-rules-alert-config
+  namespace: monitoring
+  labels:
+    release: monitoring
+spec:
+  route:
+    repeatInterval: 30m
+    receiver: 'null'
+    routes:
+    - matchers:
+      - name: alertname
+        value: HighCpuUsage
+      receiver: 'send-email'
+    - matchers:
+      - name: alertname
+        value: PodRestart
+      receiver: 'send-email'
+      repeatInterval: 5m
+  receivers:
+  - name: 'send-email'
+    emailConfigs:
+    - to: 'ankitjodhani1903@gmail.com'
+      from: 'ankitjodhani1903@gmail.com'
+      sendResolved: false
+      smarthost: smtp.gmail.com:587
+      authUsername: 'ankitjodhani1903@gmail.com'
+      authIdentity: 'ankitjodhani1903@gmail.com'
+      authPassword:
+        name: mail-pass
+        key: gmail-pass
+  - name: 'null'

day-4/alerts-alertmanager-servicemonitor-manifest/alerts.yml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: custom-alert-rules
+  namespace: monitoring
+  labels:
+    release: monitoring # if you installed through then you've to mention the release name of helm, otherwise prometheus will not recognize it
+spec:
+  groups:
+  - name: custom.rules
+    rules:
+    - alert: HighCpuUsage
+      expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 50
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "High CPU usage on instance {{ $labels.instance }}"
+        description: "CPU usage is above 50% (current value: {{ $value }}%)"
+    - alert: PodRestart
+      expr: kube_pod_container_status_restarts_total > 2
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: "Pod restart detected in namespace {{ $labels.namespace }}"
+        description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has restarted {{ $value }} times"

day-4/alerts-alertmanager-servicemonitor-manifest/email-secrets.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: mail-pass
+  namespace: monitoring
+  labels:
+    release: monitoring
+data:
+  gmail-pass: <<ENTER_YOUR_APP PASSWORDS_IN_BASE64_ENCODED_FORMAT>>
+

day-4/alerts-alertmanager-servicemonitor-manifest/kustomization.yml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+resources:
+  - alerts.yml
+  - email-secrets.yml
+  - alertmangerconfig.yml
+  - serviceMonitor.yml

day-4/alerts-alertmanager-servicemonitor-manifest/serviceMonitor.yml

@@ -0,0 +1,21 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app: prometheus-service-service-monitor
+    release: monitoring
+  name: prometheus-service-service-monitor
+  namespace: monitoring
+spec:
+  jobLabel: job
+  endpoints:
+    - interval: 2s
+      port: prometheus-service-port
+      path: /metrics
+  selector:
+    matchLabels:
+      app: prometheus-service
+  namespaceSelector:
+    matchNames:
+      - default
+      

day-4/app-code/Dockerfile

@@ -0,0 +1,11 @@
+FROM node:18-alpine
+
+COPY package*.json /usr/app/
+
+COPY index.js /usr/app/
+
+WORKDIR /usr/app
+
+RUN npm install
+
+CMD ["node", "index.js"]

day-4/app-code/index.js

@@ -0,0 +1,132 @@
+const express = require('express');
+const morgan = require('morgan');
+const pino = require('pino');
+const promClient = require('prom-client');
+
+const app = express();
+
+const logger = pino();
+
+const logging = () => {
+    logger.info("Here are the logs")
+    logger.info("Please have a look ")
+    logger.info("This is just for testing")
+}
+
+app.use(morgan('common'))
+
+
+// Prometheus metrics
+const httpRequestCounter = new promClient.Counter({
+    name: 'http_requests_total',
+    help: 'Total number of HTTP requests',
+    labelNames: ['method', 'path', 'status_code'],
+});
+
+const requestDurationHistogram = new promClient.Histogram({
+    name: 'http_request_duration_seconds',
+    help: 'Duration of HTTP requests in seconds',
+    labelNames: ['method', 'path', 'status_code'],
+    buckets: [0.1, 0.5, 1, 5, 10], // Buckets for the histogram in seconds
+});
+
+const requestDurationSummary = new promClient.Summary({
+    name: 'http_request_duration_summary_seconds',
+    help: 'Summary of the duration of HTTP requests in seconds',
+    labelNames: ['method', 'path', 'status_code'],
+    percentiles: [0.5, 0.9, 0.99], // Define your percentiles here
+});
+
+
+
+// Gauge metric
+const gauge = new promClient.Gauge({
+    name: 'node_gauge_example',
+    help: 'Example of a gauge tracking async task duration',
+    labelNames: ['method', 'status']
+});
+
+// Define an async function that simulates a task taking random time
+const simulateAsyncTask = async () => {
+    const randomTime = Math.random() * 5; // Random time between 0 and 5 seconds
+    return new Promise((resolve) => setTimeout(resolve, randomTime * 1000));
+};
+
+app.disable('etag');
+
+// Middleware to track metrics
+app.use((req, res, next) => {
+    const start = Date.now();
+    res.on('finish', () => {
+        const duration = (Date.now() - start) / 1000; // Duration in seconds
+        const { method, url } = req;
+        const statusCode = res.statusCode; // Get the actual HTTP status code
+        httpRequestCounter.labels({ method, path: url, status_code: statusCode }).inc();
+        requestDurationHistogram.labels({ method, path: url, status_code: statusCode }).observe(duration);
+        requestDurationSummary.labels({ method, path: url, status_code: statusCode }).observe(duration);
+    });
+    next();
+});
+
+app.get('/', (req, res) => {
+    res.status(200).json({
+        status: "🏃- Running"
+    });
+});
+
+app.get('/healthy', (req, res) => {
+    res.status(200).json({
+        name: "👀 - Obserability 🔥- Abhishek Veeramalla",
+        status: "healthy"
+    })
+});
+
+app.get('/serverError', (req, res) => {
+    res.status(500).json({
+        error: " Internal server error",
+        statusCode: 500
+    })
+});
+
+app.get('/notFound', (req, res) => {
+    res.status(404).json({
+        error: "Not Found",
+        statusCode: "404"
+    })
+});
+
+app.get('/logs', (req, res) => {
+    logging();
+    res.status(200).json({
+        objective: "To generate logs"
+    })
+});
+
+
+// Simulate a crash by throwing an error
+app.get('/crash', (req, res) => {
+    console.log('Intentionally crashing the server...');
+    process.exit(1);
+});
+
+
+// Define the /example route
+app.get('/example', async (req, res) => {
+    const endGauge = gauge.startTimer({ method: req.method, status: res.statusCode });
+    await simulateAsyncTask();
+    endGauge();
+    res.send('Async task completed');
+});
+
+// Expose metrics for Prometheus to scrape
+app.get('/metrics', async (req, res) => {
+    res.set('Content-Type', promClient.register.contentType);
+    res.end(await promClient.register.metrics());
+});
+
+
+const PORT = 3000;
+app.listen(PORT, () => {
+    console.log(`listening on port ${PORT}`)
+})
+