From 4a33456a711a73e8cefba2e1f269400619c54204 Mon Sep 17 00:00:00 2001
From: matthew-pilot <matthew@vulturelabs.io>
Date: Sun, 31 May 2026 07:36:34 +0000
Subject: [PATCH] fix(webhook): add dlq_dropped metric + warn on DLQ ring
 eviction (PILOT-313)

The dead-letter queue silently dropped the oldest entry when the
100-entry ring was full, with no metric or log to detect the loss.
This adds:
- atomic.Uint64 dlqDropped counter on the dispatcher
- slog.Warn when an entry is evicted from the DLQ ring
- dlq_dropped field in HandleGetWebhook response for monitoring
- Test assertion verifying the counter after ring eviction

Closes PILOT-313
---
 webhook/webhook.go      | 27 ++++++++++++++++-----------
 webhook/zz_more_test.go |  4 ++++
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/webhook/webhook.go b/webhook/webhook.go
index 300e81c..324f3c7 100644
--- a/webhook/webhook.go
+++ b/webhook/webhook.go
@@ -51,9 +51,10 @@ type dispatcher struct {
 	secret         string // HMAC-SHA256 pre-shared secret (empty = no sig)
 
 	// Dead letter queue: stores last N failed events for retry/inspection
-	dlqMu    sync.Mutex
-	dlqItems []*Event
-	dlqMax   int
+	dlqMu     sync.Mutex
+	dlqItems  []*Event
+	dlqMax    int
+	dlqDropped atomic.Uint64 // events evicted from DLQ ring
 }
 
 const (
@@ -199,6 +200,8 @@ func (d *dispatcher) addToDLQ(ev *Event) {
 	defer d.dlqMu.Unlock()
 	if len(d.dlqItems) >= d.dlqMax {
 		d.dlqItems = d.dlqItems[1:] // drop oldest
+		d.dlqDropped.Add(1)
+		slog.Warn("registry webhook DLQ full, dropping oldest event", "dlq_max", d.dlqMax)
 	}
 	d.dlqItems = append(d.dlqItems, ev)
 }
@@ -360,21 +363,23 @@ func (st *Store) HandleGetWebhook() map[string]interface{} {
 	st.mu.RUnlock()
 
 	url := ""
-	var delivered, failed, dropped uint64
+	var delivered, failed, dropped, dlqDropped uint64
 	dlqLen := 0
 	if d != nil {
 		url = d.url
 		delivered, failed, dropped = d.stats()
+		dlqDropped = d.dlqDropped.Load()
 		dlqLen = len(d.getDLQ())
 	}
 	return map[string]interface{}{
-		"type":      "get_webhook_ok",
-		"enabled":   d != nil,
-		"url":       url,
-		"delivered": delivered,
-		"failed":    failed,
-		"dropped":   dropped,
-		"dlq_size":  dlqLen,
+		"type":        "get_webhook_ok",
+		"enabled":     d != nil,
+		"url":         url,
+		"delivered":   delivered,
+		"failed":      failed,
+		"dropped":     dropped,
+		"dlq_size":    dlqLen,
+		"dlq_dropped": dlqDropped,
 	}
 }
 
diff --git a/webhook/zz_more_test.go b/webhook/zz_more_test.go
index 9469f3b..430d330 100644
--- a/webhook/zz_more_test.go
+++ b/webhook/zz_more_test.go
@@ -139,6 +139,10 @@ func TestDispatcher_AddToDLQ_DropsOldestWhenFull(t *testing.T) {
 	if got[0].EventID != 3 || got[1].EventID != 4 {
 		t.Errorf("DLQ contents = %v, want oldest dropped", got)
 	}
+	// Verify the dlqDropped counter tracks evictions (5 pushes into cap 2 = 3 drops).
+	if n := d.dlqDropped.Load(); n != 3 {
+		t.Errorf("dlqDropped = %d, want 3", n)
+	}
 }
 
 func TestDispatcher_EmitAfterCloseIsNoop(t *testing.T) {