feat(addie): anonymous→Sonnet, daily cap →, ritual-phrase stripper

.changeset/addie-quality-anon-sonnet-stripper.md

@@ -0,0 +1,4 @@
+---
+---
+
+Closes most of the Addie redteam baseline failures with two changes: (1) anonymous chat now defaults to Sonnet rather than Haiku — the daily-cap and per-IP rate limit already bound total spend, and Sonnet substantially better follows the response-style.md negative instructions ("don't say 'great question'") and avoids the fabrication patterns we saw in flagged threads; override via `ADDIE_ANONYMOUS_MODEL` if cost forces a downgrade. (2) A deterministic post-processor strips banned ritual phrases ("the honest answer is", "great question", "to be clear,", etc.) from assistant text before it reaches the user, so even Haiku-mode anonymous responses (and any future regressions) can't leak the phrases. Strip is applied outside fenced code blocks, re-capitalizes following sentences, and is idempotent.

server/src/addie/claude-client.ts

@@ -19,6 +19,7 @@ import { formatTokenCount, getConversationTokenLimit, buildDroppedMessagesSummar
 import { notifyToolError } from './error-notifier.js';
 import { ToolError } from './tool-error.js';
 import { checkCostCap, recordCost, formatCapExceededMessage } from './claude-cost-tracker.js';
+import { stripBannedRituals } from './response-postprocess.js';
 
 type ToolHandler = (input: Record<string, unknown>) => Promise<string>;
 
@@ -895,7 +896,8 @@ export class AddieClaudeClient {
 
         if (toolUseBlocks.length === 0 && serverToolBlocks.length === 0) {
           const textContent = response.content.find((c) => c.type === 'text');
-          const text = textContent && textContent.type === 'text' ? textContent.text : "I'm not sure how to help with that.";
+          const rawText = textContent && textContent.type === 'text' ? textContent.text : "I'm not sure how to help with that.";
+          const text = stripBannedRituals(rawText);
           totalToolExecutionMs = toolExecutions.reduce((sum, t) => sum + t.duration_ms, 0);
           return {
             text,
@@ -1417,7 +1419,7 @@ export class AddieClaudeClient {
           yield {
             type: 'done',
             response: {
-              text: fullText,
+              text: stripBannedRituals(fullText),
               tools_used: toolsUsed,
               tool_executions: toolExecutions,
               flagged: !!hallucinationReason,
@@ -1448,7 +1450,7 @@ export class AddieClaudeClient {
             yield {
               type: 'done',
               response: {
-                text: fullText,
+                text: stripBannedRituals(fullText),
                 tools_used: toolsUsed,
                 tool_executions: toolExecutions,
                 flagged: false,

server/src/addie/claude-cost-tracker.ts

@@ -58,9 +58,13 @@ const WINDOW_MS = 24 * 60 * 60 * 1000;
  * Explorer users get a smaller ceiling than paying members.
  *
  * Rationales:
- * - `anonymous`: $1/day. Covers a few exploratory chats (Haiku at
- *   ~$0.01/turn, Sonnet at ~$0.05/turn) without enabling a scripted
- *   chat spam that burns real money on our free surface.
+ * - `anonymous`: $3/day. Sized to slightly exceed the per-IP message
+ *   rate limit (50/day) at the current Sonnet ~$0.05/turn rate, so the
+ *   message-count cap binds first for a legitimate user and the dollar
+ *   cap only fires on scripted abuse. Was $1 when anonymous chat was
+ *   on Haiku (~$0.01/turn → ~100 turns/day, well above rate limit);
+ *   bumped on the Haiku→Sonnet move so legit users don't hit the
+ *   dollar ceiling at ~20 turns.
  * - `member_free`: $5/day. Free tier with an account — slightly more
  *   trust than anonymous, same floor a real user couldn't reach in
  *   a day of genuine conversational use.
@@ -69,7 +73,7 @@ const WINDOW_MS = 24 * 60 * 60 * 1000;
  *   trips it within an hour of sustained abuse.
  */
 export const DAILY_BUDGET_USD: Record<'anonymous' | 'member_free' | 'member_paid', number> = {
-  anonymous: 1,
+  anonymous: 3,
   member_free: 5,
   member_paid: 25,
 };

server/src/addie/response-postprocess.ts

@@ -0,0 +1,114 @@
+/**
+ * Response post-processor for Addie's assistant text.
+ *
+ * The model — particularly Haiku — leaks ritual phrases ("the honest answer
+ * is", "great question", "to be clear,") despite response-style.md banning
+ * them. This module strips those phrases deterministically before the
+ * response reaches the user.
+ *
+ * Why post-process rather than tighten the prompt:
+ *
+ * 1. Haiku has demonstrated ~10-20% adherence loss on negative instructions
+ *    in our redteam runs. Telling the model "don't say X" is unreliable.
+ * 2. The same prompt reaches every channel (web, Slack, email). One
+ *    deterministic post-processor enforces the rule everywhere.
+ * 3. The phrase list is already maintained in `BANNED_RITUAL_PHRASES`
+ *    (testing/redteam-scenarios.ts) — single source of truth.
+ *
+ * Safety notes:
+ *
+ * - Strips only outside fenced code blocks, so quoted snippets ("here the
+ *   user said 'great question'") inside ```…``` remain untouched.
+ * - Strips with surrounding punctuation/whitespace and re-capitalizes the
+ *   next sentence so output reads cleanly.
+ * - Idempotent: running twice is the same as once.
+ * - No external state, no allocations beyond the result string.
+ */
+
+/**
+ * Phrases removed wherever they appear (outside code blocks). Each entry
+ * is the literal substring to remove; the regex below adds tolerant
+ * surrounding punctuation/whitespace handling.
+ *
+ * Keep this in sync with BANNED_RITUAL_PHRASES in redteam-scenarios.ts.
+ * The redteam suite asserts presence; this module asserts absence in
+ * produced output.
+ */
+const BANNED_RITUAL_LITERALS: readonly string[] = [
+  "here's the honest answer",
+  "the honest answer is",
+  "let me be honest",
+  "that's a great question",
+  "that's a sharp question",
+  "that's a fair question",
+  "fair question",
+  "great question",
+  "sharp question",
+  "this is a sharp point",
+  "to be clear",
+  "to be direct",
+];
+
+/**
+ * Compile the literal list into a single case-insensitive regex that
+ * captures the phrase plus tolerant trailing separator (punctuation and/or
+ * whitespace, in any order Haiku throws at us).
+ *
+ *  - `\b` — word boundary so "great question" doesn't match "ungreater question"
+ *  - alternation of the literal phrases (longer-first to prevent partial steals)
+ *  - then EITHER:
+ *      - optional whitespace + one or more separator chars + optional whitespace
+ *        (covers ", ", " — ", " - ", ": ", ". ", " : ")
+ *      - OR pure whitespace alone (covers "phrase next-word")
+ *      - OR nothing (covers "phrase" at end of buffer)
+ *
+ * Separator class: comma, colon, semicolon, em-dash, en-dash, hyphen, period.
+ */
+function buildBannedRitualRegex(): RegExp {
+  // Escape regex metacharacters in literals before alternation.
+  const escaped = BANNED_RITUAL_LITERALS.map(p =>
+    p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  );
+  // Sort longer first so "that's a great question" matches before "great question"
+  // (alternation is left-to-right; longest-first prevents partial steals).
+  escaped.sort((a, b) => b.length - a.length);
+  const phrases = escaped.join('|');
+  // Trailing separator: optional whitespace + one or more sep chars + optional whitespace,
+  // OR just whitespace, OR nothing.
+  const trailingSep = `(?:\\s*[,.:;—–-]+\\s*|\\s+|)`;
+  return new RegExp(`\\b(?:${phrases})${trailingSep}`, 'gi');
+}
+
+const BANNED_RITUAL_REGEX = buildBannedRitualRegex();
+
+/**
+ * Strip banned ritual phrases from outside fenced code blocks and
+ * re-capitalize the first letter of any sentence whose opener was removed.
+ *
+ * @param text Raw assistant text from the model.
+ * @returns Cleaned text safe to send to the user.
+ */
+export function stripBannedRituals(text: string): string {
+  if (!text) return text;
+
+  // Split into [non-code, code, non-code, code, ...] segments. Code blocks
+  // are at odd indices after the split.
+  const parts = text.split(/(```[\s\S]*?```)/g);
+  for (let i = 0; i < parts.length; i++) {
+    if (i % 2 === 1) continue; // skip code blocks
+    parts[i] = parts[i].replace(BANNED_RITUAL_REGEX, '');
+    // Re-capitalize the first alphabetical character of any sentence whose
+    // opener was just removed. Pattern: start-of-string or end-of-sentence
+    // punctuation followed by lowercase letter.
+    parts[i] = parts[i].replace(/(^|[.!?]\s+)([a-z])/g, (_, prefix, ch) =>
+      prefix + ch.toUpperCase()
+    );
+  }
+  return parts.join('');
+}
+
+/**
+ * Test-only export of the literal list so the unit test can assert
+ * that every literal would actually be stripped by the regex.
+ */
+export const __test_BANNED_RITUAL_LITERALS = BANNED_RITUAL_LITERALS;

server/src/config/models.ts

@@ -94,10 +94,20 @@ export const AddieModelConfig = {
   chat: process.env.ADDIE_ANTHROPIC_MODEL || ModelConfig.primary,
 
   /**
-   * Model for anonymous web chat (cost-controlled)
-   * Override: ADDIE_ANONYMOUS_MODEL (falls back to fast/Haiku)
+   * Model for anonymous web chat.
+   *
+   * Defaults to Sonnet (`primary`). Anonymous traffic exposes Addie's worst
+   * failure modes — ritual phrases, length blow-out on short questions,
+   * fabrication of integration details — which trace to Haiku's poor
+   * adherence to negative instructions and conservative tool-call gating.
+   * Sonnet handles those substantially better at ~10x per-turn cost.
+   * Total spend is bounded by `anonymousDailyLimiter` (50 messages/IP/day)
+   * + the per-IP $5 daily Claude API cap, both unchanged by this default.
+   *
+   * Override: ADDIE_ANONYMOUS_MODEL — set to Haiku/`fast` if cost pressure
+   * forces a downgrade.
    */
-  anonymousChat: process.env.ADDIE_ANONYMOUS_MODEL || ModelConfig.fast,
+  anonymousChat: process.env.ADDIE_ANONYMOUS_MODEL || ModelConfig.primary,
 
   /**
    * Model for voice/video conversations

server/tests/unit/addie/response-postprocess.test.ts

@@ -0,0 +1,96 @@
+/**
+ * Unit tests for the Addie response post-processor.
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  stripBannedRituals,
+  __test_BANNED_RITUAL_LITERALS,
+} from '../../../src/addie/response-postprocess.js';
+
+describe('stripBannedRituals', () => {
+  it('strips a leading "the honest answer is"', () => {
+    const input = "The honest answer is, AdCP standardizes flows that already exist.";
+    expect(stripBannedRituals(input)).toBe("AdCP standardizes flows that already exist.");
+  });
+
+  it('strips "great question" with em-dash', () => {
+    const input = "Great question — the principal is liable for spend.";
+    expect(stripBannedRituals(input)).toBe("The principal is liable for spend.");
+  });
+
+  it('strips "that\'s a great question." sentence opener', () => {
+    const input = "That's a great question. The principal — the brand or agency — is responsible.";
+    // Strip leaves "The principal..." which already starts capitalized.
+    expect(stripBannedRituals(input)).toBe("The principal — the brand or agency — is responsible.");
+  });
+
+  it('strips mid-sentence "the honest answer is" and re-capitalizes', () => {
+    const input = "There are multiple angles. But the honest answer is that Scope3 was a founding contributor.";
+    // After strip: "There are multiple angles. But that Scope3 was..."
+    // The "But" is still capitalized after "."; the inner phrase is removed.
+    const output = stripBannedRituals(input);
+    expect(output).not.toMatch(/honest answer/i);
+    expect(output).toContain("Scope3 was a founding contributor");
+  });
+
+  it('strips "to be clear," at sentence start', () => {
+    const input = "To be clear, AdCP does not introduce new identifiers.";
+    expect(stripBannedRituals(input)).toBe("AdCP does not introduce new identifiers.");
+  });
+
+  it('does NOT strip phrases inside fenced code blocks', () => {
+    const input = "Here is the example log:\n```\nThe honest answer is that this user said \"great question\"\n```\nAnd that's the format.";
+    const output = stripBannedRituals(input);
+    expect(output).toContain("The honest answer is that this user said");
+    expect(output).toContain("great question");
+    // "And that's the format" should remain
+    expect(output).toMatch(/that's the format/);
+  });
+
+  it('is idempotent — running twice equals running once', () => {
+    const input = "Great question — that's a sharp question. The honest answer is, no.";
+    const once = stripBannedRituals(input);
+    const twice = stripBannedRituals(once);
+    expect(twice).toBe(once);
+  });
+
+  it('preserves text with no banned phrases unchanged', () => {
+    const input = "AdCP operates at the campaign layer. Buyers and sellers negotiate terms over the protocol.";
+    expect(stripBannedRituals(input)).toBe(input);
+  });
+
+  it('handles empty input', () => {
+    expect(stripBannedRituals('')).toBe('');
+  });
+
+  it('handles input that is ONLY a banned phrase', () => {
+    const input = "Great question.";
+    expect(stripBannedRituals(input)).toBe("");
+  });
+
+  it('every literal in the banned list is actually stripped by the regex', () => {
+    // Forward-parity: any literal we declare banned must be removed when present.
+    for (const phrase of __test_BANNED_RITUAL_LITERALS) {
+      const input = `${phrase}. The substance.`;
+      const output = stripBannedRituals(input);
+      expect(output, `failed to strip "${phrase}"`).not.toMatch(
+        new RegExp(phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'i')
+      );
+    }
+  });
+
+  it('strips with case insensitivity', () => {
+    expect(stripBannedRituals("THE HONEST ANSWER IS, no.")).toBe("No.");
+    expect(stripBannedRituals("Great Question - here's the deal.")).toBe("Here's the deal.");
+  });
+
+  it('strips multiple banned phrases in the same response', () => {
+    const input = "Great question. To be clear, AdCP is a campaign-layer protocol. Sharp question — let me explain.";
+    const output = stripBannedRituals(input);
+    expect(output).not.toMatch(/great question/i);
+    expect(output).not.toMatch(/to be clear/i);
+    expect(output).not.toMatch(/sharp question/i);
+    expect(output).toContain("AdCP is a campaign-layer protocol");
+  });
+});

server/tests/unit/claude-client-cost-gate.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect, beforeEach, vi } from 'vitest';
 import {
   recordCost,
+  DAILY_BUDGET_USD,
   __setCostTrackerStore,
   __createInMemoryCostStore,
 } from '../../src/addie/claude-cost-tracker.js';
@@ -43,8 +44,10 @@ beforeEach(() => {
 
 describe('claude-client entry-gate behavior (#2790)', () => {
   it('processMessage short-circuits with cost_cap_exceeded when the user is over budget', async () => {
-    // Burn the anonymous cap for `user-x`.
-    await recordCost('user-x', 'claude-opus-4-7', { input_tokens: 66_667, output_tokens: 0 });
+    // Burn the anonymous cap for `user-x`. Opus is $15/M-token input,
+    // so we need >cap × 1M / 15 tokens to exceed the cap with one charge.
+    const tokensToExceedCap = Math.ceil((DAILY_BUDGET_USD.anonymous * 1_000_000) / 15) + 1;
+    await recordCost('user-x', 'claude-opus-4-7', { input_tokens: tokensToExceedCap, output_tokens: 0 });
 
     const client = new AddieClaudeClient('sk-fake-unused', 'claude-sonnet-4-6');
     const response = await client.processMessage(
@@ -64,7 +67,8 @@ describe('claude-client entry-gate behavior (#2790)', () => {
   });
 
   it('processMessageStream yields a single cost_cap_exceeded done event when over budget', async () => {
-    await recordCost('user-y', 'claude-opus-4-7', { input_tokens: 66_667, output_tokens: 0 });
+    const tokensToExceedCap = Math.ceil((DAILY_BUDGET_USD.anonymous * 1_000_000) / 15) + 1;
+    await recordCost('user-y', 'claude-opus-4-7', { input_tokens: tokensToExceedCap, output_tokens: 0 });
 
     const client = new AddieClaudeClient('sk-fake-unused', 'claude-sonnet-4-6');
     const events: Array<{ type: string; response?: { flagged: boolean; flag_reason?: string } }> = [];