fix: fall back to compact tool preamble before rejecting oversized tool payloads

src/handlers/chat.js

@@ -80,6 +80,81 @@ export function chatStreamError(message, type = 'upstream_error', code = null) {
   return { error: { message: sanitizeText(message || 'Upstream stream error'), type, code } };
 }
 
+export function applyToolPreambleBudget(toolPreamble, tools, toolChoice, callerEnv, limits = {}) {
+  const softBytes = parseInt(
+    limits.softBytes ?? process.env.TOOL_PREAMBLE_SOFT_BYTES ?? '24000',
+    10,
+  );
+  const hardBytes = parseInt(
+    limits.hardBytes ?? process.env.TOOL_PREAMBLE_HARD_BYTES ?? '48000',
+    10,
+  );
+  const full = typeof toolPreamble === 'string' ? toolPreamble : '';
+  const fullBytes = Buffer.byteLength(full, 'utf8');
+
+  if (!full) {
+    return {
+      toolPreamble: full,
+      fullBytes,
+      compact: '',
+      compactBytes: 0,
+      softBytes,
+      hardBytes,
+      decision: 'none',
+    };
+  }
+
+  if (fullBytes <= softBytes) {
+    return {
+      toolPreamble: full,
+      fullBytes,
+      compact: '',
+      compactBytes: 0,
+      softBytes,
+      hardBytes,
+      decision: 'full',
+    };
+  }
+
+  const compact = buildCompactToolPreambleForProto(tools || [], toolChoice, callerEnv);
+  const compactBytes = Buffer.byteLength(compact, 'utf8');
+  const compactUsable = !!compact && compactBytes > 0 && compactBytes <= hardBytes && compactBytes < fullBytes;
+
+  if (compactUsable) {
+    return {
+      toolPreamble: compact,
+      fullBytes,
+      compact,
+      compactBytes,
+      softBytes,
+      hardBytes,
+      decision: 'compact',
+    };
+  }
+
+  if (fullBytes <= hardBytes) {
+    return {
+      toolPreamble: full,
+      fullBytes,
+      compact,
+      compactBytes,
+      softBytes,
+      hardBytes,
+      decision: 'full_over_soft',
+    };
+  }
+
+  return {
+    toolPreamble: full,
+    fullBytes,
+    compact,
+    compactBytes,
+    softBytes,
+    hardBytes,
+    decision: 'reject',
+  };
+}
+
 /**
  * Extract a clean JSON payload from a model response. Handles three common
  * shapes a non-constrained-decoding model produces when asked for JSON:
@@ -607,29 +682,36 @@ export async function handleChatCompletions(body, context = {}) {
   // knows what tools exist and how to invoke them, just not parameter
   // shapes. Past the hard cap we abort with a 413-style 400 so callers
   // can trim instead of failing in panel-state retries.
-  const TOOL_PREAMBLE_SOFT_BYTES = parseInt(process.env.TOOL_PREAMBLE_SOFT_BYTES || '24000', 10);
-  const TOOL_PREAMBLE_HARD_BYTES = parseInt(process.env.TOOL_PREAMBLE_HARD_BYTES || '48000', 10);
   if (emulateTools && toolPreamble) {
-    const fullBytes = Buffer.byteLength(toolPreamble, 'utf8');
-    if (fullBytes > TOOL_PREAMBLE_HARD_BYTES) {
-      log.warn(`Probe[${reqId}]: toolPreamble ${Math.round(fullBytes / 1024)}KB exceeds hard cap ${Math.round(TOOL_PREAMBLE_HARD_BYTES / 1024)}KB; rejecting (${(tools || []).length} tools)`);
+    const budget = applyToolPreambleBudget(toolPreamble, tools || [], tool_choice, callerEnv);
+    if (budget.decision === 'reject') {
+      const compactInfo = budget.compactBytes
+        ? `; compact fallback still ${Math.round(budget.compactBytes / 1024)}KB`
+        : '; compact fallback unavailable';
+      log.warn(
+        `Probe[${reqId}]: toolPreamble ${Math.round(budget.fullBytes / 1024)}KB exceeds hard cap ${Math.round(budget.hardBytes / 1024)}KB${compactInfo}; rejecting (${(tools || []).length} tools)`,
+      );
       return {
         status: 400,
         body: {
           error: {
-            message: `Tool definitions are too large (${Math.round(fullBytes / 1024)}KB > ${Math.round(TOOL_PREAMBLE_HARD_BYTES / 1024)}KB). Reduce the number of tools or shorten parameter schemas.`,
+            message: `Tool definitions are too large (${Math.round(budget.fullBytes / 1024)}KB > ${Math.round(budget.hardBytes / 1024)}KB). Reduce the number of tools or shorten parameter schemas.`,
             type: 'invalid_request_error',
             param: 'tools',
             code: 'tool_preamble_too_large',
           },
         },
       };
     }
-    if (fullBytes > TOOL_PREAMBLE_SOFT_BYTES) {
-      const compact = buildCompactToolPreambleForProto(tools || [], tool_choice, callerEnv);
-      const compactBytes = Buffer.byteLength(compact, 'utf8');
-      log.warn(`Probe[${reqId}]: toolPreamble ${Math.round(fullBytes / 1024)}KB exceeds soft cap ${Math.round(TOOL_PREAMBLE_SOFT_BYTES / 1024)}KB; falling back to names-only preamble (${Math.round(compactBytes / 1024)}KB, ${(tools || []).length} tools)`);
-      toolPreamble = compact;
+    if (budget.decision === 'compact') {
+      log.warn(
+        `Probe[${reqId}]: toolPreamble ${Math.round(budget.fullBytes / 1024)}KB exceeds soft/hard budget; falling back to names-only preamble (${Math.round(budget.compactBytes / 1024)}KB, ${(tools || []).length} tools)`,
+      );
+      toolPreamble = budget.toolPreamble;
+    } else if (budget.decision === 'full_over_soft') {
+      log.warn(
+        `Probe[${reqId}]: toolPreamble ${Math.round(budget.fullBytes / 1024)}KB exceeds soft cap ${Math.round(budget.softBytes / 1024)}KB, but compact fallback was not smaller/usable; keeping full preamble (${(tools || []).length} tools)`,
+      );
     }
   }
   // Diagnostic: surface whether environment lifting actually fired so a real

test/chat-tool-preamble-budget.test.js

@@ -0,0 +1,51 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { applyToolPreambleBudget } from '../src/handlers/chat.js';
+import { buildToolPreambleForProto } from '../src/handlers/tool-emulation.js';
+
+describe('applyToolPreambleBudget', () => {
+  const bigTools = Array.from({ length: 30 }, (_, i) => ({
+    type: 'function',
+    function: {
+      name: `tool_${i}`,
+      description: `Description for tool ${i} that goes on for a while to bulk up the schema.`,
+      parameters: {
+        type: 'object',
+        properties: Object.fromEntries(
+          Array.from({ length: 15 }, (_, j) => [`param_${j}`, {
+            type: 'string',
+            description: `Parameter ${j} of tool ${i}, with verbose explanation that runs long.`,
+            enum: ['option_a', 'option_b', 'option_c', 'option_d', 'option_e'],
+          }]),
+        ),
+        required: Array.from({ length: 15 }, (_, j) => `param_${j}`),
+      },
+    },
+  }));
+
+  it('falls back to compact preamble before rejecting when full schema exceeds the hard cap', () => {
+    const full = buildToolPreambleForProto(bigTools, 'auto');
+    const result = applyToolPreambleBudget(full, bigTools, 'auto', '- Working directory: /repo', {
+      softBytes: 24_000,
+      hardBytes: 48_000,
+    });
+
+    assert.equal(result.decision, 'compact');
+    assert.ok(result.fullBytes > result.hardBytes, `expected fullBytes > hardBytes, got ${result.fullBytes} <= ${result.hardBytes}`);
+    assert.ok(result.compactBytes > 0 && result.compactBytes <= result.hardBytes, `expected compactBytes <= hardBytes, got ${result.compactBytes}`);
+    assert.ok(result.toolPreamble.includes('Available functions:'));
+    assert.ok(!result.toolPreamble.includes('param_0'), 'compact preamble must omit parameter schema details');
+  });
+
+  it('rejects only when neither full nor compact form fits within the hard cap', () => {
+    const full = buildToolPreambleForProto(bigTools, 'auto');
+    const result = applyToolPreambleBudget(full, bigTools, 'auto', '- Working directory: /repo', {
+      softBytes: 1,
+      hardBytes: 512,
+    });
+
+    assert.equal(result.decision, 'reject');
+    assert.ok(result.fullBytes > result.hardBytes);
+    assert.ok(result.compactBytes > result.hardBytes, `expected compact fallback to exceed hard cap, got ${result.compactBytes}`);
+  });
+});