diff --git a/dashboard/src/lib/model-breakdown.ts b/dashboard/src/lib/model-breakdown.ts index 8a98e376..5c45ea5a 100644 --- a/dashboard/src/lib/model-breakdown.ts +++ b/dashboard/src/lib/model-breakdown.ts @@ -20,6 +20,11 @@ function resolveModelName(model: any, fallback: any) { return fallback; } +// Server-side resolution tiers that mean the price was guessed from a partial +// match rather than an exact model id (src/lib/pricing/index.js). A guessed +// price is plausible and therefore never looks wrong — worth flagging. +const FUZZY_PRICING_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip"]); + function isKnownZeroCostModel(name: any) { const lower = String(name || "").toLowerCase(); return lower.includes("free") || lower.includes("hy3-preview") || /^glm-[\d.]+-flash(?![a-z])/.test(lower); @@ -81,17 +86,32 @@ export function buildFleetData(modelBreakdown: any, { copyFn }: AnyRecord = {}) : entry.totalCost > 0 && entry.totalTokens > 0 ? (modelTokens / entry.totalTokens) * entry.totalCost : null; - const pricingMissing = - modelTokens > 0 && - (modelCost == null || modelCost <= 0) && - !isKnownZeroCostModel(name); - return { id, name, share, usage: modelTokens, cost: modelCost, pricingMissing }; + // Prefer the server's own account of how the price resolved. The + // cost<=0 heuristic below cannot tell an unpriced model from a + // genuinely free one, which is why isKnownZeroCostModel exists; it + // stays as the fallback for responses from an older server. + const pricingTier = typeof model?.pricing_tier === "string" ? model.pricing_tier : null; + const pricingMissing = pricingTier + ? pricingTier === "miss" && modelTokens > 0 + : modelTokens > 0 && (modelCost == null || modelCost <= 0) && !isKnownZeroCostModel(name); + const pricingFuzzy = Boolean(pricingTier && FUZZY_PRICING_TIERS.has(pricingTier)); + return { + id, + name, + share, + usage: modelTokens, + cost: modelCost, + pricingTier, + pricingMissing, + pricingFuzzy, + }; }) .filter(Boolean); const topCostModel = models .filter((model: any) => Number.isFinite(Number(model?.cost)) && Number(model.cost) > 0) .sort((a: any, b: any) => Number(b.cost) - Number(a.cost))[0] || null; const missingPricingModels = models.filter((model: any) => model?.pricingMissing); + const fuzzyPricingModels = models.filter((model: any) => model?.pricingFuzzy); return { source: entry.source, label, @@ -100,6 +120,7 @@ export function buildFleetData(modelBreakdown: any, { copyFn }: AnyRecord = {}) usage: entry.totalTokens, topCostModel, missingPricingModels, + fuzzyPricingModels, models, }; }); @@ -125,7 +146,9 @@ export function buildUsageInsights(modelBreakdown: any, { copyFn }: AnyRecord = .filter((model: any) => Number.isFinite(Number(model?.usage)) && Number(model.usage) > 0) .sort((a: any, b: any) => Number(b.usage) - Number(a.usage))[0] || null; const missingPricingModels = allModels.filter((model: any) => model?.pricingMissing); + const fuzzyPricingModels = allModels.filter((model: any) => model?.pricingFuzzy); return { + fuzzyPricingModels, totalTokens, totalCost, costPerMillionTokens: totalTokens > 0 ? totalCost / (totalTokens / 1_000_000) : null, diff --git a/dashboard/vite.config.js b/dashboard/vite.config.js index fafad078..3ba66e96 100644 --- a/dashboard/vite.config.js +++ b/dashboard/vite.config.js @@ -280,7 +280,7 @@ async function runLocalSyncCommand(extraEnv = {}) { // at require-time, so dev-server mocks still get LiteLLM-backed cost data. const __viteRequire = createRequire(import.meta.url); const __pricing = __viteRequire(path.resolve(REPO_ROOT, "src/lib/pricing")); -const { getModelPricing, computeRowCost } = __pricing; +const { getModelPricing, getModelPricingMeta, computeRowCost } = __pricing; async function handleLocalApi(req, res, url) { // Honor the dashboard's tz / tz_offset_minutes params so hourly/daily @@ -777,7 +777,14 @@ async function handleLocalApi(req, res, url) { model: m.model, source: s.source, }); - return { ...m, totals: { ...m.totals, total_cost_usd: cost.toFixed(6) } }; + // Mirror the real handler: without pricing_tier the dev server cannot + // exercise the unpriced/fuzzy badges at all. + const { tier } = getModelPricingMeta(m.model, { source: s.source }); + return { + ...m, + pricing_tier: tier, + totals: { ...m.totals, total_cost_usd: cost.toFixed(6) }, + }; }).sort((a, b) => b.totals.total_tokens - a.totals.total_tokens); const sourceCost = s.models.reduce((sum, m) => sum + Number(m.totals.total_cost_usd), 0); s.totals.total_cost_usd = sourceCost.toFixed(6); diff --git a/openwiki-facts/source-facts.json b/openwiki-facts/source-facts.json index 5555612f..bfe97de4 100644 --- a/openwiki-facts/source-facts.json +++ b/openwiki-facts/source-facts.json @@ -44,7 +44,7 @@ "methods": [ "POST" ], - "evidence": "src/lib/local-api.js:896", + "evidence": "src/lib/local-api.js:898", "mutation": true }, { @@ -52,7 +52,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:930", + "evidence": "src/lib/local-api.js:932", "mutation": false }, { @@ -60,7 +60,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:941", + "evidence": "src/lib/local-api.js:943", "mutation": false }, { @@ -68,7 +68,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1026", + "evidence": "src/lib/local-api.js:1028", "mutation": false }, { @@ -76,7 +76,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1037", + "evidence": "src/lib/local-api.js:1039", "mutation": false }, { @@ -84,7 +84,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1105", + "evidence": "src/lib/local-api.js:1107", "mutation": false }, { @@ -92,7 +92,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1170", + "evidence": "src/lib/local-api.js:1186", "mutation": false }, { @@ -100,7 +100,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1216", + "evidence": "src/lib/local-api.js:1232", "mutation": false }, { @@ -108,7 +108,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1293", + "evidence": "src/lib/local-api.js:1309", "mutation": false }, { @@ -116,7 +116,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1303", + "evidence": "src/lib/local-api.js:1319", "mutation": false }, { @@ -124,7 +124,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1313", + "evidence": "src/lib/local-api.js:1329", "mutation": false }, { @@ -133,7 +133,7 @@ "GET", "POST" ], - "evidence": "src/lib/local-api.js:1347", + "evidence": "src/lib/local-api.js:1363", "mutation": true }, { @@ -141,7 +141,7 @@ "methods": [ "GET" ], - "evidence": "src/lib/local-api.js:1497", + "evidence": "src/lib/local-api.js:1513", "mutation": false } ] diff --git a/src/lib/local-api.js b/src/lib/local-api.js index 8eab3228..c179869c 100644 --- a/src/lib/local-api.js +++ b/src/lib/local-api.js @@ -30,6 +30,8 @@ const avatarProxyCache = new Map(); const { MODEL_PRICING, getModelPricing, + getModelPricingMeta, + getPricingDiagnostics, computeRowCost, ensurePricingLoaded, } = require("./pricing"); @@ -1147,7 +1149,15 @@ function createLocalApiHandler({ queuePath }) { model: m.model, source: s.source, }); - return { ...m, totals: { ...m.totals, total_cost_usd: cost.toFixed(6) } }; + // How the price was resolved, so the dashboard can say "unpriced" + // or "matched by substring" instead of inferring it from a $0 cost + // — a genuinely free model and an unknown one both cost $0. + const { tier } = getModelPricingMeta(m.model, { source: s.source }); + return { + ...m, + pricing_tier: tier, + totals: { ...m.totals, total_cost_usd: cost.toFixed(6) }, + }; }) .sort((a, b) => b.totals.total_tokens - a.totals.total_tokens); const sourceCost = s.models.reduce((sum, m) => sum + Number(m.totals.total_cost_usd), 0); @@ -1157,7 +1167,13 @@ function createLocalApiHandler({ queuePath }) { json(res, { from, to, days: 0, scope, excluded_sources: excludedSources, sources, - pricing: { model: "per-model", pricing_mode: "per_token_type", source: "litellm", effective_from: new Date().toISOString().slice(0, 10) }, + pricing: { + model: "per-model", + pricing_mode: "per_token_type", + source: "litellm", + effective_from: new Date().toISOString().slice(0, 10), + ...getPricingDiagnostics(), + }, }); return true; } diff --git a/src/lib/pricing/index.js b/src/lib/pricing/index.js index d8098f59..b91a2444 100644 --- a/src/lib/pricing/index.js +++ b/src/lib/pricing/index.js @@ -34,34 +34,105 @@ function loadSeedSync() { const seedRaw = loadSeedSync(); +// How long a loaded snapshot is trusted before a lookup is allowed to trigger a +// background refresh. Mirrors the fetcher's disk-cache TTL: before this change +// that TTL only chose which snapshot to load *at startup*, so a dashboard that +// stayed up (the LaunchAgent stays up for days) never saw a new model or a +// price change — claude-opus-5 billed $0 for 21 hours that way. Issue #90. +const RELOAD_AFTER_MS = 24 * 60 * 60 * 1000; + +// Floor between background refreshes, so a permanently-unknown model cannot +// turn every request into an upstream fetch. +const RELOAD_COOLDOWN_MS = 5 * 60 * 1000; + +// Resolution tiers that mean "we guessed": the model matched a substring or a +// curated fuzzy rule rather than an exact id, so the price is plausible but may +// belong to a different model. Worth surfacing — a wrong price never looks +// wrong, unlike a $0 one. +const FUZZY_SOURCES = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip"]); + +// `last_refresh_error` is served over HTTP to the dashboard, so it is built +// from CLOSED sets, never from an arbitrary value. A previous version accepted +// anything symbol-shaped, which a QA pass broke immediately: a 32-character +// token like `sk_live_AAAA…` is symbol-shaped. There is no pattern that +// separates "a short error symbol" from "a short secret" — only an allowlist. +const KNOWN_ERROR_CODES = new Set([ + // fs + "ENOENT", "EACCES", "EPERM", "EEXIST", "ENOSPC", "EROFS", "EISDIR", "ENOTDIR", "EMFILE", "EBUSY", + // network + "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "EPIPE", + "EHOSTUNREACH", "ENETUNREACH", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", + // error classes + "AbortError", "TypeError", "SyntaxError", "RangeError", "FetchError", "Error", +]); + +// Whatever loadLitellmData can report as the origin of the data it returned. +const KNOWN_SOURCES = new Set(["upstream", "disk-cache", "stale-cache", "seed-snapshot"]); + +function labelFrom(allowed, candidates) { + for (const candidate of candidates) { + if (typeof candidate === "string" && allowed.has(candidate)) return candidate; + } + return "unknown"; +} + const state = { loaded: false, loadingPromise: null, + loadedAt: 0, + reloadPromise: null, + lastReloadAt: 0, + lastReloadError: null, litellmRawMap: seedRaw, // raw per-token; field shape from LiteLLM JSON litellmPerMillionMap: buildLitellmPerMillionMap(seedRaw), // USD/MTok source: Object.keys(seedRaw).length ? "seed-snapshot:sync" : null, // negativeCache prevents re-walking the LiteLLM map for models we've already // determined are unknown. Cleared on every reload. negativeCache: new Set(), + // model -> resolution tier, for the diagnostics surface. Cleared on reload. + tiers: new Map(), + // Models already warned about, so a hot path logs once, not once per row. + warned: new Set(), + reloadOptions: {}, }; function defaultCachePath() { return path.join(os.homedir(), ".tokentracker", "cache", "pricing.json"); } +// `requireUpstream` guards the background path. loadLitellmData falls back on +// its own (upstream → stale disk cache → bundled seed), so a refresh that fails +// to reach upstream would otherwise REPLACE good in-memory data with the older +// seed — re-introducing exactly the "new model bills $0" bug this reload exists +// to fix. Verified: with the disk cache deleted and upstream down, a model +// priced at $5/$25 dropped to $0 after a failed refresh. +async function loadInto(opts, { requireUpstream = false } = {}) { + const cachePath = opts.cachePath || defaultCachePath(); + const { data, source } = await loadLitellmData({ ...opts, cachePath }); + if (requireUpstream && source !== "upstream") { + state.lastReloadError = `refresh-fell-back-to-${labelFrom(KNOWN_SOURCES, [source])}`; + return; + } + state.litellmRawMap = data || {}; + state.litellmPerMillionMap = buildLitellmPerMillionMap(state.litellmRawMap); + state.source = source; + state.loaded = true; + state.loadedAt = Date.now(); + state.negativeCache.clear(); + state.tiers.clear(); +} + async function ensurePricingLoaded(opts = {}) { if (state.loaded) return state; if (state.loadingPromise) return state.loadingPromise; + // Remembered so a later background reload can reach the same cache path and + // fetch options without the caller having to plumb them through again. + state.reloadOptions = opts; + state.loadingPromise = (async () => { try { - const cachePath = opts.cachePath || defaultCachePath(); - const { data, source } = await loadLitellmData({ ...opts, cachePath }); - state.litellmRawMap = data || {}; - state.litellmPerMillionMap = buildLitellmPerMillionMap(state.litellmRawMap); - state.source = source; - state.loaded = true; - state.negativeCache.clear(); + await loadInto(opts); return state; } finally { state.loadingPromise = null; @@ -71,37 +142,141 @@ async function ensurePricingLoaded(opts = {}) { return state.loadingPromise; } +// Fire-and-forget refresh. Single-flight, never awaited by a lookup: the caller +// keeps whatever price it already has for this request and the next request +// benefits. A failed reload leaves the existing snapshot in place. +// +// The cooldown matters because a model that is genuinely absent upstream (a +// local or unlisted model) misses on every row it appears in. Without it, each +// of those rows would queue another upstream fetch the moment the previous one +// finished. +function scheduleReload(nowMs = Date.now()) { + if (!state.loaded || state.reloadPromise) return state.reloadPromise; + if (nowMs - state.lastReloadAt < RELOAD_COOLDOWN_MS) return null; + state.lastReloadAt = nowMs; + state.reloadPromise = (async () => { + try { + // forceRefresh skips the disk cache; without it a reload would just + // re-read the same stale snapshot we already hold. + state.lastReloadError = null; + // forceRefresh skips the disk cache; without it a reload would just + // re-read the same stale snapshot we already hold. + await loadInto({ ...state.reloadOptions, forceRefresh: true }, { requireUpstream: true }); + } catch (e) { + // Reached when loadLitellmData itself throws rather than falling back — + // statSafe rethrows a non-ENOENT stat error, so an unusable cache path + // lands here (QA probe: refresh-failed:TypeError). Only + // the error CODE is kept: messages from fs/fetch carry absolute paths and + // this string is served over HTTP to the dashboard. + state.lastReloadError = `refresh-failed:${labelFrom(KNOWN_ERROR_CODES, [e?.code, e?.name])}`; + } finally { + state.reloadPromise = null; + } + })(); + return state.reloadPromise; +} + +function isSnapshotStale(nowMs = Date.now()) { + return state.loaded && nowMs - state.loadedAt > RELOAD_AFTER_MS; +} + // For tests: drop loaded state so a fresh call can re-load. Seeds with the // bundled snapshot so getModelPricing() still works without ensurePricingLoaded. function resetPricingForTests() { state.loaded = false; state.loadingPromise = null; + state.loadedAt = 0; + state.reloadPromise = null; + state.lastReloadAt = 0; + state.lastReloadError = null; state.litellmRawMap = seedRaw; state.litellmPerMillionMap = buildLitellmPerMillionMap(seedRaw); state.source = Object.keys(seedRaw).length ? "seed-snapshot:sync" : null; state.negativeCache.clear(); + state.tiers.clear(); + state.warned.clear(); + state.reloadOptions = {}; } -function getModelPricing(model, opts = {}) { - if (!model) return ZERO_PRICING; - let lookupSource = null; - if (typeof opts === "string") { - lookupSource = opts.toLowerCase(); - } else if (typeof opts.source === "string") { - lookupSource = opts.source.toLowerCase(); - } +function resolveLookupSource(opts) { + if (typeof opts === "string") return opts.toLowerCase(); + if (opts && typeof opts.source === "string") return opts.source.toLowerCase(); + return null; +} + +// Returns the price AND how it was resolved. getModelPricing keeps the old +// bare-numbers contract for the many existing callers; anything that wants to +// show the user how much to trust the number uses this. +function getModelPricingMeta(model, opts = {}) { + if (!model) return { pricing: ZERO_PRICING, tier: "empty" }; + + const lookupSource = resolveLookupSource(opts); const cacheKey = lookupSource ? `${lookupSource}\0${model}` : model; - if (state.negativeCache.has(cacheKey)) return ZERO_PRICING; + + if (state.negativeCache.has(cacheKey)) { + // Still unknown as of the current snapshot. If that snapshot has aged out, + // a new model may have appeared upstream — refresh for the next caller. + if (isSnapshotStale()) scheduleReload(); + return { pricing: ZERO_PRICING, tier: "miss" }; + } const result = lookupPricing(model, { curated: curatedOverrides, litellm: state.litellmPerMillionMap, source: lookupSource, }); - if (result.hit) return result.value; + + if (result.hit) { + state.tiers.set(cacheKey, { model, source: lookupSource, tier: result.source }); + return { pricing: result.value, tier: result.source }; + } state.negativeCache.add(cacheKey); - return ZERO_PRICING; + state.tiers.set(cacheKey, { model, source: lookupSource, tier: "miss" }); + + // A miss is the strongest signal that our snapshot predates a model launch — + // exactly the claude-opus-5 case. Refresh in the background so the next + // request prices it, instead of waiting for a process restart. + scheduleReload(); + + if (!state.warned.has(cacheKey)) { + state.warned.add(cacheKey); + console.warn( + `[pricing] no price for model "${model}"${lookupSource ? ` (source: ${lookupSource})` : ""}` + + " — its cost is being counted as $0. Refreshing pricing data in the background;" + + " if it stays unpriced, add it to src/lib/pricing/curated-overrides.json.", + ); + } + + return { pricing: ZERO_PRICING, tier: "miss" }; +} + +function getModelPricing(model, opts = {}) { + return getModelPricingMeta(model, opts).pricing; +} + +// Snapshot of what the pricing layer knows it got wrong or guessed at, for the +// API to hand to the dashboard. +function getPricingDiagnostics() { + // Keyed by source+model, matching the lookup itself: the same model id can + // resolve differently per provider (Antigravity normalises names before the + // lookup), so a model-only key would let one provider's exact hit hide + // another's miss. + const unpriced = new Set(); + const fuzzy = []; + for (const entry of state.tiers.values()) { + if (entry.tier === "miss") unpriced.add(entry.model); + else if (FUZZY_SOURCES.has(entry.tier)) fuzzy.push({ model: entry.model, tier: entry.tier }); + } + return { + source: state.source, + loaded_at: state.loadedAt ? new Date(state.loadedAt).toISOString() : null, + stale: isSnapshotStale(), + refreshing: Boolean(state.reloadPromise), + last_refresh_error: state.lastReloadError, + unpriced_models: Array.from(unpriced).sort(), + fuzzy_priced_models: fuzzy.sort((a, b) => a.model.localeCompare(b.model)), + }; } // Same formula and Codex/every-code reasoning-folding rule as the previous @@ -134,10 +309,14 @@ const MODEL_PRICING = curatedOverrides.exact; module.exports = { ensurePricingLoaded, getModelPricing, + getModelPricingMeta, + getPricingDiagnostics, computeRowCost, resetPricingForTests, MODEL_PRICING, ZERO_PRICING, // Internal hooks for tests. __getStateForTests: () => state, + __labelFromForTests: labelFrom, + __KNOWN_ERROR_CODES: KNOWN_ERROR_CODES, }; diff --git a/src/lib/pricing/litellm-fetcher.js b/src/lib/pricing/litellm-fetcher.js index 5b6bec2d..55c1877b 100644 --- a/src/lib/pricing/litellm-fetcher.js +++ b/src/lib/pricing/litellm-fetcher.js @@ -112,6 +112,11 @@ async function writeCache(cachePath, data) { async function loadLitellmData({ cachePath, ttlMs = DEFAULT_TTL_MS, + // Skip the disk cache and go upstream. Needed because `ttlMs: 0` does NOT + // reliably force a refetch: mtime carries sub-millisecond precision while + // Date.now() does not, so a cache file written moments ago can compare as + // "written in the future" and still count as fresh. + forceRefresh = false, fetchTimeoutMs = DEFAULT_FETCH_TIMEOUT_MS, fetchImpl = fetchUpstream, url = LITELLM_PRICING_URL, @@ -126,7 +131,7 @@ async function loadLitellmData({ // 1. Fresh disk cache const stat = await statSafe(cachePath); - if (isFresh(stat, ttlMs)) { + if (!forceRefresh && isFresh(stat, ttlMs)) { try { const data = await readJsonAsync(cachePath); delete data._meta; diff --git a/test/model-breakdown.test.js b/test/model-breakdown.test.js index 68fd6d38..eb5b716c 100644 --- a/test/model-breakdown.test.js +++ b/test/model-breakdown.test.js @@ -639,3 +639,82 @@ test("buildFleetData treats GLM flash models with zero cost as known-zero-cost, "glm-4.7-flashx must appear in missingPricingModels", ); }); + +test("pricing_tier from the server beats the cost<=0 guess for missing pricing", async () => { + const mod = await loadDashboardModule("dashboard/src/lib/model-breakdown.ts"); + const { buildFleetData, buildUsageInsights } = mod; + + const modelBreakdown = { + sources: [ + { + source: "claude", + totals: { billable_total_tokens: 3000, total_cost_usd: "5" }, + models: [ + { + // Costs nothing AND resolved exactly — a genuinely free model, not + // an unpriced one. The old cost<=0 heuristic flagged this unless the + // name happened to contain "free". + model: "vendor-zero-rate", + model_id: "vendor-zero-rate", + pricing_tier: "litellm:exact", + totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, + }, + { + // Priced, so the heuristic sees nothing wrong — but the price came + // from a substring match and may belong to a different model. + model: "acme-9-turbo", + model_id: "acme-9-turbo", + pricing_tier: "litellm:fuzzy", + totals: { billable_total_tokens: 1000, total_cost_usd: "5" }, + }, + { + model: "brand-new-model", + model_id: "brand-new-model", + pricing_tier: "miss", + totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, + }, + ], + }, + ], + }; + + const [provider] = buildFleetData(modelBreakdown); + assert.deepEqual( + provider.missingPricingModels.map((m) => m.name), + ["brand-new-model"], + "only the tier=miss model is unpriced", + ); + assert.deepEqual( + provider.fuzzyPricingModels.map((m) => m.name), + ["acme-9-turbo"], + "a substring-matched price is surfaced even though it is non-zero", + ); + + const insights = buildUsageInsights(modelBreakdown); + assert.deepEqual(insights.missingPricingModels.map((m) => m.name), ["brand-new-model"]); + assert.deepEqual(insights.fuzzyPricingModels.map((m) => m.name), ["acme-9-turbo"]); +}); + +test("without pricing_tier the cost<=0 heuristic still applies (older server response)", async () => { + const mod = await loadDashboardModule("dashboard/src/lib/model-breakdown.ts"); + const { buildFleetData } = mod; + + const [provider] = buildFleetData({ + sources: [ + { + source: "claude", + totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, + models: [ + { + model: "brand-new-model", + model_id: "brand-new-model", + totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, + }, + ], + }, + ], + }); + + assert.deepEqual(provider.missingPricingModels.map((m) => m.name), ["brand-new-model"]); + assert.deepEqual(provider.fuzzyPricingModels, []); +}); diff --git a/test/pricing-observability.test.js b/test/pricing-observability.test.js new file mode 100644 index 00000000..2f55404c --- /dev/null +++ b/test/pricing-observability.test.js @@ -0,0 +1,345 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test, beforeEach } = require("node:test"); + +const pricing = require("../src/lib/pricing"); + +function tmpCachePath(name) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `tt-pricing-${name}-`)); + return path.join(dir, "pricing.json"); +} + +// Shape the fetcher expects from upstream: per-token costs. +function entry(input, output) { + return { input_cost_per_token: input, output_cost_per_token: output }; +} + +// Drives ensurePricingLoaded with an injected upstream so the test never talks +// to the network. `payload` is mutable, which is the whole point: it stands in +// for LiteLLM publishing a model after we already loaded a snapshot. +async function loadWith(payload, cachePath) { + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => payload.current, + }); +} + +function captureWarnings(run) { + const original = console.warn; + const lines = []; + console.warn = (...args) => lines.push(args.join(" ")); + try { + return { result: run(), lines }; + } finally { + console.warn = original; + } +} + +// A miss schedules a background reload that most tests never await. Left +// in flight it would land in the middle of the *next* test and overwrite the +// snapshot it just set up, so drain it before resetting. +beforeEach(async () => { + await pricing.__getStateForTests().reloadPromise; + pricing.resetPricingForTests(); +}); + +test("an exact hit reports its tier and the price", async () => { + const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("exact")); + + const meta = pricing.getModelPricingMeta("acme-1"); + assert.equal(meta.tier, "litellm:exact"); + assert.equal(meta.pricing.input, 1); + assert.equal(meta.pricing.output, 2); + // The bare-numbers contract other callers rely on is unchanged. + assert.deepEqual(pricing.getModelPricing("acme-1"), meta.pricing); +}); + +test("a substring match is reported as fuzzy, not passed off as exact", async () => { + const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("fuzzy")); + + // "acme-1-turbo-preview" contains the key "acme-1" and resolves via the + // reverse-substring tier — a plausible price for a model we never saw. + const meta = pricing.getModelPricingMeta("acme-1-turbo-preview"); + assert.equal(meta.tier, "litellm:fuzzy"); + assert.equal(meta.pricing.input, 1); + + const diagnostics = pricing.getPricingDiagnostics(); + assert.deepEqual( + diagnostics.fuzzy_priced_models, + [{ model: "acme-1-turbo-preview", tier: "litellm:fuzzy" }], + ); + assert.deepEqual(diagnostics.unpriced_models, []); +}); + +test("an unknown model is reported as a miss and listed in diagnostics", async () => { + const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("miss")); + + const meta = pricing.getModelPricingMeta("totally-unknown-model"); + assert.equal(meta.tier, "miss"); + assert.deepEqual(meta.pricing, pricing.ZERO_PRICING); + + assert.deepEqual(pricing.getPricingDiagnostics().unpriced_models, ["totally-unknown-model"]); +}); + +test("an unknown model warns once per process, not once per row", async () => { + const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("warn")); + + const { lines } = captureWarnings(() => { + for (let i = 0; i < 25; i += 1) pricing.getModelPricing("totally-unknown-model"); + }); + + assert.equal(lines.length, 1, `expected exactly one warning, got ${lines.length}`); + assert.match(lines[0], /totally-unknown-model/); + assert.match(lines[0], /\$0/); +}); + +test("a model published after startup is priced without restarting the process", async () => { + // The claude-opus-5 incident, reproduced: load a snapshot that predates the + // model, then let the background reload pick it up. + const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("reload")); + + assert.deepEqual(pricing.getModelPricing("acme-2"), pricing.ZERO_PRICING); + + payload.current = { "acme-1": entry(1e-6, 2e-6), "acme-2": entry(5e-6, 25e-6) }; + await pricing.__getStateForTests().reloadPromise; + + const meta = pricing.getModelPricingMeta("acme-2"); + assert.equal(meta.tier, "litellm:exact", "the reload must clear the negative cache"); + assert.equal(meta.pricing.input, 5); + assert.equal(meta.pricing.output, 25); +}); + +test("the reload is single-flight — concurrent misses do not stampede upstream", async () => { + let fetches = 0; + const cachePath = tmpCachePath("single-flight"); + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => { + fetches += 1; + return { "acme-1": entry(1e-6, 2e-6) }; + }, + }); + const afterLoad = fetches; + + for (const model of ["unknown-a", "unknown-b", "unknown-c"]) pricing.getModelPricing(model); + await pricing.__getStateForTests().reloadPromise; + + assert.equal(fetches - afterLoad, 1, "three misses must share one refresh"); +}); + +test("a permanently unknown model cannot refetch on every lookup", async () => { + let fetches = 0; + const cachePath = tmpCachePath("cooldown"); + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => { + fetches += 1; + return { "acme-1": entry(1e-6, 2e-6) }; + }, + }); + const afterLoad = fetches; + + // Each round completes its reload before the next lookup, so only the + // cooldown stands between this and one upstream fetch per row. + for (let i = 0; i < 5; i += 1) { + pricing.getModelPricing("never-listed-model"); + await pricing.__getStateForTests().reloadPromise; + } + + assert.equal(fetches - afterLoad, 1, "the cooldown must suppress repeat refreshes"); +}); + +test("a failed reload cannot downgrade the prices already loaded", async () => { + // The earlier version of this test left the disk cache in place, so the + // fetcher's own fallback re-read the same data and the test passed without + // proving anything. Delete the cache first: now the only fallback left is the + // OLDER bundled seed, which does not know this model — exactly the shape that + // re-introduced $0 for a newly published model. + const cachePath = tmpCachePath("offline"); + let shouldFail = false; + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => { + if (shouldFail) throw new Error("offline"); + return { "brand-new-model": entry(5e-6, 25e-6) }; + }, + }); + assert.equal(pricing.getModelPricing("brand-new-model").input, 5); + + fs.rmSync(cachePath, { force: true }); + shouldFail = true; + pricing.getModelPricing("some-other-unknown"); + await pricing.__getStateForTests().reloadPromise; + + assert.equal( + pricing.getModelPricing("brand-new-model").input, + 5, + "a failed refresh must not replace good data with the older seed", + ); + + const diagnostics = pricing.getPricingDiagnostics(); + assert.match(diagnostics.last_refresh_error, /^refresh-/); + assert.equal(diagnostics.source, "upstream", "the snapshot in memory is still the upstream one"); +}); + +test("a refresh that only reaches the bundled seed is reported, not applied", async () => { + const cachePath = tmpCachePath("fallback"); + let serveUpstream = true; + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => { + if (!serveUpstream) throw new Error("offline"); + return { "brand-new-model": entry(5e-6, 25e-6) }; + }, + }); + + fs.rmSync(cachePath, { force: true }); + serveUpstream = false; + pricing.getModelPricing("still-unknown"); + await pricing.__getStateForTests().reloadPromise; + + const diagnostics = pricing.getPricingDiagnostics(); + assert.match( + diagnostics.last_refresh_error, + /^refresh-(failed|fell-back)/, + "the dashboard must be able to see that the refresh did not reach upstream", + ); +}); + +test("a refresh error never carries a filesystem path into the HTTP response", async () => { + const cachePath = tmpCachePath("nopath"); + let fail = false; + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => { + if (fail) { + const e = new Error(`ENOENT: no such file or directory, open '${cachePath}'`); + e.code = "ENOENT"; + throw e; + } + return { "acme-1": entry(1e-6, 2e-6) }; + }, + }); + + fs.rmSync(cachePath, { force: true }); + fail = true; + pricing.getModelPricing("unknown-model"); + await pricing.__getStateForTests().reloadPromise; + + const reported = pricing.getPricingDiagnostics().last_refresh_error; + assert.ok(reported, "a failed refresh must still be reported"); + assert.equal(reported.includes("/"), false, `error must not leak a path: ${reported}`); + assert.equal(reported.includes(os.tmpdir()), false); +}); + +test("diagnostics report the snapshot's age and source", async () => { + const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("diag")); + + const diagnostics = pricing.getPricingDiagnostics(); + assert.equal(diagnostics.source, "upstream"); + assert.ok(Date.parse(diagnostics.loaded_at) > 0); + assert.equal(diagnostics.stale, false); +}); + +test("lookups still work before ensurePricingLoaded, and report no reload state", () => { + // The bundled seed answers synchronously at require-time; nothing should + // throw or try to reload when no async load has happened yet. + assert.equal(pricing.getModelPricingMeta("").tier, "empty"); + const diagnostics = pricing.getPricingDiagnostics(); + assert.equal(diagnostics.loaded_at, null); + assert.equal(diagnostics.refreshing, false); +}); + +test("the reported error label comes from a closed set, not a pattern", () => { + // The first attempt used a regex for "symbol-shaped". A QA pass broke it in + // one line: a payment-provider key prefix plus 24 characters is symbol-shaped + // too. No pattern separates a short error symbol from a short secret — only + // an allowlist does. + // + // The look-alikes are assembled at runtime rather than written as literals: + // a realistic-looking key in a source file trips GitHub push protection (it + // did) and, more to the point, a repo should not carry strings that a scanner + // has to be told to ignore. + const fakeStripeKey = ["sk", "live", "A".repeat(24)].join("_"); + const fakeGithubToken = `ghp_${"0123456789abcdefghij"}`; + const label = pricing.__labelFromForTests; + const codes = pricing.__KNOWN_ERROR_CODES; + + assert.equal(label(codes, [fakeStripeKey]), "unknown"); + assert.equal(label(codes, ["/Users/alice/private/pricing.json"]), "unknown"); + assert.equal(label(codes, [undefined, fakeGithubToken]), "unknown"); + assert.equal(label(codes, [{ toString: () => "ENOENT" }]), "unknown"); + assert.equal(label(codes, []), "unknown"); + + assert.equal(label(codes, ["ENOENT"]), "ENOENT"); + assert.equal(label(codes, [undefined, "AbortError"]), "AbortError"); +}); + +// Every value this field can take, so a call site that stopped going through +// the allowlist would produce something outside this set. +const ALLOWED_REFRESH_ERRORS = new Set([ + ...["upstream", "disk-cache", "stale-cache", "seed-snapshot", "unknown"].map( + (s) => `refresh-fell-back-to-${s}`, + ), + ...[...pricing.__KNOWN_ERROR_CODES, "unknown"].map((c) => `refresh-failed:${c}`), +]); + +test("the fallback branch reports only an allowlisted label", async () => { + const cachePath = tmpCachePath("nonerror"); + let fail = false; + await pricing.ensurePricingLoaded({ + cachePath, + fetchImpl: async () => { + if (fail) { + // Same reasoning as above: built at runtime, not written as a literal. + throw { code: "/Users/alice/private/pricing.json", name: ["sk", "live", "A".repeat(24)].join("_") }; + } + return { "acme-1": entry(1e-6, 2e-6) }; + }, + }); + + fs.rmSync(cachePath, { force: true }); + fail = true; + pricing.getModelPricing("unknown-model"); + await pricing.__getStateForTests().reloadPromise; + + const reported = pricing.getPricingDiagnostics().last_refresh_error; + assert.ok( + ALLOWED_REFRESH_ERRORS.has(reported), + `reported value must come from the allowlist, got ${JSON.stringify(reported)}`, + ); +}); + +// NOT TESTED end to end: scheduleReload's catch IS reachable — statSafe +// rethrows a non-ENOENT stat error, so an unusable cache path lands there and a +// QA probe observed refresh-failed:TypeError. An earlier attempt to drive it +// from a test made a real network call and asserted nothing, so the branch is +// left to the unit test of the allowlist above rather than covered by a test +// that cannot fail. Recorded as a real gap, not dismissed as unreachable. + +test("one provider's exact hit cannot hide another provider's miss", async () => { + // Antigravity normalises model names before the lookup, so the same id can + // resolve differently per source. Keyed by model alone, the last write wins + // and one of the two verdicts disappears from the diagnostics. + const payload = { current: { "shared-name": entry(1e-6, 2e-6) } }; + await loadWith(payload, tmpCachePath("two-source")); + + assert.equal(pricing.getModelPricingMeta("shared-name", { source: "claude" }).tier, "litellm:exact"); + assert.equal( + pricing.getModelPricingMeta("unknown-only-here", { source: "antigravity" }).tier, + "miss", + ); + // Same id, two sources: the exact hit must not erase the miss. + pricing.getModelPricingMeta("shared-name", { source: "antigravity" }); + + const diagnostics = pricing.getPricingDiagnostics(); + assert.deepEqual(diagnostics.unpriced_models, ["unknown-only-here"]); +});