diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e8db1952..ea9c5764f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Fixed - CLI server: retain timed-out route and provider work until it actually exits, preventing repeated requests or config changes from stacking background fetches. Thanks @Yuxin-Qiao! +- Token costs: coalesce bounded pricing-catalog refreshes when a newly observed model is still unpriced, preserving its exact usage until pricing arrives. Thanks @iam-brain! - Ollama: validate API keys against an authenticated endpoint instead of the public model catalog while preserving refresh cancellation. Thanks @joeVenner! - Claude CLI: resolve yearless and time-only reset timestamps against their quota window and exact calendar occurrence, keeping recently stale resets current without moving future, leap-day, or repeated-hour resets into the past. Thanks @fanwenlin! - Catalan: complete current strings, align instructional voice, and enforce catalog parity. Thanks @pmontp19! diff --git a/Sources/CodexBarCore/CostUsageFetcher.swift b/Sources/CodexBarCore/CostUsageFetcher.swift index aefd885a4b..09c7a63655 100644 --- a/Sources/CodexBarCore/CostUsageFetcher.swift +++ b/Sources/CodexBarCore/CostUsageFetcher.swift @@ -117,7 +117,9 @@ public struct CostUsageFetcher: Sendable { refreshPricingInBackground: Bool = true, scannerOptions overrideScannerOptions: CostUsageScanner.Options? = nil, piScannerOptions overridePiScannerOptions: PiSessionCostScanner - .Options? = nil) async throws -> CostUsageTokenSnapshot + .Options? = nil, + modelsDevClient: ModelsDevClient = ModelsDevClient(), + retryUnknownPricing: Bool = true) async throws -> CostUsageTokenSnapshot { guard provider == .codex || provider == .claude || provider == .vertexai || provider == .bedrock else { throw CostUsageError.unsupportedProvider(provider) @@ -148,14 +150,20 @@ public struct CostUsageFetcher: Sendable { options.codexSessionsRoot = URL(fileURLWithPath: codexHomePath, isDirectory: true) .appendingPathComponent("sessions", isDirectory: true) } - if provider == .codex || provider == .claude { + if retryUnknownPricing, provider == .codex || provider == .claude { let pricingCacheRoot = options.cacheRoot if refreshPricingInBackground { Task.detached(priority: .utility) { - await ModelsDevPricingPipeline.refreshIfNeeded(now: now, cacheRoot: pricingCacheRoot) + await ModelsDevPricingPipeline.refreshIfNeeded( + now: now, + cacheRoot: pricingCacheRoot, + client: modelsDevClient) } } else { - await ModelsDevPricingPipeline.refreshIfNeeded(now: now, cacheRoot: pricingCacheRoot) + await ModelsDevPricingPipeline.refreshIfNeeded( + now: now, + cacheRoot: pricingCacheRoot, + client: modelsDevClient) } } @@ -239,6 +247,30 @@ public struct CostUsageFetcher: Sendable { return (daily: daily, projects: projects) } + if retryUnknownPricing, + let request = Self.unknownPricingRefreshRequest( + provider: provider, + daily: scanResult.daily, + now: now, + cacheRoot: options.cacheRoot, + client: modelsDevClient), + await Self.refreshUnknownPricingIfNeeded(request, inBackground: refreshPricingInBackground) + { + return try await self.loadTokenSnapshot( + provider: provider, + environment: environment, + now: now, + forceRefresh: forceRefresh, + allowVertexClaudeFallback: allowVertexClaudeFallback, + codexHomePath: codexHomePath, + historyDays: historyDays, + refreshPricingInBackground: false, + scannerOptions: options, + piScannerOptions: piOptions, + modelsDevClient: modelsDevClient, + retryUnknownPricing: false) + } + return Self.tokenSnapshot( from: scanResult.daily, now: now, @@ -246,6 +278,60 @@ public struct CostUsageFetcher: Sendable { projects: scanResult.projects) } + private struct UnknownPricingRefreshRequest: Sendable { + let providerID: String + let modelIDs: Set + let now: Date + let cacheRoot: URL? + let client: ModelsDevClient + } + + private static func unknownPricingRefreshRequest( + provider: UsageProvider, + daily: CostUsageDailyReport, + now: Date, + cacheRoot: URL?, + client: ModelsDevClient) -> UnknownPricingRefreshRequest? + { + guard provider == .codex || provider == .claude else { return nil } + let unknownModelIDs = Set(daily.data.flatMap { entry in + entry.modelBreakdowns?.compactMap { breakdown in + breakdown.costUSD == nil ? breakdown.modelName : nil + } ?? [] + }) + guard !unknownModelIDs.isEmpty else { return nil } + + return UnknownPricingRefreshRequest( + providerID: provider == .codex ? "openai" : "anthropic", + modelIDs: unknownModelIDs, + now: now, + cacheRoot: cacheRoot, + client: client) + } + + private static func refreshUnknownPricingIfNeeded( + _ request: UnknownPricingRefreshRequest, + inBackground: Bool) async -> Bool + { + if inBackground { + Task.detached(priority: .utility) { + _ = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: request.providerID, + modelIDs: request.modelIDs, + now: request.now, + cacheRoot: request.cacheRoot, + client: request.client) + } + return false + } + return await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: request.providerID, + modelIDs: request.modelIDs, + now: request.now, + cacheRoot: request.cacheRoot, + client: request.client) == .pricingAvailable + } + static func loadCachedCodexTokenSnapshot( now: Date = Date(), codexHomePath: String? = nil, @@ -470,10 +556,14 @@ public struct CostUsageFetcher: Sendable { .sorted { lhs, rhs in let lhsCost = lhs.totalCostUSD ?? -1 let rhsCost = rhs.totalCostUSD ?? -1 - if lhsCost != rhsCost { return lhsCost > rhsCost } + if lhsCost != rhsCost { + return lhsCost > rhsCost + } let lhsTokens = lhs.totalTokens ?? -1 let rhsTokens = rhs.totalTokens ?? -1 - if lhsTokens != rhsTokens { return lhsTokens > rhsTokens } + if lhsTokens != rhsTokens { + return lhsTokens > rhsTokens + } return lhs.name.localizedStandardCompare(rhs.name) == .orderedAscending } } @@ -495,10 +585,14 @@ public struct CostUsageFetcher: Sendable { .sorted { lhs, rhs in let lhsCost = lhs.totalCostUSD ?? -1 let rhsCost = rhs.totalCostUSD ?? -1 - if lhsCost != rhsCost { return lhsCost > rhsCost } + if lhsCost != rhsCost { + return lhsCost > rhsCost + } let lhsTokens = lhs.totalTokens ?? -1 let rhsTokens = rhs.totalTokens ?? -1 - if lhsTokens != rhsTokens { return lhsTokens > rhsTokens } + if lhsTokens != rhsTokens { + return lhsTokens > rhsTokens + } return lhs.name.localizedStandardCompare(rhs.name) == .orderedAscending } } @@ -546,10 +640,14 @@ public struct CostUsageFetcher: Sendable { .sorted { lhs, rhs in let lhsCost = lhs.costUSD ?? -1 let rhsCost = rhs.costUSD ?? -1 - if lhsCost != rhsCost { return lhsCost > rhsCost } + if lhsCost != rhsCost { + return lhsCost > rhsCost + } let lhsTokens = lhs.totalTokens ?? -1 let rhsTokens = rhs.totalTokens ?? -1 - if lhsTokens != rhsTokens { return lhsTokens > rhsTokens } + if lhsTokens != rhsTokens { + return lhsTokens > rhsTokens + } return lhs.modelName > rhs.modelName } } @@ -557,17 +655,25 @@ public struct CostUsageFetcher: Sendable { static func selectCurrentSession(from sessions: [CostUsageSessionReport.Entry]) -> CostUsageSessionReport.Entry? { - if sessions.isEmpty { return nil } + if sessions.isEmpty { + return nil + } return sessions.max { lhs, rhs in let lDate = CostUsageDateParser.parse(lhs.lastActivity) ?? .distantPast let rDate = CostUsageDateParser.parse(rhs.lastActivity) ?? .distantPast - if lDate != rDate { return lDate < rDate } + if lDate != rDate { + return lDate < rDate + } let lCost = lhs.costUSD ?? -1 let rCost = rhs.costUSD ?? -1 - if lCost != rCost { return lCost < rCost } + if lCost != rCost { + return lCost < rCost + } let lTokens = lhs.totalTokens ?? -1 let rTokens = rhs.totalTokens ?? -1 - if lTokens != rTokens { return lTokens < rTokens } + if lTokens != rTokens { + return lTokens < rTokens + } return lhs.session < rhs.session } } @@ -575,17 +681,25 @@ public struct CostUsageFetcher: Sendable { static func selectMostRecentMonth(from months: [CostUsageMonthlyReport.Entry]) -> CostUsageMonthlyReport.Entry? { - if months.isEmpty { return nil } + if months.isEmpty { + return nil + } return months.max { lhs, rhs in let lDate = CostUsageDateParser.parseMonth(lhs.month) ?? .distantPast let rDate = CostUsageDateParser.parseMonth(rhs.month) ?? .distantPast - if lDate != rDate { return lDate < rDate } + if lDate != rDate { + return lDate < rDate + } let lCost = lhs.costUSD ?? -1 let rCost = rhs.costUSD ?? -1 - if lCost != rCost { return lCost < rCost } + if lCost != rCost { + return lCost < rCost + } let lTokens = lhs.totalTokens ?? -1 let rTokens = rhs.totalTokens ?? -1 - if lTokens != rTokens { return lTokens < rTokens } + if lTokens != rTokens { + return lTokens < rTokens + } return lhs.month < rhs.month } } diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index b5ac76cffc..488fa95e8b 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "1745966d77d9dae8" + static let value = "3edc97b8139ae59e" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/ModelsDevPricing.swift b/Sources/CodexBarCore/Vendored/CostUsage/ModelsDevPricing.swift index 282c6e55c3..6915086291 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/ModelsDevPricing.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/ModelsDevPricing.swift @@ -499,22 +499,24 @@ enum ModelsDevCache { } } - static func save(catalog: ModelsDevCatalog, fetchedAt: Date = Date(), cacheRoot: URL? = nil) { + @discardableResult + static func save(catalog: ModelsDevCatalog, fetchedAt: Date = Date(), cacheRoot: URL? = nil) -> Bool { let artifact = ModelsDevCacheArtifact( version: Self.artifactVersion, fetchedAt: fetchedAt, catalog: catalog) - self.save(artifact: artifact, cacheRoot: cacheRoot) + return self.save(artifact: artifact, cacheRoot: cacheRoot) } - static func save(artifact: ModelsDevCacheArtifact, cacheRoot: URL? = nil) { + @discardableResult + static func save(artifact: ModelsDevCacheArtifact, cacheRoot: URL? = nil) -> Bool { let url = self.cacheFileURL(cacheRoot: cacheRoot) let dir = url.deletingLastPathComponent() try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) let encoder = JSONEncoder() encoder.dateEncodingStrategy = .iso8601 - guard let data = try? encoder.encode(artifact) else { return } + guard let data = try? encoder.encode(artifact) else { return false } let tmp = dir.appendingPathComponent(".tmp-\(UUID().uuidString).json", isDirectory: false) do { @@ -526,8 +528,10 @@ enum ModelsDevCache { } // The on-disk catalog changed; drop the memo so the next load decodes the fresh file. Self.memo.invalidate(path: url.path) + return true } catch { try? FileManager.default.removeItem(at: tmp) + return false } } } @@ -542,7 +546,7 @@ struct URLSessionModelsDevTransport: ModelsDevHTTPTransport { } } -struct ModelsDevClient { +struct ModelsDevClient: Sendable { enum Error: Swift.Error, Equatable { case invalidResponse case httpStatus(Int) @@ -577,7 +581,16 @@ struct ModelsDevClient { } } +enum ModelsDevUnknownModelRefreshOutcome: Equatable { + case pricingAvailable + case unavailable +} + +private let modelsDevCatalogRetryInterval: TimeInterval = 15 * 60 + enum ModelsDevPricingPipeline { + private static let refreshCoordinator = ModelsDevRefreshCoordinator() + static func lookup( providerID: String, modelID: String, @@ -598,14 +611,107 @@ enum ModelsDevPricingPipeline { let load = ModelsDevCache.load(now: now, cacheRoot: cacheRoot) guard load.isStale else { return } + let cachePath = ModelsDevCache.cacheFileURL(cacheRoot: cacheRoot).standardizedFileURL.path + _ = await self.refreshCoordinator.refresh( + cachePath: cachePath, + now: now) + { + await self.refreshStaleCache(now: now, cacheRoot: cacheRoot, client: client) + } + } + + static func refreshForUnknownModelsIfNeeded( + providerID: String, + modelIDs: Set, + now: Date = Date(), + cacheRoot: URL? = nil, + client: ModelsDevClient = ModelsDevClient()) async -> ModelsDevUnknownModelRefreshOutcome + { + guard !modelIDs.isEmpty else { return .unavailable } + let load = ModelsDevCache.load(now: now, cacheRoot: cacheRoot) + let unknownModelIDs = modelIDs.filter { + load.artifact?.catalog.pricing(providerID: providerID, modelID: $0) == nil + } + guard !unknownModelIDs.isEmpty else { return .pricingAvailable } + if let fetchedAt = load.artifact?.fetchedAt, + now.timeIntervalSince(fetchedAt) < modelsDevCatalogRetryInterval + { + return .unavailable + } + + let cachePath = ModelsDevCache.cacheFileURL(cacheRoot: cacheRoot).standardizedFileURL.path + _ = await self.refreshCoordinator.refresh( + cachePath: cachePath, + now: now) + { + await self.performRefresh(now: now, cacheRoot: cacheRoot, client: client) + } + + let refreshedCatalog = ModelsDevCache.load(now: now, cacheRoot: cacheRoot).artifact?.catalog + let pricingBecameAvailable = unknownModelIDs.contains { + refreshedCatalog?.pricing(providerID: providerID, modelID: $0) != nil + } + return pricingBecameAvailable ? .pricingAvailable : .unavailable + } + + private static func performRefresh( + now: Date, + cacheRoot: URL?, + client: ModelsDevClient) async -> Bool + { do { let catalog = try await client.fetchCatalog() - let oldCatalog = load.artifact?.catalog - guard catalog.isPlausibleRefresh() else { return } + guard catalog.isPlausibleRefresh() else { return false } + let oldCatalog = ModelsDevCache.load(now: now, cacheRoot: cacheRoot).artifact?.catalog let refreshedCatalog = oldCatalog.map { catalog.mergingFallbackPricing(from: $0) } ?? catalog - ModelsDevCache.save(catalog: refreshedCatalog, fetchedAt: now, cacheRoot: cacheRoot) + return ModelsDevCache.save(catalog: refreshedCatalog, fetchedAt: now, cacheRoot: cacheRoot) } catch { - // Best-effort refresh only. Future scanner integration should keep using the last valid cache. + return false + } + } + + static func refreshStaleCache( + now: Date, + cacheRoot: URL?, + client: ModelsDevClient) async -> Bool + { + guard ModelsDevCache.load(now: now, cacheRoot: cacheRoot).isStale else { return true } + return await self.performRefresh(now: now, cacheRoot: cacheRoot, client: client) + } +} + +private actor ModelsDevRefreshCoordinator { + private struct InFlightRefresh { + let id: UUID + let task: Task + } + + private var inFlightByCachePath: [String: InFlightRefresh] = [:] + private var lastCatalogAttemptByCachePath: [String: Date] = [:] + + func refresh( + cachePath: String, + now: Date, + operation: @escaping @Sendable () async -> Bool) async -> Bool + { + if let inFlight = self.inFlightByCachePath[cachePath] { + return await inFlight.task.value + } + if let lastAttempt = self.lastCatalogAttemptByCachePath[cachePath], + now.timeIntervalSince(lastAttempt) < modelsDevCatalogRetryInterval + { + return false + } + self.lastCatalogAttemptByCachePath[cachePath] = now + + let inFlight = InFlightRefresh( + id: UUID(), + task: Task { await operation() }) + self.inFlightByCachePath[cachePath] = inFlight + let result = await inFlight.task.value + if self.inFlightByCachePath[cachePath]?.id == inFlight.id { + self.inFlightByCachePath[cachePath] = nil } + return result } } diff --git a/Tests/CodexBarTests/CostUsageFetcherUnknownModelPricingTests.swift b/Tests/CodexBarTests/CostUsageFetcherUnknownModelPricingTests.swift new file mode 100644 index 0000000000..29828dd8d6 --- /dev/null +++ b/Tests/CodexBarTests/CostUsageFetcherUnknownModelPricingTests.swift @@ -0,0 +1,213 @@ +import Foundation +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif +import Testing +@testable import CodexBarCore + +struct CostUsageFetcherUnknownModelPricingTests { + @Test + func `fetcher reprices an unknown model after an on demand catalog refresh`() async throws { + let fixture = try UnknownModelPricingFixture() + defer { fixture.environment.cleanup() } + + let snapshot = try await CostUsageFetcher.loadTokenSnapshot( + provider: .codex, + now: fixture.day, + refreshPricingInBackground: false, + scannerOptions: fixture.options, + modelsDevClient: ModelsDevClient(transport: CostUsageFetcherModelsDevTransport( + data: fixture.refreshedCatalog))) + + let breakdown = try #require(snapshot.daily.first?.modelBreakdowns?.first) + #expect(breakdown.modelName == "gpt-new") + #expect(abs((breakdown.costUSD ?? 0) - 0.00028) < 0.0000001) + } + + @Test + func `background pricing refresh returns unpriced usage before catalog download finishes`() async throws { + let fixture = try UnknownModelPricingFixture() + defer { fixture.environment.cleanup() } + let gate = UnknownModelPricingTransportGate() + let completion = UnknownModelPricingCompletionProbe() + let task = Task { + let snapshot = try await CostUsageFetcher.loadTokenSnapshot( + provider: .codex, + now: fixture.day, + refreshPricingInBackground: true, + scannerOptions: fixture.options, + modelsDevClient: ModelsDevClient(transport: CostUsageFetcherGatedModelsDevTransport( + data: fixture.refreshedCatalog, + gate: gate))) + await completion.markCompleted() + return snapshot + } + + await gate.waitUntilStarted() + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: .seconds(1)) + while await !(completion.isCompleted), clock.now < deadline { + try await Task.sleep(for: .milliseconds(10)) + } + let returnedBeforeRelease = await completion.isCompleted + await gate.release() + let snapshot = try await task.value + + #expect(returnedBeforeRelease) + let breakdown = try #require(snapshot.daily.first?.modelBreakdowns?.first) + #expect(breakdown.modelName == "gpt-new") + #expect(breakdown.totalTokens == 110) + #expect(breakdown.costUSD == nil) + + let refreshDeadline = clock.now.advanced(by: .seconds(1)) + while ModelsDevPricingPipeline.lookup( + providerID: "openai", + modelID: "gpt-new", + cacheRoot: fixture.environment.cacheRoot) == nil, + clock.now < refreshDeadline + { + try await Task.sleep(for: .milliseconds(10)) + } + #expect(ModelsDevPricingPipeline.lookup( + providerID: "openai", + modelID: "gpt-new", + cacheRoot: fixture.environment.cacheRoot) != nil) + } +} + +private struct UnknownModelPricingFixture { + let environment: CostUsageTestEnvironment + let day: Date + let options: CostUsageScanner.Options + let refreshedCatalog: Data + + init() throws { + let environment = try CostUsageTestEnvironment() + self.environment = environment + self.day = try environment.makeLocalNoon(year: 2026, month: 4, day: 12) + let oldCatalog = try JSONDecoder().decode(ModelsDevCatalog.self, from: Data(""" + { + "openai": { + "id": "openai", + "models": { "gpt-old": { "id": "gpt-old", "cost": { "input": 1, "output": 4 } } } + }, + "anthropic": { + "id": "anthropic", + "models": { "claude-old": { "id": "claude-old", "cost": { "input": 3, "output": 15 } } } + } + } + """.utf8)) + ModelsDevCache.save( + catalog: oldCatalog, + fetchedAt: self.day.addingTimeInterval(-901), + cacheRoot: environment.cacheRoot) + + self.refreshedCatalog = Data(""" + { + "openai": { + "id": "openai", + "models": { "gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } } } + }, + "anthropic": { + "id": "anthropic", + "models": { "claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } } } + } + } + """.utf8) + let turnContext: [String: Any] = [ + "type": "turn_context", + "timestamp": environment.isoString(for: self.day), + "payload": ["model": "gpt-new"], + ] + let tokenCount: [String: Any] = [ + "type": "event_msg", + "timestamp": environment.isoString(for: self.day.addingTimeInterval(1)), + "payload": [ + "type": "token_count", + "info": [ + "total_token_usage": [ + "input_tokens": 100, + "cached_input_tokens": 20, + "output_tokens": 10, + ], + ], + ], + ] + _ = try environment.writeCodexSessionFile( + day: self.day, + filename: "unknown-model.jsonl", + contents: environment.jsonl([turnContext, tokenCount])) + self.options = CostUsageScanner.Options( + codexSessionsRoot: environment.codexSessionsRoot, + claudeProjectsRoots: [environment.claudeProjectsRoot], + cacheRoot: environment.cacheRoot) + } +} + +private struct CostUsageFetcherModelsDevTransport: ModelsDevHTTPTransport { + let data: Data + + func data(for request: URLRequest) async throws -> (Data, URLResponse) { + let response = HTTPURLResponse( + url: request.url!, + statusCode: 200, + httpVersion: nil, + headerFields: nil)! + return (self.data, response) + } +} + +private struct CostUsageFetcherGatedModelsDevTransport: ModelsDevHTTPTransport { + let data: Data + let gate: UnknownModelPricingTransportGate + + func data(for request: URLRequest) async throws -> (Data, URLResponse) { + await self.gate.markStartedAndWaitForRelease() + let response = HTTPURLResponse( + url: request.url!, + statusCode: 200, + httpVersion: nil, + headerFields: nil)! + return (self.data, response) + } +} + +private actor UnknownModelPricingTransportGate { + private var started = false + private var released = false + private var startWaiters: [CheckedContinuation] = [] + private var releaseWaiters: [CheckedContinuation] = [] + + func markStartedAndWaitForRelease() async { + self.started = true + let startWaiters = self.startWaiters + self.startWaiters.removeAll() + startWaiters.forEach { $0.resume() } + guard !self.released else { return } + await withCheckedContinuation { continuation in + self.releaseWaiters.append(continuation) + } + } + + func waitUntilStarted() async { + guard !self.started else { return } + await withCheckedContinuation { continuation in + self.startWaiters.append(continuation) + } + } + + func release() { + self.released = true + let releaseWaiters = self.releaseWaiters + self.releaseWaiters.removeAll() + releaseWaiters.forEach { $0.resume() } + } +} + +private actor UnknownModelPricingCompletionProbe { + private(set) var isCompleted = false + + func markCompleted() { + self.isCompleted = true + } +} diff --git a/Tests/CodexBarTests/ModelsDevPricingTests.swift b/Tests/CodexBarTests/ModelsDevPricingTests.swift index d1a2fdcf91..a0da2f6370 100644 --- a/Tests/CodexBarTests/ModelsDevPricingTests.swift +++ b/Tests/CodexBarTests/ModelsDevPricingTests.swift @@ -153,6 +153,305 @@ struct ModelsDevPricingTests { } extension ModelsDevPricingTests { + @Test + func `unknown model refresh makes newly published pricing available`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 10000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-901), + cacheRoot: root) + let refreshed = Data(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } } + } + }, + "anthropic": { + "id": "anthropic", + "models": { + "claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } } + } + } + } + """.utf8) + let transport = TrackingTransport(result: .success((refreshed, Self.response(status: 200)))) + let client = ModelsDevClient(transport: transport) + + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["gpt-new"], + now: now, + cacheRoot: root, + client: client) + #expect(outcome == .pricingAvailable) + #expect(transport.calls == 1) + #expect(ModelsDevPricingPipeline.lookup( + providerID: "openai", + modelID: "gpt-new", + cacheRoot: root) != nil) + } + + @Test + func `unknown model refresh is bounded per provider cache`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 20000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-901), + cacheRoot: root) + let transport = try TrackingTransport(result: .success(( + JSONEncoder().encode(Self.fixtureCatalog()), + Self.response(status: 200)))) + let client = ModelsDevClient(transport: transport) + + let first = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["still-unknown"], + now: now, + cacheRoot: root, + client: client) + let second = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["another-unknown-model"], + now: now.addingTimeInterval(60), + cacheRoot: root, + client: client) + + #expect(first == .unavailable) + #expect(second == .unavailable) + #expect(transport.calls == 1) + } + + @Test + func `known requested model does not mask an unresolved unknown model`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 25000) + let catalog = try Self.catalog(""" + { + "openai": { + "id": "openai", + "models": { + "already-priced": { "id": "already-priced", "cost": { "input": 1, "output": 2 } } + } + }, + "anthropic": { + "id": "anthropic", + "models": { + "catalog-anchor": { "id": "catalog-anchor", "cost": { "input": 3, "output": 4 } } + } + } + } + """) + ModelsDevCache.save( + catalog: catalog, + fetchedAt: now.addingTimeInterval(-901), + cacheRoot: root) + let transport = try TrackingTransport(result: .success(( + JSONEncoder().encode(catalog), + Self.response(status: 200)))) + + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["already-priced", "still-unknown"], + now: now, + cacheRoot: root, + client: ModelsDevClient(transport: transport)) + + #expect(outcome == .unavailable) + #expect(transport.calls == 1) + } + + @Test + func `pricing added by a completed background refresh requests a rescan`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 30000) + let refreshed = Data(""" + { + "openai": { + "id": "openai", + "models": { "gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } } } + }, + "anthropic": { + "id": "anthropic", + "models": { "claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } } } + } + } + """.utf8) + let refreshedCatalog = try JSONDecoder().decode(ModelsDevCatalog.self, from: refreshed) + ModelsDevCache.save(catalog: refreshedCatalog, fetchedAt: now, cacheRoot: root) + let transport = TrackingTransport(result: .failure(MockError.failed)) + + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["gpt-new"], + now: now, + cacheRoot: root, + client: ModelsDevClient(transport: transport)) + + #expect(outcome == .pricingAvailable) + #expect(transport.calls == 0) + } + + @Test + func `ttl and unknown model refreshes share one download`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 40000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1), + cacheRoot: root) + let transport = try TrackingTransport( + result: .success((JSONEncoder().encode(Self.fixtureCatalog()), Self.response(status: 200))), + delayNanoseconds: 100_000_000) + let client = ModelsDevClient(transport: transport) + + async let ttl: Void = ModelsDevPricingPipeline.refreshIfNeeded( + now: now, + cacheRoot: root, + client: client) + async let unknown = ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["still-unknown"], + now: now, + cacheRoot: root, + client: client) + _ = await (ttl, unknown) + + #expect(transport.calls == 1) + } + + @Test + func `completed ttl refresh bounds a following unknown model refresh`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 45000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1), + cacheRoot: root) + let transport = try TrackingTransport(result: .success(( + JSONEncoder().encode(Self.fixtureCatalog()), + Self.response(status: 200)))) + let client = ModelsDevClient(transport: transport) + + await ModelsDevPricingPipeline.refreshIfNeeded( + now: now, + cacheRoot: root, + client: client) + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["still-unknown"], + now: now, + cacheRoot: root, + client: client) + + #expect(outcome == .unavailable) + #expect(transport.calls == 1) + } + + @Test + func `failed ttl refresh bounds a following unknown model refresh within cooldown`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 46000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1), + cacheRoot: root) + let transport = TrackingTransport(result: .failure(MockError.failed)) + let client = ModelsDevClient(transport: transport) + + await ModelsDevPricingPipeline.refreshIfNeeded( + now: now, + cacheRoot: root, + client: client) + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["still-unknown"], + now: now, + cacheRoot: root, + client: client) + + #expect(outcome == .unavailable) + #expect(transport.calls == 1) + } + + @Test + func `failed unknown model refresh bounds a following ttl refresh within cooldown`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 47000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1), + cacheRoot: root) + let transport = TrackingTransport(result: .failure(MockError.failed)) + let client = ModelsDevClient(transport: transport) + + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["still-unknown"], + now: now, + cacheRoot: root, + client: client) + await ModelsDevPricingPipeline.refreshIfNeeded( + now: now, + cacheRoot: root, + client: client) + + #expect(outcome == .unavailable) + #expect(transport.calls == 1) + } + + @Test + func `ttl refresh rechecks cache freshness after coordination`() async throws { + let root = try Self.cacheRoot() + let now = Date(timeIntervalSince1970: 48000) + try ModelsDevCache.save( + catalog: Self.fixtureCatalog(), + fetchedAt: now.addingTimeInterval(-ModelsDevCache.ttlSeconds - 1), + cacheRoot: root) + #expect(ModelsDevCache.load(now: now, cacheRoot: root).isStale) + + try ModelsDevCache.save(catalog: Self.fixtureCatalog(), fetchedAt: now, cacheRoot: root) + let transport = TrackingTransport(result: .failure(MockError.failed)) + let cacheIsCurrent = await ModelsDevPricingPipeline.refreshStaleCache( + now: now, + cacheRoot: root, + client: ModelsDevClient(transport: transport)) + + #expect(cacheIsCurrent) + #expect(transport.calls == 0) + } + + @Test + func `failed cache save does not report pricing available`() async { + let root = URL(fileURLWithPath: "/dev/null", isDirectory: true) + let now = Date(timeIntervalSince1970: 50000) + let refreshed = Data(""" + { + "openai": { + "id": "openai", + "models": { "gpt-new": { "id": "gpt-new", "cost": { "input": 2, "output": 8 } } } + }, + "anthropic": { + "id": "anthropic", + "models": { "claude-new": { "id": "claude-new", "cost": { "input": 3, "output": 15 } } } + } + } + """.utf8) + + let outcome = await ModelsDevPricingPipeline.refreshForUnknownModelsIfNeeded( + providerID: "openai", + modelIDs: ["gpt-new"], + now: now, + cacheRoot: root, + client: ModelsDevClient(transport: MockTransport( + result: .success((refreshed, Self.response(status: 200)))))) + + #expect(outcome == .unavailable) + } + @Test func `refresh accepts model churn and preserves removed pricing as fallback`() async throws { let root = try Self.cacheRoot() @@ -1044,15 +1343,25 @@ private struct MockTransport: ModelsDevHTTPTransport { } private final class TrackingTransport: ModelsDevHTTPTransport, @unchecked Sendable { - private(set) var calls = 0 + private let lock = NSLock() + private var callCount = 0 let result: Result<(Data, URLResponse), Error> + let delayNanoseconds: UInt64 + + var calls: Int { + self.lock.withLock { self.callCount } + } - init(result: Result<(Data, URLResponse), Error>) { + init(result: Result<(Data, URLResponse), Error>, delayNanoseconds: UInt64 = 0) { self.result = result + self.delayNanoseconds = delayNanoseconds } func data(for _: URLRequest) async throws -> (Data, URLResponse) { - self.calls += 1 + self.lock.withLock { self.callCount += 1 } + if self.delayNanoseconds > 0 { + try await Task.sleep(nanoseconds: self.delayNanoseconds) + } return try self.result.get() } }