diff --git a/CHANGELOG.md b/CHANGELOG.md index 027b51eaec..e2ec911000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## 0.49.5 — Unreleased +### Fixed +- Codex: keep large Usage & Spend history indexing bounded, durable across relaunches, and resumable after appends without replaying completed sessions (#2815, #2849). Thanks @Quicksaver for the fix and @Yoroin, @xiehaibin18, and @Astro-Han for reports and diagnostics! +- Codex: make Finish now accelerate the full configured cost-history window while retaining the 30-day floor (#2861, #2864). Thanks @thomaschow19 for the report and fix! +- Codex: detect semantic cost-history progress across bounded passes and stop cyclic catch-up without treating live appends as progress (#2815, #2861, #2844). Thanks @pavbar! + ## 0.49.4 — 2026-08-13 ### Added diff --git a/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift b/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift index 39de988e04..d473029996 100644 --- a/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift +++ b/Sources/CodexBar/UsageStore+CodexCostCatchUp.swift @@ -22,7 +22,13 @@ extension UsageStore { if self.codexCostCatchUpTask != nil, self.codexCostCatchUpScopeSignature == scopeSignature { - guard self.codexCostCatchUpMode != mode else { return } + if self.codexCostCatchUpMode == mode { + // Hydration can observe a complete cache while the foreground refresh that follows it + // creates new tail work. Keep one restart queued so that refresh cannot lose the race + // with the existing task's completion cleanup. + self.codexCostCatchUpRestartRequested = true + return + } self.codexCostCatchUpMode = mode // Never cancel a pass while it may be committing a resume checkpoint. The new mode // applies immediately after that bounded pass completes. @@ -53,6 +59,10 @@ extension UsageStore { self.codexCostCatchUpTask = nil self.codexCostCatchUpToken = nil self.codexCostCatchUpScopeSignature = nil + if self.codexCostCatchUpRestartRequested { + self.codexCostCatchUpRestartRequested = false + self.startCodexCostCatchUpIfNeeded(mode: self.codexCostCatchUpMode) + } } } await self.runCodexCostCatchUp(context: context) @@ -66,6 +76,7 @@ extension UsageStore { self.codexCostCatchUpScopeSignature = nil self.codexCostCatchUpStopRequested = false self.codexCostCatchUpPassIsRunning = false + self.codexCostCatchUpRestartRequested = false self.codexCostCatchUpActivity = nil } @@ -80,6 +91,7 @@ extension UsageStore { func stopCodexCostCatchUp() { guard self.codexCostCatchUpTask != nil else { return } self.codexCostCatchUpStopRequested = true + self.codexCostCatchUpRestartRequested = false guard !self.codexCostCatchUpPassIsRunning else { return } if let activity = self.codexCostCatchUpActivity { self.codexCostCatchUpActivity = CodexCostCatchUpActivity( @@ -106,7 +118,8 @@ extension UsageStore { context: context, phase: status.pending ? .indexing : .complete) var didAdvance = false - var previousActiveDuration: TimeInterval? + var (previousActiveDuration, seenProgressKeys): (TimeInterval?, Set) = + (nil, [status.progressKey]) while status.pending { do { guard self.codexCostCatchUpContextIsCurrent(context) else { return } @@ -183,7 +196,7 @@ extension UsageStore { pauseReason: .user) return } - if nextStatus.pending, nextStatus.progressKey == status.progressKey { + if nextStatus.pending, !seenProgressKeys.insert(nextStatus.progressKey).inserted { self.publishCodexCostCatchUpActivity( status: nextStatus, context: context, diff --git a/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift b/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift index 0ee8c911ec..86758961c5 100644 --- a/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift +++ b/Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift @@ -31,13 +31,20 @@ extension UsageStore { return } - let scopeSignature = accounts + let historyDays = max(SpendDashboardSource.scanDays, self.settings.costUsageHistoryDays) + let accountScopeSignature = accounts .map { "\($0.id)|\($0.cacheIdentity)" } .joined(separator: "\u{0}") + let scopeSignature = "\(historyDays)\u{0}\(accountScopeSignature)" if self.spendDashboardCodexCostCatchUpTask != nil, self.spendDashboardCodexCostCatchUpScopeSignature == scopeSignature { - guard self.spendDashboardCodexCostCatchUpMode != mode else { return } + if self.spendDashboardCodexCostCatchUpMode == mode { + // A dashboard reload can discover fresh tail work while the previous task is + // completing. Queue one restart so that the new pending status retains a worker. + self.spendDashboardCodexCostCatchUpRestartRequested = true + return + } self.spendDashboardCodexCostCatchUpMode = mode // A bounded parser pass may be committing a resume checkpoint. Let it finish and // apply the new mode before scheduling the next account instead of cancelling it. @@ -51,7 +58,7 @@ extension UsageStore { let context = SpendDashboardCodexCostCatchUpContext( token: token, accounts: accounts, - historyDays: SpendDashboardSource.scanDays, + historyDays: historyDays, scopeSignature: scopeSignature, providerConfigRevision: self.settings.providerConfigRevision(for: .codex), costUsageSettingsRevision: self.settings.costUsageSettingsRevision) @@ -68,6 +75,12 @@ extension UsageStore { self.spendDashboardCodexCostCatchUpTask = nil self.spendDashboardCodexCostCatchUpToken = nil self.spendDashboardCodexCostCatchUpScopeSignature = nil + if self.spendDashboardCodexCostCatchUpRestartRequested { + self.spendDashboardCodexCostCatchUpRestartRequested = false + self.startSpendDashboardCodexCostCatchUpIfNeeded( + accounts: context.accounts, + mode: self.spendDashboardCodexCostCatchUpMode) + } } } await self.runSpendDashboardCodexCostCatchUp(context: context) @@ -77,6 +90,7 @@ extension UsageStore { func stopSpendDashboardCodexCostCatchUp() { guard self.spendDashboardCodexCostCatchUpTask != nil else { return } self.spendDashboardCodexCostCatchUpStopRequested = true + self.spendDashboardCodexCostCatchUpRestartRequested = false guard !self.spendDashboardCodexCostCatchUpPassIsRunning else { return } if let activity = self.spendDashboardCodexCostCatchUpActivity { self.spendDashboardCodexCostCatchUpActivity = CodexCostCatchUpActivity( @@ -102,6 +116,7 @@ extension UsageStore { self.spendDashboardCodexCostCatchUpScopeSignature = nil self.spendDashboardCodexCostCatchUpStopRequested = false self.spendDashboardCodexCostCatchUpPassIsRunning = false + self.spendDashboardCodexCostCatchUpRestartRequested = false self.spendDashboardCodexCostCatchUpActivity = nil } @@ -117,7 +132,8 @@ extension UsageStore { var didChangeCache = false var previousActiveDuration: TimeInterval? - var stalledCacheIdentities: Set = [] + var (stalledCacheIdentities, seenKeysByCache) = + (Set(), statuses.mapValues { Set([$0.progressKey]) }) while Self.spendDashboardCodexCatchUpIsPending(statuses) { do { guard self.spendDashboardCodexCostCatchUpContextIsCurrent(context) else { return } @@ -197,7 +213,7 @@ extension UsageStore { didChangeCache = didChangeCache || nextStatus.progressKey != previousStatus?.progressKey statuses[account.cacheIdentity] = nextStatus if nextStatus.pending, - nextStatus.progressKey == previousStatus?.progressKey + !seenKeysByCache[account.cacheIdentity, default: []].insert(nextStatus.progressKey).inserted { stalledCacheIdentities.insert(account.cacheIdentity) } else { @@ -245,6 +261,7 @@ extension UsageStore { && self.spendDashboardCodexCostCatchUpScopeSignature == context.scopeSignature && self.settings.providerConfigRevision(for: .codex) == context.providerConfigRevision && self.settings.costUsageSettingsRevision == context.costUsageSettingsRevision + && max(SpendDashboardSource.scanDays, self.settings.costUsageHistoryDays) == context.historyDays && self.settings.isCostUsageEffectivelyEnabled(for: .codex) && self.isEnabled(.codex) && context.accounts.allSatisfy(SpendDashboardSource.codexAuthFingerprintMatches) diff --git a/Sources/CodexBar/UsageStore.swift b/Sources/CodexBar/UsageStore.swift index ef45a3dd4d..c12fcfefbc 100644 --- a/Sources/CodexBar/UsageStore.swift +++ b/Sources/CodexBar/UsageStore.swift @@ -357,12 +357,14 @@ final class UsageStore { @ObservationIgnored var codexCostCatchUpMode: CodexCostCatchUpMode = .automatic @ObservationIgnored var codexCostCatchUpStopRequested = false @ObservationIgnored var codexCostCatchUpPassIsRunning = false + @ObservationIgnored var codexCostCatchUpRestartRequested = false @ObservationIgnored var spendDashboardCodexCostCatchUpTask: Task? @ObservationIgnored var spendDashboardCodexCostCatchUpToken: UUID? @ObservationIgnored var spendDashboardCodexCostCatchUpScopeSignature: String? @ObservationIgnored var spendDashboardCodexCostCatchUpMode: CodexCostCatchUpMode = .automatic @ObservationIgnored var spendDashboardCodexCostCatchUpStopRequested = false @ObservationIgnored var spendDashboardCodexCostCatchUpPassIsRunning = false + @ObservationIgnored var spendDashboardCodexCostCatchUpRestartRequested = false @ObservationIgnored var forcedRefreshEnrichmentTask: Task? @ObservationIgnored var forcedRefreshEnrichmentToken: UUID? @ObservationIgnored var pendingForcedRefreshEnrichmentTask: Task? diff --git a/Sources/CodexBarCore/CostUsageFetcher.swift b/Sources/CodexBarCore/CostUsageFetcher.swift index d2da21b55e..e1d0e74703 100644 --- a/Sources/CodexBarCore/CostUsageFetcher.swift +++ b/Sources/CodexBarCore/CostUsageFetcher.swift @@ -312,19 +312,12 @@ public struct CostUsageFetcher: Sendable { } let scoped = CostUsageScanner.codexCache(cache, scopedTo: roots) - var progressHasher = Hasher() - for (path, usage) in scoped.files.sorted(by: { $0.key < $1.key }) { - progressHasher.combine(path) - progressHasher.combine(usage.codexScanFileId) - progressHasher.combine(usage.parsedBytes) - progressHasher.combine(usage.size) - progressHasher.combine(usage.codexScanComplete) - } + let progressKey = self.codexScanProgressKey(cache: cache, scopedFiles: scoped.files) let hasIncompleteFile = scoped.files.values.contains { $0.codexScanComplete == false } let pending = cache.codexScanCatchUpPending == true || hasIncompleteFile return CodexScanCatchUpStatus( pending: pending, - progressKey: "\(scoped.files.count):\(progressHasher.finalize())", + progressKey: progressKey, processedBytes: cache.codexScanProcessedBytes ?? 0, totalBytes: cache.codexScanTotalBytes ?? 0, completedFiles: cache.codexScanCompletedFiles ?? 0, @@ -1348,6 +1341,110 @@ public struct CostUsageFetcher: Sendable { } extension CostUsageFetcher { + static func codexScanProgressKey( + cache: CostUsageCache, + scopedFiles: [String: CostUsageFileUsage]) -> String + { + var progressHasher = Hasher() + progressHasher.combine(cache.codexScanCompletedFiles) + + for (path, usage) in scopedFiles.sorted(by: { $0.key < $1.key }) { + progressHasher.combine(path) + progressHasher.combine(usage.codexScanFileId) + progressHasher.combine(usage.codexScanComplete) + if usage.codexScanComplete == false { + progressHasher.combine(usage.parsedBytes) + progressHasher.combine(usage.size) + progressHasher.combine(usage.codexJSONLResumeState?.offset) + } + let hasBufferedRetry = usage.hasBufferedCodexForkRetryLines + progressHasher.combine(hasBufferedRetry) + if hasBufferedRetry { + progressHasher.combine(usage.forkedFromId) + progressHasher.combine(usage.forkBaselineDependencyKey) + progressHasher.combine(usage.codexBufferedSubagentLines?.isEmpty == false) + progressHasher.combine(usage.codexBufferedUnresolvedForkLines?.isEmpty == false) + } + } + + if let discovery = cache.codexSessionDiscovery { + progressHasher.combine(discovery.generation) + progressHasher.combine(discovery.directoryPaths.count) + progressHasher.combine(discovery.nextDirectoryIndex) + progressHasher.combine(discovery.filePaths.count) + progressHasher.combine(discovery.nextFileIndex) + progressHasher.combine(discovery.headScan?.path) + progressHasher.combine(discovery.headScan?.offset) + progressHasher.combine(discovery.headScan?.resumeState?.offset) + progressHasher.combine(discovery.filePathBySessionId.count) + progressHasher.combine(discovery.missingSessionIds.sorted()) + progressHasher.combine(discovery.pendingSessionIds.sorted()) + progressHasher.combine(discovery.validationDirectoryIndex) + progressHasher.combine(discovery.isComplete) + } else { + progressHasher.combine("no-discovery") + } + + if let lookback = cache.codexActiveLookbackState { + progressHasher.combine(lookback.scanSinceKey) + progressHasher.combine(lookback.rootPaths.sorted()) + progressHasher.combine("next-day") + for (root, dayKey) in lookback.nextDayKeyByRoot.sorted(by: { $0.key < $1.key }) { + progressHasher.combine(root) + progressHasher.combine(dayKey) + } + progressHasher.combine("next-directory-offset") + progressHasher.combine(lookback.nextDirectoryOffsetByRoot == nil) + for (root, offset) in (lookback.nextDirectoryOffsetByRoot ?? [:]).sorted(by: { $0.key < $1.key }) { + progressHasher.combine(root) + progressHasher.combine(offset) + } + progressHasher.combine(lookback.completedRootPaths.sorted()) + progressHasher.combine(lookback.pendingFilePaths.sorted()) + progressHasher.combine(lookback.legacyRecursivePendingRootPaths.sorted()) + progressHasher.combine("current-window-next-day") + progressHasher.combine(lookback.currentWindowNextDayKeyByRoot == nil) + for (root, dayKey) in (lookback.currentWindowNextDayKeyByRoot ?? [:]).sorted(by: { $0.key < $1.key }) { + progressHasher.combine(root) + progressHasher.combine(dayKey) + } + progressHasher.combine("current-window-directory-offset") + progressHasher.combine(lookback.currentWindowDirectoryOffsetByRoot == nil) + for (root, offset) in (lookback.currentWindowDirectoryOffsetByRoot ?? [:]) + .sorted(by: { $0.key < $1.key }) + { + progressHasher.combine(root) + progressHasher.combine(offset) + } + progressHasher.combine("completed-current-window-roots") + progressHasher.combine(lookback.completedCurrentWindowRootPaths == nil) + progressHasher.combine((lookback.completedCurrentWindowRootPaths ?? []).sorted()) + progressHasher.combine("current-window-flat-directory-offset") + progressHasher.combine(lookback.currentWindowFlatDirectoryOffsetByRoot == nil) + for (root, offset) in (lookback.currentWindowFlatDirectoryOffsetByRoot ?? [:]) + .sorted(by: { $0.key < $1.key }) + { + progressHasher.combine(root) + progressHasher.combine(offset) + } + progressHasher.combine("completed-current-window-flat-roots") + progressHasher.combine(lookback.completedCurrentWindowFlatRootPaths == nil) + progressHasher.combine((lookback.completedCurrentWindowFlatRootPaths ?? []).sorted()) + progressHasher.combine(lookback.cacheWideMigrationQueueActive) + } else { + progressHasher.combine("no-lookback") + } + + if let inventoryPaths = cache.codexScanInventoryPaths { + progressHasher.combine("inventory") + progressHasher.combine(inventoryPaths.sorted()) + } else { + progressHasher.combine("no-inventory") + } + + return "v2:\(scopedFiles.count):\(progressHasher.finalize())" + } + fileprivate static func loadRemoteTokenSnapshot( provider: UsageProvider, environment: [String: String], diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 2510cbdb36..40e7d25b1b 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "47144baa8daccf52" + static let value = "e2899fcb0234e5c1" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCacheModels.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCacheModels.swift index ec1775183c..8484476d6d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCacheModels.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCacheModels.swift @@ -18,6 +18,7 @@ struct CostUsageCache: Codable, Equatable, @unchecked Sendable { var codexScanTotalBytes: Int64? var codexScanCompletedFiles: Int? var codexScanTotalFiles: Int? + var codexScanInventoryPaths: [String]? var codexPreviousReport: CostUsageCodexPreviousReport? var codexSessionDiscovery: CostUsageCodexSessionDiscovery? var codexActiveLookbackState: CostUsageCodexActiveLookbackState? @@ -30,9 +31,16 @@ struct CostUsageCodexActiveLookbackState: Codable, Equatable { var scanSinceKey: String var rootPaths: [String] var nextDayKeyByRoot: [String: String] = [:] + var nextDirectoryOffsetByRoot: [String: Int64]? var completedRootPaths: [String] = [] var pendingFilePaths: [String] = [] var legacyRecursivePendingRootPaths: [String] = [] + var currentWindowNextDayKeyByRoot: [String: String]? + var currentWindowDirectoryOffsetByRoot: [String: Int64]? + var completedCurrentWindowRootPaths: [String]? + var currentWindowFlatDirectoryOffsetByRoot: [String: Int64]? + var completedCurrentWindowFlatRootPaths: [String]? + var cacheWideMigrationQueueActive: Bool? } struct CostUsageCodexSessionDiscovery: Codable, Equatable { diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index cce75ec95c..c36513fe23 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -5,6 +5,11 @@ import Crypto #endif import Dispatch import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif // swiftlint:disable type_body_length file_length enum CostUsageScanner { @@ -13,11 +18,16 @@ enum CostUsageScanner { static let log = CodexBarLog.logger(LogCategories.tokenCost) static let codexActiveSessionLookbackDays = 30 + static let codexCatchUpScanCandidateLimit = 512 static let costScale = 1_000_000_000.0 /// Reserved cache marker. Resolver-produced dependencies use `file|...` or `missing:...`; /// this value records that lineage exists but this rollout owns its counter or suffix. static let codexForkDependencyNotRequiredKey = "mode:lineage-only:v1" + static func resetCodexDirectoryCursorsForTesting() { + self.codexDirectoryCursorRegistry.reset() + } + final class CodexSessionHeadParseObserverStore: @unchecked Sendable { let observer: () -> Void @@ -46,12 +56,29 @@ enum CostUsageScanner { struct CodexScanWorkMetrics: Equatable, Sendable { var usageRowsProcessed: Int var usageRowsRepriced: Int + var cacheAliasEntriesIndexed: Int + var cacheAliasLookups: Int + var cacheAliasCandidatesVisited: Int + var activeLookbackCompletionCandidates: Int + var codexDiscoveryVisits: Int + var codexCandidateSelectionVisits: Int + var codexFileScanAttempts: Int + var codexProgressAccountingVisits: Int } final class CodexScanWorkRecorder: @unchecked Sendable { private let lock = NSLock() private var processed = 0 private var repriced = 0 + private var cacheAliasEntriesIndexed = 0 + private var cacheAliasLookups = 0 + private var cacheAliasCandidatesVisited = 0 + private var activeLookbackCompletionCandidates = 0 + private var codexDiscoveryVisits = 0 + private var codexCandidateSelectionVisits = 0 + private var codexFileScanAttempts = 0 + private var codexFileScanAttemptPaths: Set = [] + private var codexProgressAccountingVisits = 0 func record(processed: Int, repriced: Int) { self.lock.lock() @@ -60,12 +87,68 @@ enum CostUsageScanner { self.lock.unlock() } + func recordCacheAliasIndex(entries: Int) { + self.lock.lock() + self.cacheAliasEntriesIndexed += max(0, entries) + self.lock.unlock() + } + + func recordCacheAliasLookup(candidatesVisited: Int) { + self.lock.lock() + self.cacheAliasLookups += 1 + self.cacheAliasCandidatesVisited += max(0, candidatesVisited) + self.lock.unlock() + } + + func recordActiveLookbackFinalization(completionCandidates: Int) { + self.lock.lock() + self.activeLookbackCompletionCandidates += max(0, completionCandidates) + self.lock.unlock() + } + + func recordCodexDiscoveryVisit() { + self.lock.lock() + self.codexDiscoveryVisits += 1 + self.lock.unlock() + } + + func recordCodexCandidateSelectionVisit() { + self.lock.lock() + self.codexCandidateSelectionVisits += 1 + self.lock.unlock() + } + + func recordCodexFileScanAttempt(path: String) { + self.lock.lock() + self.codexFileScanAttempts += 1 + self.codexFileScanAttemptPaths.insert(path) + self.lock.unlock() + } + + func attemptedCodexFilePaths() -> Set { + self.lock.withLock { self.codexFileScanAttemptPaths } + } + + func recordCodexProgressAccountingVisit() { + self.lock.lock() + self.codexProgressAccountingVisits += 1 + self.lock.unlock() + } + func snapshot() -> CodexScanWorkMetrics { self.lock.lock() defer { self.lock.unlock() } return CodexScanWorkMetrics( usageRowsProcessed: self.processed, - usageRowsRepriced: self.repriced) + usageRowsRepriced: self.repriced, + cacheAliasEntriesIndexed: self.cacheAliasEntriesIndexed, + cacheAliasLookups: self.cacheAliasLookups, + cacheAliasCandidatesVisited: self.cacheAliasCandidatesVisited, + activeLookbackCompletionCandidates: self.activeLookbackCompletionCandidates, + codexDiscoveryVisits: self.codexDiscoveryVisits, + codexCandidateSelectionVisits: self.codexCandidateSelectionVisits, + codexFileScanAttempts: self.codexFileScanAttempts, + codexProgressAccountingVisits: self.codexProgressAccountingVisits) } } @@ -209,6 +292,10 @@ enum CostUsageScanner { } return true } + + func shouldStopBeforeNextFile() -> Bool { + self.shouldYield(additionalBytes: 1) + } } struct CodexParseResult { @@ -765,12 +852,54 @@ enum CostUsageScanner { struct CodexScanResources { let fileIndex: CodexSessionFileIndex let inheritedResolver: CodexInheritedTotalsResolver + let cachePathAliasIndex: CodexCachePathAliasIndex let projectPathResolver: CodexCanonicalProjectPathResolver let modelsDevCatalog: ModelsDevCatalog? let modelsDevCacheRoot: URL? let priorityTurns: [String: CodexPriorityTurnMetadata] } + final class CodexCachePathAliasIndex { + private var pathsByFileID: [String: Set] = [:] + private var fileIDByPath: [String: String] = [:] + private let workRecorder: CodexScanWorkRecorder? + + init(files: [String: CostUsageFileUsage], workRecorder: CodexScanWorkRecorder? = nil) { + self.workRecorder = workRecorder + var indexedEntries = 0 + for (path, usage) in files { + guard let fileID = usage.codexScanFileId else { continue } + self.pathsByFileID[fileID, default: []].insert(path) + self.fileIDByPath[path] = fileID + indexedEntries += 1 + } + workRecorder?.recordCacheAliasIndex(entries: indexedEntries) + } + + func aliases(fileID: String, excludingPath path: String) -> [String] { + let candidates = self.pathsByFileID[fileID] ?? [] + self.workRecorder?.recordCacheAliasLookup(candidatesVisited: candidates.count) + return candidates.filter { $0 != path }.sorted() + } + + func update(path: String, fileID: String?) { + if let previousFileID = self.fileIDByPath[path], previousFileID != fileID { + self.pathsByFileID[previousFileID]?.remove(path) + if self.pathsByFileID[previousFileID]?.isEmpty == true { + self.pathsByFileID.removeValue(forKey: previousFileID) + } + self.fileIDByPath.removeValue(forKey: path) + } + guard let fileID else { return } + self.pathsByFileID[fileID, default: []].insert(path) + self.fileIDByPath[path] = fileID + } + + func remove(path: String) { + self.update(path: path, fileID: nil) + } + } + struct CodexFileScanContext { let range: CostUsageDayRange let forceFullScan: Bool @@ -878,6 +1007,7 @@ enum CostUsageScanner { let rootsFingerprint: [String: Int64] let rootsChanged: Bool let windowExpanded: Bool + let needsPricingMetadataMigration: Bool let needsProjectMetadataMigration: Bool let modelsDevCatalog: ModelsDevCatalog? let codexPricingKey: String @@ -890,6 +1020,9 @@ enum CostUsageScanner { let priorityTurnsChanged: Bool let needsTurnIDCacheMigration: Bool let changedPriorityTurnIDs: Set + let requiresAllFilesForCacheWideMigration: Bool + let cacheWideMigrationPendingPathKeys: Set + let requiresCacheWideFileReprocessing: Bool let shouldRefresh: Bool } @@ -912,6 +1045,8 @@ enum CostUsageScanner { private let scanBudget: CodexScanBudget? private let headParseObserver: (() -> Void)? private var discovery: CostUsageCodexSessionDiscovery + private var knownFilePaths: Set = [] + private var knownDirectoryPaths: Set = [] init( files: [URL], @@ -933,6 +1068,8 @@ enum CostUsageScanner { cachedDiscovery.filePathBySessionId[sessionId] = fileURL.standardizedFileURL.path } self.discovery = cachedDiscovery + self.knownFilePaths = Set(cachedDiscovery.filePaths) + self.knownDirectoryPaths = Set(cachedDiscovery.directoryPaths) if !cachedDiscovery.isComplete { self.enqueueCurrentFiles() } @@ -942,6 +1079,8 @@ enum CostUsageScanner { files: files, cachedSessionFiles: cachedSessionFiles, retaining: nil) + self.knownFilePaths = Set(self.discovery.filePaths) + self.knownDirectoryPaths = Set(self.discovery.directoryPaths) } } @@ -984,6 +1123,8 @@ enum CostUsageScanner { files: self.files, cachedSessionFiles: self.cachedSessionFiles(), retaining: self.discovery) + self.knownFilePaths = Set(self.discovery.filePaths) + self.knownDirectoryPaths = Set(self.discovery.directoryPaths) case .deferred: return .deferred } @@ -1154,13 +1295,13 @@ enum CostUsageScanner { private func enqueueFile(_ fileURL: URL) { let path = fileURL.standardizedFileURL.path - guard !self.discovery.filePaths.contains(path) else { return } + guard self.knownFilePaths.insert(path).inserted else { return } self.discovery.filePaths.append(path) } private func enqueueDirectory(_ directoryURL: URL) { let path = directoryURL.standardizedFileURL.path - guard !self.discovery.directoryPaths.contains(path) else { return } + guard self.knownDirectoryPaths.insert(path).inserted else { return } self.discovery.directoryPaths.append(path) } @@ -1879,8 +2020,8 @@ enum CostUsageScanner { let recursive = includeRecursive ? self.listCodexLegacySessionFilesRecursive(root: root) : [] var seen: Set = [] var out: [URL] = [] - for item in partitioned + flat + recursive where !seen.contains(item.path) { - seen.insert(item.path) + for item in partitioned + flat + recursive where !seen.contains(Self.codexPathKey(item)) { + seen.insert(Self.codexPathKey(item)) out.append(item) } return out @@ -1893,11 +2034,12 @@ enum CostUsageScanner { excludingPaths: Set) -> [URL] { cache.files.compactMap { path, usage in - guard !excludingPaths.contains(path) else { return nil } + guard !excludingPaths.contains(Self.codexPathKey(URL(fileURLWithPath: path))) else { return nil } let hasRelevantDay = usage.days.keys.contains { CostUsageDayRange.isInRange(dayKey: $0, since: range.scanSinceKey, until: range.scanUntilKey) } - guard hasRelevantDay else { return nil } + let hasPendingWork = usage.codexScanComplete == false || usage.hasBufferedCodexForkRetryLines + guard hasRelevantDay || hasPendingWork else { return nil } guard FileManager.default.fileExists(atPath: path) else { return nil } let fileURL = URL(fileURLWithPath: path) guard Self.isWithinCodexRoots(fileURL: fileURL, roots: roots) else { return nil } @@ -1913,7 +2055,7 @@ enum CostUsageScanner { var out: [String: URL] = [:] for (path, usage) in cache.files { guard let sessionId = usage.sessionId, !sessionId.isEmpty else { continue } - if knownExistingPaths.contains(path) { + if knownExistingPaths.contains(Self.codexPathKey(URL(fileURLWithPath: path))) { out[sessionId] = URL(fileURLWithPath: path) continue } @@ -2250,12 +2392,203 @@ enum CostUsageScanner { return path } + private static func codexPathKey(_ url: URL) -> String { + let path = url.standardizedFileURL.path + if path.hasPrefix("/private/var/") { + return String(path.dropFirst("/private".count)) + } + return path + } + private struct CodexDatePartitionListing { let files: [URL] let isComplete: Bool let nextDayKey: String? } + private struct CodexDirectoryPage { + let files: [URL] + let nextOffset: Int64? + let visits: Int + } + + private struct CodexPartitionPage { + let files: [URL] + let nextDayKey: String? + let nextDirectoryOffset: Int64? + let visits: Int + + var isComplete: Bool { + self.nextDayKey == nil && self.nextDirectoryOffset == nil + } + } + + #if os(Linux) + private typealias CodexDirectoryHandle = OpaquePointer + #else + private typealias CodexDirectoryHandle = UnsafeMutablePointer + #endif + + private final class CodexDirectoryCursor: @unchecked Sendable { + let directory: CodexDirectoryHandle + var logicalOffset: Int64 + + init(directory: CodexDirectoryHandle, logicalOffset: Int64 = 0) { + self.directory = directory + self.logicalOffset = logicalOffset + } + + deinit { + closedir(self.directory) + } + } + + private final class CodexDirectoryCursorRegistry: @unchecked Sendable { + private let lock = NSLock() + private var cursors: [String: CodexDirectoryCursor] = [:] + + func page( + directoryURL: URL, + resumeOffset: Int64, + visitLimit: Int, + filter: (String) -> Bool, + workRecorder: CodexScanWorkRecorder?) -> CodexDirectoryPage + { + self.lock.lock() + defer { self.lock.unlock() } + + let path = directoryURL.path + let resumeOffset = max(0, resumeOffset) + if resumeOffset == 0 || (self.cursors[path]?.logicalOffset ?? 0) > resumeOffset { + self.cursors.removeValue(forKey: path) + } + if self.cursors[path] == nil { + guard let directory = opendir(path) else { + return CodexDirectoryPage(files: [], nextOffset: nil, visits: 0) + } + self.cursors[path] = CodexDirectoryCursor(directory: directory) + } + guard let cursor = self.cursors[path] else { + return CodexDirectoryPage(files: [], nextOffset: nil, visits: 0) + } + + var files: [URL] = [] + var visits = 0 + while visits < visitLimit { + guard let entry = readdir(cursor.directory) else { + self.cursors.removeValue(forKey: path) + return CodexDirectoryPage(files: files, nextOffset: nil, visits: visits) + } + let name = withUnsafePointer(to: entry.pointee.d_name) { pointer in + pointer.withMemoryRebound(to: CChar.self, capacity: 1024) { String(cString: $0) } + } + guard name != ".", name != ".." else { continue } + cursor.logicalOffset += 1 + visits += 1 + workRecorder?.recordCodexDiscoveryVisit() + guard cursor.logicalOffset > resumeOffset, filter(name) else { continue } + files.append(directoryURL.appendingPathComponent(name, isDirectory: false)) + } + return CodexDirectoryPage( + files: files, + nextOffset: max(resumeOffset, cursor.logicalOffset), + visits: visits) + } + + func reset() { + self.lock.lock() + self.cursors.removeAll() + self.lock.unlock() + } + } + + private static let codexDirectoryCursorRegistry = CodexDirectoryCursorRegistry() + + private static func listCodexDirectoryPage( + directoryURL: URL, + resumeOffset: Int64, + visitLimit: Int, + filter: (String) -> Bool, + workRecorder: CodexScanWorkRecorder?) -> CodexDirectoryPage + { + guard visitLimit > 0 else { + return CodexDirectoryPage(files: [], nextOffset: max(0, resumeOffset), visits: 0) + } + return self.codexDirectoryCursorRegistry.page( + directoryURL: directoryURL, + resumeOffset: resumeOffset, + visitLimit: visitLimit, + filter: filter, + workRecorder: workRecorder) + } + + // swiftlint:disable:next function_parameter_count + private static func listCodexSessionFilesByDatePartitionPage( + root: URL, + scanSinceKey: String, + scanUntilKey: String, + resumeDayKey: String?, + resumeDirectoryOffset: Int64, + visitLimit: Int, + preferNewest: Bool, + calendar: Calendar, + workRecorder: CodexScanWorkRecorder?) -> CodexPartitionPage + { + guard FileManager.default.fileExists(atPath: root.path) else { + return CodexPartitionPage(files: [], nextDayKey: nil, nextDirectoryOffset: nil, visits: 0) + } + let calendar = CostUsageDayRange.localGregorianCalendar(matching: calendar) + let sinceDate = Self.parseDayKey(scanSinceKey, calendar: calendar) ?? Date() + let untilDate = Self.parseDayKey(scanUntilKey, calendar: calendar) ?? sinceDate + let resumedDate = resumeDayKey.flatMap { Self.parseDayKey($0, calendar: calendar) } + var date = if let resumedDate, resumedDate >= sinceDate, resumedDate <= untilDate { + resumedDate + } else { + preferNewest ? untilDate : sinceDate + } + var directoryOffset = max(0, resumeDirectoryOffset) + var remainingVisits = max(0, visitLimit) + var totalVisits = 0 + var files: [URL] = [] + + while date >= sinceDate, date <= untilDate { + guard remainingVisits > 0 else { + return CodexPartitionPage( + files: files, + nextDayKey: CostUsageDayRange.dayKey(from: date, calendar: calendar), + nextDirectoryOffset: directoryOffset, + visits: totalVisits) + } + let comps = calendar.dateComponents([.year, .month, .day], from: date) + let dayDirectory = root + .appendingPathComponent(String(format: "%04d", comps.year ?? 1970), isDirectory: true) + .appendingPathComponent(String(format: "%02d", comps.month ?? 1), isDirectory: true) + .appendingPathComponent(String(format: "%02d", comps.day ?? 1), isDirectory: true) + let page = Self.listCodexDirectoryPage( + directoryURL: dayDirectory, + resumeOffset: directoryOffset, + visitLimit: remainingVisits, + filter: { $0.lowercased().hasSuffix(".jsonl") }, + workRecorder: workRecorder) + files.append(contentsOf: page.files) + totalVisits += page.visits + remainingVisits -= page.visits + if let nextOffset = page.nextOffset { + return CodexPartitionPage( + files: files, + nextDayKey: CostUsageDayRange.dayKey(from: date, calendar: calendar), + nextDirectoryOffset: nextOffset, + visits: totalVisits) + } + directoryOffset = 0 + guard let nextDate = calendar.date(byAdding: .day, value: preferNewest ? -1 : 1, to: date) else { + break + } + date = nextDate + } + return CodexPartitionPage(files: files, nextDayKey: nil, nextDirectoryOffset: nil, visits: totalVisits) + } + private static func listCodexSessionFilesByDatePartition( root: URL, scanSinceKey: String, @@ -2332,12 +2665,113 @@ enum CostUsageScanner { { return cached } + let retainedPendingFilePaths = cache.codexScanCatchUpPending == true + ? cache.codexActiveLookbackState?.pendingFilePaths ?? [] + : [] return CostUsageCodexActiveLookbackState( scanSinceKey: scanSinceKey, rootPaths: rootPaths, + pendingFilePaths: retainedPendingFilePaths, legacyRecursivePendingRootPaths: includeLegacyRecursiveScan ? rootPaths : []) } + private static func codexBoundedDiscoveryIsComplete( + _ state: CostUsageCodexActiveLookbackState) -> Bool + { + let rootPaths = Set(state.rootPaths) + return Set(state.completedRootPaths) == rootPaths + && Set(state.completedCurrentWindowRootPaths ?? []) == rootPaths + && Set(state.completedCurrentWindowFlatRootPaths ?? []) == rootPaths + && state.pendingFilePaths.isEmpty + } + + // swiftlint:disable:next function_parameter_count + private static func advanceCodexCurrentWindow( + root: URL, + range: CostUsageDayRange, + preferNewest: Bool, + remainingDiscoveryVisits: inout Int, + excludedPendingPathKeys: Set, + workRecorder: CodexScanWorkRecorder?, + state: inout CostUsageCodexActiveLookbackState) + { + let rootPath = Self.codexResolvedPath(root) + state.currentWindowNextDayKeyByRoot = state.currentWindowNextDayKeyByRoot ?? [:] + state.currentWindowDirectoryOffsetByRoot = state.currentWindowDirectoryOffsetByRoot ?? [:] + state.currentWindowFlatDirectoryOffsetByRoot = state.currentWindowFlatDirectoryOffsetByRoot ?? [:] + var completedPartitionRoots = Set(state.completedCurrentWindowRootPaths ?? []) + var completedFlatRoots = Set(state.completedCurrentWindowFlatRootPaths ?? []) + var discoveredFilePaths: [String] = [] + + if !completedPartitionRoots.contains(rootPath), remainingDiscoveryVisits > 0 { + let page = Self.listCodexSessionFilesByDatePartitionPage( + root: root, + scanSinceKey: range.scanSinceKey, + scanUntilKey: range.scanUntilKey, + resumeDayKey: state.currentWindowNextDayKeyByRoot?[rootPath], + resumeDirectoryOffset: state.currentWindowDirectoryOffsetByRoot?[rootPath] ?? 0, + visitLimit: remainingDiscoveryVisits, + preferNewest: preferNewest, + calendar: range.calendar, + workRecorder: workRecorder) + remainingDiscoveryVisits -= page.visits + discoveredFilePaths.append(contentsOf: page.files.compactMap { fileURL in + let path = Self.codexResolvedPath(fileURL) + return excludedPendingPathKeys.contains(Self.codexPathKey(URL(fileURLWithPath: path))) + ? nil + : path + }) + if page.isComplete { + completedPartitionRoots.insert(rootPath) + state.currentWindowNextDayKeyByRoot?.removeValue(forKey: rootPath) + state.currentWindowDirectoryOffsetByRoot?.removeValue(forKey: rootPath) + } else { + state.currentWindowNextDayKeyByRoot?[rootPath] = page.nextDayKey + state.currentWindowDirectoryOffsetByRoot?[rootPath] = page.nextDirectoryOffset + } + } + + if completedPartitionRoots.contains(rootPath), + !completedFlatRoots.contains(rootPath), + remainingDiscoveryVisits > 0 + { + let page = Self.listCodexDirectoryPage( + directoryURL: root, + resumeOffset: state.currentWindowFlatDirectoryOffsetByRoot?[rootPath] ?? 0, + visitLimit: remainingDiscoveryVisits, + filter: { name in + guard name.lowercased().hasSuffix(".jsonl") else { return false } + guard let dayKey = Self.dayKeyFromFilename(name) else { return true } + return CostUsageDayRange.isInRange( + dayKey: dayKey, + since: range.scanSinceKey, + until: range.scanUntilKey) + }, + workRecorder: workRecorder) + remainingDiscoveryVisits -= page.visits + discoveredFilePaths.append(contentsOf: page.files.compactMap { fileURL in + let path = Self.codexResolvedPath(fileURL) + return excludedPendingPathKeys.contains(Self.codexPathKey(URL(fileURLWithPath: path))) + ? nil + : path + }) + if let nextOffset = page.nextOffset { + state.currentWindowFlatDirectoryOffsetByRoot?[rootPath] = nextOffset + } else { + completedFlatRoots.insert(rootPath) + state.currentWindowFlatDirectoryOffsetByRoot?.removeValue(forKey: rootPath) + } + } + + if !discoveredFilePaths.isEmpty { + var pendingFilePaths = Set(state.pendingFilePaths) + pendingFilePaths.formUnion(discoveredFilePaths) + state.pendingFilePaths = pendingFilePaths.sorted() + } + state.completedCurrentWindowRootPaths = completedPartitionRoots.sorted() + state.completedCurrentWindowFlatRootPaths = completedFlatRoots.sorted() + } + private static func advanceCodexActiveLookback( root: URL, range: CostUsageDayRange, @@ -2347,7 +2781,6 @@ enum CostUsageScanner { { let rootPath = Self.codexResolvedPath(root) var completedRootPaths = Set(state.completedRootPaths) - var pendingFilePaths = Set(state.pendingFilePaths) if !completedRootPaths.contains(rootPath) { let listing = Self.listCodexRecentlyModifiedPartitionFiles( root: root, @@ -2356,7 +2789,7 @@ enum CostUsageScanner { scanBudget: scanBudget, resumeDayKey: state.nextDayKeyByRoot[rootPath], calendar: range.calendar) - pendingFilePaths.formUnion(listing.files.map(Self.codexResolvedPath)) + Self.appendCodexActiveLookbackPaths(listing.files, state: &state) if listing.isComplete { completedRootPaths.insert(rootPath) state.nextDayKeyByRoot.removeValue(forKey: rootPath) @@ -2372,53 +2805,298 @@ enum CostUsageScanner { let legacy = Self.listCodexRecentlyModifiedFilesRecursive( root: root, modifiedSince: modifiedSince) - pendingFilePaths.formUnion(legacy.map(Self.codexResolvedPath)) + Self.appendCodexActiveLookbackPaths(legacy, state: &state) } state.completedRootPaths = completedRootPaths.sorted() - state.pendingFilePaths = pendingFilePaths.sorted() state.legacyRecursivePendingRootPaths = legacyPendingRoots.sorted() } + // swiftlint:disable:next function_parameter_count + private static func advanceCodexActiveLookbackPage( + root: URL, + range: CostUsageDayRange, + modifiedSince: Date, + preferNewest: Bool, + remainingDiscoveryVisits: inout Int, + excludedPendingPathKeys: Set, + workRecorder: CodexScanWorkRecorder?, + state: inout CostUsageCodexActiveLookbackState) + { + let rootPath = Self.codexResolvedPath(root) + var completedRootPaths = Set(state.completedRootPaths) + guard !completedRootPaths.contains(rootPath), remainingDiscoveryVisits > 0 else { return } + state.nextDirectoryOffsetByRoot = state.nextDirectoryOffsetByRoot ?? [:] + let lookbackSinceKey = Self.dayKey( + range.scanSinceKey, + addingDays: -Self.codexActiveSessionLookbackDays, + calendar: range.calendar) ?? range.scanSinceKey + let lookbackUntilKey = Self.dayKey( + range.scanSinceKey, + addingDays: -1, + calendar: range.calendar) ?? lookbackSinceKey + let page = Self.listCodexSessionFilesByDatePartitionPage( + root: root, + scanSinceKey: lookbackSinceKey, + scanUntilKey: lookbackUntilKey, + resumeDayKey: state.nextDayKeyByRoot[rootPath], + resumeDirectoryOffset: state.nextDirectoryOffsetByRoot?[rootPath] ?? 0, + visitLimit: remainingDiscoveryVisits, + preferNewest: preferNewest, + calendar: range.calendar, + workRecorder: workRecorder) + remainingDiscoveryVisits -= page.visits + let discoveredFilePaths = Self.filterRecentlyModified( + files: page.files, + modifiedSince: modifiedSince).compactMap { fileURL in + let path = Self.codexResolvedPath(fileURL) + return excludedPendingPathKeys.contains(Self.codexPathKey(URL(fileURLWithPath: path))) + ? nil + : path + } + if !discoveredFilePaths.isEmpty { + var pendingFilePaths = Set(state.pendingFilePaths) + pendingFilePaths.formUnion(discoveredFilePaths) + state.pendingFilePaths = pendingFilePaths.sorted() + } + if page.isComplete { + completedRootPaths.insert(rootPath) + state.nextDayKeyByRoot.removeValue(forKey: rootPath) + state.nextDirectoryOffsetByRoot?.removeValue(forKey: rootPath) + } else { + state.nextDayKeyByRoot[rootPath] = page.nextDayKey + state.nextDirectoryOffsetByRoot?[rootPath] = page.nextDirectoryOffset + } + state.completedRootPaths = completedRootPaths.sorted() + } + + private static func appendCodexActiveLookbackPaths( + _ files: some Sequence, + normalizeExisting: Bool = false, + state: inout CostUsageCodexActiveLookbackState) + { + let files = Array(files) + guard !files.isEmpty else { return } + var queuedPaths: Set + if normalizeExisting { + var normalizedPaths: Set = [] + state.pendingFilePaths = state.pendingFilePaths.compactMap { path in + let resolvedPath = Self.codexResolvedPath(URL(fileURLWithPath: path)) + return normalizedPaths.insert(resolvedPath).inserted ? resolvedPath : nil + } + queuedPaths = normalizedPaths + } else { + queuedPaths = Set(state.pendingFilePaths) + } + for fileURL in files { + let resolvedPath = Self.codexResolvedPath(fileURL) + guard queuedPaths.insert(resolvedPath).inserted else { continue } + state.pendingFilePaths.append(resolvedPath) + } + } + + private struct CodexActiveLookbackQueueUpdateContext { + let seedFiles: [URL] + let migrationSeedPathKeys: [String]? + let discoveredFiles: [URL] + let previousDiscovery: CostUsageCodexSessionDiscovery? + let shouldBoundCatchUp: Bool + let shouldSeedBoundedQueue: Bool + } + + private static func seedOrExtendCodexActiveLookbackQueue( + context: CodexActiveLookbackQueueUpdateContext, + state: inout CostUsageCodexActiveLookbackState) + { + guard context.shouldBoundCatchUp else { return } + if context.shouldSeedBoundedQueue { + if let migrationSeedPathKeys = context.migrationSeedPathKeys { + self.reseedCodexActiveLookbackPathKeys(migrationSeedPathKeys, state: &state) + } else { + self.appendCodexActiveLookbackPaths( + context.seedFiles, + normalizeExisting: true, + state: &state) + } + return + } + guard let previousDiscovery = context.previousDiscovery else { return } + let previousPaths = Set(previousDiscovery.fileStamps.keys.map { + Self.codexResolvedPath(URL(fileURLWithPath: $0)) + }) + let newFiles = context.discoveredFiles.filter { !previousPaths.contains(Self.codexResolvedPath($0)) } + Self.appendCodexActiveLookbackPaths(newFiles, state: &state) + } + + private static func reseedCodexActiveLookbackPathKeys( + _ pathKeys: some Sequence, + state: inout CostUsageCodexActiveLookbackState) + { + var queuedPaths: Set = [] + var reseededPaths: [String] = [] + func append(_ path: String) { + let pathKey = Self.codexPathKey(URL(fileURLWithPath: path)) + guard queuedPaths.insert(pathKey).inserted else { return } + reseededPaths.append(pathKey) + } + for path in pathKeys { + append(path) + } + for path in state.pendingFilePaths { + append(path) + } + state.pendingFilePaths = reseededPaths + } + + private static func cacheWideMigrationNeedsQueueReseed( + plan: CodexRefreshPlan, + inventoryPathKeys: Set, + state: CostUsageCodexActiveLookbackState) -> Bool + { + guard plan.requiresCacheWideFileReprocessing else { return false } + let queuedPathKeys = Set(state.pendingFilePaths.map { + Self.codexPathKey(URL(fileURLWithPath: $0)) + }) + let requiredPathKeys = plan.requiresAllFilesForCacheWideMigration + ? inventoryPathKeys + : plan.cacheWideMigrationPendingPathKeys.intersection(inventoryPathKeys) + return !requiredPathKeys.isSubset(of: queuedPathKeys) + } + + private struct CodexPendingLookbackAppendContext { + let roots: [URL] + let maxCount: Int? + let validateRoots: Bool + } + private static func appendPendingCodexActiveLookbackFiles( state: inout CostUsageCodexActiveLookbackState, - roots: [URL], + context: CodexPendingLookbackAppendContext, seenPaths: inout Set, - files: inout [URL]) + fileURLsByPathKey: inout [String: URL], + files: inout [URL]) -> Int { - state.pendingFilePaths = state.pendingFilePaths.filter { path in - FileManager.default.fileExists(atPath: path) - && Self.isWithinCodexRoots(fileURL: URL(fileURLWithPath: path), roots: roots) - } - var seenFileIDs = Set(files.compactMap { Self.codexFileMetadata(fileURL: $0).fileId }) - for path in state.pendingFilePaths where !seenPaths.contains(path) { - let fileID = Self.codexFileMetadata(fileURL: URL(fileURLWithPath: path)).fileId - if let fileID, !seenFileIDs.insert(fileID).inserted { - continue + if context.validateRoots { + state.pendingFilePaths = state.pendingFilePaths.filter { path in + Self.isWithinCodexRoots(fileURL: URL(fileURLWithPath: path), roots: context.roots) } - seenPaths.insert(path) - files.append(URL(fileURLWithPath: path)) } + let pendingCount = min(context.maxCount ?? state.pendingFilePaths.count, state.pendingFilePaths.count) + var normalizedPathSet: Set = [] + let normalizedPrefix = state.pendingFilePaths.prefix(pendingCount).compactMap { path in + let resolvedPath = Self.codexResolvedPath(URL(fileURLWithPath: path)) + return normalizedPathSet.insert(resolvedPath).inserted ? resolvedPath : nil + } + state.pendingFilePaths.replaceSubrange(0.. CodexRefreshCandidateSelection + { + guard context.shouldBoundCatchUp else { + return CodexRefreshCandidateSelection( + files: context.preferNewest ? self.sortedCodexSessionFilesNewestFirst(files) : files, + exhaustedVisitBudget: false) + } + + let candidateLimit = Self.codexCatchUpScanCandidateLimit + var candidates: [URL] = [] + candidates.reserveCapacity(candidateLimit) + var selectionVisits = 0 + + func appendPendingCandidate(path: String) { + guard selectionVisits < candidateLimit else { return } + selectionVisits += 1 + context.workRecorder?.recordCodexCandidateSelectionVisit() + let pendingURL = URL(fileURLWithPath: path) + let pathKey = Self.codexPathKey(pendingURL) + candidates.append(context.fileURLsByPathKey[pathKey] ?? pendingURL) + } + + let pendingPaths = activeLookbackState.pendingFilePaths.prefix(context.boundedQueuePathCount) + for path in pendingPaths { + appendPendingCandidate(path: path) + if selectionVisits == candidateLimit { + break + } + } + return CodexRefreshCandidateSelection( + files: context.preferNewest ? self.sortedCodexSessionFilesNewestFirst(candidates) : candidates, + exhaustedVisitBudget: activeLookbackState.pendingFilePaths.count > candidates.count) } + // swiftlint:disable:next function_parameter_count private static func finalizedCodexActiveLookbackState( _ state: CostUsageCodexActiveLookbackState, - cache: CostUsageCache) -> CostUsageCodexActiveLookbackState? + completedFilePaths: Set, + completionCandidateCount: Int, + requiresBoundedDiscoveryCompletion: Bool, + retainCompletedStateForExactValidation: Bool, + workRecorder: CodexScanWorkRecorder?) -> CostUsageCodexActiveLookbackState? { var state = state - state.pendingFilePaths.removeAll { path in - guard FileManager.default.fileExists(atPath: path) else { return true } - let fileID = Self.codexFileMetadata(fileURL: URL(fileURLWithPath: path)).fileId - if let fileID { - return cache.files.values.contains { - $0.codexScanFileId == fileID && $0.codexScanComplete == true - } - } - return cache.files[path]?.codexScanComplete == true - } - let isComplete = Set(state.completedRootPaths) == Set(state.rootPaths) + workRecorder?.recordActiveLookbackFinalization(completionCandidates: completedFilePaths.count) + let prefixCount = min(completionCandidateCount, state.pendingFilePaths.count) + let retainedPrefix = state.pendingFilePaths.prefix(prefixCount).filter { path in + completedFilePaths.contains(path) + == false + } + state.pendingFilePaths.replaceSubrange(0.., + attemptedPaths: Set, + cache: CostUsageCache) -> Set + { + Set(scheduledFiles.compactMap { fileURL -> String? in + guard attemptedPaths.contains(fileURL.path) else { return nil } + let resolvedPath = Self.codexResolvedPath(fileURL) + guard pendingPaths.contains(resolvedPath) else { return nil } + let metadata = Self.codexFileMetadata(fileURL: fileURL) + if metadata.fileId == nil, !FileManager.default.fileExists(atPath: fileURL.path) { + return resolvedPath + } + guard let usage = cache.files[fileURL.path], + usage.codexScanComplete == true, + !usage.hasBufferedCodexForkRetryLines, + usage.codexScanFileId == metadata.fileId, + usage.mtimeUnixMs == metadata.mtimeUnixMs, + usage.size == metadata.size + else { return nil } + return resolvedPath + }) } private static func listCodexSessionFilesFlat(root: URL, scanSinceKey: String, scanUntilKey: String) -> [URL] { @@ -4238,11 +4916,19 @@ enum CostUsageScanner { { try context.checkCancellation?() let metadata = Self.codexFileMetadata(fileURL: fileURL) + defer { + context.resources.cachePathAliasIndex.update( + path: metadata.path, + fileID: cache.files[metadata.path]?.codexScanFileId) + } if let fileId = metadata.fileId, state.seenFileIds.contains(fileId) { Self.dropCachedCodexFile(path: metadata.path, cached: cache.files[metadata.path], cache: &cache) return } - Self.reconcileCodexCachePathAliases(metadata: metadata, cache: &cache) + Self.reconcileCodexCachePathAliases( + metadata: metadata, + cache: &cache, + aliasIndex: context.resources.cachePathAliasIndex) let cached = cache.files[metadata.path] @@ -4363,23 +5049,30 @@ enum CostUsageScanner { let rootsFingerprint = Self.codexRootsFingerprint(roots) let rootsChanged = cache.roots != rootsFingerprint let windowExpanded = Self.requestedWindowExpandsCache(range: range, cache: cache) - let needsPricingMetadataMigration = cache.files.values.contains { - Self.needsCodexPricingMetadata($0, range: range) - } + let pricingMetadataMigrationPathKeys = Set(cache.files.compactMap { path, usage in + Self.needsCodexPricingMetadata(usage, range: range) + ? Self.codexPathKey(URL(fileURLWithPath: path)) + : nil + }) + let needsPricingMetadataMigration = !pricingMetadataMigrationPathKeys.isEmpty let needsProjectMetadataMigration = cache.codexProjectMetadataVersion != Self.codexProjectMetadataVersion let modelsDevLoad = ModelsDevCache.load(now: now, cacheRoot: options.cacheRoot) let modelsDevCatalog = modelsDevLoad.artifact?.catalog let codexPricingKey = Self.codexPricingKey(modelsDevArtifact: modelsDevLoad.artifact) + let pricingKeyChanged = cache.codexPricingKey != codexPricingKey let codexPriorityMetadataKey = Self.codexPriorityMetadataKey(databaseURL: options.codexTraceDatabaseURL) let hasPriorityMetadata = codexPriorityMetadataKey.hasPrefix("sqlite:") let priorityMetadataChanged = Self.codexPriorityMetadataChanged( old: cache.codexPriorityMetadataKey, new: codexPriorityMetadataKey) - let needsTurnIDCacheMigration = hasPriorityMetadata && cache.files.values.contains { - $0.codexTurnIDs == nil && $0.touchesCodexScanWindow( + let turnIDCacheMigrationPathKeys = hasPriorityMetadata ? Set(cache.files.compactMap { path, usage in + usage.codexTurnIDs == nil && usage.touchesCodexScanWindow( sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) - } + ? Self.codexPathKey(URL(fileURLWithPath: path)) + : nil + }) : [] + let needsTurnIDCacheMigration = !turnIDCacheMigrationPathKeys.isEmpty let shouldInspectPriorityTurns = options.forceRescan || windowExpanded || rootsChanged @@ -4410,10 +5103,20 @@ enum CostUsageScanner { newKeys: priorityTurnKeys, range: range) : [] + let requiresAllFilesForCacheWideMigration = !cache.files.isEmpty + && (pricingKeyChanged + || needsProjectMetadataMigration + || priorityMetadataChanged + || priorityTurnsChanged) + let cacheWideMigrationPendingPathKeys = pricingMetadataMigrationPathKeys + .union(turnIDCacheMigrationPathKeys) + let requiresCacheWideFileReprocessing = requiresAllFilesForCacheWideMigration + || !cacheWideMigrationPendingPathKeys.isEmpty let shouldRefresh = options.forceRescan || windowExpanded || rootsChanged || needsPricingMetadataMigration + || pricingKeyChanged || needsProjectMetadataMigration || needsTurnIDCacheMigration || priorityMetadataChanged @@ -4428,6 +5131,7 @@ enum CostUsageScanner { rootsFingerprint: rootsFingerprint, rootsChanged: rootsChanged, windowExpanded: windowExpanded, + needsPricingMetadataMigration: needsPricingMetadataMigration, needsProjectMetadataMigration: needsProjectMetadataMigration, modelsDevCatalog: modelsDevCatalog, codexPricingKey: codexPricingKey, @@ -4440,6 +5144,9 @@ enum CostUsageScanner { priorityTurnsChanged: priorityTurnsChanged, needsTurnIDCacheMigration: needsTurnIDCacheMigration, changedPriorityTurnIDs: changedPriorityTurnIDs, + requiresAllFilesForCacheWideMigration: requiresAllFilesForCacheWideMigration, + cacheWideMigrationPendingPathKeys: cacheWideMigrationPendingPathKeys, + requiresCacheWideFileReprocessing: requiresCacheWideFileReprocessing, shouldRefresh: shouldRefresh) } @@ -4508,7 +5215,28 @@ enum CostUsageScanner { return previous } - // swiftlint:disable:next function_body_length + private static func saveCodexCache( + _ cache: inout CostUsageCache, + store: CostUsageStore, + range: CostUsageDayRange, + previousReport: CostUsageCodexPreviousReport?) + { + // The serial scan queue remains the per-process writer boundary. The store actor owns + // the sole writable connection; app and CLI readers take independent WAL snapshots. + let saveResult = CostUsageStoreAccess.save( + store: store, + cache: cache, + calendar: range.calendar, + requestedScanWindow: (sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey), + reportWindow: (sinceKey: range.sinceKey, untilKey: range.untilKey), + skipIdenticalContent: true) + if saveResult.catchUpRequired { + cache.codexScanCatchUpPending = true + cache.codexPreviousReport = previousReport + } + } + + // swiftlint:disable:next cyclomatic_complexity function_body_length private static func loadCodexDaily( range: CostUsageDayRange, now: Date, @@ -4544,18 +5272,58 @@ enum CostUsageScanner { roots: plan.roots, scanSinceKey: range.scanSinceKey, includeLegacyRecursiveScan: shouldRunColdCacheLookback) + let activeLookbackStateWasReset = cache.codexActiveLookbackState.map { + $0.scanSinceKey != activeLookbackState.scanSinceKey + || $0.rootPaths != activeLookbackState.rootPaths + } ?? true + let isExactInventoryProofPass = scanBudget.hasTimeLimit + && !options.forceRescan + && cache.codexActiveLookbackState != nil + && Self.codexBoundedDiscoveryIsComplete(activeLookbackState) + && !plan.requiresCacheWideFileReprocessing + let shouldBoundCatchUp = scanBudget.hasTimeLimit + && !options.forceRescan + && (cache.files.isEmpty + || cache.codexScanCatchUpPending == true + || cache.files.values.contains { + $0.codexScanComplete == false || $0.hasBufferedCodexForkRetryLines + } + || cache.codexActiveLookbackState != nil + || plan.requiresCacheWideFileReprocessing) + let shouldPageDiscovery = shouldBoundCatchUp && !isExactInventoryProofPass + let migrationQueueOwnsCachedPaths = plan.requiresCacheWideFileReprocessing + || activeLookbackState.cacheWideMigrationQueueActive == true + let discoveryExcludedPathKeys = migrationQueueOwnsCachedPaths + ? Set(cache.files.keys.map { Self.codexPathKey(URL(fileURLWithPath: $0)) }) + : [] var seenPaths: Set = [] + var fileURLsByPathKey: [String: URL] = [:] var files: [URL] = [] + var remainingDiscoveryVisits = Self.codexCatchUpScanCandidateLimit for root in plan.roots { - let rootFiles = Self.listCodexSessionFiles( - root: root, - scanSinceKey: range.scanSinceKey, - scanUntilKey: range.scanUntilKey, - includeRecursive: options.forceRescan, - calendar: options.calendar) - for fileURL in rootFiles.sorted(by: { $0.path < $1.path }) where !seenPaths.contains(fileURL.path) { - seenPaths.insert(fileURL.path) - files.append(fileURL) + if shouldPageDiscovery { + Self.advanceCodexCurrentWindow( + root: root, + range: range, + preferNewest: options.preferNewestCodexSessionsFirst, + remainingDiscoveryVisits: &remainingDiscoveryVisits, + excludedPendingPathKeys: discoveryExcludedPathKeys, + workRecorder: options.codexScanWorkRecorderForTesting, + state: &activeLookbackState) + } else { + let rootFiles = Self.listCodexSessionFiles( + root: root, + scanSinceKey: range.scanSinceKey, + scanUntilKey: range.scanUntilKey, + includeRecursive: options.forceRescan || isExactInventoryProofPass, + calendar: options.calendar) + for fileURL in rootFiles.sorted(by: { $0.path < $1.path }) { + let pathKey = Self.codexPathKey(fileURL) + guard seenPaths.insert(pathKey).inserted else { continue } + let canonicalFileURL = URL(fileURLWithPath: pathKey) + fileURLsByPathKey[pathKey] = canonicalFileURL + files.append(canonicalFileURL) + } } // The lookback runs on every refresh, not just cold ones: a session @@ -4567,45 +5335,121 @@ enum CostUsageScanner { // Partition discovery and any discovered candidates persist across bounded // passes. That prevents a small budget from restarting at the oldest day or // rediscovering a file without ever leaving enough budget to parse it. - if let coldCacheLookbackStart { - Self.advanceCodexActiveLookback( - root: root, - range: range, - modifiedSince: coldCacheLookbackStart, - scanBudget: scanBudget, - state: &activeLookbackState) + if isExactInventoryProofPass { + let rootPath = Self.codexResolvedPath(root) + activeLookbackState.completedRootPaths = Array( + Set(activeLookbackState.completedRootPaths).union([rootPath])).sorted() + activeLookbackState.legacyRecursivePendingRootPaths.removeAll { $0 == rootPath } + activeLookbackState.nextDayKeyByRoot.removeValue(forKey: rootPath) + activeLookbackState.nextDirectoryOffsetByRoot?.removeValue(forKey: rootPath) + } else if let coldCacheLookbackStart { + if shouldPageDiscovery { + Self.advanceCodexActiveLookbackPage( + root: root, + range: range, + modifiedSince: coldCacheLookbackStart, + preferNewest: options.preferNewestCodexSessionsFirst, + remainingDiscoveryVisits: &remainingDiscoveryVisits, + excludedPendingPathKeys: discoveryExcludedPathKeys, + workRecorder: options.codexScanWorkRecorderForTesting, + state: &activeLookbackState) + } else { + Self.advanceCodexActiveLookback( + root: root, + range: range, + modifiedSince: coldCacheLookbackStart, + scanBudget: scanBudget, + state: &activeLookbackState) + } } } + let discoveredFiles = files - Self.appendPendingCodexActiveLookbackFiles( + let materializedPendingPathCount = Self.appendPendingCodexActiveLookbackFiles( state: &activeLookbackState, - roots: plan.roots, + context: CodexPendingLookbackAppendContext( + roots: plan.roots, + maxCount: shouldBoundCatchUp ? Self.codexCatchUpScanCandidateLimit : nil, + validateRoots: activeLookbackStateWasReset), seenPaths: &seenPaths, + fileURLsByPathKey: &fileURLsByPathKey, files: &files) - for fileURL in Self.cachedCodexSessionFiles( - cache: cache, - range: range, - roots: plan.roots, - excludingPaths: seenPaths) - .sorted(by: { $0.path < $1.path }) - { - seenPaths.insert(fileURL.path) - files.append(fileURL) - } - - if options.preferNewestCodexSessionsFirst { - files = Self.sortedCodexSessionFilesNewestFirst(files) + if !shouldPageDiscovery { + for fileURL in Self.cachedCodexSessionFiles( + cache: cache, + range: range, + roots: plan.roots, + excludingPaths: seenPaths) + .sorted(by: { $0.path < $1.path }) + { + let pathKey = Self.codexPathKey(fileURL) + seenPaths.insert(pathKey) + fileURLsByPathKey[pathKey] = fileURL + files.append(fileURL) + } } - var filePathsInScan = Set(files.map(\.path)) + let inventoryPathKeys = shouldPageDiscovery + ? Set(cache.files.keys.map { Self.codexPathKey(URL(fileURLWithPath: $0)) }) + .union(fileURLsByPathKey.keys) + : Set(fileURLsByPathKey.keys) + let cacheWideMigrationNeedsQueueReseed = Self.cacheWideMigrationNeedsQueueReseed( + plan: plan, + inventoryPathKeys: inventoryPathKeys, + state: activeLookbackState) + let migrationSeedPathKeys = cacheWideMigrationNeedsQueueReseed + ? (options.preferNewestCodexSessionsFirst + ? Self.sortedCodexSessionFilesNewestFirst( + inventoryPathKeys.map { URL(fileURLWithPath: $0) }) + : inventoryPathKeys.sorted().map { URL(fileURLWithPath: $0) }) + .map(Self.codexPathKey) + : nil + if cacheWideMigrationNeedsQueueReseed { + activeLookbackState.cacheWideMigrationQueueActive = true + } + // One-shot metadata keys can advance in this pass because the durable queue now owns + // every required revisit. Later passes observe the new key and drain the queue without reseeding. + let shouldSeedBoundedQueue = activeLookbackStateWasReset || cacheWideMigrationNeedsQueueReseed + var filePathsInScan = Set(files.map(Self.codexPathKey)) + if activeLookbackState.cacheWideMigrationQueueActive == true { + filePathsInScan.formUnion(inventoryPathKeys) + } + Self.seedOrExtendCodexActiveLookbackQueue( + context: CodexActiveLookbackQueueUpdateContext( + seedFiles: files, + migrationSeedPathKeys: migrationSeedPathKeys, + discoveredFiles: discoveredFiles, + previousDiscovery: cache.codexSessionDiscovery, + shouldBoundCatchUp: shouldBoundCatchUp, + shouldSeedBoundedQueue: shouldSeedBoundedQueue), + state: &activeLookbackState) + let boundedQueuePathCount = shouldSeedBoundedQueue + ? min(Self.codexCatchUpScanCandidateLimit, activeLookbackState.pendingFilePaths.count) + : materializedPendingPathCount + let refreshSelection = Self.codexFilesScheduledForRefresh( + files, + activeLookbackState: &activeLookbackState, + context: CodexRefreshCandidateSelectionContext( + fileURLsByPathKey: fileURLsByPathKey, + shouldBoundCatchUp: shouldBoundCatchUp, + boundedQueuePathCount: boundedQueuePathCount, + preferNewest: options.preferNewestCodexSessionsFirst, + workRecorder: options.codexScanWorkRecorderForTesting)) + let filesScheduledForRefresh = refreshSelection.files + let completionStatesBeforeScan = Self.codexCompletionStates( + files: filesScheduledForRefresh.prefix(Self.codexCatchUpScanCandidateLimit), + cache: cache, + includePreviouslyCompletedSnapshots: true) let fileIndex = CodexSessionFileIndex( files: files, roots: plan.roots, - cachedSessionFiles: Self.cachedCodexSessionIndex( - cache: cache, - roots: plan.roots, - knownExistingPaths: filePathsInScan), + cachedSessionFiles: shouldPageDiscovery + ? [:] + : Self.cachedCodexSessionIndex( + cache: cache, + roots: plan.roots, + knownExistingPaths: filePathsInScan), cachedDiscovery: plan.rootsChanged ? nil : cache.codexSessionDiscovery, scanBudget: scanBudget, headParseObserver: self.codexSessionHeadParseObserverStore?.observer, @@ -4615,9 +5459,13 @@ enum CostUsageScanner { checkCancellation: checkCancellation, scanBudget: scanBudget, cachedFiles: cache.files) + let cachePathAliasIndex = CodexCachePathAliasIndex( + files: cache.files, + workRecorder: options.codexScanWorkRecorderForTesting) let resources = CodexScanResources( fileIndex: fileIndex, inheritedResolver: inheritedResolver, + cachePathAliasIndex: cachePathAliasIndex, projectPathResolver: CodexCanonicalProjectPathResolver(), modelsDevCatalog: plan.modelsDevCatalog, modelsDevCacheRoot: options.cacheRoot, @@ -4629,14 +5477,30 @@ enum CostUsageScanner { resources: resources, checkCancellation: checkCancellation, scanBudget: scanBudget) - try filePathsInScan.formUnion(Self.scanCodexFiles( - files, + let scanResult = try Self.scanCodexFiles( + filesScheduledForRefresh, context: scanContext, cache: &cache, - inheritedResolver: inheritedResolver)) + inheritedResolver: inheritedResolver) + filePathsInScan.formUnion(scanResult.scannedPaths.map { + Self.codexPathKey(URL(fileURLWithPath: $0)) + }) + let pendingLookbackPathCount = shouldBoundCatchUp + ? boundedQueuePathCount + : activeLookbackState.pendingFilePaths.count + let pendingLookbackPaths = Set(activeLookbackState.pendingFilePaths.prefix(pendingLookbackPathCount)) + let completedScheduledPaths = Self.completedCodexActiveLookbackPaths( + scheduledFiles: filesScheduledForRefresh, + pendingPaths: pendingLookbackPaths, + attemptedPaths: scanResult.attemptedPaths, + cache: cache) cache.codexActiveLookbackState = Self.finalizedCodexActiveLookbackState( activeLookbackState, - cache: cache) + completedFilePaths: completedScheduledPaths, + completionCandidateCount: pendingLookbackPathCount, + requiresBoundedDiscoveryCompletion: shouldPageDiscovery, + retainCompletedStateForExactValidation: shouldBoundCatchUp && pendingLookbackPathCount > 0, + workRecorder: options.codexScanWorkRecorderForTesting) if scanBudget.resumedPartialFileCount > 0 || scanBudget.deferredByBudgetFileCount > 0 || scanBudget.deferredByTimeBudgetFileCount > 0 @@ -4661,17 +5525,20 @@ enum CostUsageScanner { let shouldDropAllUnscannedFiles = options.forceRescan || plan.rootsChanged || cache.files.isEmpty || plan.needsProjectMetadataMigration - for key in cache.files.keys where !filePathsInScan.contains(key) { - guard let old = cache.files[key] else { continue } - let shouldDrop = shouldDropAllUnscannedFiles || - old.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) - guard shouldDrop else { continue } - Self.applyFileDays(cache: &cache, fileDays: old.days, sign: -1) - cache.files.removeValue(forKey: key) - } + if !shouldPageDiscovery { + for key in cache.files.keys + where !filePathsInScan.contains(Self.codexPathKey(URL(fileURLWithPath: key))) + { + guard let old = cache.files[key] else { continue } + let shouldDrop = shouldDropAllUnscannedFiles || + old.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) + guard shouldDrop else { continue } + Self.applyFileDays(cache: &cache, fileDays: old.days, sign: -1) + cache.files.removeValue(forKey: key) + } - if !shouldDropAllUnscannedFiles { for key in cache.files.keys { + guard !shouldDropAllUnscannedFiles else { break } guard let old = cache.files[key] else { continue } guard old.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) else { continue } @@ -4691,6 +5558,13 @@ enum CostUsageScanner { let retainedUntilKey = shouldRetainWiderWindow ? [cachedUntilKey, range.scanUntilKey].compactMap(\.self).max() ?? range.scanUntilKey : range.scanUntilKey + let canReuseApproximateProgress = !options.forceRescan + && !plan.rootsChanged + && !plan.windowExpanded + && !plan.requiresAllFilesForCacheWideMigration + && !cacheWideMigrationNeedsQueueReseed + && cachedSinceKey == retainedSinceKey + && cachedUntilKey == retainedUntilKey Self.pruneDays(cache: &cache, sinceKey: retainedSinceKey, untilKey: retainedUntilKey) cache.roots = plan.rootsFingerprint cache.scanSinceKey = retainedSinceKey @@ -4698,20 +5572,32 @@ enum CostUsageScanner { cache.codexPricingKey = plan.codexPricingKey cache.codexPriorityMetadataKey = plan.codexPriorityMetadataKey cache.codexProjectMetadataVersion = Self.codexProjectMetadataVersion - let scanProgress = Self.codexScanProgress(paths: filePathsInScan, cache: cache) + let hasKnownBoundedWork = scanBudget.resumedPartialFileCount > 0 + || scanBudget.deferredByBudgetFileCount > 0 + || scanBudget.deferredByTimeBudgetFileCount > 0 + || refreshSelection.exhaustedVisitBudget + || cache.codexActiveLookbackState != nil + || fileIndex.hasPendingDiscovery + let progressUpdate = Self.updateCodexScanProgress( + cache: &cache, + context: CodexScanProgressUpdateContext( + inventoryPaths: filePathsInScan, + hasKnownBoundedWork: hasKnownBoundedWork, + canReuseApproximateProgress: canReuseApproximateProgress, + pendingQueuePathCount: cache.codexActiveLookbackState?.pendingFilePaths.count, + completionStatesBeforeScan: completionStatesBeforeScan, + workRecorder: options.codexScanWorkRecorderForTesting)) + let scanProgress = progressUpdate.summary + let canValidateExactInventory = progressUpdate.isExact cache.codexScanProcessedBytes = scanProgress.processedBytes cache.codexScanTotalBytes = scanProgress.totalBytes cache.codexScanCompletedFiles = scanProgress.completedFiles cache.codexScanTotalFiles = scanProgress.totalFiles cache.codexSessionDiscovery = fileIndex.persistedState - let catchUpPending = scanBudget.resumedPartialFileCount > 0 - || scanBudget.deferredByBudgetFileCount > 0 - || scanBudget.deferredByTimeBudgetFileCount > 0 + let catchUpPending = !canValidateExactInventory || scanProgress.completedFiles < scanProgress.totalFiles || cache.files.values.contains { $0.codexScanComplete == false } || cache.files.values.contains { $0.hasBufferedCodexForkRetryLines } - || fileIndex.hasPendingDiscovery - || cache.codexActiveLookbackState != nil cache.codexScanCatchUpPending = catchUpPending cache.codexPreviousReport = catchUpPending ? previousReport : nil if plan.hasPriorityMetadata { @@ -4730,19 +5616,11 @@ enum CostUsageScanner { } cache.lastScanUnixMs = nowMs try checkCancellation?() - // The serial scan queue remains the per-process writer boundary. The store actor owns - // the sole writable connection; app and CLI readers take independent WAL snapshots. - let saveResult = CostUsageStoreAccess.save( + Self.saveCodexCache( + &cache, store: loadedCache.store, - cache: cache, - calendar: range.calendar, - requestedScanWindow: (sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey), - reportWindow: (sinceKey: range.sinceKey, untilKey: range.untilKey), - skipIdenticalContent: true) - if saveResult.catchUpRequired { - cache.codexScanCatchUpPending = true - cache.codexPreviousReport = previousReport - } + range: range, + previousReport: previousReport) } if let previous = Self.codexPreviousReport( @@ -4767,9 +5645,108 @@ enum CostUsageScanner { let totalFiles: Int } + private struct CodexScanProgressUpdateContext { + let inventoryPaths: Set + let hasKnownBoundedWork: Bool + let canReuseApproximateProgress: Bool + let pendingQueuePathCount: Int? + let completionStatesBeforeScan: [String: Bool] + let workRecorder: CodexScanWorkRecorder? + } + + private static func updateCodexScanProgress( + cache: inout CostUsageCache, + context: CodexScanProgressUpdateContext) -> (summary: CodexScanProgressSummary, isExact: Bool) + { + if !context.hasKnownBoundedWork { + let summary = Self.codexScanProgress( + paths: context.inventoryPaths, + cache: cache, + workRecorder: context.workRecorder) + guard summary.completedFiles == summary.totalFiles else { + cache.codexScanInventoryPaths = nil + return (CodexScanProgressSummary( + processedBytes: 0, + totalBytes: 0, + completedFiles: summary.completedFiles, + totalFiles: summary.totalFiles), false) + } + cache.codexScanInventoryPaths = context.inventoryPaths.sorted() + return (summary, true) + } + + let statesBeforeScan = context.canReuseApproximateProgress + ? context.completionStatesBeforeScan + : context.completionStatesBeforeScan.mapValues { _ in false } + let statesAfterScan = Self.codexCompletionStates( + paths: context.completionStatesBeforeScan.keys, + cache: cache, + includePreviouslyCompletedSnapshots: false) + let completionDelta = statesBeforeScan.reduce(into: 0) { delta, entry in + let after = statesAfterScan[entry.key] ?? false + delta += (after ? 1 : 0) - (entry.value ? 1 : 0) + } + let previousCompletedFiles = context.canReuseApproximateProgress + ? max(0, cache.codexScanCompletedFiles ?? 0) + : 0 + let previousTotalFiles = context.canReuseApproximateProgress + ? max(0, cache.codexScanTotalFiles ?? 0) + : 0 + var completedFiles = max(0, previousCompletedFiles + completionDelta) + let totalFiles = max(previousTotalFiles, context.inventoryPaths.count, 1) + if let pendingQueuePathCount = context.pendingQueuePathCount { + completedFiles = max(completedFiles, max(0, totalFiles - pendingQueuePathCount)) + } + + // A bounded-work signal proves at least one pass remains even if this slice happened + // to visit the final 512 candidates. Keep one conservative slot open until an exact + // identity-deduplicated traversal validates the inventory. + let incompleteSelectedFiles = statesAfterScan.values.count(where: { !$0 }) + completedFiles = min(completedFiles, max(0, totalFiles - max(1, incompleteSelectedFiles))) + + cache.codexScanInventoryPaths = nil + return (CodexScanProgressSummary( + processedBytes: 0, + totalBytes: 0, + completedFiles: completedFiles, + totalFiles: totalFiles), false) + } + + private static func codexCompletionStates( + files: some Sequence, + cache: CostUsageCache, + includePreviouslyCompletedSnapshots: Bool) -> [String: Bool] + { + self.codexCompletionStates( + paths: files.map(\.path), + cache: cache, + includePreviouslyCompletedSnapshots: includePreviouslyCompletedSnapshots) + } + + private static func codexCompletionStates( + paths: some Sequence, + cache: CostUsageCache, + includePreviouslyCompletedSnapshots: Bool) -> [String: Bool] + { + paths.reduce(into: [String: Bool]()) { result, path in + let standardizedPath = URL(fileURLWithPath: path).standardizedFileURL.path + guard let usage = cache.files[path] ?? cache.files[standardizedPath], + !usage.hasBufferedCodexForkRetryLines + else { + result[path] = false + return + } + let isComplete = usage.codexScanComplete != false + let wasCompletedSnapshot = includePreviouslyCompletedSnapshots + && (usage.parsedBytes ?? -1) >= max(0, usage.size) + result[path] = isComplete || wasCompletedSnapshot + } + } + private static func codexScanProgress( paths: Set, - cache: CostUsageCache) -> CodexScanProgressSummary + cache: CostUsageCache, + workRecorder: CodexScanWorkRecorder? = nil) -> CodexScanProgressSummary { var processedBytes: Int64 = 0 var totalBytes: Int64 = 0 @@ -4778,6 +5755,7 @@ enum CostUsageScanner { var seenIdentities: Set = [] for path in paths.sorted() { + workRecorder?.recordCodexProgressAccountingVisit() let fileURL = URL(fileURLWithPath: path) let metadata = Self.codexFileMetadata(fileURL: fileURL) let identity = metadata.fileId ?? fileURL.standardizedFileURL.path @@ -4788,7 +5766,10 @@ enum CostUsageScanner { let usage = cache.files[path] ?? cache.files[fileURL.standardizedFileURL.path] guard let usage else { continue } let identityMatches = usage.codexScanFileId == nil || usage.codexScanFileId == metadata.fileId - guard identityMatches else { continue } + guard identityMatches, + usage.mtimeUnixMs == metadata.mtimeUnixMs, + usage.size == metadata.size + else { continue } let parsedBytes = min( max(0, metadata.size), max(0, usage.parsedBytes ?? (usage.codexScanComplete == false ? 0 : usage.size))) @@ -4808,17 +5789,28 @@ enum CostUsageScanner { totalFiles: totalFiles) } + private struct CodexFileScanResult { + let scannedPaths: Set + let attemptedPaths: Set + } + private static func scanCodexFiles( _ files: [URL], context: CodexFileScanContext, cache: inout CostUsageCache, - inheritedResolver: CodexInheritedTotalsResolver) throws -> Set + inheritedResolver: CodexInheritedTotalsResolver) throws -> CodexFileScanResult { var scanState = CodexScanState() var bufferedForkRetries: [URL] = [] var visitedPaths = Set(files.map(\.standardizedFileURL.path)) var scannedPaths = Set(files.map(\.path)) + var attemptedPaths: Set = [] for fileURL in files { + if context.scanBudget?.shouldStopBeforeNextFile() == true { + break + } + context.workRecorder?.recordCodexFileScanAttempt(path: Self.codexPathKey(fileURL)) + attemptedPaths.insert(fileURL.path) try Self.scanCodexFile( fileURL: fileURL, context: context, @@ -4835,13 +5827,18 @@ enum CostUsageScanner { // children. Scan those dependencies through the same budgeted path and retain their cache // entries so later passes can resume instead of restarting from byte zero. var dependencyState = CodexScanState() - while true { + dependencyScan: while true { let pendingParents = inheritedResolver.takePendingParentFiles().filter { visitedPaths.insert($0.standardizedFileURL.path).inserted } guard !pendingParents.isEmpty else { break } for fileURL in pendingParents { + if context.scanBudget?.shouldStopBeforeNextFile() == true { + break dependencyScan + } + context.workRecorder?.recordCodexFileScanAttempt(path: Self.codexPathKey(fileURL)) scannedPaths.insert(fileURL.path) + attemptedPaths.insert(fileURL.path) try Self.scanCodexFile( fileURL: fileURL, context: context, @@ -4871,7 +5868,7 @@ enum CostUsageScanner { fileURL: fileURL, usage: cache.files[fileURL.path]) } - return scannedPaths + return CodexFileScanResult(scannedPaths: scannedPaths, attemptedPaths: attemptedPaths) } private static func shouldRetryBufferedCodexFork(_ usage: CostUsageFileUsage?) -> Bool { @@ -4921,22 +5918,25 @@ enum CostUsageScanner { private static func reconcileCodexCachePathAliases( metadata: CodexFileMetadata, - cache: inout CostUsageCache) + cache: inout CostUsageCache, + aliasIndex: CodexCachePathAliasIndex) { guard let fileID = metadata.fileId else { return } - var aliases = cache.files.compactMap { path, usage in - path != metadata.path && usage.codexScanFileId == fileID ? path : nil - }.sorted() + var aliases = aliasIndex.aliases(fileID: fileID, excludingPath: metadata.path) guard !aliases.isEmpty else { return } if cache.files[metadata.path] == nil, let migratedPath = aliases.first { cache.files[metadata.path] = cache.files.removeValue(forKey: migratedPath) + aliasIndex.remove(path: migratedPath) + aliasIndex.update(path: metadata.path, fileID: fileID) aliases.removeFirst() } for alias in aliases { - guard let stale = cache.files[alias] else { continue } - Self.applyFileDays(cache: &cache, fileDays: stale.days, sign: -1) - cache.files.removeValue(forKey: alias) + if let stale = cache.files[alias] { + Self.applyFileDays(cache: &cache, fileDays: stale.days, sign: -1) + cache.files.removeValue(forKey: alias) + } + aliasIndex.remove(path: alias) } } } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift index 763ae50198..2bb0991e7d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift @@ -65,6 +65,8 @@ extension CostUsageStore { fileBudgetBytes: Int64 = CostUsageStore.defaultFileBudgetBytes, skipIdenticalContent: Bool = false) -> CostUsageStoreBudgetResult { + var cache = cache + Self.reconcileCompletedCodexCatchUp(cache: &cache) let previous = self.readSnapshot() if skipIdenticalContent, Self.persistedContentMatches( @@ -242,6 +244,17 @@ extension CostUsageStore { var canReuseRows: Bool } + private struct CurrentCodexRootDevice { + var path: String + var device: String + } + + private struct RestoredCodexScanState { + var identity: String? + var isComplete: Bool + var validatedCurrentSnapshot = false + } + private static func cache(from snapshot: CostUsageStoreSnapshot) -> CostUsageCache { var cache = CostUsageCache() let metadata = snapshot.metadata @@ -256,6 +269,7 @@ extension CostUsageStore { cache.codexScanTotalBytes = metadata.totalBytes cache.codexScanCompletedFiles = metadata.completedFiles cache.codexScanTotalFiles = metadata.totalFiles + cache.codexScanInventoryPaths = metadata.scanInventoryPaths cache.roots = metadata.rootMtimes cache.codexProjectMetadataVersion = metadata.projectMetadataVersion cache.codexPreviousReport = metadata.previousReportPayload.flatMap { @@ -276,6 +290,11 @@ extension CostUsageStore { let lineageByPath = Dictionary(uniqueKeysWithValues: snapshot.forkLineage.map { ($0.path, $0) }) let buffersByPath = Dictionary(grouping: snapshot.bufferedLines, by: \.path) let accumulatorByPath = Dictionary(uniqueKeysWithValues: snapshot.accumulators.map { ($0.path, $0) }) + let currentRootDevices = Self.currentCodexRootDevices(rootMtimes: metadata.rootMtimes) + var remainingIdentityValidationVisits = CostUsageScanner.codexCatchUpScanCandidateLimit + var deferredIdentityValidationPaths: [String] = [] + var completedIdentityValidationPaths: [String] = [] + var invalidatedIdentityValidationPaths: [String] = [] for file in snapshot.files { guard let detailsData = file.scanState.detailsPayload, @@ -290,6 +309,33 @@ extension CostUsageStore { let lineage = lineageByPath[file.path] let accumulator = accumulatorByPath[file.path] let buffers = buffersByPath[file.path] ?? [] + let normalizedIdentity = Self.normalizedCodexFileIdentity( + file: file, + currentRootDevices: currentRootDevices) + let identityNeedsValidation = normalizedIdentity != file.scanState.fileIdentity + let restoredScanState: RestoredCodexScanState + if identityNeedsValidation, remainingIdentityValidationVisits > 0 { + Self.codexCatchUpReconciliationVisitForTesting?() + remainingIdentityValidationVisits -= 1 + restoredScanState = Self.restoredCodexScanState( + file: file, + currentRootDevices: currentRootDevices, + validateMetadata: true) + if restoredScanState.isComplete, restoredScanState.validatedCurrentSnapshot { + completedIdentityValidationPaths.append(file.path) + } else { + invalidatedIdentityValidationPaths.append(file.path) + } + } else if identityNeedsValidation { + deferredIdentityValidationPaths.append(file.path) + restoredScanState = RestoredCodexScanState( + identity: file.scanState.fileIdentity, + isComplete: file.scanState.isComplete) + } else { + restoredScanState = RestoredCodexScanState( + identity: normalizedIdentity, + isComplete: file.scanState.isComplete) + } let usage = CostUsageFileUsage( mtimeUnixMs: file.mtimeUnixMs, size: file.size, @@ -333,9 +379,9 @@ extension CostUsageStore { sha256: $0.sha256) }, claudeRows: nil, - codexScanFileId: file.scanState.fileIdentity, + codexScanFileId: restoredScanState.identity, codexScanTargetSize: file.scanState.targetSize, - codexScanComplete: file.scanState.isComplete, + codexScanComplete: restoredScanState.isComplete, codexJSONLResumeState: file.scanState.resumePayload.flatMap { try? JSONDecoder().decode(CostUsageJsonl.ResumeState.self, from: $0) }, @@ -343,10 +389,311 @@ extension CostUsageStore { codexBufferedUnresolvedForkLines: Self.bufferedLines(buffers, kind: .unresolvedFork)) cache.files[file.path] = usage } + Self.enqueueDeferredCodexIdentityValidation( + deferredIdentityValidationPaths + invalidatedIdentityValidationPaths, + metadata: metadata, + cache: &cache) + Self.removeCompletedCodexIdentityValidation( + completedIdentityValidationPaths, + cache: &cache) + Self.reconcileCompletedCodexCatchUp( + cache: &cache, + visitLimit: remainingIdentityValidationVisits) cache.days = Self.days(from: snapshot.dayAggregates) return cache } + private static func enqueueDeferredCodexIdentityValidation( + _ paths: [String], + metadata: CostUsageStoreMetadata, + cache: inout CostUsageCache) + { + guard !paths.isEmpty, let scanSinceKey = metadata.scanSinceDay else { return } + let rootPaths = (metadata.rootMtimes ?? [:]).keys.map { path in + Self.normalizedCodexPath( + URL(fileURLWithPath: path, isDirectory: true) + .resolvingSymlinksInPath() + .standardizedFileURL.path) + }.sorted() + guard !rootPaths.isEmpty else { return } + var lookback = cache.codexActiveLookbackState ?? CostUsageCodexActiveLookbackState( + scanSinceKey: scanSinceKey, + rootPaths: rootPaths, + completedRootPaths: rootPaths, + currentWindowNextDayKeyByRoot: [:], + currentWindowDirectoryOffsetByRoot: [:], + completedCurrentWindowRootPaths: rootPaths, + currentWindowFlatDirectoryOffsetByRoot: [:], + completedCurrentWindowFlatRootPaths: rootPaths) + var pendingPaths = Set(lookback.pendingFilePaths) + pendingPaths.formUnion(paths) + lookback.pendingFilePaths = pendingPaths.sorted() + cache.codexActiveLookbackState = lookback + cache.codexScanCatchUpPending = true + } + + private static func removeCompletedCodexIdentityValidation( + _ paths: [String], + cache: inout CostUsageCache) + { + guard !paths.isEmpty, var lookback = cache.codexActiveLookbackState else { return } + let completedPathKeys = Set(paths.map(Self.normalizedCodexPath)) + lookback.pendingFilePaths.removeAll { path in + completedPathKeys.contains(Self.normalizedCodexPath(path)) + } + cache.codexActiveLookbackState = lookback + } + + private static func currentCodexRootDevices( + rootMtimes: [String: Int64]?) -> [CurrentCodexRootDevice] + { + (rootMtimes ?? [:]).keys.compactMap { path in + let rootURL = URL(fileURLWithPath: path, isDirectory: true).standardizedFileURL + let metadata = CostUsageScanner.codexFileMetadata(fileURL: rootURL) + guard let device = Self.device(from: metadata.fileId) else { return nil } + return CurrentCodexRootDevice(path: Self.normalizedCodexPath(rootURL.path), device: device) + }.sorted { $0.path.count > $1.path.count } + } + + private static func normalizedCodexFileIdentity( + file: CostUsageStoreFile, + currentRootDevices: [CurrentCodexRootDevice]) -> String? + { + guard let identity = file.scanState.fileIdentity, + let inode = Self.inode(from: identity) + else { return file.scanState.fileIdentity } + if let persistedInode = file.inode, persistedInode != inode { + return identity + } + let filePath = Self.normalizedCodexPath(file.path) + guard let root = currentRootDevices.first(where: { root in + if filePath == root.path { + return true + } + let prefix = root.path.hasSuffix("/") ? root.path : root.path + "/" + return filePath.hasPrefix(prefix) + }) else { return identity } + return "\(root.device):\(inode)" + } + + private static func restoredCodexScanState( + file: CostUsageStoreFile, + currentRootDevices: [CurrentCodexRootDevice], + validateMetadata: Bool) -> RestoredCodexScanState + { + let identity = Self.normalizedCodexFileIdentity( + file: file, + currentRootDevices: currentRootDevices) + guard validateMetadata else { + return RestoredCodexScanState(identity: identity, isComplete: file.scanState.isComplete) + } + + let fileURL = URL(fileURLWithPath: file.path) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + guard let currentIdentity = metadata.fileId else { + return RestoredCodexScanState(identity: identity, isComplete: file.scanState.isComplete) + } + let metadataIsUnchanged = identity == currentIdentity + && file.mtimeUnixMs == metadata.mtimeUnixMs + && file.size == metadata.size + if metadataIsUnchanged { + return RestoredCodexScanState( + identity: identity, + isComplete: file.scanState.isComplete, + validatedCurrentSnapshot: true) + } + + let isAppend = identity == currentIdentity && metadata.size > file.size + return RestoredCodexScanState( + identity: isAppend ? identity : nil, + isComplete: false) + } + + private static func normalizedCodexPath(_ path: String) -> String { + let path = URL(fileURLWithPath: path).standardizedFileURL.path + if path.hasPrefix("/private/var/") { + return String(path.dropFirst("/private".count)) + } + return path + } + + private static func reconcileCompletedCodexCatchUp( + cache: inout CostUsageCache, + visitLimit: Int = CostUsageScanner.codexCatchUpScanCandidateLimit) + { + if var lookback = cache.codexActiveLookbackState { + let reconciliationLimit = max(0, min( + visitLimit, + CostUsageScanner.codexCatchUpScanCandidateLimit)) + let candidatePaths = lookback.pendingFilePaths.prefix(reconciliationLimit) + var completedIdentityValidationPathKeys: Set = [] + for path in candidatePaths { + Self.codexCatchUpReconciliationVisitForTesting?() + let fileURL = URL(fileURLWithPath: path) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + guard let fileId = metadata.fileId, + let cachedEntry = Self.cachedCodexUsageEntry(for: path, cache: cache), + cachedEntry.usage.codexScanComplete != false, + !cachedEntry.usage.hasBufferedCodexForkRetryLines + else { + continue + } + + guard Self.matchesCompletedCodexFileSnapshot( + usage: cachedEntry.usage, + metadata: metadata, + fileURL: fileURL) + else { + continue + } + + // APFS can expose the same volume with a different st_dev value after relaunch. + // Retain the inode and validate the indexed content before adopting the current identity. + if cachedEntry.usage.codexScanFileId != fileId, + var normalized = cache.files[cachedEntry.path] + { + normalized.codexScanFileId = fileId + cache.files[cachedEntry.path] = normalized + completedIdentityValidationPathKeys.insert(Self.normalizedCodexPath(path)) + } + } + if !completedIdentityValidationPathKeys.isEmpty { + lookback.pendingFilePaths.removeAll { path in + completedIdentityValidationPathKeys.contains(Self.normalizedCodexPath(path)) + } + } + let rootPaths = Set(lookback.rootPaths) + let lookbackIsComplete = Set(lookback.completedRootPaths) == rootPaths + && Set(lookback.completedCurrentWindowRootPaths ?? []) == rootPaths + && Set(lookback.completedCurrentWindowFlatRootPaths ?? []) == rootPaths + && lookback.pendingFilePaths.isEmpty + && lookback.legacyRecursivePendingRootPaths.isEmpty + let awaitingExactValidation = cache.codexScanCatchUpPending == true + && cache.codexScanInventoryPaths == nil + cache.codexActiveLookbackState = lookbackIsComplete && !awaitingExactValidation ? nil : lookback + } + + guard cache.codexScanCatchUpPending == true, + cache.codexActiveLookbackState == nil + else { return } + let discoveryHasPendingWork = cache.codexSessionDiscovery.map { + !$0.isComplete && (!$0.pendingSessionIds.isEmpty || $0.headScan != nil) + } ?? false + guard !discoveryHasPendingWork else { return } + let filesHavePendingWork = cache.files.values.contains { + $0.codexScanComplete == false || $0.hasBufferedCodexForkRetryLines + } + guard !filesHavePendingWork else { return } + let expectedTotalFiles = max(0, cache.codexScanTotalFiles ?? 0) + let reconciliationLimit = CostUsageScanner.codexCatchUpScanCandidateLimit + guard expectedTotalFiles <= reconciliationLimit, + (cache.codexScanInventoryPaths?.count ?? 0) <= reconciliationLimit + else { return } + guard let completedInventory = Self.completedCodexScanInventory( + cache: cache, + expectedTotalFiles: expectedTotalFiles) + else { return } + + cache.codexScanCatchUpPending = false + cache.codexScanProcessedBytes = completedInventory.totalBytes + cache.codexScanTotalBytes = completedInventory.totalBytes + cache.codexScanCompletedFiles = completedInventory.fileCount + cache.codexScanTotalFiles = completedInventory.fileCount + cache.codexPreviousReport = nil + } + + private static func cachedCodexUsageEntry( + for path: String, + cache: CostUsageCache) -> (path: String, usage: CostUsageFileUsage)? + { + let normalizedPath = Self.normalizedCodexPath(path) + var candidatePaths = [path] + if normalizedPath != path { + candidatePaths.append(normalizedPath) + } + if normalizedPath.hasPrefix("/var/") { + candidatePaths.append("/private" + normalizedPath) + } + var seenPaths: Set = [] + for candidatePath in candidatePaths where seenPaths.insert(candidatePath).inserted { + if let usage = cache.files[candidatePath] { + return (candidatePath, usage) + } + } + return nil + } + + private static func completedCodexScanInventory( + cache: CostUsageCache, + expectedTotalFiles: Int) -> (fileCount: Int, totalBytes: Int64)? + { + guard expectedTotalFiles > 0, + let inventoryPaths = cache.codexScanInventoryPaths, + !inventoryPaths.isEmpty + else { return nil } + + let cachedFilesByIdentity = cache.files.values.reduce( + into: [String: CostUsageFileUsage]()) + { result, usage in + guard let identity = usage.codexScanFileId else { return } + result[identity] = usage + } + let cachedFilesByNormalizedPath = cache.files.reduce( + into: [String: CostUsageFileUsage]()) + { result, entry in + result[Self.normalizedCodexPath(entry.key)] = entry.value + } + + var seenIdentities: Set = [] + var totalBytes: Int64 = 0 + for path in inventoryPaths { + let fileURL = URL(fileURLWithPath: path) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + guard let fileId = metadata.fileId else { return nil } + guard seenIdentities.insert(fileId).inserted else { continue } + guard let usage = cache.files[path] + ?? cachedFilesByNormalizedPath[Self.normalizedCodexPath(path)] + ?? cachedFilesByIdentity[fileId], + usage.codexScanComplete != false, + !usage.hasBufferedCodexForkRetryLines, + Self.matchesCompletedCodexFileSnapshot( + usage: usage, + metadata: metadata, + fileURL: fileURL) + else { return nil } + totalBytes += max(0, metadata.size) + } + + guard seenIdentities.count == expectedTotalFiles else { return nil } + return (seenIdentities.count, totalBytes) + } + + private static func matchesCompletedCodexFileSnapshot( + usage: CostUsageFileUsage, + metadata: CostUsageScanner.CodexFileMetadata, + fileURL: URL) -> Bool + { + guard usage.mtimeUnixMs == metadata.mtimeUnixMs, + usage.size == metadata.size, + let cachedIdentity = usage.codexScanFileId, + let currentIdentity = metadata.fileId + else { return false } + if cachedIdentity == currentIdentity { + return true + } + guard Self.inode(from: cachedIdentity) == Self.inode(from: currentIdentity) else { + return false + } + if metadata.size == 0 { + return true + } + guard let anchor = usage.codexTokenIndexAnchor else { return false } + return CostUsageScanner.codexTokenIndexAnchorMatches( + anchor, + fileURL: fileURL, + metadata: metadata) + } + private func persistFile( path: String, usage: CostUsageFileUsage, @@ -483,6 +830,7 @@ extension CostUsageStore { totalBytes: cache.codexScanTotalBytes, completedFiles: cache.codexScanCompletedFiles, totalFiles: cache.codexScanTotalFiles, + scanInventoryPaths: cache.codexScanInventoryPaths, rootMtimes: cache.roots, previousReportPayload: cache.codexPreviousReport.flatMap { try? JSONEncoder().encode($0) }, priorityTurnStatePayload: try? JSONEncoder().encode(priority), @@ -714,9 +1062,16 @@ extension CostUsageStore { scanSinceDay: $0.scanSinceKey, rootPaths: $0.rootPaths, nextDayByRoot: $0.nextDayKeyByRoot, + nextDirectoryOffsetByRoot: $0.nextDirectoryOffsetByRoot, completedRootPaths: $0.completedRootPaths, pendingFilePaths: $0.pendingFilePaths, - legacyRecursivePendingRootPaths: $0.legacyRecursivePendingRootPaths) + legacyRecursivePendingRootPaths: $0.legacyRecursivePendingRootPaths, + currentWindowNextDayKeyByRoot: $0.currentWindowNextDayKeyByRoot, + currentWindowDirectoryOffsetByRoot: $0.currentWindowDirectoryOffsetByRoot, + completedCurrentWindowRootPaths: $0.completedCurrentWindowRootPaths, + currentWindowFlatDirectoryOffsetByRoot: $0.currentWindowFlatDirectoryOffsetByRoot, + completedCurrentWindowFlatRootPaths: $0.completedCurrentWindowFlatRootPaths, + cacheWideMigrationQueueActive: $0.cacheWideMigrationQueueActive) } } @@ -725,9 +1080,16 @@ extension CostUsageStore { scanSinceKey: value.scanSinceDay, rootPaths: value.rootPaths, nextDayKeyByRoot: value.nextDayByRoot, + nextDirectoryOffsetByRoot: value.nextDirectoryOffsetByRoot, completedRootPaths: value.completedRootPaths, pendingFilePaths: value.pendingFilePaths, - legacyRecursivePendingRootPaths: value.legacyRecursivePendingRootPaths) + legacyRecursivePendingRootPaths: value.legacyRecursivePendingRootPaths, + currentWindowNextDayKeyByRoot: value.currentWindowNextDayKeyByRoot, + currentWindowDirectoryOffsetByRoot: value.currentWindowDirectoryOffsetByRoot, + completedCurrentWindowRootPaths: value.completedCurrentWindowRootPaths, + currentWindowFlatDirectoryOffsetByRoot: value.currentWindowFlatDirectoryOffsetByRoot, + completedCurrentWindowFlatRootPaths: value.completedCurrentWindowFlatRootPaths, + cacheWideMigrationQueueActive: value.cacheWideMigrationQueueActive) } private static func tokenSnapshot( @@ -799,6 +1161,10 @@ extension CostUsageStore { identity?.split(separator: ":").last.flatMap { Int64($0) } } + private static func device(from identity: String?) -> String? { + identity?.split(separator: ":", maxSplits: 1).first.map(String.init) + } + private static func totals(_ value: CostUsageCodexTotals?) -> CostUsageStoreTotals? { value.map { CostUsageStoreTotals( input: Int64($0.input), diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift index 02d1a79f53..2a1069c162 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift @@ -412,6 +412,7 @@ extension CostUsageStore { table: "scan_metadata") ?? .empty metadata.catchUpPending = true metadata.lastScanUnixMs = 0 + metadata.scanInventoryPaths = nil try self.writeSingleton(metadata, database: database, table: "scan_metadata") } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift index 244105246e..90c03d14d1 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift @@ -80,6 +80,7 @@ actor CostUsageStore { "98da5914d2f6a9cd", // Pushed PR producer before retry signaling; persisted rows unchanged. "43609cc56f76a003", // 0.49.3 request-tier pricing; persisted row shape unchanged. "b975eb705f905b9a", // 0.49.0-0.49.2 SQLite producer with compatible rows. + "47144baa8daccf52", // This branch changes only scan scheduling, discovery, and persistence bookkeeping. ] /// Test-only crash injection: invoked inside `saveCodexCache`'s transaction after each @@ -89,6 +90,9 @@ actor CostUsageStore { /// Test-only interleaving point after optimistic identity succeeds and before its writer lock. nonisolated(unsafe) static var identicalContentPreLockCheckpointForTesting: (() -> Void)? + /// Test-only traversal proof for persisted Codex catch-up reconciliation. Never set in production. + nonisolated(unsafe) static var codexCatchUpReconciliationVisitForTesting: (() -> Void)? + /// Process-wide serialization keeps every writable store connection on the same queue. /// This matches the scan pipeline's single-writer contract without multiplying executor /// threads when tests or short-lived readers create several store actors. diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStoreModels.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStoreModels.swift index 95778ffcab..fb18ebe46d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStoreModels.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStoreModels.swift @@ -146,9 +146,16 @@ struct CostUsageStoreLookbackState: Codable, Equatable, Sendable { var scanSinceDay: String var rootPaths: [String] var nextDayByRoot: [String: String] + var nextDirectoryOffsetByRoot: [String: Int64]? var completedRootPaths: [String] var pendingFilePaths: [String] var legacyRecursivePendingRootPaths: [String] + var currentWindowNextDayKeyByRoot: [String: String]? + var currentWindowDirectoryOffsetByRoot: [String: Int64]? + var completedCurrentWindowRootPaths: [String]? + var currentWindowFlatDirectoryOffsetByRoot: [String: Int64]? + var completedCurrentWindowFlatRootPaths: [String]? + var cacheWideMigrationQueueActive: Bool? } struct CostUsageStoreAccumulator: Codable, Equatable, Sendable { @@ -176,6 +183,7 @@ struct CostUsageStoreMetadata: Codable, Equatable, Sendable { var totalBytes: Int64? var completedFiles: Int? var totalFiles: Int? + var scanInventoryPaths: [String]? var rootMtimes: [String: Int64]? var previousReportPayload: Data? var priorityTurnStatePayload: Data? @@ -193,6 +201,7 @@ struct CostUsageStoreMetadata: Codable, Equatable, Sendable { totalBytes: nil, completedFiles: nil, totalFiles: nil, + scanInventoryPaths: nil, rootMtimes: nil, previousReportPayload: nil, priorityTurnStatePayload: nil, diff --git a/Tests/CodexBarTests/CostUsageBoundedProgressTests.swift b/Tests/CodexBarTests/CostUsageBoundedProgressTests.swift new file mode 100644 index 0000000000..2699ef4e90 --- /dev/null +++ b/Tests/CodexBarTests/CostUsageBoundedProgressTests.swift @@ -0,0 +1,915 @@ +import Foundation +import Testing +@testable import CodexBarCore + +@Suite(.serialized) +// swiftlint:disable:next type_body_length +struct CostUsageBoundedProgressTests { + @Test + func `bounded progress accumulates while retaining a wider scan window`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + var options = Self.boundedOptions(env: env) + let priorDay = try #require(options.calendar.date(byAdding: .day, value: -1, to: day)) + options.maxCodexScanDurationPerRefresh = nil + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: priorDay, + until: day, + now: day, + options: options) + + let corpusSize = 600 + try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + options.maxCodexScanDurationPerRefresh = 60 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(firstCache.files.count == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstCache.codexScanCompletedFiles == CostUsageScanner.codexCatchUpScanCandidateLimit - 1) + #expect(firstCache.codexScanTotalFiles == CostUsageScanner.codexCatchUpScanCandidateLimit) + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(secondRecorder.snapshot().codexProgressAccountingVisits == 0) + #expect(secondCache.files.count == corpusSize) + #expect(secondCache.codexScanCompletedFiles == CostUsageScanner.codexCatchUpScanCandidateLimit - 1) + #expect(secondCache.codexScanTotalFiles == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(secondCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(secondCache.codexScanInventoryPaths == nil) + #expect(secondCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(finalRecorder.snapshot().codexCandidateSelectionVisits == 0) + #expect(finalRecorder.snapshot().codexFileScanAttempts == 0) + #expect(finalRecorder.snapshot().codexProgressAccountingVisits == corpusSize) + #expect(finalCache.codexActiveLookbackState == nil) + #expect(finalCache.codexScanCompletedFiles == corpusSize) + #expect(finalCache.codexScanTotalFiles == corpusSize) + #expect(finalCache.codexScanInventoryPaths?.count == corpusSize) + #expect(finalCache.codexScanCatchUpPending == false) + } + + @Test + func `narrow bounded catch-up completes a retained-window pending file`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let priorDay = try #require(Calendar.current.date(byAdding: .day, value: -1, to: day)) + let retainedURL = try #require(Self.writeSyntheticCorpus(env: env, day: priorDay, fileCount: 1).first) + + var options = Self.boundedOptions(env: env) + options.maxCodexScanDurationPerRefresh = nil + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: priorDay, + until: day, + now: day, + options: options) + + let handle = try FileHandle(forWritingTo: retainedURL) + try handle.seekToEnd() + let iso = env.isoString(for: day) + let appendedRow = + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":250,"cached_input_tokens":80,"output_tokens":30},"# + + #""model":"openai/gpt-5.2-codex"}}}"# + try handle.write(contentsOf: Data((appendedRow + "\n").utf8)) + try handle.close() + + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let pendingPath = try #require(pendingCache.files.keys.first { $0.hasSuffix(retainedURL.lastPathComponent) }) + pendingCache.files[pendingPath]?.codexScanComplete = false + pendingCache.codexActiveLookbackState = nil + pendingCache.codexScanInventoryPaths = nil + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = 60 + let boundedRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = boundedRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let boundedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let retainedPath = try #require(boundedCache.files.keys.first { $0.hasSuffix(retainedURL.lastPathComponent) }) + #expect(boundedRecorder.snapshot().codexCandidateSelectionVisits == 1) + #expect(boundedRecorder.snapshot().codexFileScanAttempts == 1) + #expect(boundedRecorder.snapshot().codexProgressAccountingVisits == 0) + #expect(boundedCache.files[retainedPath]?.lastCountedTotals?.input == 250) + #expect(boundedCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(boundedCache.codexScanCatchUpPending == true) + + let exactRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = exactRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let exactCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(exactRecorder.snapshot().codexCandidateSelectionVisits == 0) + #expect(exactRecorder.snapshot().codexFileScanAttempts == 0) + #expect(exactRecorder.snapshot().codexProgressAccountingVisits == 1) + #expect(exactCache.codexActiveLookbackState == nil) + #expect(exactCache.codexScanCatchUpPending == false) + } + + @Test + func `time limited catch-up keeps bounded progress until exact validation`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = 1500 + try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + + var options = Self.boundedOptions(env: env) + let saveCounter = BoundedProgressCounter() + CostUsageStore.codexCatchUpReconciliationVisitForTesting = { saveCounter.increment() } + let firstRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = firstRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let firstMetrics = firstRecorder.snapshot() + + let loadCounter = BoundedProgressCounter() + CostUsageStore.codexCatchUpReconciliationVisitForTesting = { loadCounter.increment() } + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + CostUsageStore.codexCatchUpReconciliationVisitForTesting = nil + #expect(saveCounter.value == 0) + #expect(loadCounter.value == 0) + #expect(firstMetrics.codexFileScanAttempts == 512) + #expect(firstMetrics.codexCandidateSelectionVisits == 512) + #expect(firstMetrics.activeLookbackCompletionCandidates == 512) + #expect(firstMetrics.codexProgressAccountingVisits == 0) + #expect(firstCache.files.count == 512) + #expect(firstMetrics.codexDiscoveryVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(firstCache.codexScanProcessedBytes == 0) + #expect(firstCache.codexScanTotalBytes == 0) + #expect(firstCache.codexScanCompletedFiles == CostUsageScanner.codexCatchUpScanCandidateLimit - 1) + #expect(firstCache.codexScanTotalFiles == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstCache.codexScanInventoryPaths == nil) + #expect(firstCache.codexScanCatchUpPending == true) + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let secondMetrics = secondRecorder.snapshot() + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(secondMetrics.codexFileScanAttempts == 512) + #expect(secondMetrics.codexCandidateSelectionVisits == 512) + #expect(secondMetrics.activeLookbackCompletionCandidates == 512) + #expect(secondMetrics.codexProgressAccountingVisits == 0) + #expect(secondCache.files.count == 1024) + #expect(secondMetrics.codexDiscoveryVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(secondCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(secondCache.codexScanProcessedBytes == 0) + #expect(secondCache.codexScanTotalBytes == 0) + #expect(secondCache.codexScanCompletedFiles == CostUsageScanner.codexCatchUpScanCandidateLimit - 1) + #expect(secondCache.codexScanTotalFiles == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(secondCache.codexScanInventoryPaths == nil) + #expect(secondCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + options.maxCodexScanDurationPerRefresh = nil + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let finalMetrics = finalRecorder.snapshot() + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let exactTotalBytes = finalCache.files.values.reduce(Int64(0)) { $0 + max(0, $1.size) } + #expect(finalMetrics.codexProgressAccountingVisits == corpusSize) + #expect(finalCache.codexActiveLookbackState == nil) + #expect(finalCache.codexScanCatchUpPending == false) + #expect(finalCache.files.count == corpusSize) + #expect(Set(finalCache.codexScanInventoryPaths ?? []) == Set(finalCache.files.keys)) + #expect(finalCache.codexScanProcessedBytes == exactTotalBytes) + #expect(finalCache.codexScanTotalBytes == exactTotalBytes) + #expect(finalCache.codexScanCompletedFiles == corpusSize) + #expect(finalCache.codexScanTotalFiles == corpusSize) + + var deferredCompletionCache = finalCache + deferredCompletionCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: deferredCompletionCache) + let restoredPendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(restoredPendingCache.codexScanCatchUpPending == true) + } + + @Test + func `bounded queue advances past a cached complete prefix`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 1 + let fileURLs = try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + let oldModificationDate = day.addingTimeInterval(-24 * 60 * 60) + for fileURL in fileURLs { + try FileManager.default.setAttributes( + [.modificationDate: oldModificationDate], + ofItemAtPath: fileURL.path) + } + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let incompleteFilename = try #require(fileURLs.last?.lastPathComponent) + let incompletePath = try #require(pendingCache.files.keys.first { $0.hasSuffix(incompleteFilename) }) + pendingCache.files[incompletePath]?.codexScanComplete = false + pendingCache.codexActiveLookbackState = try Self.completedLookbackState( + cache: pendingCache, + options: options, + pendingFilePaths: fileURLs.map(\.path.resolvingTemporaryPath)) + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = 60 + let recorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = recorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + + let firstMetrics = recorder.snapshot() + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(firstMetrics.codexCandidateSelectionVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstMetrics.codexFileScanAttempts == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstMetrics.codexProgressAccountingVisits == 0) + #expect(firstCache.codexActiveLookbackState?.pendingFilePaths == [incompletePath.resolvingTemporaryPath]) + #expect(firstCache.files[incompletePath]?.codexScanComplete == false) + #expect(firstCache.codexScanCompletedFiles == corpusSize - 1) + #expect(firstCache.codexScanTotalFiles == corpusSize) + #expect(firstCache.codexScanInventoryPaths == nil) + #expect(firstCache.codexScanCatchUpPending == true) + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondMetrics = secondRecorder.snapshot() + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(secondMetrics.codexCandidateSelectionVisits == 1) + #expect(secondMetrics.codexFileScanAttempts == 1) + #expect(secondMetrics.codexProgressAccountingVisits == 0) + #expect(secondCache.files[incompletePath]?.codexScanComplete == true) + #expect(secondCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(secondCache.codexScanCompletedFiles == corpusSize - 1) + #expect(secondCache.codexScanTotalFiles == corpusSize) + #expect(secondCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(finalRecorder.snapshot().codexCandidateSelectionVisits == 0) + #expect(finalRecorder.snapshot().codexFileScanAttempts == 0) + #expect(finalRecorder.snapshot().codexProgressAccountingVisits == corpusSize) + #expect(finalCache.codexActiveLookbackState == nil) + #expect(finalCache.codexScanCompletedFiles == corpusSize) + #expect(finalCache.codexScanTotalFiles == corpusSize) + #expect(finalCache.codexScanCatchUpPending == false) + } + + @Test + func `bounded queue rescans an appended cached complete path outside the first slice`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 2 + let fileURLs = try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + let oldModificationDate = day.addingTimeInterval(-24 * 60 * 60) + for fileURL in fileURLs { + try FileManager.default.setAttributes( + [.modificationDate: oldModificationDate], + ofItemAtPath: fileURL.path) + } + + var options = Self.boundedOptions(env: env) + options.maxCodexScanDurationPerRefresh = nil + options.preferNewestCodexSessionsFirst = false + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + let appendedURL = fileURLs[CostUsageScanner.codexCatchUpScanCandidateLimit] + let incompleteURL = try #require(fileURLs.last) + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let appendedPath = try #require(pendingCache.files.keys.first { $0.hasSuffix(appendedURL.lastPathComponent) }) + let incompletePath = try #require(pendingCache.files.keys + .first { $0.hasSuffix(incompleteURL.lastPathComponent) }) + let beforeTotals = try #require(pendingCache.files[appendedPath]?.lastCountedTotals) + #expect(beforeTotals.input == 100) + #expect(beforeTotals.cached == 20) + pendingCache.files[incompletePath]?.codexScanComplete = false + pendingCache.codexActiveLookbackState = try Self.completedLookbackState( + cache: pendingCache, + options: options, + pendingFilePaths: fileURLs.map(\.path.resolvingTemporaryPath)) + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = 60 + let firstRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = firstRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let firstMetrics = firstRecorder.snapshot() + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(firstMetrics.codexCandidateSelectionVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstMetrics.codexFileScanAttempts == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstMetrics.codexProgressAccountingVisits == 0) + #expect(firstCache.codexActiveLookbackState?.pendingFilePaths.count == 2) + + let iso = env.isoString(for: day.addingTimeInterval(2)) + let appendedLine = [ + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"#, + #"{"total_token_usage":{"input_tokens":250,"cached_input_tokens":80,"output_tokens":30},"#, + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined() + let handle = try FileHandle(forWritingTo: appendedURL) + try handle.seekToEnd() + try handle.write(contentsOf: Data((appendedLine + "\n").utf8)) + try handle.close() + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondMetrics = secondRecorder.snapshot() + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let afterTotals = try #require(secondCache.files[appendedPath]?.lastCountedTotals) + #expect(secondMetrics.codexCandidateSelectionVisits == 2) + #expect(secondMetrics.codexFileScanAttempts == 2) + #expect(secondMetrics.codexProgressAccountingVisits == 0) + #expect(afterTotals.input == 250) + #expect(afterTotals.cached == 80) + #expect(afterTotals.output == 30) + #expect(afterTotals != beforeTotals) + #expect(secondCache.files[incompletePath]?.codexScanComplete == true) + #expect(secondCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(secondCache.codexScanCompletedFiles == corpusSize - 1) + #expect(secondCache.codexScanTotalFiles == corpusSize) + #expect(secondCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(finalRecorder.snapshot().codexCandidateSelectionVisits == 0) + #expect(finalRecorder.snapshot().codexFileScanAttempts == 0) + #expect(finalRecorder.snapshot().codexProgressAccountingVisits == corpusSize) + #expect(finalCache.codexActiveLookbackState == nil) + #expect(finalCache.codexScanCompletedFiles == corpusSize) + #expect(finalCache.codexScanTotalFiles == corpusSize) + #expect(finalCache.codexScanCatchUpPending == false) + } + + @Test + func `active bounded queue appends a newly discovered tail path`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 1 + try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + + var options = Self.boundedOptions(env: env) + options.preferNewestCodexSessionsFirst = false + let firstRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = firstRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(firstRecorder.snapshot().codexFileScanAttempts == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + + let iso = env.isoString(for: day.addingTimeInterval(1)) + _ = try env.writeCodexSessionFile( + day: day, + filename: "progress-new-tail.jsonl", + contents: [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"progress-new-tail"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":300,"cached_input_tokens":40,"output_tokens":20},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n") + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let secondMetrics = secondRecorder.snapshot() + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(secondMetrics.codexCandidateSelectionVisits == 1) + #expect(secondMetrics.codexFileScanAttempts == 1) + #expect(secondMetrics.codexProgressAccountingVisits == 0) + #expect(secondCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(secondCache.files.count == corpusSize) + #expect(secondCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let finalMetrics = finalRecorder.snapshot() + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(finalMetrics.codexCandidateSelectionVisits == 0) + #expect(finalMetrics.codexFileScanAttempts == 0) + #expect(finalMetrics.codexProgressAccountingVisits == 0) + #expect(finalCache.codexActiveLookbackState?.pendingFilePaths.count == 1) + #expect(finalCache.codexScanCatchUpPending == true) + + let validationRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = validationRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let validatedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(validationRecorder.snapshot().codexCandidateSelectionVisits == 1) + #expect(validationRecorder.snapshot().codexFileScanAttempts == 1) + #expect(validationRecorder.snapshot().codexProgressAccountingVisits == 0) + #expect(validatedCache.files.count == corpusSize + 1) + #expect(validatedCache.codexScanCatchUpPending == true) + + let exactRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = exactRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(4), + options: options) + let exactCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(exactRecorder.snapshot().codexCandidateSelectionVisits == 0) + #expect(exactRecorder.snapshot().codexFileScanAttempts == 0) + #expect(exactRecorder.snapshot().codexProgressAccountingVisits == corpusSize + 1) + #expect(exactCache.codexActiveLookbackState == nil) + #expect(exactCache.codexScanCompletedFiles == corpusSize + 1) + #expect(exactCache.codexScanTotalFiles == corpusSize + 1) + #expect(exactCache.codexScanCatchUpPending == false) + } + + @Test + func `exact validation requeues a completed prefix path rewritten after its slice`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 1 + let fileURLs = try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + + var options = Self.boundedOptions(env: env) + options.maxCodexScanDurationPerRefresh = nil + options.preferNewestCodexSessionsFirst = false + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + let rewrittenURL = fileURLs[0] + let incompleteURL = try #require(fileURLs.last) + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let rewrittenPath = try #require(pendingCache.files.keys.first { $0.hasSuffix(rewrittenURL.lastPathComponent) }) + let incompletePath = try #require(pendingCache.files.keys + .first { $0.hasSuffix(incompleteURL.lastPathComponent) }) + pendingCache.files[incompletePath]?.codexScanComplete = false + pendingCache.codexActiveLookbackState = nil + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = 60 + let firstRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = firstRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + #expect(firstRecorder.snapshot().codexFileScanAttempts == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstRecorder.snapshot().codexProgressAccountingVisits == 0) + + let original = try String(contentsOf: rewrittenURL, encoding: .utf8) + let rewritten = original.replacingOccurrences(of: #""input_tokens":100"#, with: #""input_tokens":900"#) + #expect(rewritten != original) + #expect(rewritten.utf8.count == original.utf8.count) + try rewritten.write(to: rewrittenURL, atomically: false, encoding: .utf8) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(120)], + ofItemAtPath: rewrittenURL.path) + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondMetrics = secondRecorder.snapshot() + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(secondMetrics.codexCandidateSelectionVisits == 1) + #expect(secondMetrics.codexFileScanAttempts == 1) + #expect(secondMetrics.codexProgressAccountingVisits == 0) + #expect(secondCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(secondCache.codexScanCompletedFiles == corpusSize - 1) + #expect(secondCache.codexScanInventoryPaths == nil) + #expect(secondCache.codexScanCatchUpPending == true) + + let thirdRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = thirdRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let thirdMetrics = thirdRecorder.snapshot() + let thirdCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(thirdMetrics.codexCandidateSelectionVisits == 0) + #expect(thirdMetrics.codexFileScanAttempts == 0) + #expect(thirdMetrics.codexProgressAccountingVisits == corpusSize) + #expect(thirdCache.codexActiveLookbackState == nil) + #expect(thirdCache.codexScanCompletedFiles == corpusSize - 1) + #expect(thirdCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + options.maxCodexScanDurationPerRefresh = nil + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(4), + options: options) + let finalMetrics = finalRecorder.snapshot() + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(finalMetrics.codexCandidateSelectionVisits == 0) + #expect(finalMetrics.codexFileScanAttempts == corpusSize) + #expect(finalMetrics.codexProgressAccountingVisits == corpusSize) + #expect(finalCache.files[rewrittenPath]?.lastCountedTotals?.input == 900) + #expect(finalCache.codexActiveLookbackState == nil) + #expect(finalCache.codexScanCompletedFiles == corpusSize) + #expect(finalCache.codexScanTotalFiles == corpusSize) + #expect(finalCache.codexScanCatchUpPending == false) + } + + @Test + func `missing queue prefix advances after scanner validation`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 1 + let fileURLs = try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + + var options = Self.boundedOptions(env: env) + options.maxCodexScanDurationPerRefresh = nil + options.preferNewestCodexSessionsFirst = false + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let validURL = try #require(fileURLs.last) + let validPath = try #require(pendingCache.files.keys.first { $0.hasSuffix(validURL.lastPathComponent) }) + let roots = CostUsageScanner.codexSessionsRoots(options: options) + .map { $0.resolvingSymlinksInPath().standardizedFileURL.path } + .sorted() + let missingURLs = fileURLs.prefix(CostUsageScanner.codexCatchUpScanCandidateLimit) + for fileURL in missingURLs { + try FileManager.default.removeItem(at: fileURL) + } + pendingCache.files[validPath]?.codexScanComplete = false + pendingCache.codexScanInventoryPaths = nil + pendingCache.codexActiveLookbackState = try CostUsageCodexActiveLookbackState( + scanSinceKey: #require(pendingCache.scanSinceKey), + rootPaths: roots, + completedRootPaths: roots, + pendingFilePaths: missingURLs.map(\.path.resolvingTemporaryPath) + [validURL.path.resolvingTemporaryPath]) + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = 60 + let firstRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = firstRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let firstMetrics = firstRecorder.snapshot() + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(firstMetrics.codexCandidateSelectionVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstMetrics.codexFileScanAttempts == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(firstMetrics.codexProgressAccountingVisits == 0) + #expect(firstCache.codexActiveLookbackState?.pendingFilePaths == [validURL.path.resolvingTemporaryPath]) + #expect(firstCache.codexScanCatchUpPending == true) + + let finalRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = finalRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let finalMetrics = finalRecorder.snapshot() + let finalCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(finalMetrics.codexCandidateSelectionVisits == 1) + #expect(finalMetrics.codexFileScanAttempts == 1) + #expect(finalMetrics.codexProgressAccountingVisits == 0) + #expect(finalCache.codexActiveLookbackState?.pendingFilePaths.isEmpty == true) + #expect(finalCache.codexScanCompletedFiles == corpusSize - 1) + #expect(finalCache.codexScanTotalFiles == corpusSize) + #expect(finalCache.codexScanCatchUpPending == true) + + let validationRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = validationRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(3), + options: options) + let validatedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(validationRecorder.snapshot().codexCandidateSelectionVisits == 0) + #expect(validationRecorder.snapshot().codexFileScanAttempts == 0) + #expect(validationRecorder.snapshot().codexProgressAccountingVisits == 1) + #expect(validatedCache.codexActiveLookbackState == nil) + #expect(validatedCache.codexScanCompletedFiles == 1) + #expect(validatedCache.codexScanTotalFiles == 1) + #expect(validatedCache.codexScanCatchUpPending == false) + } + + @Test + func `time budget stop retains selected paths that were not scanned`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 1 + let fileURLs = try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + + var options = Self.boundedOptions(env: env) + options.maxCodexScanDurationPerRefresh = nil + options.preferNewestCodexSessionsFirst = false + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + let incompleteURL = try #require(fileURLs.last) + let incompletePath = try #require(pendingCache.files.keys + .first { $0.hasSuffix(incompleteURL.lastPathComponent) }) + pendingCache.files[incompletePath]?.codexScanComplete = false + pendingCache.codexScanInventoryPaths = nil + pendingCache.codexActiveLookbackState = nil + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = .leastNonzeroMagnitude + let recorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = recorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let metrics = recorder.snapshot() + let stoppedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(metrics.codexCandidateSelectionVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(metrics.codexFileScanAttempts == 0) + #expect(metrics.activeLookbackCompletionCandidates == 0) + #expect(metrics.codexProgressAccountingVisits == 0) + #expect(stoppedCache.codexActiveLookbackState?.pendingFilePaths.count + == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(stoppedCache.files[incompletePath]?.codexScanComplete == false) + #expect(stoppedCache.codexScanCatchUpPending == true) + } + + @Test + func `reset progress baseline counts validated cached snapshots`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = CostUsageScanner.codexCatchUpScanCandidateLimit + 1 + try Self.writeSyntheticCorpus(env: env, day: day, fileCount: corpusSize) + + var options = Self.boundedOptions(env: env) + options.maxCodexScanDurationPerRefresh = nil + options.preferNewestCodexSessionsFirst = false + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + var pendingCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + for path in pendingCache.files.keys { + pendingCache.files[path]?.codexCostCacheComplete = false + } + pendingCache.codexScanInventoryPaths = nil + pendingCache.codexActiveLookbackState = nil + pendingCache.codexScanCatchUpPending = true + CostUsageStoreAccess.replace(cacheRoot: env.cacheRoot, cache: pendingCache) + + options.maxCodexScanDurationPerRefresh = 60 + let recorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = recorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let metrics = recorder.snapshot() + let migratedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + #expect(metrics.codexCandidateSelectionVisits == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(metrics.codexFileScanAttempts == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(metrics.codexProgressAccountingVisits == 0) + #expect(migratedCache.codexScanCompletedFiles == CostUsageScanner.codexCatchUpScanCandidateLimit) + #expect(migratedCache.codexScanTotalFiles == corpusSize) + #expect(migratedCache.codexActiveLookbackState?.pendingFilePaths.count == 1) + #expect(migratedCache.codexScanCatchUpPending == true) + } + + private static func completedLookbackState( + cache: CostUsageCache, + options: CostUsageScanner.Options, + pendingFilePaths: [String]) throws -> CostUsageCodexActiveLookbackState + { + let roots = CostUsageScanner.codexSessionsRoots(options: options) + .map { $0.resolvingSymlinksInPath().standardizedFileURL.path } + .sorted() + return try CostUsageCodexActiveLookbackState( + scanSinceKey: #require(cache.scanSinceKey), + rootPaths: roots, + completedRootPaths: roots, + pendingFilePaths: pendingFilePaths, + currentWindowNextDayKeyByRoot: [:], + currentWindowDirectoryOffsetByRoot: [:], + completedCurrentWindowRootPaths: roots, + currentWindowFlatDirectoryOffsetByRoot: [:], + completedCurrentWindowFlatRootPaths: roots) + } + + private static func boundedOptions(env: CostUsageTestEnvironment) -> CostUsageScanner.Options { + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0, + maxCodexScanDurationPerRefresh: 60) + options.refreshMinIntervalSeconds = 0 + return options + } + + @discardableResult + private static func writeSyntheticCorpus( + env: CostUsageTestEnvironment, + day: Date, + fileCount: Int) throws -> [URL] + { + let iso = env.isoString(for: day) + var fileURLs: [URL] = [] + fileURLs.reserveCapacity(fileCount) + for index in 0.. CostUsageScanner.Options { + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0, + maxCodexScanDurationPerRefresh: 60) + options.refreshMinIntervalSeconds = 0 + return options + } + + private static func writeSyntheticCorpus( + env: CostUsageTestEnvironment, + day: Date, + fileCount: Int) throws -> [URL] + { + let iso = env.isoString(for: day) + return try (0.. firstResumeOffset) + #expect(CostUsageFetcher.codexScanProgressKey(cache: secondCache, scopedFiles: [:]) + != CostUsageFetcher.codexScanProgressKey(cache: firstCache, scopedFiles: [:])) + } + + @Test + func `progress key includes active lookback cursor and ignores dictionary insertion order`() { + var initialCache = CostUsageCache() + initialCache.codexActiveLookbackState = CostUsageCodexActiveLookbackState( + scanSinceKey: "2026-07-01", + rootPaths: ["/sessions", "/archived_sessions"], + nextDayKeyByRoot: [ + "/sessions": "2026-07-02", + "/archived_sessions": "2026-07-03", + ]) + var advancedCache = initialCache + advancedCache.codexActiveLookbackState?.nextDayKeyByRoot["/sessions"] = "2026-07-01" + + let first = CostUsageScanner.makeFileUsage( + mtimeUnixMs: 1, + size: 10, + days: [:], + parsedBytes: 10, + codexScanFileId: "1:1", + codexScanComplete: true) + let second = CostUsageScanner.makeFileUsage( + mtimeUnixMs: 1, + size: 20, + days: [:], + parsedBytes: 10, + codexScanFileId: "2:2", + codexScanComplete: false) + var forward: [String: CostUsageFileUsage] = [:] + forward["/sessions/a.jsonl"] = first + forward["/sessions/b.jsonl"] = second + var reverse: [String: CostUsageFileUsage] = [:] + reverse["/sessions/b.jsonl"] = second + reverse["/sessions/a.jsonl"] = first + + let initial = CostUsageFetcher.codexScanProgressKey(cache: initialCache, scopedFiles: forward) + let advanced = CostUsageFetcher.codexScanProgressKey(cache: advancedCache, scopedFiles: forward) + let reordered = CostUsageFetcher.codexScanProgressKey(cache: initialCache, scopedFiles: reverse) + + #expect(advanced != initial) + #expect(reordered == initial) + } + + @Test + func `progress key includes bounded current window directory cursor`() { + var initialCache = CostUsageCache() + initialCache.codexActiveLookbackState = CostUsageCodexActiveLookbackState( + scanSinceKey: "2026-07-01", + rootPaths: ["/sessions"], + currentWindowDirectoryOffsetByRoot: ["/sessions": 512]) + var advancedCache = initialCache + advancedCache.codexActiveLookbackState?.currentWindowDirectoryOffsetByRoot?["/sessions"] = 1024 + + #expect(CostUsageFetcher.codexScanProgressKey(cache: advancedCache, scopedFiles: [:]) + != CostUsageFetcher.codexScanProgressKey(cache: initialCache, scopedFiles: [:])) + } + + @Test + func `progress key changes when exact proof inventory is installed`() { + var beforeProof = CostUsageCache() + beforeProof.codexScanCatchUpPending = true + beforeProof.codexScanCompletedFiles = 1 + var afterProof = beforeProof + afterProof.codexScanInventoryPaths = ["/sessions/complete.jsonl"] + + #expect(CostUsageFetcher.codexScanProgressKey(cache: afterProof, scopedFiles: [:]) + != CostUsageFetcher.codexScanProgressKey(cache: beforeProof, scopedFiles: [:])) + } +} diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index 28b0fcd740..41b082c9f7 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -1,3 +1,4 @@ +// swiftlint:disable file_length import Foundation #if canImport(Darwin) import Darwin @@ -10,7 +11,6 @@ import Testing @testable import CodexBarCore // The performance corpus and its fixtures intentionally stay together so timing gates share setup. -// swiftlint:disable file_length /// Regression gates for the two cost-usage scan-storm classes that have shipped before: /// re-parsing unchanged session files on every refresh (#1387, #1392) and re-running the @@ -18,6 +18,149 @@ import Testing @Suite(.serialized) // swiftlint:disable:next type_body_length struct CostUsagePerformanceGateTests { + @Test + func `time limited codex catch-up bounds oversized active day discovery`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + CostUsageScanner.resetCodexDirectoryCursorsForTesting() + defer { CostUsageScanner.resetCodexDirectoryCursorsForTesting() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = 1500 + let candidateLimit = CostUsageScanner.codexCatchUpScanCandidateLimit + _ = try Self.writeSyntheticCodexCorpus( + env: env, + day: day, + files: corpusSize, + turnsPerFile: 1) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0, + maxCodexScanDurationPerRefresh: 60) + options.refreshMinIntervalSeconds = 0 + + let firstRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = firstRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let firstMetrics = firstRecorder.snapshot() + let firstCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + print( + "[discovery-proof] first=\(firstCache.files.count), " + + "discovery=\(firstMetrics.codexDiscoveryVisits), " + + "attempts=\(firstMetrics.codexFileScanAttempts)") + + #expect(firstMetrics.codexDiscoveryVisits == candidateLimit) + #expect(firstMetrics.codexFileScanAttempts == candidateLimit) + #expect(firstCache.files.count == candidateLimit) + #expect(firstCache.codexScanCatchUpPending == true) + + CostUsageScanner.resetCodexDirectoryCursorsForTesting() + let relaunchedRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = relaunchedRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let relaunchedMetrics = relaunchedRecorder.snapshot() + let relaunchedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + print( + "[discovery-proof] relaunched=\(relaunchedCache.files.count), " + + "discovery=\(relaunchedMetrics.codexDiscoveryVisits), " + + "attempts=\(relaunchedMetrics.codexFileScanAttempts)") + + #expect(relaunchedMetrics.codexDiscoveryVisits == candidateLimit) + #expect(relaunchedMetrics.codexFileScanAttempts == 0) + #expect(relaunchedCache.files.count == candidateLimit) + #expect(relaunchedCache.codexScanCatchUpPending == true) + + let secondRecorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = secondRecorder + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondMetrics = secondRecorder.snapshot() + let secondCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) + print( + "[discovery-proof] second=\(secondCache.files.count), " + + "discovery=\(secondMetrics.codexDiscoveryVisits), " + + "visits=\(secondMetrics.codexCandidateSelectionVisits), " + + "attempts=\(secondMetrics.codexFileScanAttempts), " + + "accounting=\(secondMetrics.codexProgressAccountingVisits)") + + #expect(secondMetrics.codexDiscoveryVisits == candidateLimit) + #expect(secondMetrics.codexCandidateSelectionVisits == candidateLimit) + #expect(secondMetrics.codexFileScanAttempts == candidateLimit) + #expect(secondMetrics.codexProgressAccountingVisits == 0) + #expect(secondCache.files.count == candidateLimit * 2) + #expect(secondCache.codexScanCatchUpPending == true) + } + + @Test + func `warm codex refresh indexes cache aliases once at incident corpus scale`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let corpusSize = 1500 + _ = try Self.writeSyntheticCodexCorpus( + env: env, + day: day, + files: corpusSize, + turnsPerFile: 1) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite")) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + let recorder = CostUsageScanner.CodexScanWorkRecorder() + options.codexScanWorkRecorderForTesting = recorder + let started = ContinuousClock.now + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let elapsed = ContinuousClock.now - started + let metrics = recorder.snapshot() + + // The cache identity index may inspect only the matching identity bucket per file. + // With unique files that is one candidate per lookup, not corpusSize² cache visits. + #expect(metrics.cacheAliasEntriesIndexed == corpusSize) + #expect(metrics.cacheAliasLookups == corpusSize) + #expect(metrics.cacheAliasCandidatesVisited == corpusSize) + #expect(metrics.usageRowsProcessed == 0) + #expect(elapsed < TestTimingBudget.scaled(.seconds(10))) + let elapsedComponents = elapsed.components + let elapsedMilliseconds = elapsedComponents.seconds * 1000 + + elapsedComponents.attoseconds / 1_000_000_000_000_000 + print( + "[alias-index-proof] warm refresh \(corpusSize) files: \(elapsedMilliseconds) ms, " + + "lookups=\(metrics.cacheAliasLookups), candidates=\(metrics.cacheAliasCandidatesVisited)") + } + @Test func `warm codex refresh over an unchanged session corpus must not re-parse it`() throws { let env = try CostUsageTestEnvironment() @@ -841,10 +984,10 @@ struct CostUsagePerformanceGateTests { let fetcher = CostUsageFetcher(scannerOptions: options) var status = await fetcher.codexScanCatchUpStatus() #expect(status.pending) - var progressKeys = [status.progressKey] + var progressStates = [(pending: status.pending, key: status.progressKey)] for _ in 0..<12 where status.pending { status = try await fetcher.advanceCodexScanCatchUp(now: day, historyDays: 1) - progressKeys.append(status.progressKey) + progressStates.append((pending: status.pending, key: status.progressKey)) } let completedCache = CostUsageStoreAccess.read(cacheRoot: env.cacheRoot) @@ -855,7 +998,9 @@ struct CostUsagePerformanceGateTests { #expect(!status.pending) #expect(completedUsage.codexScanComplete == true) #expect(completedUsage.parsedBytes == metadata.size) - #expect(zip(progressKeys, progressKeys.dropFirst()).allSatisfy(!=)) + #expect(zip(progressStates, progressStates.dropFirst()).allSatisfy { previous, next in + previous.key != next.key || (previous.pending && !next.pending) + }) #expect(completedReport.summary?.totalTokens == baseline.summary?.totalTokens) #expect(completedReport.data.map(\.totalTokens) == baseline.data.map(\.totalTokens)) } @@ -1529,7 +1674,6 @@ extension CostUsagePerformanceGateTests { maxCodexSessionFileBytes: 1024, maxCodexScanBytesPerRefresh: 64 * 1024 * 1024) options.refreshMinIntervalSeconds = 0 - let started = Date() _ = CostUsageScanner.loadDailyReport( provider: .codex, diff --git a/Tests/CodexBarTests/CostUsageStoreTests.swift b/Tests/CodexBarTests/CostUsageStoreTests.swift index 22e3494d6b..3e6c3e4da9 100644 --- a/Tests/CodexBarTests/CostUsageStoreTests.swift +++ b/Tests/CodexBarTests/CostUsageStoreTests.swift @@ -1008,6 +1008,7 @@ extension CostUsageStoreTests { "98da5914d2f6a9cd", "43609cc56f76a003", "b975eb705f905b9a", + "47144baa8daccf52", ]) let predecessorHash = "43609cc56f76a003" let predecessorVersion = CostUsageStore.combinedSchemaVersion( @@ -1119,6 +1120,31 @@ extension CostUsageStoreTests { #expect(await store.rebuildCount == 1) } + @Test + func `v0_49_2 parser hash upgrades without rebuilding completed files`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let previousParserHash = "b975eb705f905b9a" + let previousSchemaVersion = CostUsageStore.combinedSchemaVersion( + base: CostUsageStore.baseSchemaVersion, + parserHash: previousParserHash) + let previousStore = CostUsageStore( + cacheRoot: fixture.root, + schemaVersion: previousSchemaVersion, + parserHash: previousParserHash) + let file = Self.file(path: "/rollouts/completed.jsonl", day: "2026-08-01") + #expect(await previousStore.upsertFile(file)) + + let upgradedStore = CostUsageStore(cacheRoot: fixture.root) + + #expect(await upgradedStore.fetchFile(path: file.path) == file) + #expect(await upgradedStore.rebuildCount == 0) + #expect(await upgradedStore.configuration()?.userVersion == Int(CostUsageStore.schemaVersion)) + let connection = try SQLiteTestConnection(url: fixture.databaseURL, readOnly: true) + #expect(try connection.scalarInt( + "SELECT COUNT(*) FROM meta WHERE key = 'parser_hash' AND value = '\(CodexParserHash.value)'") == 1) + } + @Test func `compatible predecessor hash with mismatched version still rebuilds`() async throws { let fixture = try StoreFixture() @@ -2018,6 +2044,7 @@ extension CostUsageStoreTests { totalBytes: 200, completedFiles: 2, totalFiles: 4, + scanInventoryPaths: ["/root/2026/08/01/session.jsonl"], rootMtimes: ["/root": 123], previousReportPayload: Data([2, 4, 6]), priorityTurnStatePayload: Data([1, 3, 5]), diff --git a/Tests/CodexBarTests/ProviderArchitectureGatekeeperTests.swift b/Tests/CodexBarTests/ProviderArchitectureGatekeeperTests.swift index 6ede691a27..bac0e58634 100644 --- a/Tests/CodexBarTests/ProviderArchitectureGatekeeperTests.swift +++ b/Tests/CodexBarTests/ProviderArchitectureGatekeeperTests.swift @@ -998,37 +998,37 @@ struct ProviderArchitectureGatekeeperTests { reason: "This provider-owned integration passes its fixed identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 41, + line: 47, anchor: "providerConfigRevision: self.settings.providerConfigRevision(for: .codex),", expectedProviderIDs: ["codex"], reason: "This provider-owned integration passes its fixed identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 242, + line: 255, anchor: "provider: .codex,", expectedProviderIDs: ["codex"], reason: "This provider-owned integration passes its fixed identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 255, + line: 268, anchor: "self.publishConfirmedEmptyTokenSnapshot(for: .codex)", expectedProviderIDs: ["codex"], reason: "This provider-owned integration passes its fixed identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 258, + line: 271, anchor: "self.publishTokenSnapshot(snapshot, for: .codex)", expectedProviderIDs: ["codex"], reason: "This provider-owned integration passes its fixed identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 278, + line: 291, anchor: "&& self.settings.isCostUsageEffectivelyEnabled(for: .codex)", expectedProviderIDs: ["codex"], reason: "This provider-owned integration passes its fixed identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 279, + line: 292, anchor: "&& self.isEnabled(.codex)", expectedProviderIDs: ["codex"], reason: "This provider-owned integration passes its fixed identity to a shared helper."), @@ -1118,19 +1118,19 @@ struct ProviderArchitectureGatekeeperTests { reason: "This Codex account projection passes its fixed provider identity to shared spend infrastructure."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift", - line: 56, + line: 63, anchor: "providerConfigRevision: self.settings.providerConfigRevision(for: .codex),", expectedProviderIDs: ["codex"], reason: "This Codex account projection passes its fixed provider identity to shared spend infrastructure."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift", - line: 248, + line: 265, anchor: "&& self.settings.isCostUsageEffectivelyEnabled(for: .codex)", expectedProviderIDs: ["codex"], reason: "This Codex account projection passes its fixed provider identity to shared spend infrastructure."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift", - line: 249, + line: 266, anchor: "&& self.isEnabled(.codex)", expectedProviderIDs: ["codex"], reason: "This Codex account projection passes its fixed provider identity to shared spend infrastructure."), @@ -1275,19 +1275,19 @@ struct ProviderArchitectureGatekeeperTests { reason: "Claude widget quota ownership uses the selected Claude account's isolated snapshot key."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore.swift", - line: 1048, + line: 1050, anchor: "provider: .deepseek,", expectedProviderIDs: ["deepseek"], reason: "This provider-specific app branch passes its already-selected identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore.swift", - line: 1150, + line: 1152, anchor: "let sourceMode = self.sourceMode(for: .claude)", expectedProviderIDs: ["claude"], reason: "This provider-specific app branch passes its already-selected identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBar/UsageStore.swift", - line: 1154, + line: 1156, anchor: "provider: .claude,", expectedProviderIDs: ["claude"], reason: "This provider-specific app branch passes its already-selected identity to a shared helper."), @@ -1353,19 +1353,19 @@ struct ProviderArchitectureGatekeeperTests { reason: "This provider-specific core branch passes its already-selected identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 715, + line: 708, anchor: "provider: .codex,", expectedProviderIDs: ["codex"], reason: "This provider-specific core branch passes its already-selected identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 790, + line: 783, anchor: "provider: .codex,", expectedProviderIDs: ["codex"], reason: "This provider-specific core branch passes its already-selected identity to a shared helper."), SuppressedProviderReference( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 865, + line: 858, anchor: "provider: .codex,", expectedProviderIDs: ["codex"], reason: "This provider-specific core branch passes its already-selected identity to a shared helper."), @@ -2602,7 +2602,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 252, + line: 265, anchor: "self.lastTokenFetchAt[.codex] = now", expectedProviderIDs: ["codex"], expectedReferenceCount: 6, @@ -2610,7 +2610,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore+CodexCostCatchUp.swift", - line: 275, + line: 288, anchor: "&& self.settings.providerConfigRevision(for: .codex) == context.providerConfigRevision", expectedProviderIDs: ["codex"], expectedReferenceCount: 3, @@ -2915,7 +2915,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore+SpendDashboardCodexCostCatchUp.swift", - line: 246, + line: 262, anchor: "&& self.settings.providerConfigRevision(for: .codex) == context.providerConfigRevision", expectedProviderIDs: ["codex"], expectedReferenceCount: 1, @@ -3225,7 +3225,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 589, + line: 591, anchor: "self.metadata(for: .codex).browserCookieOrder ?? Browser.defaultImportOrder", expectedProviderIDs: ["codex"], expectedReferenceCount: 1, @@ -3233,7 +3233,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 641, + line: 643, anchor: "self.providerSpecs[provider]?.style ?? .codex", expectedProviderIDs: ["codex"], expectedReferenceCount: 1, @@ -3241,7 +3241,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 674, + line: 676, anchor: "guard provider != .codex else { return true }", expectedProviderIDs: ["codex"], expectedReferenceCount: 1, @@ -3249,7 +3249,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 1022, + line: 1024, anchor: "let claudeDebugConfiguration: ClaudeDebugLogConfiguration? = if provider == .claude {", expectedProviderIDs: ["claude"], expectedReferenceCount: 1, @@ -3257,7 +3257,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 1045, + line: 1047, anchor: "let deepSeekHasTokenAccount = self.settings.selectedTokenAccount(for: .deepseek) != nil", expectedProviderIDs: ["deepseek"], expectedReferenceCount: 1, @@ -3265,7 +3265,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 1102, + line: 1104, anchor: "case .amp:", expectedProviderIDs: ["amp", "deepseek", "notion", "ollama", "warp"], expectedReferenceCount: 7, @@ -3281,7 +3281,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact app-runtime bridge coordinates provider-owned state through the shared controller."), AllowedProviderConstruct( path: "Sources/CodexBar/UsageStore.swift", - line: 1157, + line: 1159, anchor: "let claudeSettings = snapshot.claude ?? ProviderSettingsSnapshot.ClaudeProviderSettings(", expectedProviderIDs: ["claude"], expectedReferenceCount: 1, @@ -3385,7 +3385,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact shared construct dispatches a provider-owned capability at the generic integration boundary."), AllowedProviderConstruct( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 539, + line: 532, anchor: "if provider == .codex {", expectedProviderIDs: ["codex"], expectedReferenceCount: 1, @@ -3393,7 +3393,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact cost scanner dispatch selects a provider-owned transcript, cache, or pricing format."), AllowedProviderConstruct( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 567, + line: 560, anchor: "provider == .claude || (provider == .codex && options.shouldMergePiUsage)", expectedProviderIDs: ["claude", "codex"], expectedReferenceCount: 5, @@ -3401,7 +3401,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact cost scanner dispatch selects a provider-owned transcript, cache, or pricing format."), AllowedProviderConstruct( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 614, + line: 607, anchor: "options.provider == .codex || options.provider == .claude", expectedProviderIDs: ["claude", "codex"], expectedReferenceCount: 2, @@ -3409,7 +3409,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact cost scanner dispatch selects a provider-owned transcript, cache, or pricing format."), AllowedProviderConstruct( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 641, + line: 634, anchor: "guard provider == .codex || provider == .claude else { return nil }", expectedProviderIDs: ["claude", "codex", "openai"], expectedReferenceCount: 5, @@ -3417,7 +3417,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact cost scanner dispatch selects a provider-owned transcript, cache, or pricing format."), AllowedProviderConstruct( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 1122, + line: 1115, anchor: "if provider == .vertexai {", expectedProviderIDs: ["claude", "vertexai"], expectedReferenceCount: 2, @@ -3425,7 +3425,7 @@ struct ProviderArchitectureGatekeeperTests { reason: "This exact cost scanner dispatch selects a provider-owned transcript, cache, or pricing format."), AllowedProviderConstruct( path: "Sources/CodexBarCore/CostUsageFetcher.swift", - line: 1373, + line: 1470, anchor: "if provider == .cursor {", expectedProviderIDs: ["cursor"], expectedReferenceCount: 1, diff --git a/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift b/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift index bf6e8c25be..c2641ff963 100644 --- a/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift +++ b/Tests/CodexBarTests/UsageStoreCodexCostCatchUpTests.swift @@ -87,6 +87,139 @@ struct UsageStoreCodexCostCatchUpTests { #expect(store.codexCostCatchUpActivity?.pauseReason == .noProgress) } + @Test + func `catch-up stops when bounded progress revisits an earlier semantic state`() async throws { + let store = try Self.makeStore(suite: "cyclic-progress") + let progressKeys = ["validation-1", "validation-2", "validation-0"] + var advanceCount = 0 + store._test_codexCostCatchUpStatusOverride = { _ in + CostUsageFetcher.CodexScanCatchUpStatus( + pending: true, + progressKey: "validation-0") + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: true, + progressKey: progressKeys[min(advanceCount - 1, progressKeys.count - 1)]) + } + store._test_codexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startCodexCostCatchUpIfNeeded(mode: .accelerated) + await Self.waitUntil { + store.codexCostCatchUpTask == nil + } + + #expect(advanceCount == 3) + #expect(store.codexCostCatchUpActivity?.phase == .paused) + #expect(store.codexCostCatchUpActivity?.pauseReason == .noProgress) + } + + @Test + func `catch-up continues when existing complete file backlog advances`() async throws { + let store = try Self.makeStore(suite: "existing-complete-backlog") + let first = CostUsageScanner.makeFileUsage( + mtimeUnixMs: 1, + size: 125, + days: [:], + parsedBytes: 125, + codexScanFileId: "1:1", + codexScanComplete: true) + let second = CostUsageScanner.makeFileUsage( + mtimeUnixMs: 1, + size: 125, + days: [:], + parsedBytes: 125, + codexScanFileId: "2:2", + codexScanComplete: true) + let files = [ + "/sessions/first.jsonl": first, + "/sessions/second.jsonl": second, + ] + var caches = [CostUsageCache(), CostUsageCache(), CostUsageCache()] + caches[0].codexScanCompletedFiles = 0 + caches[1].codexScanCompletedFiles = 1 + caches[2].codexScanCompletedFiles = 2 + let keys = caches.map { + CostUsageFetcher.codexScanProgressKey(cache: $0, scopedFiles: files) + } + var statusLoadCount = 0 + var advanceCount = 0 + store._test_tokenUsageSnapshotLoaderOverride = { _, _, now, _, _ in + Self.tokenSnapshot(cost: 1, now: now) + } + store._test_codexCostCatchUpStatusOverride = { _ in + statusLoadCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: statusLoadCount == 1, + progressKey: statusLoadCount == 1 ? keys[0] : keys[2]) + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: advanceCount < 2, + progressKey: keys[advanceCount]) + } + store._test_codexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startCodexCostCatchUpIfNeeded(mode: .accelerated) + await Self.waitUntil { + store.codexCostCatchUpTask == nil + } + + #expect(Set(keys).count == 3) + #expect(advanceCount == 2) + #expect(store.codexCostCatchUpActivity?.phase == .complete) + } + + @Test + func `a same-mode refresh queues a worker after the completing task`() async throws { + let store = try Self.makeStore(suite: "same-mode-restart") + var statusLoadCount = 0 + var advanceCount = 0 + store._test_tokenUsageSnapshotLoaderOverride = { _, _, now, _, _ in + Self.tokenSnapshot(cost: 1, now: now) + } + store._test_codexCostCatchUpStatusOverride = { _ in + statusLoadCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: statusLoadCount == 2, + progressKey: "status-\(statusLoadCount)") + } + store._test_codexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return CostUsageFetcher.CodexScanCatchUpStatus( + pending: false, + progressKey: "complete") + } + store._test_codexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_codexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startCodexCostCatchUpIfNeeded() + store.startCodexCostCatchUpIfNeeded() + await Self.waitUntil { + store.codexCostCatchUpTask == nil && statusLoadCount == 3 + } + + #expect(statusLoadCount == 3) + #expect(advanceCount == 1) + #expect(store.codexCostCatchUpActivity?.phase == .complete) + } + @Test func `accelerated catch-up runs without an inter-pass delay and publishes progress`() async throws { let store = try Self.makeStore(suite: "accelerated") @@ -167,6 +300,20 @@ struct UsageStoreCodexCostCatchUpTests { #expect(store.codexCostCatchUpActivity?.fractionCompleted == 0.5) } + @Test + func `stopping an active pass clears a queued restart`() throws { + let store = try Self.makeStore(suite: "stop-clears-restart") + store.codexCostCatchUpTask = Task {} + store.codexCostCatchUpPassIsRunning = true + store.codexCostCatchUpRestartRequested = true + + store.stopCodexCostCatchUp() + + #expect(store.codexCostCatchUpStopRequested) + #expect(!store.codexCostCatchUpRestartRequested) + store.cancelCodexCostCatchUp() + } + private static func makeStore(suite: String) throws -> UsageStore { let settings = testSettingsStore(suiteName: "UsageStoreCodexCostCatchUpTests-\(suite)") settings.costUsageEnabled = true diff --git a/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift b/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift index 93a5419a75..ca63d14f21 100644 --- a/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift +++ b/Tests/CodexBarTests/UsageStoreSpendDashboardCodexCostCatchUpTests.swift @@ -58,6 +58,47 @@ struct UsageStoreSpendDashboardCodexCostCatchUpTests { #expect(store.spendDashboardCodexCostCatchUpActivity?.fractionCompleted == 1) } + @Test(arguments: [123, 248, 365]) + func `dashboard catch-up accelerates the configured history window`(historyDays: Int) async throws { + let receivedHistoryDays = try await Self.receivedHistoryDays( + configuredHistoryDays: historyDays, + suite: "configured-\(historyDays)") + + #expect(receivedHistoryDays == historyDays) + } + + @Test(arguments: [1, 7, 29]) + func `dashboard catch-up retains its thirty day floor`(historyDays: Int) async throws { + let receivedHistoryDays = try await Self.receivedHistoryDays( + configuredHistoryDays: historyDays, + suite: "floor-\(historyDays)") + + #expect(receivedHistoryDays == SpendDashboardSource.scanDays) + } + + @Test + func `changing the history window replaces the active catch-up context`() throws { + let store = try Self.makeStore(suite: "history-context") + let accounts = [Self.account(id: "account", cacheIdentity: "cache-account")] + store.settings.costUsageHistoryDays = 30 + store._test_spendDashboardCodexCostCatchUpStatusOverride = { _ in + Self.status(pending: true, key: "pending", processedBytes: 25) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + try await Task.sleep(for: .seconds(60)) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: .accelerated) + let originalToken = try #require(store.spendDashboardCodexCostCatchUpToken) + + store.settings.costUsageHistoryDays = 123 + store.synchronizeSpendDashboardCodexCostCatchUp(accounts: accounts) + let replacementToken = try #require(store.spendDashboardCodexCostCatchUpToken) + + #expect(replacementToken != originalToken) + store.cancelSpendDashboardCodexCostCatchUp() + } + @Test func `a stalled account cache does not prevent a sibling cache from advancing`() async throws { let store = try Self.makeStore(suite: "stalled-sibling") @@ -130,6 +171,74 @@ struct UsageStoreSpendDashboardCodexCostCatchUpTests { #expect(store.spendDashboardCodexCostCatchUpActivity?.pauseReason == .noProgress) } + @Test + func `dashboard catch-up stalls a cache that revisits an earlier semantic state`() async throws { + let store = try Self.makeStore(suite: "cyclic-progress") + let accounts = [Self.account(id: "cyclic", cacheIdentity: "cache-cyclic")] + let progressKeys = ["validation-1", "validation-2", "validation-0"] + var advanceCount = 0 + store._test_spendDashboardCodexCostCatchUpStatusOverride = { _ in + Self.status(pending: true, key: "validation-0", processedBytes: 25) + } + store._test_spendDashboardCodexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return Self.status( + pending: true, + key: progressKeys[min(advanceCount - 1, progressKeys.count - 1)], + processedBytes: 25) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_spendDashboardCodexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts, mode: .accelerated) + await Self.waitUntil { + store.spendDashboardCodexCostCatchUpTask == nil + } + + #expect(advanceCount == 3) + #expect(store.spendDashboardCodexCostCatchUpActivity?.phase == .paused) + #expect(store.spendDashboardCodexCostCatchUpActivity?.pauseReason == .noProgress) + } + + @Test + func `a same-mode dashboard reload queues a worker after the completing task`() async throws { + let store = try Self.makeStore(suite: "same-mode-restart") + let accounts = [Self.account(id: "account", cacheIdentity: "cache-account")] + var statusLoadCount = 0 + var advanceCount = 0 + store._test_spendDashboardCodexCostCatchUpStatusOverride = { _ in + statusLoadCount += 1 + return Self.status( + pending: statusLoadCount == 2, + key: "status-\(statusLoadCount)", + processedBytes: statusLoadCount == 2 ? 25 : 100) + } + store._test_spendDashboardCodexCostCatchUpAdvanceOverride = { _, _, _ in + advanceCount += 1 + return Self.status(pending: false, key: "complete", processedBytes: 100) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_spendDashboardCodexCostCatchUpResourceStateOverride = { + (.ac, false, .nominal) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts) + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: accounts) + await Self.waitUntil { + store.spendDashboardCodexCostCatchUpTask == nil && statusLoadCount == 2 + } + + #expect(statusLoadCount == 2) + #expect(advanceCount == 1) + #expect(store.spendDashboardCodexCostCatchUpActivity?.phase == .complete) + } + @Test func `dashboard synchronization keeps an accelerated account queue accelerated`() throws { let store = try Self.makeStore(suite: "preserve-mode") @@ -145,6 +254,20 @@ struct UsageStoreSpendDashboardCodexCostCatchUpTests { store.cancelSpendDashboardCodexCostCatchUp() } + @Test + func `stopping an active pass clears a queued restart`() throws { + let store = try Self.makeStore(suite: "stop-clears-restart") + store.spendDashboardCodexCostCatchUpTask = Task {} + store.spendDashboardCodexCostCatchUpPassIsRunning = true + store.spendDashboardCodexCostCatchUpRestartRequested = true + + store.stopSpendDashboardCodexCostCatchUp() + + #expect(store.spendDashboardCodexCostCatchUpStopRequested) + #expect(!store.spendDashboardCodexCostCatchUpRestartRequested) + store.cancelSpendDashboardCodexCostCatchUp() + } + private static func makeStore(suite: String) throws -> UsageStore { let settings = testSettingsStore( suiteName: "UsageStoreSpendDashboardCodexCostCatchUpTests-\(suite)") @@ -159,6 +282,41 @@ struct UsageStoreSpendDashboardCodexCostCatchUpTests { environmentBase: [:]) } + private static func receivedHistoryDays( + configuredHistoryDays: Int, + suite: String) async throws -> Int + { + let store = try Self.makeStore(suite: suite) + let account = Self.account(id: "account", cacheIdentity: "cache-account") + store.settings.costUsageHistoryDays = configuredHistoryDays + var completed = false + var receivedHistoryDays: Int? + store._test_spendDashboardCodexCostCatchUpStatusOverride = { _ in + Self.status( + pending: !completed, + key: completed ? "complete" : "pending", + processedBytes: completed ? 100 : 25) + } + store._test_spendDashboardCodexCostCatchUpAdvanceOverride = { _, _, historyDays in + receivedHistoryDays = historyDays + completed = true + return Self.status(pending: false, key: "complete", processedBytes: 100) + } + store._test_spendDashboardCodexCostCatchUpSleepOverride = { _ in + await Task.yield() + } + store._test_spendDashboardCodexCostCatchUpResourceStateOverride = { + (.battery, true, .serious) + } + + store.startSpendDashboardCodexCostCatchUpIfNeeded(accounts: [account], mode: .accelerated) + await Self.waitUntil { + store.spendDashboardCodexCostCatchUpTask == nil + } + + return try #require(receivedHistoryDays) + } + private static func account(id: String, cacheIdentity: String) -> CodexSpendScanRequest { CodexSpendScanRequest( id: id,