Skip to content

Commit 6170f5c

Browse files
iam-brainiam-brain
committed
Add shadow Codex lineage ledger
1 parent c852c13 commit 6170f5c

2 files changed

Lines changed: 358 additions & 0 deletions

File tree

‎Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
import Foundation
2+
3+
/// Experimental accounting model for Codex rollout families.
4+
///
5+
/// Rollout files are overlapping physical views of a logical lineage. The ledger builds the
6+
/// transitive lineage first, then admits each complete token observation once per lineage.
7+
/// It intentionally does not participate in production cost totals yet.
8+
enum CodexLineageLedger {
9+
struct Totals: Equatable, Hashable, Sendable {
10+
var input: Int
11+
var cached: Int
12+
var output: Int
13+
14+
static let zero = Self(input: 0, cached: 0, output: 0)
15+
16+
mutating func add(_ other: Self) {
17+
self.input += other.input
18+
self.cached += other.cached
19+
self.output += other.output
20+
}
21+
}
22+
23+
struct Observation: Equatable, Sendable {
24+
let timestamp: String
25+
let last: Totals
26+
let total: Totals
27+
}
28+
29+
struct Document: Equatable, Sendable {
30+
/// Canonical owner from the rollout filename when available.
31+
let ownerID: String
32+
/// Session identity persisted in metadata. Fork copies may retain an ancestor identity.
33+
let metadataSessionID: String?
34+
let parentSessionID: String?
35+
let observations: [Observation]
36+
}
37+
38+
struct Report: Equatable, Sendable {
39+
let utcDays: [String: Totals]
40+
let localDays: [String: Totals]
41+
let componentCount: Int
42+
let acceptedObservationCount: Int
43+
let duplicateObservationCount: Int
44+
}
45+
46+
enum LedgerError: Error, Equatable {
47+
case emptyOwnerID
48+
case invalidTimestamp(String)
49+
}
50+
51+
static func reconcile(documents: [Document], localTimeZone: TimeZone) throws -> Report {
52+
var graph = DisjointSet()
53+
for document in documents {
54+
guard !document.ownerID.isEmpty else { throw LedgerError.emptyOwnerID }
55+
graph.insert(document.ownerID)
56+
if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) {
57+
graph.union(document.ownerID, metadataSessionID)
58+
}
59+
if let parentSessionID = Self.nonEmpty(document.parentSessionID) {
60+
graph.union(document.ownerID, parentSessionID)
61+
}
62+
}
63+
64+
var acceptedByComponent: [String: [Fingerprint: AcceptedObservation]] = [:]
65+
var physicalObservationCount = 0
66+
for document in documents {
67+
let componentID = graph.find(document.ownerID)
68+
var accepted = acceptedByComponent[componentID] ?? [:]
69+
for observation in document.observations {
70+
physicalObservationCount += 1
71+
let date = try Self.date(from: observation.timestamp)
72+
let fingerprint = Fingerprint(last: observation.last, total: observation.total)
73+
if let existing = accepted[fingerprint], existing.date <= date {
74+
continue
75+
}
76+
accepted[fingerprint] = AcceptedObservation(date: date, last: observation.last)
77+
}
78+
acceptedByComponent[componentID] = accepted
79+
}
80+
81+
var utcDays: [String: Totals] = [:]
82+
var localDays: [String: Totals] = [:]
83+
var acceptedObservationCount = 0
84+
for accepted in acceptedByComponent.values {
85+
acceptedObservationCount += accepted.count
86+
for observation in accepted.values {
87+
Self.add(observation.last, on: observation.date, timeZone: .gmt, to: &utcDays)
88+
Self.add(observation.last, on: observation.date, timeZone: localTimeZone, to: &localDays)
89+
}
90+
}
91+
92+
return Report(
93+
utcDays: utcDays,
94+
localDays: localDays,
95+
componentCount: Set(documents.map { graph.find($0.ownerID) }).count,
96+
acceptedObservationCount: acceptedObservationCount,
97+
duplicateObservationCount: physicalObservationCount - acceptedObservationCount)
98+
}
99+
100+
private struct Fingerprint: Equatable, Hashable {
101+
let last: Totals
102+
let total: Totals
103+
}
104+
105+
private struct AcceptedObservation {
106+
let date: Date
107+
let last: Totals
108+
}
109+
110+
private static func nonEmpty(_ value: String?) -> String? {
111+
guard let value, !value.isEmpty else { return nil }
112+
return value
113+
}
114+
115+
private static func date(from timestamp: String) throws -> Date {
116+
guard let date = CostUsageScanner.dateFromTimestamp(timestamp) else {
117+
throw LedgerError.invalidTimestamp(timestamp)
118+
}
119+
return date
120+
}
121+
122+
private static func add(
123+
_ totals: Totals,
124+
on date: Date,
125+
timeZone: TimeZone,
126+
to days: inout [String: Totals])
127+
{
128+
var calendar = Calendar(identifier: .gregorian)
129+
calendar.timeZone = timeZone
130+
let components = calendar.dateComponents([.year, .month, .day], from: date)
131+
guard let year = components.year, let month = components.month, let day = components.day else { return }
132+
let key = String(format: "%04d-%02d-%02d", year, month, day)
133+
var dayTotals = days[key] ?? .zero
134+
dayTotals.add(totals)
135+
days[key] = dayTotals
136+
}
137+
138+
private struct DisjointSet {
139+
private var parents: [String: String] = [:]
140+
141+
mutating func insert(_ item: String) {
142+
if self.parents[item] == nil {
143+
self.parents[item] = item
144+
}
145+
}
146+
147+
mutating func find(_ item: String) -> String {
148+
self.insert(item)
149+
guard let parent = self.parents[item], parent != item else { return item }
150+
let root = self.find(parent)
151+
self.parents[item] = root
152+
return root
153+
}
154+
155+
mutating func union(_ first: String, _ second: String) {
156+
let firstRoot = self.find(first)
157+
let secondRoot = self.find(second)
158+
if firstRoot != secondRoot {
159+
self.parents[secondRoot] = firstRoot
160+
}
161+
}
162+
}
163+
}

‎Tests/CodexBarTests/CodexLineageLedgerTests.swift‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
import Foundation
2+
import Testing
3+
@testable import CodexBarCore
4+
5+
struct CodexLineageLedgerTests {
6+
@Test
7+
func `transitive lineage counts copied observations once`() throws {
8+
let first = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100)
9+
let second = Self.observation(timestamp: "2026-07-09T12:01:00Z", input: 50, totalInput: 150)
10+
let third = Self.observation(timestamp: "2026-07-09T12:02:00Z", input: 25, totalInput: 175)
11+
let documents = [
12+
Self.document(owner: "root", observations: [first]),
13+
Self.document(owner: "child", metadata: "root", parent: "root", observations: [first, second]),
14+
Self.document(owner: "grandchild", metadata: "child", parent: "child", observations: [
15+
first,
16+
second,
17+
third,
18+
]),
19+
]
20+
21+
let report = try CodexLineageLedger.reconcile(
22+
documents: documents,
23+
localTimeZone: #require(TimeZone(identifier: "America/New_York")))
24+
25+
#expect(report.utcDays["2026-07-09"]?.input == 175)
26+
#expect(report.localDays["2026-07-09"]?.input == 175)
27+
#expect(report.componentCount == 1)
28+
#expect(report.acceptedObservationCount == 3)
29+
#expect(report.duplicateObservationCount == 3)
30+
}
31+
32+
@Test
33+
func `equal observations in disconnected lineages remain additive`() throws {
34+
let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100)
35+
let report = try CodexLineageLedger.reconcile(
36+
documents: [
37+
Self.document(owner: "first", observations: [observation]),
38+
Self.document(owner: "second", observations: [observation]),
39+
],
40+
localTimeZone: #require(TimeZone(identifier: "America/New_York")))
41+
42+
#expect(report.utcDays["2026-07-09"]?.input == 200)
43+
#expect(report.componentCount == 2)
44+
#expect(report.acceptedObservationCount == 2)
45+
#expect(report.duplicateObservationCount == 0)
46+
}
47+
48+
@Test
49+
func `unchanged state reemissions within a lineage count once`() throws {
50+
let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100)
51+
let report = try CodexLineageLedger.reconcile(
52+
documents: [Self.document(owner: "root", observations: [observation, observation, observation])],
53+
localTimeZone: #require(TimeZone(identifier: "America/New_York")))
54+
55+
#expect(report.utcDays["2026-07-09"]?.input == 100)
56+
#expect(report.acceptedObservationCount == 1)
57+
#expect(report.duplicateObservationCount == 2)
58+
}
59+
60+
@Test
61+
func `complete token state distinguishes observations within a lineage`() throws {
62+
let first = Self.observation(
63+
timestamp: "2026-07-09T12:00:00Z",
64+
input: 100,
65+
cached: 40,
66+
output: 10,
67+
totalInput: 100)
68+
let changedTotal = Self.observation(
69+
timestamp: "2026-07-09T12:01:00Z",
70+
input: 100,
71+
cached: 40,
72+
output: 10,
73+
totalInput: 200)
74+
let changedLast = Self.observation(
75+
timestamp: "2026-07-09T12:02:00Z",
76+
input: 125,
77+
cached: 50,
78+
output: 15,
79+
totalInput: 200)
80+
let report = try CodexLineageLedger.reconcile(
81+
documents: [
82+
Self.document(owner: "root", observations: [first, changedTotal, changedLast]),
83+
],
84+
localTimeZone: #require(TimeZone(identifier: "America/New_York")))
85+
86+
#expect(report.utcDays["2026-07-09"] == .init(input: 325, cached: 130, output: 35))
87+
#expect(report.acceptedObservationCount == 3)
88+
#expect(report.duplicateObservationCount == 0)
89+
}
90+
91+
@Test
92+
func `UTC and local projections preserve their distinct day boundaries`() throws {
93+
let observation = Self.observation(timestamp: "2026-07-10T02:00:00Z", input: 100, totalInput: 100)
94+
let report = try CodexLineageLedger.reconcile(
95+
documents: [Self.document(owner: "root", observations: [observation])],
96+
localTimeZone: #require(TimeZone(identifier: "America/New_York")))
97+
98+
#expect(report.utcDays["2026-07-10"]?.input == 100)
99+
#expect(report.localDays["2026-07-09"]?.input == 100)
100+
}
101+
102+
@Test(arguments: [
103+
("archived-fork-33ce-3869", 15_309_178),
104+
("live-fork-4d90-52bf", 26_801_911),
105+
])
106+
func `sanitized fork fixtures collapse copied prefixes and unchanged reemissions`(
107+
fixtureName: String,
108+
expectedTokens: Int) throws
109+
{
110+
let fixture = try SanitizedForkFamilyFixture.load(named: fixtureName)
111+
let parentMetadata = try fixture.sessionMetadata(named: "parent")
112+
let childMetadata = try fixture.sessionMetadata(named: "child")
113+
let parent = try fixture.events(named: "parent")
114+
let child = try fixture.events(named: "child")
115+
let documents = [
116+
CodexLineageLedger.Document(
117+
ownerID: "parent-owner",
118+
metadataSessionID: parentMetadata.id,
119+
parentSessionID: parentMetadata.forkedFromID,
120+
observations: parent.map(Self.observation)),
121+
CodexLineageLedger.Document(
122+
ownerID: "child-owner",
123+
metadataSessionID: childMetadata.id,
124+
parentSessionID: childMetadata.forkedFromID,
125+
observations: child.map(Self.observation)),
126+
]
127+
128+
let report = try CodexLineageLedger.reconcile(
129+
documents: documents,
130+
localTimeZone: #require(TimeZone(identifier: "America/New_York")))
131+
let total = report.utcDays.values.reduce(0) { partial, totals in
132+
partial + totals.input + totals.output
133+
}
134+
135+
#expect(total == expectedTokens)
136+
#expect(total < fixture.manifest.oracle.dedupedLastTokens)
137+
}
138+
139+
@Test
140+
func `document order does not change lineage totals or attribution`() throws {
141+
let copiedLater = Self.observation(timestamp: "2026-07-10T00:05:00Z", input: 100, totalInput: 100)
142+
let original = Self.observation(timestamp: "2026-07-09T23:55:00Z", input: 100, totalInput: 100)
143+
let root = Self.document(owner: "root", observations: [original])
144+
let child = Self.document(owner: "child", parent: "root", observations: [copiedLater])
145+
let timeZone = try #require(TimeZone(identifier: "America/New_York"))
146+
147+
let forward = try CodexLineageLedger.reconcile(documents: [root, child], localTimeZone: timeZone)
148+
let reversed = try CodexLineageLedger.reconcile(documents: [child, root], localTimeZone: timeZone)
149+
150+
#expect(forward == reversed)
151+
#expect(forward.utcDays["2026-07-09"]?.input == 100)
152+
#expect(forward.utcDays["2026-07-10"] == nil)
153+
}
154+
155+
private static func document(
156+
owner: String,
157+
metadata: String? = nil,
158+
parent: String? = nil,
159+
observations: [CodexLineageLedger.Observation]) -> CodexLineageLedger.Document
160+
{
161+
.init(
162+
ownerID: owner,
163+
metadataSessionID: metadata,
164+
parentSessionID: parent,
165+
observations: observations)
166+
}
167+
168+
private static func observation(
169+
timestamp: String,
170+
input: Int,
171+
cached: Int = 0,
172+
output: Int = 0,
173+
totalInput: Int) -> CodexLineageLedger.Observation
174+
{
175+
.init(
176+
timestamp: timestamp,
177+
last: .init(input: input, cached: cached, output: output),
178+
total: .init(input: totalInput, cached: cached, output: output))
179+
}
180+
181+
private static func observation(
182+
_ event: SanitizedForkFamilyFixture.TokenEvent) -> CodexLineageLedger.Observation
183+
{
184+
.init(
185+
timestamp: event.timestamp,
186+
last: .init(
187+
input: event.last.inputTokens,
188+
cached: event.last.cachedInputTokens,
189+
output: event.last.outputTokens),
190+
total: .init(
191+
input: event.total.inputTokens,
192+
cached: event.total.cachedInputTokens,
193+
output: event.total.outputTokens))
194+
}
195+
}

0 commit comments

Comments
 (0)