From 9c5010693d1c281b5848608346814f2d89eae9ef Mon Sep 17 00:00:00 2001 From: tyulyukov Date: Tue, 5 May 2026 14:48:28 +0300 Subject: [PATCH] feat(analytics): record provider flows as duration spans - collapse provider turn, tool call, and pr/mr create events into span-based analytics - export duration metadata and span events through otlp - tighten codex turn-completed notifications around active turn ids --- apps/server/src/git/Layers/GitManager.ts | 60 +- .../provider/Layers/ProviderService.test.ts | 89 +- .../src/provider/Layers/ProviderService.ts | 188 ++- .../src/telemetry/Layers/AnalyticsService.ts | 14 +- apps/server/src/telemetry/OtlpProduct.test.ts | 49 + apps/server/src/telemetry/OtlpProduct.ts | 34 +- .../telemetry/Services/AnalyticsService.ts | 14 +- apps/web/src/components/chat/ChatComposer.tsx | 4 - apps/web/src/routes/__root.tsx | 10 +- apps/web/src/turnNotification.test.ts | 73 +- apps/web/src/turnNotification.ts | 40 +- .../marcode-product-analytics-dashboard.json | 1465 ----------------- 12 files changed, 484 insertions(+), 1556 deletions(-) delete mode 100644 grafana/marcode-product-analytics-dashboard.json diff --git a/apps/server/src/git/Layers/GitManager.ts b/apps/server/src/git/Layers/GitManager.ts index ef84d153bff..eb8ab691e3a 100644 --- a/apps/server/src/git/Layers/GitManager.ts +++ b/apps/server/src/git/Layers/GitManager.ts @@ -502,8 +502,11 @@ export const makeGitManager = Effect.fn("makeGitManager")(function* () { flush: Effect.void, })); - const recordGitAnalytics = (event: string, properties: Record) => - analytics.record(event, properties); + const recordGitAnalytics = ( + event: string, + properties: Record, + options?: Parameters[2], + ) => analytics.record(event, properties, options); const resolveJiraTickets = ( threadId: ThreadId | undefined, @@ -1310,12 +1313,41 @@ export const makeGitManager = Effect.fn("makeGitManager")(function* () { label: `Creating ${prOrMr}...`, }); const originRepo = headContext.originRepositoryNameWithOwner; - yield* recordGitAnalytics("marcode.git.pr_mr.create_requested", { + const createStartedAtMs = Date.now(); + const baseAnalyticsProperties = { "git.host.provider": detectedHostProvider ?? "unknown", "git.change_request.kind": changeRequestKind, "git.branch": headContext.headBranch, ...(repositoryName ? { "repository.name": repositoryName } : {}), - }); + }; + const recordCreateResult = (outcome: "created" | "error", hasUrl: boolean) => { + const completedAtMs = Date.now(); + const durationMs = Math.max(1, completedAtMs - createStartedAtMs); + return recordGitAnalytics( + "marcode.git.pr_mr.create", + { + ...baseAnalyticsProperties, + outcome, + has_url: hasUrl, + "duration.ms": durationMs, + }, + { + durationMs, + startedAt: createStartedAtMs, + spanEvents: [ + { + name: "marcode.git.pr_mr.create.requested", + at: createStartedAtMs, + }, + { + name: "marcode.git.pr_mr.create.completed", + at: completedAtMs, + attributes: { outcome, has_url: hasUrl }, + }, + ], + }, + ); + }; yield* gitHostCli .createPullRequest({ cwd, @@ -1331,27 +1363,11 @@ export const makeGitManager = Effect.fn("makeGitManager")(function* () { schedule: Schedule.exponential(PR_CREATE_RETRY_BASE_DELAY, 2), while: isBranchNotReadyError, }), - Effect.tapError(() => - recordGitAnalytics("marcode.git.pr_mr.create_completed", { - "git.host.provider": detectedHostProvider ?? "unknown", - "git.change_request.kind": changeRequestKind, - "git.branch": headContext.headBranch, - outcome: "error", - has_url: false, - ...(repositoryName ? { "repository.name": repositoryName } : {}), - }), - ), + Effect.tapError(() => recordCreateResult("error", false)), ); const created = yield* findOpenPr(cwd, headContext); - yield* recordGitAnalytics("marcode.git.pr_mr.create_completed", { - "git.host.provider": detectedHostProvider ?? "unknown", - "git.change_request.kind": changeRequestKind, - "git.branch": headContext.headBranch, - outcome: "created", - has_url: created?.url ? true : false, - ...(repositoryName ? { "repository.name": repositoryName } : {}), - }); + yield* recordCreateResult("created", Boolean(created?.url)); if (!created) { return { status: "created" as const, diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index ba94eab41a3..3b0686c0e5d 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -43,6 +43,10 @@ import { } from "../../persistence/Layers/Sqlite.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; import { AnalyticsServiceNoopLive } from "../../telemetry/Layers/AnalyticsService.ts"; +import { + AnalyticsService, + type AnalyticsServiceShape, +} from "../../telemetry/Services/AnalyticsService.ts"; const defaultServerSettingsLayer = ServerSettingsService.layerTest(); @@ -242,6 +246,12 @@ const hasMetricSnapshot = ( Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value), ); +interface AnalyticsRecord { + readonly event: string; + readonly properties?: Record | undefined; + readonly options?: Parameters[2] | undefined; +} + function makeProviderServiceLayer() { const codex = makeFakeCodexAdapter(); const claude = makeFakeCodexAdapter("claudeAgent"); @@ -382,7 +392,7 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se state: "completed", }, }); - yield* sleep(20); + yield* sleep(100); }).pipe(Effect.provide(providerLayer)); assert.equal(canonicalEvents.length, 1); @@ -391,6 +401,83 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se }).pipe(Effect.provide(NodeServices.layer)), ); +it.effect("ProviderServiceLive records provider turns as one duration product span", () => + Effect.gen(function* () { + const codex = makeFakeCodexAdapter(); + const analyticsRecords: AnalyticsRecord[] = []; + const registry: typeof ProviderAdapterRegistry.Service = { + getByProvider: (provider) => + provider === "codex" + ? Effect.succeed(codex.adapter) + : Effect.fail(new ProviderUnsupportedError({ provider })), + listProviders: () => Effect.succeed(["codex"]), + }; + const runtimeRepositoryLayer = ProviderSessionRuntimeRepositoryLive.pipe( + Layer.provide(SqlitePersistenceMemory), + ); + const directoryLayer = ProviderSessionDirectoryLive.pipe(Layer.provide(runtimeRepositoryLayer)); + const analyticsLayer = Layer.succeed(AnalyticsService, { + record: (event, properties, options) => + Effect.sync(() => { + analyticsRecords.push({ event, properties, options }); + }), + flush: Effect.void, + }); + const providerLayer = makeProviderServiceLive().pipe( + Layer.provide(Layer.succeed(ProviderAdapterRegistry, registry)), + Layer.provide(directoryLayer), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(analyticsLayer), + ); + + yield* Effect.gen(function* () { + const provider = yield* ProviderService; + yield* sleep(10); + const threadId = asThreadId("thread-turn-analytics"); + yield* provider.startSession(threadId, { + provider: "codex", + threadId, + runtimeMode: "full-access", + }); + const turn = yield* provider.sendTurn({ + threadId, + input: "hello", + interactionMode: "default", + }); + codex.emit({ + eventId: asEventId("evt-turn-analytics-completed"), + provider: "codex", + threadId, + turnId: turn.turnId, + createdAt: new Date(Date.now() + 250).toISOString(), + type: "turn.completed", + payload: { + state: "completed", + }, + }); + yield* sleep(100); + }).pipe(Effect.provide(providerLayer)); + + const turnRecord = analyticsRecords.find((record) => record.event === "marcode.provider.turn"); + + assert.isDefined(turnRecord); + assert.isUndefined( + analyticsRecords.find((record) => record.event === "marcode.provider.turn.sent"), + ); + assert.isUndefined( + analyticsRecords.find((record) => record.event === "marcode.provider.turn.completed"), + ); + assert.equal(turnRecord?.properties?.provider, "codex"); + assert.equal(turnRecord?.properties?.outcome, "success"); + assert.equal(turnRecord?.properties?.interaction_mode, "default"); + assert.isAtLeast(Number(turnRecord?.options?.durationMs ?? 0), 1); + assert.deepEqual( + turnRecord?.options?.spanEvents?.map((event) => event.name), + ["marcode.provider.turn.sent", "marcode.provider.turn.completed"], + ); + }).pipe(Effect.provide(NodeServices.layer)), +); + it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", () => Effect.gen(function* () { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "marcode-provider-service-")); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 6894ed4c97e..2b4590552a8 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -22,7 +22,7 @@ import { type ProviderRuntimeEvent, type ProviderSession, } from "@marcode/contracts"; -import { Effect, Layer, Option, PubSub, Schema, SchemaIssue, Stream } from "effect"; +import { Effect, Layer, Option, PubSub, Ref, Schema, SchemaIssue, Stream } from "effect"; import { increment, @@ -142,6 +142,30 @@ function readPersistedCwd( return trimmed.length > 0 ? trimmed : undefined; } +const toolCallKey = (event: ProviderRuntimeEvent): string | undefined => + event.itemId ? `${event.provider}:${event.threadId}:${event.itemId}` : undefined; + +const turnKey = (input: { + readonly provider: string; + readonly threadId: unknown; + readonly turnId: unknown; +}): string => `${input.provider}:${String(input.threadId)}:${String(input.turnId)}`; + +const eventTimeMs = (event: ProviderRuntimeEvent): number => { + const parsed = Date.parse(event.createdAt); + return Number.isFinite(parsed) ? parsed : Date.now(); +}; + +const isToolCallEvent = (event: ProviderRuntimeEvent): boolean => + "itemType" in event.payload && + typeof event.payload.itemType === "string" && + event.payload.itemType.includes("tool_call"); + +interface ProviderTurnAnalyticsStart { + readonly startedAtMs: number; + readonly properties: Readonly>; +} + const makeProviderService = Effect.fn("makeProviderService")(function* ( options?: ProviderServiceLiveOptions, ) { @@ -196,33 +220,44 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const adapters = yield* Effect.forEach(providers, (provider) => registry.getByProvider(provider)); const hashId = (id: unknown) => hashTelemetryIdentifier(String(id)).pipe(Effect.orElseSucceed(() => String(id))); + const toolCallStarts = yield* Ref.make>(new Map()); + const providerTurnStarts = yield* Ref.make>( + new Map(), + ); const recordRuntimeAnalytics = (event: ProviderRuntimeEvent): Effect.Effect => Effect.gen(function* () { const threadIdHash = yield* hashId(event.threadId); if ( (event.type === "item.started" || event.type === "item.updated") && - "itemType" in event.payload && - typeof event.payload.itemType === "string" && - event.payload.itemType.includes("tool_call") + isToolCallEvent(event) ) { - yield* analytics.record("marcode.tool.call.started", { - provider: event.provider, - "thread.id_hash": threadIdHash, - "tool.kind": event.payload.itemType, - "tool.name_normalized": - "title" in event.payload && typeof event.payload.title === "string" - ? event.payload.title.toLowerCase() - : event.payload.itemType, - }); + const key = toolCallKey(event); + if (key) { + const startedAtMs = eventTimeMs(event); + yield* Ref.update(toolCallStarts, (current) => { + if (current.has(key)) return current; + const next = new Map(current); + next.set(key, startedAtMs); + return next; + }); + } } - if ( - event.type === "item.completed" && - "itemType" in event.payload && - typeof event.payload.itemType === "string" && - event.payload.itemType.includes("tool_call") - ) { - yield* analytics.record("marcode.tool.call.completed", { + if (event.type === "item.completed" && isToolCallEvent(event)) { + const key = toolCallKey(event); + const completedAtMs = eventTimeMs(event); + const startedAtMs = key + ? yield* Ref.modify(toolCallStarts, (current) => { + const started = current.get(key); + if (started === undefined) return [undefined, current] as const; + const next = new Map(current); + next.delete(key); + return [started, next] as const; + }) + : undefined; + const durationMs = + startedAtMs !== undefined ? Math.max(1, completedAtMs - startedAtMs) : undefined; + const properties = { provider: event.provider, "thread.id_hash": threadIdHash, "tool.kind": event.payload.itemType, @@ -231,7 +266,29 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ? event.payload.title.toLowerCase() : event.payload.itemType, outcome: "success", - }); + ...(durationMs !== undefined ? { "duration.ms": durationMs } : {}), + }; + yield* analytics.record( + "marcode.tool.call", + properties, + startedAtMs !== undefined && durationMs !== undefined + ? { + durationMs, + startedAt: startedAtMs, + spanEvents: [ + { + name: "marcode.tool.call.started", + at: startedAtMs, + }, + { + name: "marcode.tool.call.completed", + at: completedAtMs, + attributes: { outcome: "success" }, + }, + ], + } + : undefined, + ); } if (event.type === "request.opened") { yield* analytics.record("marcode.approval.requested", { @@ -249,11 +306,57 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); } if (event.type === "turn.completed" || event.type === "turn.aborted") { - yield* analytics.record("marcode.provider.turn.completed", { - provider: event.provider, - "thread.id_hash": threadIdHash, - outcome: event.type === "turn.completed" ? "success" : "failure", - }); + const completedAtMs = eventTimeMs(event); + const started = + event.turnId !== undefined + ? yield* Ref.modify(providerTurnStarts, (current) => { + const key = turnKey({ + provider: event.provider, + threadId: event.threadId, + turnId: event.turnId, + }); + const value = current.get(key); + if (value === undefined) return [undefined, current] as const; + const next = new Map(current); + next.delete(key); + return [value, next] as const; + }) + : undefined; + const durationMs = + started !== undefined ? Math.max(1, completedAtMs - started.startedAtMs) : undefined; + const outcome = event.type === "turn.completed" ? "success" : "failure"; + const options = + started !== undefined && durationMs !== undefined + ? { + durationMs, + startedAt: started.startedAtMs, + spanEvents: [ + { + name: "marcode.provider.turn.sent", + at: started.startedAtMs, + }, + { + name: + event.type === "turn.completed" + ? "marcode.provider.turn.completed" + : "marcode.provider.turn.aborted", + at: completedAtMs, + attributes: { outcome }, + }, + ], + } + : undefined; + yield* analytics.record( + "marcode.provider.turn", + { + provider: event.provider, + "thread.id_hash": threadIdHash, + ...started?.properties, + outcome, + ...(durationMs !== undefined ? { "duration.ms": durationMs } : {}), + }, + options, + ); } }).pipe(Effect.ignoreCause({ log: true })); @@ -478,12 +581,6 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* upsertSessionBinding(session, threadId, { modelSelection: input.modelSelection, }); - yield* analytics.record("marcode.provider.session.started", { - provider: adapter.provider, - "thread.id_hash": yield* hashId(threadId), - runtime_mode: input.runtimeMode, - ...(input.modelSelection?.model ? { "model.family": input.modelSelection.model } : {}), - }); return session; }).pipe( @@ -534,13 +631,28 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.kind": routed.adapter.provider, ...(input.modelSelection?.model ? { "provider.model": input.modelSelection.model } : {}), }); + const startedAtMs = Date.now(); const turn = yield* routed.adapter.sendTurn(input); - yield* analytics.record("marcode.provider.turn.sent", { - provider: routed.adapter.provider, - "thread.id_hash": yield* hashId(input.threadId), - interaction_mode: input.interactionMode, - "attachment.count": input.attachments.length, - ...(input.modelSelection?.model ? { "model.family": input.modelSelection.model } : {}), + yield* Ref.update(providerTurnStarts, (current) => { + const next = new Map(current); + next.set( + turnKey({ + provider: routed.adapter.provider, + threadId: input.threadId, + turnId: turn.turnId, + }), + { + startedAtMs, + properties: { + interaction_mode: input.interactionMode, + "attachment.count": input.attachments.length, + ...(input.modelSelection?.model + ? { "model.family": input.modelSelection.model } + : {}), + }, + }, + ); + return next; }); yield* directory.upsert({ threadId: input.threadId, diff --git a/apps/server/src/telemetry/Layers/AnalyticsService.ts b/apps/server/src/telemetry/Layers/AnalyticsService.ts index 04cae85facb..af6f283164a 100644 --- a/apps/server/src/telemetry/Layers/AnalyticsService.ts +++ b/apps/server/src/telemetry/Layers/AnalyticsService.ts @@ -2,11 +2,12 @@ import { Data, DateTime, Effect, Layer, Option, Ref } from "effect"; import { ServerConfig } from "../../config.ts"; import { JiraTokenService } from "../../jira/Services/JiraTokenService.ts"; -import { AnalyticsService } from "../Services/AnalyticsService.ts"; +import { AnalyticsService, type AnalyticsServiceShape } from "../Services/AnalyticsService.ts"; import { getTelemetryIdentifier } from "../Identify.ts"; import { JIRA_ACCESS_TOKEN_HEADER, makeProductSpanBatchPayload, + type ProductAnalyticsSpanEvent, productAnalyticsUrlFromConfig, shouldAttachJiraProof, } from "../OtlpProduct.ts"; @@ -20,6 +21,9 @@ interface BufferedAnalyticsEvent { readonly event: string; readonly properties?: Readonly>; readonly capturedAt: string; + readonly durationMs?: number; + readonly startedAt?: string | number | Date; + readonly spanEvents?: ReadonlyArray; } const MAX_BUFFERED_EVENTS = 1_000; @@ -103,6 +107,9 @@ const makeAnalyticsService = Effect.gen(function* () { events.map((event) => ({ event: event.event, capturedAt: event.capturedAt, + ...(event.durationMs !== undefined ? { durationMs: event.durationMs } : {}), + ...(event.startedAt !== undefined ? { startedAt: event.startedAt } : {}), + ...(event.spanEvents !== undefined ? { spanEvents: event.spanEvents } : {}), attributes: { ...makeBaseAttributes(), ...event.properties, @@ -168,7 +175,7 @@ const makeAnalyticsService = Effect.gen(function* () { } }).pipe(Effect.catchCause(() => Effect.void)); - const record = (event: string, properties?: Record) => + const record: AnalyticsServiceShape["record"] = (event, properties, options) => Effect.gen(function* () { yield* Effect.annotateCurrentSpan({ ...makeBaseAttributes(), @@ -183,6 +190,9 @@ const makeAnalyticsService = Effect.gen(function* () { event, ...(properties ? { properties } : {}), capturedAt: DateTime.formatIso(now), + ...(options?.durationMs !== undefined ? { durationMs: options.durationMs } : {}), + ...(options?.startedAt !== undefined ? { startedAt: options.startedAt } : {}), + ...(options?.spanEvents !== undefined ? { spanEvents: options.spanEvents } : {}), }, ]; return appended.length > MAX_BUFFERED_EVENTS diff --git a/apps/server/src/telemetry/OtlpProduct.test.ts b/apps/server/src/telemetry/OtlpProduct.test.ts index 828cc2c69fa..13467001064 100644 --- a/apps/server/src/telemetry/OtlpProduct.test.ts +++ b/apps/server/src/telemetry/OtlpProduct.test.ts @@ -21,4 +21,53 @@ describe("makeProductSpanBatchPayload", () => { assert.equal(payload.resourceSpans[0]?.scopeSpans[0]?.spans[0]?.name, "marcode.first"); assert.equal(payload.resourceSpans[1]?.scopeSpans[0]?.spans[0]?.name, "marcode.second"); }); + + it("uses durationMs as the exported OTLP span duration", () => { + const payload = makeProductSpanBatchPayload([ + { + event: "marcode.tool.call", + capturedAt: "2026-05-04T10:00:01.000Z", + durationMs: 250, + startedAt: "2026-05-04T10:00:00.000Z", + attributes: {}, + }, + ]); + + const span = payload.resourceSpans[0]?.scopeSpans[0]?.spans[0]; + + assert.equal(span?.startTimeUnixNano, "1777888800000000000"); + assert.equal(span?.endTimeUnixNano, "1777888800250000000"); + }); + + it("exports span events on duration spans", () => { + const payload = makeProductSpanBatchPayload([ + { + event: "marcode.provider.turn", + capturedAt: "2026-05-04T10:00:01.000Z", + durationMs: 1_000, + startedAt: "2026-05-04T10:00:00.000Z", + attributes: {}, + spanEvents: [ + { + name: "marcode.provider.turn.sent", + at: "2026-05-04T10:00:00.000Z", + }, + { + name: "marcode.provider.turn.completed", + at: "2026-05-04T10:00:01.000Z", + attributes: { outcome: "success" }, + }, + ], + }, + ]); + + const events = payload.resourceSpans[0]?.scopeSpans[0]?.spans[0]?.events; + + assert.equal(events?.length, 2); + assert.equal(events?.[0]?.name, "marcode.provider.turn.sent"); + assert.equal(events?.[1]?.name, "marcode.provider.turn.completed"); + assert.deepEqual(events?.[1]?.attributes, [ + { key: "outcome", value: { stringValue: "success" } }, + ]); + }); }); diff --git a/apps/server/src/telemetry/OtlpProduct.ts b/apps/server/src/telemetry/OtlpProduct.ts index c4cf0141d84..c335feb0239 100644 --- a/apps/server/src/telemetry/OtlpProduct.ts +++ b/apps/server/src/telemetry/OtlpProduct.ts @@ -3,6 +3,11 @@ import packageJson from "../../package.json" with { type: "json" }; export const JIRA_ACCESS_TOKEN_HEADER = "x-marcode-jira-access-token"; export type ProductAnalyticsAttributes = Readonly>; +export type ProductAnalyticsSpanEvent = Readonly<{ + name: string; + attributes?: ProductAnalyticsAttributes; + at?: string | number | Date; +}>; function valueToOtlp(value: unknown): Record | null { if (value === undefined || value === null) return null; @@ -22,12 +27,25 @@ export function productAttributesToOtlp(attributes: ProductAnalyticsAttributes) }); } +function timeToUnixNano(value: string | number | Date): bigint { + const ms = + value instanceof Date ? value.getTime() : typeof value === "number" ? value : Date.parse(value); + return BigInt(Number.isFinite(ms) ? ms : Date.now()) * 1_000_000n; +} + export function makeProductSpanPayload(input: { readonly event: string; readonly attributes: ProductAnalyticsAttributes; readonly capturedAt: string; + readonly durationMs?: number; + readonly startedAt?: string | number | Date; + readonly spanEvents?: ReadonlyArray; }) { - const now = BigInt(new Date(input.capturedAt).getTime()) * 1_000_000n; + const start = input.startedAt + ? timeToUnixNano(input.startedAt) + : timeToUnixNano(input.capturedAt); + const durationNano = BigInt(Math.max(1, input.durationMs ?? 1)) * 1_000_000n; + const end = start + durationNano; return { resourceSpans: [ { @@ -51,10 +69,15 @@ export function makeProductSpanPayload(input: { spanId: crypto.randomUUID().replaceAll("-", "").slice(0, 16), name: input.event, kind: 1, - startTimeUnixNano: String(now), - endTimeUnixNano: String(now + 1_000_000n), + startTimeUnixNano: String(start), + endTimeUnixNano: String(end), attributes: productAttributesToOtlp(input.attributes), - events: [], + events: (input.spanEvents ?? []).map((event) => ({ + timeUnixNano: String(event.at ? timeToUnixNano(event.at) : end), + name: event.name, + attributes: productAttributesToOtlp(event.attributes ?? {}), + droppedAttributesCount: 0, + })), links: [], status: { code: "STATUS_CODE_OK" }, flags: 1, @@ -72,6 +95,9 @@ export function makeProductSpanBatchPayload( readonly event: string; readonly attributes: ProductAnalyticsAttributes; readonly capturedAt: string; + readonly durationMs?: number; + readonly startedAt?: string | number | Date; + readonly spanEvents?: ReadonlyArray; }>, ) { return { diff --git a/apps/server/src/telemetry/Services/AnalyticsService.ts b/apps/server/src/telemetry/Services/AnalyticsService.ts index 74bcb213a42..a7c0568f9d3 100644 --- a/apps/server/src/telemetry/Services/AnalyticsService.ts +++ b/apps/server/src/telemetry/Services/AnalyticsService.ts @@ -2,7 +2,19 @@ import { Context } from "effect"; import type { Effect } from "effect"; export interface AnalyticsServiceShape { - readonly record: (event: string, properties?: Record) => Effect.Effect; + readonly record: ( + event: string, + properties?: Record, + options?: { + readonly durationMs?: number; + readonly startedAt?: string | number | Date; + readonly spanEvents?: ReadonlyArray<{ + readonly name: string; + readonly attributes?: Record; + readonly at?: string | number | Date; + }>; + }, + ) => Effect.Effect; readonly flush: Effect.Effect; } diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index cd826d3e126..eab67923cbc 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -640,10 +640,6 @@ export const ChatComposer = memo( ); const handleProviderModelSelect = useCallback( (provider: ProviderKind, model: string) => { - recordClientProductSpan("marcode.ui.provider.changed", { - provider, - "model.family": normalizeModelSlug(model), - }); onProviderModelSelect(provider, model); }, [onProviderModelSelect], diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsx index 7c7db5621e4..b5ec8a413be 100644 --- a/apps/web/src/routes/__root.tsx +++ b/apps/web/src/routes/__root.tsx @@ -53,7 +53,7 @@ import { getPrimaryEnvironmentConnection, startEnvironmentConnectionService, } from "../environments/runtime"; -import { configureClientTracing, recordClientProductSpan } from "../observability/clientTracing"; +import { configureClientTracing } from "../observability/clientTracing"; import { ensurePrimaryEnvironmentReady, resolveInitialServerAuthGateState, @@ -232,18 +232,10 @@ function RuntimeToolOutputBootstrap() { } function AuthenticatedTracingBootstrap() { - const pathname = useLocation({ select: (location) => location.pathname }); - useEffect(() => { void configureClientTracing(); }, []); - useEffect(() => { - recordClientProductSpan("marcode.ui.route.changed", { - route: pathname, - }); - }, [pathname]); - return null; } diff --git a/apps/web/src/turnNotification.test.ts b/apps/web/src/turnNotification.test.ts index cf34d073730..a672956d822 100644 --- a/apps/web/src/turnNotification.test.ts +++ b/apps/web/src/turnNotification.test.ts @@ -56,12 +56,15 @@ function makeActivityAppendedEvent(threadId: string, kind: string): Orchestratio } as unknown as OrchestrationEvent; } -function makeTurnDiffCompletedEvent(threadId: string): OrchestrationEvent { +function makeTurnDiffCompletedEvent( + threadId: string, + turnId = `turn-${threadId}`, +): OrchestrationEvent { return { type: "thread.turn-diff-completed", payload: { threadId: threadId as ThreadId, - turnId: `turn-${threadId}`, + turnId, }, } as unknown as OrchestrationEvent; } @@ -308,7 +311,7 @@ describe("deriveTurnNotificationTriggers", () => { () => project, ); - const events = [makeTurnDiffCompletedEvent("thread-1")]; + const events = [makeTurnDiffCompletedEvent("thread-1", "turn-1")]; const triggers = deriveTurnNotificationTriggers( events, () => thread, @@ -370,6 +373,70 @@ describe("deriveTurnNotificationTriggers", () => { expect(triggers).toHaveLength(0); }); + it("waits for turn-diff-completed before firing Codex turn-completed", () => { + const thread = makeThread({ + session: { orchestrationStatus: "running", activeTurnId: "turn-1" }, + } as Partial); + const project = makeProject(); + + deriveTurnNotificationTriggers( + [ + makeSessionSetEvent("thread-1", "running", { + activeTurnId: "turn-1", + providerName: "codex", + }), + ], + () => thread, + () => project, + ); + + const idleBatch = deriveTurnNotificationTriggers( + [ + makeSessionSetEvent("thread-1", "ready", { + activeTurnId: null, + providerName: "codex", + }), + ], + () => thread, + () => project, + ); + const completionBatch = deriveTurnNotificationTriggers( + [makeTurnDiffCompletedEvent("thread-1", "turn-1")], + () => thread, + () => project, + ); + + expect(idleBatch).toHaveLength(0); + expect(completionBatch).toHaveLength(1); + expect(completionBatch[0]!.reason).toBe("turn-completed"); + }); + + it("ignores stale Codex turn-diff-completed events for a previous active turn", () => { + const thread = makeThread({ + session: { orchestrationStatus: "running", activeTurnId: "turn-thread-1" }, + } as Partial); + const project = makeProject(); + + deriveTurnNotificationTriggers( + [ + makeSessionSetEvent("thread-1", "running", { + activeTurnId: "turn-current", + providerName: "codex", + }), + ], + () => thread, + () => project, + ); + + const triggers = deriveTurnNotificationTriggers( + [makeTurnDiffCompletedEvent("thread-1")], + () => thread, + () => project, + ); + + expect(triggers).toHaveLength(0); + }); + describe("regression: ACP (Cursor/OpenCode) thread lifecycle", () => { it("does NOT fire on session.started emitting status=ready with activeTurnId=null", () => { // Repro for Bug 1: when Cursor/OpenCode open a session, the provider diff --git a/apps/web/src/turnNotification.ts b/apps/web/src/turnNotification.ts index 5217074d57f..44b4f101b6c 100644 --- a/apps/web/src/turnNotification.ts +++ b/apps/web/src/turnNotification.ts @@ -2,6 +2,7 @@ import type { NotificationEventGroup, OrchestrationEvent, OrchestrationSessionStatus, + ProviderKind, ProjectId, ThreadId, TurnId, @@ -72,8 +73,13 @@ function isThreadSuppressed(threadId: ThreadId): boolean { // the detail stream can race each other — by the time a "ready" status arrives // in one stream, the stored session may already read "ready" from the other, so // we can't rely on reading `thread.session?.orchestrationStatus` to decide -// whether a turn *was* active. This set is authoritative across batches. -const threadsWithActiveTurn = new Set(); +// whether a turn *was* active. This map is authoritative across batches. +type ActiveTurnState = { + readonly provider: ProviderKind | null; + readonly turnId: TurnId | null; +}; + +const threadsWithActiveTurn = new Map(); // Persistent recent-completion dedup. `thread.session-set` (ready) and // `thread.turn-diff-completed` arrive as two separate single-event batches in @@ -106,6 +112,18 @@ export function __resetTurnNotificationStateForTests(): void { suppressedThreads.clear(); } +function providerKindFromSession(value: string | null | undefined): ProviderKind | null { + switch (value) { + case "claudeAgent": + case "codex": + case "cursor": + case "opencode": + return value; + default: + return null; + } +} + // Tracks turn ids the user has locally interrupted via the Stop button, so the // in-chat "Working…" indicator can clear immediately without waiting for a // provider-emitted turn.completed event (which may be slow, dropped, or hang). @@ -165,8 +183,10 @@ export function deriveTurnNotificationTriggers( const newStatus = session.status; if (newStatus === "running") { - // Arm the persistent active-turn flag. Cleared on completion below. - threadsWithActiveTurn.add(threadId); + threadsWithActiveTurn.set(threadId, { + provider: providerKindFromSession(session.providerName), + turnId: session.activeTurnId ?? null, + }); } const reason = COMPLETION_STATUS_TO_REASON[newStatus]; @@ -182,7 +202,11 @@ export function deriveTurnNotificationTriggers( // orchestrationStatus / activeTurnId / latestTurn) all had failure modes // where session.started's status=ready + stale store state misfired for // OpenCode/Cursor — the flag below is authoritative. - if (!threadsWithActiveTurn.has(threadId)) continue; + const activeTurn = threadsWithActiveTurn.get(threadId); + if (!activeTurn) continue; + + const provider = providerKindFromSession(session.providerName) ?? activeTurn.provider; + if (provider === "codex" && reason === "turn-completed") continue; const thread = getThread(threadId); if (!thread) continue; @@ -213,11 +237,13 @@ export function deriveTurnNotificationTriggers( // turn.completed runtime event. Still gated on the armed flag so we never // notify for a turn that never started from the client's perspective. if (event.type === "thread.turn-diff-completed") { - const { threadId } = event.payload; + const { threadId, turnId } = event.payload; if (completionFiredThreadIds.has(threadId)) continue; if (isThreadSuppressed(threadId)) continue; if (userInitiatedThreadIds.has(threadId)) continue; - if (!threadsWithActiveTurn.has(threadId)) continue; + const activeTurn = threadsWithActiveTurn.get(threadId); + if (!activeTurn) continue; + if (activeTurn.turnId !== null && activeTurn.turnId !== turnId) continue; if (wasRecentlyFired(threadId)) { // The primary session-set path already fired in a nearby batch. completionFiredThreadIds.add(threadId); diff --git a/grafana/marcode-product-analytics-dashboard.json b/grafana/marcode-product-analytics-dashboard.json deleted file mode 100644 index 2a19dd18515..00000000000 --- a/grafana/marcode-product-analytics-dashboard.json +++ /dev/null @@ -1,1465 +0,0 @@ -{ - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "MarCode product analytics dashboard built from OTLP spanmetrics. Active users are users with at least one marcode.message.user.sent span in the selected time range.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Product Pulse", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "description": "Distinct analytics.user.id values with at least one user message in the selected range.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 5 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "count(count by (analytics_user_id) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", analytics_user_id=~\".+\"}[$__range]) > 0))", - "instant": true, - "legendFormat": "Active users", - "range": false, - "refId": "A" - } - ], - "title": "Active Users", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", provider=~\"$provider\"}[$__range]))", - "instant": true, - "legendFormat": "Messages", - "range": false, - "refId": "A" - } - ], - "title": "Messages Sent", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.thread.created\", provider_default=~\"$provider\"}[$__range]))", - "instant": true, - "legendFormat": "Threads", - "range": false, - "refId": "A" - } - ], - "title": "Threads Created", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", provider=~\"$provider\"}[$__range])) / clamp_min(count(count by (analytics_user_id) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", analytics_user_id=~\".+\"}[$__range]) > 0)), 1)", - "instant": true, - "legendFormat": "Messages / active user", - "range": false, - "refId": "A" - } - ], - "title": "Messages / User", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.9 - }, - { - "color": "green", - "value": 0.98 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.provider.turn.completed\", provider=~\"$provider\", outcome=\"success\"}[$__range])) / clamp_min(sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.provider.turn.completed\", provider=~\"$provider\"}[$__range])), 1)", - "instant": true, - "legendFormat": "Success rate", - "range": false, - "refId": "A" - } - ], - "title": "Turn Success", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.tool.call.completed\", provider=~\"$provider\"}[$__range]))", - "instant": true, - "legendFormat": "Tool calls", - "range": false, - "refId": "A" - } - ], - "title": "Tool Calls", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 16, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "count(count by (analytics_user_id) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", analytics_user_id=~\".+\"}[$__rate_interval]) > 0))", - "legendFormat": "Active users", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", provider=~\"$provider\"}[$__rate_interval]))", - "legendFormat": "Messages", - "range": true, - "refId": "B" - } - ], - "title": "Active Users & Messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 12, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 9, - "options": { - "legend": { - "calcs": ["lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.ui.composer.submit\", provider=~\"$provider\"}[$__rate_interval]))", - "legendFormat": "Composer submits", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.provider.turn.sent\", provider=~\"$provider\"}[$__rate_interval]))", - "legendFormat": "Provider turns sent", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum(increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.provider.turn.completed\", provider=~\"$provider\", outcome=\"success\"}[$__rate_interval]))", - "legendFormat": "Provider turns completed", - "range": true, - "refId": "C" - } - ], - "title": "Prompt Funnel", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 10, - "panels": [], - "title": "Provider Quality", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 11, - "options": { - "legend": { - "calcs": ["lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum by (provider, outcome) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.provider.turn.completed\", provider=~\"$provider\"}[$__rate_interval]))", - "legendFormat": "{{provider}} / {{outcome}}", - "range": true, - "refId": "A" - } - ], - "title": "Provider Turns by Outcome", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 15 - }, - "id": 12, - "options": { - "displayLabels": ["name", "percent"], - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "values": ["value", "percent"] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum by (provider) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.provider.turn.sent\", provider=~\"$provider\"}[$__range]))", - "instant": true, - "legendFormat": "{{provider}}", - "range": false, - "refId": "A" - } - ], - "title": "Provider Mix", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 15 - }, - "id": 13, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "topk(12, sum by (model_family) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=~\"marcode.provider.(turn.sent|session.started)\", provider=~\"$provider\", model_family=~\".+\"}[$__range])))", - "instant": true, - "legendFormat": "{{model_family}}", - "range": false, - "refId": "A" - } - ], - "title": "Top Models", - "type": "bargauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 14, - "panels": [], - "title": "Agent Workflows", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 12, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 15, - "options": { - "legend": { - "calcs": ["lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum by (tool_name_normalized) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.tool.call.completed\", provider=~\"$provider\", tool_name_normalized=~\".+\"}[$__rate_interval]))", - "legendFormat": "{{tool_name_normalized}}", - "range": true, - "refId": "A" - } - ], - "title": "Tool Calls by Tool", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 70, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 24 - }, - "id": 16, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum by (decision) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.approval.resolved\", provider=~\"$provider\"}[$__rate_interval]))", - "legendFormat": "{{decision}}", - "range": true, - "refId": "A" - } - ], - "title": "Approval Decisions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 24 - }, - "id": 17, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum by (git_host_provider, outcome) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.git.pr_mr.create_completed\"}[$__range]))", - "instant": true, - "legendFormat": "{{git_host_provider}} / {{outcome}}", - "range": false, - "refId": "A" - } - ], - "title": "PR/MR Outcomes", - "type": "bargauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 18, - "panels": [], - "title": "Adoption & Navigation", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlPu" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 33 - }, - "id": 19, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "topk(15, sum by (project_name) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", project_name=~\".+\"}[$__range])))", - "instant": true, - "legendFormat": "{{project_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Projects by Message Volume", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-YlBl" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 33 - }, - "id": 20, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "topk(15, sum by (route) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.ui.route.changed\", route=~\".+\"}[$__range])))", - "instant": true, - "legendFormat": "{{route}}", - "range": false, - "refId": "A" - } - ], - "title": "Top Routes", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 33 - }, - "id": 21, - "options": { - "displayLabels": ["name", "percent"], - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "values": ["value", "percent"] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "editorMode": "code", - "expr": "sum by (analytics_user_source) (increase(traces_spanmetrics_calls_total{service=~\"$service\", span_name=\"marcode.message.user.sent\", analytics_user_source=~\".+\"}[$__range]))", - "instant": true, - "legendFormat": "{{analytics_user_source}}", - "range": false, - "refId": "A" - } - ], - "title": "User Identity Source", - "type": "piechart" - } - ], - "refresh": "1m", - "schemaVersion": 42, - "tags": ["marcode", "product-analytics", "spanmetrics"], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": "Prometheus", - "name": "prometheus", - "query": "prometheus", - "refresh": 1, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "marcode", - "value": "marcode" - }, - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "definition": "label_values(traces_spanmetrics_calls_total, service)", - "hide": 0, - "includeAll": false, - "label": "Service", - "name": "service", - "options": [], - "query": { - "query": "label_values(traces_spanmetrics_calls_total, service)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/^marcode$/", - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${prometheus}" - }, - "definition": "label_values(traces_spanmetrics_calls_total{service=~\"$service\"}, provider)", - "hide": 0, - "includeAll": true, - "label": "Provider", - "multi": true, - "name": "provider", - "options": [], - "query": { - "query": "label_values(traces_spanmetrics_calls_total{service=~\"$service\"}, provider)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": ["30s", "1m", "5m", "15m", "30m"], - "time_options": ["1h", "6h", "12h", "24h", "7d", "30d"] - }, - "timezone": "browser", - "title": "MarCode Product Analytics", - "uid": "marcode-product-analytics", - "version": 1, - "weekStart": "" -}