feat(server): record runtime mode per turn and on mode changes

apps/server/integration/providerService.integration.test.ts

@@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
 import * as Path from "effect/Path";
 import * as Queue from "effect/Queue";
+import * as Ref from "effect/Ref";
 import * as Stream from "effect/Stream";
 
 import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts";
@@ -54,35 +55,58 @@ interface IntegrationFixture {
   readonly layer: Layer.Layer<ProviderService, unknown, never>;
 }
 
-const makeIntegrationFixture = Effect.gen(function* () {
-  const cwd = yield* makeWorkspaceDirectory;
-  const harness = yield* makeTestProviderAdapterHarness();
-
-  const registry = makeAdapterRegistryMock({
-    [ProviderDriverKind.make("codex")]: harness.adapter,
-  });
+interface RecordedAnalyticsEvent {
+  readonly event: string;
+  readonly properties: Readonly<Record<string, unknown>> | undefined;
+}
 
-  const directoryLayer = ProviderSessionDirectoryLive.pipe(
-    Layer.provide(ProviderSessionRuntime.layer),
+/**
+ * Analytics layer that keeps captured events in memory so tests can assert on
+ * telemetry payloads. `AnalyticsService.layerTest` discards them.
+ */
+const makeRecordingAnalytics = Effect.gen(function* () {
+  const recorded = yield* Ref.make<ReadonlyArray<RecordedAnalyticsEvent>>([]);
+  const layer = Layer.succeed(
+    AnalyticsService,
+    AnalyticsService.of({
+      record: (event, properties) =>
+        Ref.update(recorded, (current) => [...current, { event, properties }]),
+      flush: Effect.void,
+    }),
   );
-
-  const shared = Layer.mergeAll(
-    directoryLayer,
-    Layer.succeed(ProviderAdapterRegistry, registry),
-    ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
-    AnalyticsService.layerTest,
-    Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
-  ).pipe(Layer.provide(SqlitePersistenceMemory));
-
-  const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
-
-  return {
-    cwd,
-    harness,
-    layer,
-  } satisfies IntegrationFixture;
+  return { layer, get: Ref.get(recorded) } as const;
 });
 
+const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<AnalyticsService> }) =>
+  Effect.gen(function* () {
+    const cwd = yield* makeWorkspaceDirectory;
+    const harness = yield* makeTestProviderAdapterHarness();
+
+    const registry = makeAdapterRegistryMock({
+      [ProviderDriverKind.make("codex")]: harness.adapter,
+    });
+
+    const directoryLayer = ProviderSessionDirectoryLive.pipe(
+      Layer.provide(ProviderSessionRuntime.layer),
+    );
+
+    const shared = Layer.mergeAll(
+      directoryLayer,
+      Layer.succeed(ProviderAdapterRegistry, registry),
+      ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
+      options?.analytics ?? AnalyticsService.layerTest,
+      Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
+    ).pipe(Layer.provide(SqlitePersistenceMemory));
+
+    const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
+
+    return {
+      cwd,
+      harness,
+      layer,
+    } satisfies IntegrationFixture;
+  });
+
 const collectEventsDuring = <A, E, R>(
   stream: Stream.Stream<ProviderRuntimeEvent>,
   count: number,
@@ -126,7 +150,7 @@ const runTurn = (input: {
 
 it.live("replays typed runtime fixture events", () =>
   Effect.gen(function* () {
-    const fixture = yield* makeIntegrationFixture;
+    const fixture = yield* makeIntegrationFixture();
 
     yield* Effect.gen(function* () {
       const provider = yield* ProviderService;
@@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () =>
 
 it.live("replays file-changing fixture turn events", () =>
   Effect.gen(function* () {
-    const fixture = yield* makeIntegrationFixture;
+    const fixture = yield* makeIntegrationFixture();
     const { join } = yield* Path.Path;
     const { writeFileString } = yield* FileSystem.FileSystem;
 
@@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () =>
 
 it.live("runs multi-turn tool/approval flow", () =>
   Effect.gen(function* () {
-    const fixture = yield* makeIntegrationFixture;
+    const fixture = yield* makeIntegrationFixture();
     const { join } = yield* Path.Path;
     const { writeFileString } = yield* FileSystem.FileSystem;
 
@@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () =>
 
 it.live("rolls back provider conversation state only", () =>
   Effect.gen(function* () {
-    const fixture = yield* makeIntegrationFixture;
+    const fixture = yield* makeIntegrationFixture();
     const { join } = yield* Path.Path;
     const { writeFileString, readFileString } = yield* FileSystem.FileSystem;
 
@@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () =>
     }).pipe(Effect.provide(fixture.layer));
   }).pipe(Effect.provide(NodeServices.layer)),
 );
+
+it.live("reports runtime mode per turn and on mode transitions", () =>
+  Effect.gen(function* () {
+    const analytics = yield* makeRecordingAnalytics;
+    const fixture = yield* makeIntegrationFixture({ analytics: analytics.layer });
+    const threadId = ThreadId.make("thread-integration-runtime-mode");
+
+    yield* Effect.gen(function* () {
+      const provider = yield* ProviderService;
+      const startSession = (runtimeMode: "approval-required" | "full-access") =>
+        provider.startSession(threadId, {
+          threadId,
+          provider: ProviderDriverKind.make("codex"),
+          providerInstanceId: codexInstanceId,
+          cwd: fixture.cwd,
+          runtimeMode,
+        });
+
+      yield* startSession("approval-required");
+      yield* runTurn({
+        provider,
+        harness: fixture.harness,
+        threadId,
+        userText: "supervised turn",
+        response: { events: codexTurnTextFixture },
+      });
+
+      // Toggling the mode restarts the session, which is the only place the
+      // transition is observable.
+      yield* startSession("full-access");
+      yield* runTurn({
+        provider,
+        harness: fixture.harness,
+        threadId,
+        userText: "full access turn",
+        response: { events: codexTurnTextFixture },
+      });
+
+      const recorded = yield* analytics.get;
+
+      assert.deepEqual(
+        recorded
+          .filter((entry) => entry.event === "provider.turn.sent")
+          .map((entry) => entry.properties?.runtimeMode),
+        ["approval-required", "full-access"],
+      );
+
+      assert.deepEqual(
+        recorded
+          .filter((entry) => entry.event === "provider.runtime_mode.changed")
+          .map((entry) => [entry.properties?.from, entry.properties?.to]),
+        [["approval-required", "full-access"]],
+      );
+    }).pipe(Effect.provide(fixture.layer));
+  }).pipe(Effect.provide(NodeServices.layer)),
+);

apps/server/src/provider/Layers/ProviderService.ts

@@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
         adapter,
         instanceId,
         threadId: input.threadId,
+        runtimeMode: binding.runtimeMode,
         isActive: true,
       } as const;
     }
@@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
         adapter,
         instanceId,
         threadId: input.threadId,
+        runtimeMode: binding.runtimeMode,
         isActive: false,
       } as const;
     }
@@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
       adapter: recovered.adapter,
       instanceId,
       threadId: input.threadId,
+      runtimeMode: recovered.session.runtimeMode,
       isActive: true,
     } as const;
   });
@@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
             input.modelSelection.model.trim().length > 0,
         });
 
+        // Changing runtime mode restarts the session, so the transition is only
+        // observable here, by diffing against the mode the previous session for
+        // this thread was bound to. Recording it separately is what makes the
+        // "started supervised, switched to full access" funnel answerable.
+        const previousRuntimeMode = persistedBinding?.runtimeMode;
+        if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) {
+          yield* analytics.record("provider.runtime_mode.changed", {
+            provider: sessionWithInstance.provider,
+            from: previousRuntimeMode,
+            to: input.runtimeMode,
+          });
+        }
+
         return sessionWithInstance;
       }).pipe(
         withMetrics({
@@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
         provider: routed.adapter.provider,
         model: input.modelSelection?.model,
         interactionMode: input.interactionMode,
+        // Session-start events alone skew runtime mode toward users who toggle
+        // often, since every toggle restarts the session. Recording it per turn
+        // gives a usage-weighted view and lets it cross with interactionMode.
+        runtimeMode: routed.runtimeMode,
         attachmentCount: input.attachments.length,
         hasInput: typeof input.input === "string" && input.input.trim().length > 0,
       });