diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21fbce026f5..1169a2e2a7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,3 +99,22 @@ jobs: - name: Exercise release-only workflow steps run: node scripts/release-smoke.ts + + hermes_plugin: + name: Hermes Plugin + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Lint + run: pipx run ruff check integrations/hermes-t3-gateway + + - name: Test + run: python -m unittest discover -s integrations/hermes-t3-gateway/tests -v diff --git a/.gitignore b/.gitignore index 8e1669c8115..c1f7d342c90 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ node_modules/ *.log .env* !.env.example +__pycache__/ +*.py[cod] diff --git a/apps/mobile/src/features/archive/archivedThreadList.test.ts b/apps/mobile/src/features/archive/archivedThreadList.test.ts index 697d13e7c47..59dd10b7c5c 100644 --- a/apps/mobile/src/features/archive/archivedThreadList.test.ts +++ b/apps/mobile/src/features/archive/archivedThreadList.test.ts @@ -13,6 +13,7 @@ function makeProject( return { workspaceRoot: `/workspaces/${input.id}`, repositoryIdentity: null, + agentInstanceId: null, defaultModelSelection: null, scripts: [], createdAt: "2026-06-01T00:00:00.000Z", diff --git a/apps/mobile/src/features/home/HomeRouteScreen.tsx b/apps/mobile/src/features/home/HomeRouteScreen.tsx index 9fa179f4c76..1fdd9dc9d89 100644 --- a/apps/mobile/src/features/home/HomeRouteScreen.tsx +++ b/apps/mobile/src/features/home/HomeRouteScreen.tsx @@ -3,6 +3,7 @@ import * as Order from "effect/Order"; import { useNavigation } from "@react-navigation/native"; import { useEffect, useMemo, useState } from "react"; +import { selectWorkspaceProjects } from "../../lib/repositoryGroups"; import { NativeHeaderToolbar, NativeStackScreenOptions } from "../../native/StackHeader"; import { useProjects, useThreadShells } from "../../state/entities"; import { usePendingNewTasks } from "../../state/use-pending-new-tasks"; @@ -59,10 +60,12 @@ export function HomeRouteScreen() { } = useHomeListOptions(availableEnvironmentIds); const selectedEnvironmentId = listOptions.selectedEnvironmentId; const [selectedProjectKey, setSelectedProjectKey] = useState(null); + // Filter menu = picker: agent projects are excluded here, while the thread + // list below still receives the full list so agent threads stay visible. const projectFilterOptions = useMemo( () => buildHomeProjectScopes({ - projects, + projects: selectWorkspaceProjects(projects), environmentId: selectedEnvironmentId, projectGroupingMode: listOptions.projectGroupingMode, }).map((scope) => ({ diff --git a/apps/mobile/src/features/home/HomeScreen.tsx b/apps/mobile/src/features/home/HomeScreen.tsx index 0263e74eade..335d2d1b35c 100644 --- a/apps/mobile/src/features/home/HomeScreen.tsx +++ b/apps/mobile/src/features/home/HomeScreen.tsx @@ -24,6 +24,7 @@ import { AppText as Text } from "../../components/AppText"; import { EmptyState } from "../../components/EmptyState"; import type { WorkspaceState } from "../../state/workspaceModel"; import type { SavedRemoteConnection } from "../../lib/connection"; +import { selectWorkspaceProjects } from "../../lib/repositoryGroups"; import { scopedProjectKey } from "../../lib/scopedEntities"; import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; @@ -773,7 +774,9 @@ export function HomeScreen(props: HomeScreenProps) { const shouldShowConnectionStatus = shouldShowWorkspaceConnectionStatus(props.catalogState); const emptyState = deriveEmptyState({ catalogState: props.catalogState, - projectCount: props.projects.length, + // "No projects found" counts real workspaces: a synthetic agent project + // is not something the user added, so it must not suppress the hint. + projectCount: selectWorkspaceProjects(props.projects).length, }); const connectionStatus = shouldShowConnectionStatus && Platform.OS !== "ios" ? ( diff --git a/apps/mobile/src/features/home/homeListItems.test.ts b/apps/mobile/src/features/home/homeListItems.test.ts index c5a9f2c6bbc..985b8780091 100644 --- a/apps/mobile/src/features/home/homeListItems.test.ts +++ b/apps/mobile/src/features/home/homeListItems.test.ts @@ -25,6 +25,7 @@ function makeProject(id: string, title: string): EnvironmentProject { title, workspaceRoot: `/workspaces/${id}`, repositoryIdentity: null, + agentInstanceId: null, defaultModelSelection: null, scripts: [], createdAt: "2026-06-01T00:00:00.000Z", diff --git a/apps/mobile/src/features/home/homeThreadList.test.ts b/apps/mobile/src/features/home/homeThreadList.test.ts index e791cd3b36a..44fed543319 100644 --- a/apps/mobile/src/features/home/homeThreadList.test.ts +++ b/apps/mobile/src/features/home/homeThreadList.test.ts @@ -17,6 +17,7 @@ function makeProject( return { workspaceRoot: `/workspaces/${input.id}`, repositoryIdentity: null, + agentInstanceId: null, defaultModelSelection: null, scripts: [], createdAt: "2026-06-01T00:00:00.000Z", diff --git a/apps/mobile/src/features/home/homeThreadList.ts b/apps/mobile/src/features/home/homeThreadList.ts index 21084f0f5fe..c0422f61c84 100644 --- a/apps/mobile/src/features/home/homeThreadList.ts +++ b/apps/mobile/src/features/home/homeThreadList.ts @@ -56,6 +56,11 @@ export function buildHomeProjectScopes(input: { readonly environmentId: EnvironmentId | null; readonly projectGroupingMode: SidebarProjectGroupingMode; }): ReadonlyArray { + // NOTE: agent projects are deliberately *not* filtered here. This builder + // feeds both the project filter menu (a picker — callers pass an + // agent-free list, see `selectWorkspaceProjects`) and the thread list + // itself. Excluding agent projects here would silently drop every agent + // thread from mobile, which has no Agents surface to show them in instead. const projects = input.projects.filter( (project) => input.environmentId === null || project.environmentId === input.environmentId, ); @@ -305,6 +310,9 @@ export function buildHomeThreadGroups(input: { workspaceRoot: pendingTask.creation.projectCwd ?? String(pendingTask.creation.projectId), repositoryIdentity: null, + // A queued task always targets a real workspace; agent projects + // never enqueue through this path. + agentInstanceId: null, defaultModelSelection: null, scripts: [], createdAt: pendingTask.message.createdAt, diff --git a/apps/mobile/src/features/projects/AddProjectScreen.tsx b/apps/mobile/src/features/projects/AddProjectScreen.tsx index eef40382b24..44dd2459cda 100644 --- a/apps/mobile/src/features/projects/AddProjectScreen.tsx +++ b/apps/mobile/src/features/projects/AddProjectScreen.tsx @@ -32,6 +32,7 @@ import * as Cause from "effect/Cause"; import * as Order from "effect/Order"; import { AsyncResult } from "effect/unstable/reactivity"; import { cn } from "../../lib/cn"; +import { selectWorkspaceProjects } from "../../lib/repositoryGroups"; import { useProjects, useServerConfigs } from "../../state/entities"; import { filesystemEnvironment } from "../../state/filesystem"; @@ -496,7 +497,9 @@ export function AddProjectSourceScreen() { function useCreateProject(environment: EnvironmentOption | null) { const navigation = useNavigation(); const createProject = useAtomCommand(projectEnvironment.create, { reportFailure: false }); - const projects = useProjects(); + // Dedupe against real projects only: an agent's synthetic directory should + // never make "add project" bounce the user into an agent thread. + const projects = selectWorkspaceProjects(useProjects()); return useCallback( async (workspaceRoot: string) => { diff --git a/apps/mobile/src/features/showcase/showcasePendingTasks.test.ts b/apps/mobile/src/features/showcase/showcasePendingTasks.test.ts index 8ab043d7cd8..a747dd5e56e 100644 --- a/apps/mobile/src/features/showcase/showcasePendingTasks.test.ts +++ b/apps/mobile/src/features/showcase/showcasePendingTasks.test.ts @@ -14,6 +14,7 @@ const projects: ReadonlyArray = [ title: "T3 Code", workspaceRoot: "/workspace/t3code", repositoryIdentity: null, + agentInstanceId: null, defaultModelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, scripts: [], createdAt: "2026-07-16T08:00:00.000Z", @@ -25,6 +26,7 @@ const projects: ReadonlyArray = [ title: "React", workspaceRoot: "/workspace/react", repositoryIdentity: null, + agentInstanceId: null, defaultModelSelection: null, scripts: [], createdAt: "2026-07-16T08:00:00.000Z", diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index e328bcef00e..b4fbcb549b5 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -44,6 +44,7 @@ import { } from "../../state/use-composer-drafts"; import { useProjects } from "../../state/entities"; import { deriveThreadTitleFromPrompt } from "../../lib/projectThreadStartTurn"; +import { selectWorkspaceProjects } from "../../lib/repositoryGroups"; import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/remoteRegistration"; import { enqueueThreadOutboxMessage, removeThreadOutboxMessage } from "../../state/thread-outbox"; import { useRemoteConnectionStatus } from "../../state/use-remote-environment-registry"; @@ -73,7 +74,12 @@ export function NewTaskDraftScreen(props: { /** Durable native share inbox item to merge into this project draft. */ readonly incomingShareId?: string; }) { - const projects = useProjects(); + // Filtered wholesale: mobile has no agent surface, so a route (or share + // reservation) pointing at a synthetic agent project should fall through to + // "return to the project picker" rather than open a draft against an empty + // agent directory. + const allProjects = useProjects(); + const projects = useMemo(() => selectWorkspaceProjects(allProjects), [allProjects]); const createProjectThread = useCreateProjectThread(); const flow = useNewTaskFlow(); const navigation = useNavigation(); @@ -264,7 +270,12 @@ export function NewTaskDraftScreen(props: { return; } - if (projects.length > 0) { + // Guard on the *unfiltered* catalog. `projects` drops synthetic agent + // projects, so an agent-only environment is indistinguishable from a + // catalog that has not loaded yet — and a route pointing at an agent + // project then found no directProject, skipped this branch, and left the + // screen on "Loading task" with nothing left to resolve it. + if (allProjects.length > 0) { // Never fall through to the flow provider's temporary first-project // default. Return to the picker with the share id intact so the user // can choose an available destination. @@ -284,6 +295,7 @@ export function NewTaskDraftScreen(props: { navigation.dispatch(StackActions.replace("NewTask")); }, [ + allProjects, logicalProjects, projects, props.initialProjectRef, diff --git a/apps/mobile/src/features/threads/ThreadNavigationSidebar.tsx b/apps/mobile/src/features/threads/ThreadNavigationSidebar.tsx index 3d413f9c487..20dd735a68e 100644 --- a/apps/mobile/src/features/threads/ThreadNavigationSidebar.tsx +++ b/apps/mobile/src/features/threads/ThreadNavigationSidebar.tsx @@ -21,6 +21,7 @@ import { ControlPillMenu } from "../../components/ControlPill"; import { SymbolView } from "../../components/AppSymbol"; import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; import { NativeStackScreenOptions } from "../../native/StackHeader"; +import { selectWorkspaceProjects } from "../../lib/repositoryGroups"; import { scopedProjectKey, scopedThreadKey } from "../../lib/scopedEntities"; import { useThemeColor } from "../../lib/useThemeColor"; import { useProjects, useThreadShells } from "../../state/entities"; @@ -219,13 +220,19 @@ function ThreadNavigationSidebarPane( }), [options.projectGroupingMode, options.selectedEnvironmentId, projects], ); + // The scope list above stays complete so agent threads remain visible and + // scopable; only the *picker* built from it drops agent projects. const projectFilterOptions = useMemo( () => - projectScopes.map((scope) => ({ + buildHomeProjectScopes({ + projects: selectWorkspaceProjects(projects), + environmentId: options.selectedEnvironmentId, + projectGroupingMode: options.projectGroupingMode, + }).map((scope) => ({ key: scope.key, label: scope.title, })), - [projectScopes], + [options.projectGroupingMode, options.selectedEnvironmentId, projects], ); const projectTitleByProjectKey = useMemo( () => diff --git a/apps/mobile/src/features/threads/new-task-flow-provider.tsx b/apps/mobile/src/features/threads/new-task-flow-provider.tsx index 74fe2f4852a..ed14cfdeb49 100644 --- a/apps/mobile/src/features/threads/new-task-flow-provider.tsx +++ b/apps/mobile/src/features/threads/new-task-flow-provider.tsx @@ -23,7 +23,7 @@ import type { TurnCommandMetadata } from "../../lib/commandMetadata"; import type { DraftComposerImageAttachment } from "../../lib/composerImages"; import type { ModelOption, ProviderGroup } from "../../lib/modelOptions"; import { buildModelOptions, groupByProvider } from "../../lib/modelOptions"; -import { groupProjectsByRepository } from "../../lib/repositoryGroups"; +import { groupProjectsByRepository, selectWorkspaceProjects } from "../../lib/repositoryGroups"; import { scopedProjectKey } from "../../lib/scopedEntities"; import { appAtomRegistry } from "../../state/atom-registry"; import { @@ -168,7 +168,12 @@ type NewTaskFlowContextValue = { const NewTaskFlowContext = React.createContext(null); export function NewTaskFlowProvider(props: React.PropsWithChildren) { - const projects = useProjects(); + // Every use of this list is part of choosing a destination for a new task, + // so synthetic agent projects are excluded once, here, rather than at each + // of the environment/project selection sites below. Memoized because the + // downstream grouping/selection memos take it as a dependency. + const allProjects = useProjects(); + const projects = useMemo(() => selectWorkspaceProjects(allProjects), [allProjects]); const threads = useThreadShells(); const { savedConnectionsById } = useSavedRemoteConnections(); @@ -259,6 +264,9 @@ export function NewTaskFlowProvider(props: React.PropsWithChildren) { // fabricated path. workspaceRoot: creation.projectCwd ?? "", repositoryIdentity: null, + // A queued task always targets a real workspace; agent projects never + // enqueue through this path. + agentInstanceId: null, defaultModelSelection: editingPendingTask.modelSelection ?? null, scripts: [], createdAt: editingPendingTask.createdAt, diff --git a/apps/mobile/src/lib/repositoryGroups.test.ts b/apps/mobile/src/lib/repositoryGroups.test.ts index ab4311524ce..9b00ab1526b 100644 --- a/apps/mobile/src/lib/repositoryGroups.test.ts +++ b/apps/mobile/src/lib/repositoryGroups.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vite-plus/test"; import { EnvironmentId, ProjectId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; -import { groupProjectsByRepository } from "./repositoryGroups"; +import { groupProjectsByRepository, isAgentProject } from "./repositoryGroups"; import { EnvironmentProject, EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; function makeProject( @@ -11,6 +11,7 @@ function makeProject( return { workspaceRoot: `/workspaces/${input.id}`, repositoryIdentity: null, + agentInstanceId: null, defaultModelSelection: null, scripts: [], createdAt: "2026-04-01T00:00:00.000Z", @@ -190,3 +191,50 @@ describe("groupProjectsByRepository", () => { expect(groups[0]?.subtitle).toBeNull(); }); }); + +describe("agent project exclusion", () => { + const environmentId = EnvironmentId.make("env-local"); + const agentInstanceId = ProviderInstanceId.make("hermes_workstation"); + + it("keeps a project with no reported agent instance", () => { + // Older snapshots omit `agentInstanceId` entirely; that must read as an + // ordinary project, not as an agent. + expect( + isAgentProject( + makeProject({ environmentId, id: ProjectId.make("project-legacy"), title: "Legacy" }), + ), + ).toBe(false); + }); + + it("identifies a synthetic agent project", () => { + expect( + isAgentProject( + makeProject({ + environmentId, + id: ProjectId.make("project-agent"), + title: "Workstation", + agentInstanceId, + }), + ), + ).toBe(true); + }); + + it("omits agent projects from repository groups", () => { + const workspace = makeProject({ + environmentId, + id: ProjectId.make("project-real"), + title: "Real", + }); + const agent = makeProject({ + environmentId, + id: ProjectId.make("project-agent"), + title: "Workstation", + agentInstanceId, + }); + + const groups = groupProjectsByRepository({ projects: [workspace, agent], threads: [] }); + + expect(groups).toHaveLength(1); + expect(groups[0]?.projects[0]?.project.id).toBe(workspace.id); + }); +}); diff --git a/apps/mobile/src/lib/repositoryGroups.ts b/apps/mobile/src/lib/repositoryGroups.ts index bf4c2f3fccd..48e8b2de6e8 100644 --- a/apps/mobile/src/lib/repositoryGroups.ts +++ b/apps/mobile/src/lib/repositoryGroups.ts @@ -58,10 +58,34 @@ function deriveProjectLatestActivity( return latestThread?.updatedAt ?? latestThread?.createdAt ?? project.updatedAt; } -export function groupProjectsByRepository(input: { +/** + * True for the synthetic project row backing an agent instance + * (`OrchestrationProjectShell.agentInstanceId`). Such a row is a legal project + * with a real — but empty — workspace directory, created only so an agent + * thread can satisfy `project_id NOT NULL`. It is never a workspace the user + * chose, so it must not appear in any surface that offers a project to pick. + */ +export function isAgentProject(project: Pick): boolean { + // `!= null` on purpose: the contract decodes a default of `null`, but an + // older snapshot (or a partial fixture) can omit the field entirely, and + // "not reported" must mean "ordinary project", never "agent". + return project.agentInstanceId != null; +} + +/** Drop synthetic agent projects from a list destined for a picker. */ +export function selectWorkspaceProjects< + TProject extends Pick, +>(projects: ReadonlyArray): TProject[] { + return projects.filter((project) => !isAgentProject(project)); +} + +export function groupProjectsByRepository(rawInput: { readonly projects: ReadonlyArray; readonly threads: ReadonlyArray; }): ReadonlyArray { + // This grouping only ever feeds the "choose a project" screens, so the + // exclusion lives here rather than at each call site. + const input = { ...rawInput, projects: selectWorkspaceProjects(rawInput.projects) }; const threadsByProjectKey = new Map(); for (const thread of input.threads) { diff --git a/apps/mobile/src/widgets/AgentActivity.tsx b/apps/mobile/src/widgets/AgentActivity.tsx index 88c85648271..d74337a0bb7 100644 --- a/apps/mobile/src/widgets/AgentActivity.tsx +++ b/apps/mobile/src/widgets/AgentActivity.tsx @@ -23,6 +23,7 @@ export type AgentActivityPhase = | "running" | "waiting_for_approval" | "waiting_for_input" + | "notification" | "completed" | "failed" | "stale"; @@ -81,6 +82,8 @@ export function AgentActivity( return isLightScheme ? "#d97706" : "#fcd34d"; // amber-600 / amber-300 case "waiting_for_input": return isLightScheme ? "#4f46e5" : "#a5b4fc"; // indigo-600 / indigo-300 + case "notification": + return isLightScheme ? "#7c3aed" : "#c4b5fd"; // violet-600 / violet-300 case "failed": return isLightScheme ? "#dc2626" : "#fca5a5"; // red-600 / red-300 case "completed": @@ -97,7 +100,9 @@ export function AgentActivity( const phasePriority = (phase: AgentActivityPhase): number => { if (phase === "waiting_for_approval" || phase === "waiting_for_input") return 0; if (phase === "failed") return 1; - if (phase === "running" || phase === "starting") return 2; + // A proactive delivery is unread news, so it outranks finished/stale rows + // but never a human who is actually blocked. + if (phase === "running" || phase === "starting" || phase === "notification") return 2; return 3; }; const ordered = [...props.activities].sort( @@ -163,6 +168,8 @@ export function AgentActivity( return "exclamationmark.circle.fill"; case "waiting_for_input": return "questionmark.circle.fill"; + case "notification": + return "bell.badge.fill"; case "failed": return "xmark.octagon.fill"; case "completed": diff --git a/apps/server/src/attachmentStore.ts b/apps/server/src/attachmentStore.ts index 3d5b531db21..5cef79db22a 100644 --- a/apps/server/src/attachmentStore.ts +++ b/apps/server/src/attachmentStore.ts @@ -10,7 +10,35 @@ import { } from "./attachmentPaths.ts"; import { inferImageExtension, SAFE_IMAGE_FILE_EXTENSIONS } from "./imageMime.ts"; -const ATTACHMENT_FILENAME_EXTENSIONS = [...SAFE_IMAGE_FILE_EXTENSIONS, ".bin"]; +/** + * Extensions a generic file attachment may take on disk. The extension is + * derived from the declared mime rather than the client-supplied filename — + * the asset route later re-derives the served content type from it, so this + * map is what keeps a video playing as a video and everything unknown pinned + * to inert `.bin`. + */ +const FILE_EXTENSION_BY_MIME_TYPE: Record = { + "application/gzip": ".gz", + "application/json": ".json", + "application/pdf": ".pdf", + "application/zip": ".zip", + "audio/mpeg": ".mp3", + "audio/ogg": ".ogg", + "audio/wav": ".wav", + "text/csv": ".csv", + "text/markdown": ".md", + "text/plain": ".txt", + "video/mp4": ".mp4", + "video/quicktime": ".mov", + "video/webm": ".webm", +}; +const SAFE_FILE_EXTENSIONS = new Set(Object.values(FILE_EXTENSION_BY_MIME_TYPE)); + +const ATTACHMENT_FILENAME_EXTENSIONS = [ + ...SAFE_IMAGE_FILE_EXTENSIONS, + ...SAFE_FILE_EXTENSIONS, + ".bin", +]; const ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS = 80; const ATTACHMENT_ID_THREAD_SEGMENT_PATTERN = "[a-z0-9_]+(?:-[a-z0-9_]+)*"; const ATTACHMENT_ID_UUID_PATTERN = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; @@ -63,6 +91,11 @@ export function attachmentRelativePath(attachment: ChatAttachment): string { }); return `${attachment.id}${extension}`; } + case "file": { + const extension = + FILE_EXTENSION_BY_MIME_TYPE[attachment.mimeType.trim().toLowerCase()] ?? ".bin"; + return `${attachment.id}${extension}`; + } } } diff --git a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts index 8e0e5fb74d5..55d238852a7 100644 --- a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts +++ b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts @@ -86,6 +86,7 @@ describe("CheckpointDiffQuery.layer", () => { getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.succeed(Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => @@ -106,6 +107,8 @@ describe("CheckpointDiffQuery.layer", () => { }); }), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), @@ -194,11 +197,14 @@ describe("CheckpointDiffQuery.layer", () => { getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.succeed(Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.some(threadCheckpointContext)), getFullThreadDiffContext: () => Effect.die("unused"), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), @@ -277,11 +283,14 @@ describe("CheckpointDiffQuery.layer", () => { getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.succeed(Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.some(threadCheckpointContext)), getFullThreadDiffContext: () => Effect.die("unused"), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), @@ -345,11 +354,14 @@ describe("CheckpointDiffQuery.layer", () => { getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.succeed(Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.some(threadCheckpointContext)), getFullThreadDiffContext: () => Effect.die("unused"), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), @@ -398,11 +410,14 @@ describe("CheckpointDiffQuery.layer", () => { getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.succeed(Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.none()), getFullThreadDiffContext: () => Effect.succeed(Option.none()), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts index 731002ea783..feb5ee343a7 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts @@ -124,6 +124,7 @@ describe("OrchestrationEngine", () => { model: "gpt-5-codex", }, scripts: [], + agentInstanceId: null, createdAt: "2026-03-03T00:00:00.000Z", updatedAt: "2026-03-03T00:00:01.000Z", deletedAt: null, @@ -172,6 +173,7 @@ describe("OrchestrationEngine", () => { const layer = OrchestrationEngineLive.pipe( Layer.provide( Layer.succeed(ProjectionSnapshotQuery, { + getActiveAgentProject: () => Effect.succeed(Option.none()), getCommandReadModel: () => Effect.succeed(commandReadModel), getSnapshot: () => Effect.sync(() => { @@ -201,6 +203,8 @@ describe("OrchestrationEngine", () => { getThreadCheckpointContext: () => Effect.succeed(Option.none()), getFullThreadDiffContext: () => Effect.succeed(Option.none()), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index 926182a3ef0..9b029c2199a 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -1429,6 +1429,21 @@ it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => { assert.deepEqual(settledRows, [ { state: "completed", completedAt: "2026-01-01T00:01:00.000Z" }, ]); + + // The session cleared `activeTurnId` because the turn ENDED, not + // because the thread never had one — the pointer must survive. + // + // Nulling it here used to be masked by `thread.turn-diff-completed`, + // which re-writes the id moments later. A workspaceless provider + // (Hermes) emits no diff, so the id stayed null forever, `latestTurn` + // never resolved, and every completed turn read as a queued-but- + // unadopted turn start — which blocks settling for the grace window. + const latestTurnRows = yield* sql<{ readonly latestTurnId: string | null }>` + SELECT latest_turn_id AS "latestTurnId" + FROM projection_threads + WHERE thread_id = ${threadId} + `; + assert.deepEqual(latestTurnRows, [{ latestTurnId: turnId }]); }), ); @@ -1652,6 +1667,117 @@ it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => { }), ); + it.effect("replaces an assistant snapshot between legacy appended deltas", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const now = "2026-07-24T12:00:00.000Z"; + const projectId = ProjectId.make("project-replacement"); + const threadId = ThreadId.make("thread-replacement"); + const messageId = MessageId.make("assistant-replacement"); + + yield* eventStore.append({ + type: "project.created", + eventId: EventId.make("evt-replacement-1"), + aggregateKind: "project", + aggregateId: projectId, + occurredAt: now, + commandId: CommandId.make("cmd-replacement-1"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-replacement-1"), + metadata: {}, + payload: { + projectId, + title: "Project Replacement", + workspaceRoot: "/tmp/project-replacement", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + }); + + yield* eventStore.append({ + type: "thread.created", + eventId: EventId.make("evt-replacement-2"), + aggregateKind: "thread", + aggregateId: threadId, + occurredAt: now, + commandId: CommandId.make("cmd-replacement-2"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-replacement-2"), + metadata: {}, + payload: { + threadId, + projectId, + title: "Thread Replacement", + modelSelection: { + instanceId: ProviderInstanceId.make("hermes"), + model: "hermes", + }, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: now, + updatedAt: now, + }, + }); + + const appendMessage = (input: { + eventNumber: number; + text: string; + streaming: boolean; + textOperation?: "replace"; + }) => + eventStore.append({ + type: "thread.message-sent", + eventId: EventId.make(`evt-replacement-${input.eventNumber}`), + aggregateKind: "thread", + aggregateId: threadId, + occurredAt: now, + commandId: CommandId.make(`cmd-replacement-${input.eventNumber}`), + causationEventId: null, + correlationId: CorrelationId.make(`cmd-replacement-${input.eventNumber}`), + metadata: {}, + payload: { + threadId, + messageId, + role: "assistant", + text: input.text, + turnId: null, + streaming: input.streaming, + ...(input.textOperation !== undefined ? { textOperation: input.textOperation } : {}), + createdAt: now, + updatedAt: now, + }, + }); + + yield* appendMessage({ eventNumber: 3, text: "Hello wor", streaming: true }); + yield* appendMessage({ + eventNumber: 4, + text: "Hello there", + streaming: true, + textOperation: "replace", + }); + yield* appendMessage({ eventNumber: 5, text: "!", streaming: true }); + yield* appendMessage({ eventNumber: 6, text: "", streaming: false }); + + yield* projectionPipeline.bootstrap; + + const messageRows = yield* sql<{ readonly text: string; readonly isStreaming: unknown }>` + SELECT + text, + is_streaming AS "isStreaming" + FROM projection_thread_messages + WHERE message_id = ${messageId} + `; + assert.equal(messageRows.length, 1); + assert.equal(messageRows[0]?.text, "Hello there!"); + assert.isFalse(Boolean(messageRows[0]?.isStreaming)); + }), + ); + it.effect( "resolves turn-count conflicts when checkpoint completion rewrites provisional turns", () => @@ -2257,6 +2383,193 @@ it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => { }), ); + it.effect("projects notification deliveries into the message row and shell summary", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const appendAndProject = (event: Parameters[0]) => + eventStore + .append(event) + .pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent))); + const createdAt = "2026-07-01T09:00:00.000Z"; + + yield* appendAndProject({ + type: "project.created", + eventId: EventId.make("evt-notify-project"), + aggregateKind: "project", + aggregateId: ProjectId.make("project-notify"), + occurredAt: createdAt, + commandId: CommandId.make("cmd-notify-project"), + causationEventId: null, + correlationId: CommandId.make("cmd-notify-project"), + metadata: {}, + payload: { + projectId: ProjectId.make("project-notify"), + title: "Hermes", + workspaceRoot: "/tmp/project-notify", + defaultModelSelection: null, + scripts: [], + createdAt, + updatedAt: createdAt, + }, + }); + + yield* appendAndProject({ + type: "thread.created", + eventId: EventId.make("evt-notify-thread"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-notify"), + occurredAt: createdAt, + commandId: CommandId.make("cmd-notify-thread"), + causationEventId: null, + correlationId: CommandId.make("cmd-notify-thread"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-notify"), + projectId: ProjectId.make("project-notify"), + title: "Home", + modelSelection: { + instanceId: ProviderInstanceId.make("hermes"), + model: "hermes", + }, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt, + updatedAt: createdAt, + }, + }); + + const deliveredAt = "2026-07-01T09:05:00.000Z"; + yield* appendAndProject({ + type: "thread.notification-delivered", + eventId: EventId.make("evt-notify-delivery"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-notify"), + occurredAt: deliveredAt, + commandId: CommandId.make("cmd-notify-delivery"), + causationEventId: null, + correlationId: CommandId.make("cmd-notify-delivery"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-notify"), + messageId: MessageId.make("message-notify"), + text: "Digest ready.", + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }, + createdAt: deliveredAt, + updatedAt: deliveredAt, + }, + }); + + const messageRows = yield* sql<{ + readonly messageId: string; + readonly text: string; + readonly turnId: string | null; + readonly role: string; + readonly isStreaming: number; + readonly notification: string | null; + }>` + SELECT + message_id AS "messageId", + text, + turn_id AS "turnId", + role, + is_streaming AS "isStreaming", + notification_json AS "notification" + FROM projection_thread_messages + WHERE thread_id = 'thread-notify' + `; + assert.equal(messageRows.length, 1); + assert.equal(messageRows[0]?.messageId, "message-notify"); + assert.equal(messageRows[0]?.turnId, null); + assert.equal(messageRows[0]?.role, "assistant"); + assert.equal(messageRows[0]?.isStreaming, 0); + assert.deepEqual( + // @effect-diagnostics-next-line preferSchemaOverJson:off + JSON.parse(messageRows[0]?.notification ?? "null"), + { kind: "cron", label: "Cron: daily-digest", deliveryId: "delivery-1" }, + ); + + // The shell summary is what the sidebar and the push relay read; the + // provenance is invisible to them unless it is flattened here. + const threadRows = yield* sql<{ + readonly updatedAt: string; + readonly latestNotification: string | null; + }>` + SELECT + updated_at AS "updatedAt", + latest_notification_json AS "latestNotification" + FROM projection_threads + WHERE thread_id = 'thread-notify' + `; + assert.equal(threadRows[0]?.updatedAt, deliveredAt); + assert.deepEqual( + // @effect-diagnostics-next-line preferSchemaOverJson:off + JSON.parse(threadRows[0]?.latestNotification ?? "null"), + { kind: "cron", label: "Cron: daily-digest", deliveredAt }, + ); + + // Media rides the same event: attachments land in attachments_json and + // a turn-scoped delivery keeps its turn id on the row. + const mediaDeliveredAt = "2026-07-01T09:06:00.000Z"; + const mediaAttachment = { + type: "file", + id: "thread-notify-2c8b3f1e-0000-4000-8000-000000000001", + name: "chart.mp4", + mimeType: "video/mp4", + sizeBytes: 2_048, + } as const; + yield* appendAndProject({ + type: "thread.notification-delivered", + eventId: EventId.make("evt-notify-media"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-notify"), + occurredAt: mediaDeliveredAt, + commandId: CommandId.make("cmd-notify-media"), + causationEventId: null, + correlationId: CommandId.make("cmd-notify-media"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-notify"), + messageId: MessageId.make("message-notify-media"), + text: "A caption", + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-media", + }, + attachments: [mediaAttachment], + turnId: TurnId.make("turn-live"), + createdAt: mediaDeliveredAt, + updatedAt: mediaDeliveredAt, + }, + }); + + const mediaRows = yield* sql<{ + readonly turnId: string | null; + readonly attachments: string | null; + }>` + SELECT + turn_id AS "turnId", + attachments_json AS "attachments" + FROM projection_thread_messages + WHERE message_id = 'message-notify-media' + `; + assert.equal(mediaRows.length, 1); + assert.equal(mediaRows[0]?.turnId, "turn-live"); + assert.deepEqual( + // @effect-diagnostics-next-line preferSchemaOverJson:off + JSON.parse(mediaRows[0]?.attachments ?? "null"), + [mediaAttachment], + ); + }), + ); + it.effect("does not fallback-retain messages whose turnId is removed by revert", () => Effect.gen(function* () { const projectionPipeline = yield* OrchestrationProjectionPipeline; diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 1f24a4a0200..5d5a89f6015 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -1,6 +1,7 @@ import { ApprovalRequestId, type ChatAttachment, + mergeAssistantMessageText, type OrchestrationEvent, type OrchestrationSessionStatus, ThreadId, @@ -33,7 +34,10 @@ import { type ProjectionTurn, ProjectionTurnRepository, } from "../../persistence/Services/ProjectionTurns.ts"; -import { ProjectionThreadRepository } from "../../persistence/Services/ProjectionThreads.ts"; +import { + type ProjectionThread, + ProjectionThreadRepository, +} from "../../persistence/Services/ProjectionThreads.ts"; import { ProjectionPendingApprovalRepositoryLive } from "../../persistence/Layers/ProjectionPendingApprovals.ts"; import { ProjectionProjectRepositoryLive } from "../../persistence/Layers/ProjectionProjects.ts"; import { ProjectionStateRepositoryLive } from "../../persistence/Layers/ProjectionState.ts"; @@ -338,9 +342,6 @@ function collectThreadAttachmentRelativePaths( const relativePaths = new Set(); for (const message of messages) { for (const attachment of message.attachments ?? []) { - if (attachment.type !== "image") { - continue; - } const attachmentThreadSegment = parseThreadSegmentFromAttachmentId(attachment.id); if (!attachmentThreadSegment || attachmentThreadSegment !== threadSegment) { continue; @@ -496,6 +497,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti workspaceRoot: event.payload.workspaceRoot, defaultModelSelection: event.payload.defaultModelSelection, scripts: event.payload.scripts, + agentInstanceId: event.payload.agentInstanceId, createdAt: event.payload.createdAt, updatedAt: event.payload.updatedAt, deletedAt: null, @@ -562,6 +564,11 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti ]); let latestUserMessageAt: string | null = null; + // Newest proactive delivery, flattened for shell readers that never load + // messages (sidebar rows, agent awareness). Ties break on message id so + // two deliveries stamped in the same millisecond resolve deterministically. + let latestNotification: ProjectionThread["latestNotification"] = null; + let latestNotificationKey: string | null = null; for (const message of messages) { if ( message.role === "user" && @@ -569,6 +576,17 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti ) { latestUserMessageAt = message.createdAt; } + if (message.notification !== undefined) { + const key = `${message.createdAt}${message.messageId}`; + if (latestNotificationKey === null || key > latestNotificationKey) { + latestNotificationKey = key; + latestNotification = { + kind: message.notification.kind, + label: message.notification.label, + deliveredAt: message.createdAt, + }; + } + } } const pendingApprovalCount = pendingApprovals.filter( @@ -586,6 +604,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti pendingApprovalCount, pendingUserInputCount, hasActionableProposedPlan: hasActionableProposedPlan ? 1 : 0, + latestNotification, }); }); @@ -615,6 +634,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti pendingApprovalCount: 0, pendingUserInputCount: 0, hasActionableProposedPlan: 0, + latestNotification: null, deletedAt: null, }); return; @@ -782,6 +802,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti } case "thread.message-sent": + case "thread.notification-delivered": case "thread.proposed-plan-upserted": case "thread.activity-appended": case "thread.approval-response-requested": @@ -807,9 +828,19 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti if (Option.isNone(existingRow)) { return; } + // A session clearing `activeTurnId` means the turn ENDED, not that + // the thread never had one — so retain the last known turn instead + // of nulling it. Nulling was survivable only for providers that + // emit `thread.turn-diff-completed`, which re-writes this field + // moments later. A workspaceless provider (Hermes) emits no diff, + // so the id stayed null forever and `latestTurn` never resolved — + // leaving every completed turn looking like a queued-but-unadopted + // turn start, which blocks settling for the whole grace window. + const nextLatestTurnId = + event.payload.session.activeTurnId ?? existingRow.value.latestTurnId; yield* projectionThreadRepository.upsert({ ...existingRow.value, - latestTurnId: event.payload.session.activeTurnId, + latestTurnId: nextLatestTurnId, updatedAt: event.occurredAt, }); yield* refreshThreadShellSummary(event.payload.threadId); @@ -886,15 +917,14 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti const previousMessage = Option.getOrUndefined(existingMessage); const nextText = Option.match(existingMessage, { onNone: () => event.payload.text, - onSome: (message) => { - if (event.payload.streaming) { - return `${message.text}${event.payload.text}`; - } - if (event.payload.text.length === 0) { - return message.text; - } - return event.payload.text; - }, + onSome: (message) => + mergeAssistantMessageText(message.text, { + text: event.payload.text, + streaming: event.payload.streaming, + ...(event.payload.textOperation !== undefined + ? { textOperation: event.payload.textOperation } + : {}), + }), }); const nextAttachments = event.payload.attachments !== undefined @@ -916,6 +946,29 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti return; } + case "thread.notification-delivered": { + // No merge accounting: a delivery is written whole, never streamed, + // and a replay re-emits the identical payload (the decider rebuilds + // it from the persisted row), so the upsert is idempotent. + yield* projectionThreadMessageRepository.upsert({ + messageId: event.payload.messageId, + threadId: event.payload.threadId, + // Null for proactive deliveries; turn-scoped media names its turn + // so it sequences beside that turn's text. + turnId: event.payload.turnId ?? null, + role: "assistant", + text: event.payload.text, + notification: event.payload.notification, + ...(event.payload.attachments !== undefined + ? { attachments: [...event.payload.attachments] } + : {}), + isStreaming: false, + createdAt: event.payload.createdAt, + updatedAt: event.payload.updatedAt, + }); + return; + } + case "thread.reverted": { const existingRows = yield* projectionThreadMessageRepository.listByThreadId({ threadId: event.payload.threadId, diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index d4a24a209ad..d0ea11d858d 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -275,6 +275,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { runOnWorktreeCreate: false, }, ], + agentInstanceId: null, createdAt: "2026-02-24T00:00:00.000Z", updatedAt: "2026-02-24T00:00:01.000Z", deletedAt: null, @@ -390,6 +391,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { runOnWorktreeCreate: false, }, ], + agentInstanceId: null, createdAt: "2026-02-24T00:00:00.000Z", updatedAt: "2026-02-24T00:00:01.000Z", }, @@ -439,6 +441,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { hasPendingApprovals: true, hasPendingUserInput: false, hasActionableProposedPlan: false, + latestNotification: null, }, ]); @@ -1549,6 +1552,128 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { assert.equal(shellSnapshot.threads.length, 0); }), ); + + it.effect("finds a notification delivery in an archived thread", () => + Effect.gen(function* () { + const snapshotQuery = yield* ProjectionSnapshotQuery; + const sql = yield* SqlClient.SqlClient; + + yield* sql`DELETE FROM projection_threads`; + yield* sql`DELETE FROM projection_thread_messages`; + + yield* sql` + INSERT INTO projection_threads ( + thread_id, + project_id, + title, + model_selection_json, + runtime_mode, + interaction_mode, + branch, + worktree_path, + latest_turn_id, + latest_user_message_at, + pending_approval_count, + pending_user_input_count, + has_actionable_proposed_plan, + created_at, + updated_at, + archived_at, + deleted_at + ) + VALUES ( + 'thread-archived-home', + 'project-1', + 'Home', + '{"provider":"codex","model":"gpt-5-codex"}', + 'full-access', + 'default', + NULL, + NULL, + NULL, + NULL, + 0, + 0, + 0, + '2026-04-07T00:00:00.000Z', + '2026-04-07T00:00:01.000Z', + '2026-04-07T00:00:02.000Z', + NULL + ) + `; + + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + attachments_json, + notification_json, + is_streaming, + created_at, + updated_at + ) + VALUES + ( + 'message-delivered', + 'thread-archived-home', + NULL, + 'assistant', + 'Nightly brief', + NULL, + '{"kind":"cron","label":"Cron: nightly","deliveryId":"delivery-abc"}', + 0, + '2026-04-07T00:00:03.000Z', + '2026-04-07T00:00:03.000Z' + ), + ( + 'message-ordinary', + 'thread-archived-home', + NULL, + 'user', + 'delivery-abc appears in this body, but not as provenance', + NULL, + NULL, + 0, + '2026-04-07T00:00:04.000Z', + '2026-04-07T00:00:04.000Z' + ) + `; + + // Archiving parks a thread; it does not unwrite what is in it. The + // thread-detail read filters archived rows, so a dedupe check built on + // it reports every delivery as new and the plugin's retries append a + // duplicate per reconnect. + const detail = yield* snapshotQuery.getThreadDetailById( + ThreadId.make("thread-archived-home"), + ); + assert.equal(detail._tag, "None"); + + assert.isTrue( + yield* snapshotQuery.hasThreadNotificationDelivery({ + threadId: ThreadId.make("thread-archived-home"), + deliveryId: "delivery-abc", + }), + ); + // Matched on the provenance field, not a substring of the row: a + // deliveryId a user happened to type must not suppress a real delivery. + assert.isFalse( + yield* snapshotQuery.hasThreadNotificationDelivery({ + threadId: ThreadId.make("thread-archived-home"), + deliveryId: "delivery-never-written", + }), + ); + // Scoped to the thread — one instance's delivery id cannot mask another's. + assert.isFalse( + yield* snapshotQuery.hasThreadNotificationDelivery({ + threadId: ThreadId.make("thread-elsewhere"), + deliveryId: "delivery-abc", + }), + ); + }), + ); }); it.effect( diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 3d05bef4bdf..68cccbce69c 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -5,6 +5,7 @@ import { MessageId, NonNegativeInt, OrchestrationCheckpointFile, + OrchestrationMessageNotification, OrchestrationProposedPlanId, OrchestrationReadModel, OrchestrationShellSnapshot, @@ -22,7 +23,9 @@ import { type OrchestrationThreadActivity, type OrchestrationThreadShell, ModelSelection, + OrchestrationThreadNotificationSummary, ProjectId, + ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; import * as Arr from "effect/Array"; @@ -72,12 +75,19 @@ const ProjectionThreadMessageDbRowSchema = ProjectionThreadMessage.mapFields( Struct.assign({ isStreaming: Schema.Number, attachments: Schema.NullOr(Schema.fromJsonString(Schema.Array(ChatAttachment))), + notification: Schema.NullOr(Schema.fromJsonString(OrchestrationMessageNotification)), }), ); const ProjectionThreadProposedPlanDbRowSchema = ProjectionThreadProposedPlan; const ProjectionThreadDbRowSchema = ProjectionThread.mapFields( Struct.assign({ modelSelection: Schema.fromJsonString(ModelSelection), + // Stored as a JSON string, like `modelSelection`. Every column the row + // schema does not decode here arrives as raw text and fails the field's + // object schema — which takes the whole thread read down with it. + latestNotification: Schema.NullOr( + Schema.fromJsonString(OrchestrationThreadNotificationSummary), + ), }), ); const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields( @@ -114,6 +124,9 @@ const WorkspaceRootLookupInput = Schema.Struct({ const ProjectIdLookupInput = Schema.Struct({ projectId: ProjectId, }); +const AgentInstanceLookupInput = Schema.Struct({ + agentInstanceId: ProviderInstanceId, +}); const ThreadIdLookupInput = Schema.Struct({ threadId: ThreadId, }); @@ -235,6 +248,7 @@ function mapProjectShellRow( repositoryIdentity, defaultModelSelection: row.defaultModelSelection, scripts: row.scripts, + agentInstanceId: row.agentInstanceId, createdAt: row.createdAt, updatedAt: row.updatedAt, }; @@ -308,6 +322,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { workspace_root AS "workspaceRoot", default_model_selection_json AS "defaultModelSelection", scripts_json AS "scripts", + agent_instance_id AS "agentInstanceId", created_at AS "createdAt", updated_at AS "updatedAt", deleted_at AS "deletedAt" @@ -342,6 +357,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pending_approval_count AS "pendingApprovalCount", pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", + latest_notification_json AS "latestNotification", deleted_at AS "deletedAt" FROM projection_threads ORDER BY created_at ASC, thread_id ASC @@ -374,6 +390,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pending_approval_count AS "pendingApprovalCount", pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", + latest_notification_json AS "latestNotification", deleted_at AS "deletedAt" FROM projection_threads WHERE deleted_at IS NULL @@ -408,6 +425,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pending_approval_count AS "pendingApprovalCount", pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", + latest_notification_json AS "latestNotification", deleted_at AS "deletedAt" FROM projection_threads WHERE deleted_at IS NULL @@ -428,6 +446,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { role, text, attachments_json AS "attachments", + notification_json AS "notification", is_streaming AS "isStreaming", created_at AS "createdAt", updated_at AS "updatedAt" @@ -681,6 +700,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { workspace_root AS "workspaceRoot", default_model_selection_json AS "defaultModelSelection", scripts_json AS "scripts", + agent_instance_id AS "agentInstanceId", created_at AS "createdAt", updated_at AS "updatedAt", deleted_at AS "deletedAt" @@ -703,6 +723,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { workspace_root AS "workspaceRoot", default_model_selection_json AS "defaultModelSelection", scripts_json AS "scripts", + agent_instance_id AS "agentInstanceId", created_at AS "createdAt", updated_at AS "updatedAt", deleted_at AS "deletedAt" @@ -713,6 +734,36 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + /** + * Active agent project for one provider instance. + * + * Ordered oldest-first so a duplicate — which the enrollment lock is meant + * to prevent but a crash between dispatch and commit could still leave — + * resolves to the same row every time rather than flapping between two. + */ + const getActiveAgentProjectRow = SqlSchema.findOneOption({ + Request: AgentInstanceLookupInput, + Result: ProjectionProjectLookupRowSchema, + execute: ({ agentInstanceId }) => + sql` + SELECT + project_id AS "projectId", + title, + workspace_root AS "workspaceRoot", + default_model_selection_json AS "defaultModelSelection", + scripts_json AS "scripts", + agent_instance_id AS "agentInstanceId", + created_at AS "createdAt", + updated_at AS "updatedAt", + deleted_at AS "deletedAt" + FROM projection_projects + WHERE agent_instance_id = ${agentInstanceId} + AND deleted_at IS NULL + ORDER BY created_at ASC, project_id ASC + LIMIT 1 + `, + }); + const getFirstActiveThreadIdByProject = SqlSchema.findOneOption({ Request: ProjectIdLookupInput, Result: ProjectionThreadIdLookupRowSchema, @@ -748,6 +799,44 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + // Existence, not visibility: deliberately no `archived_at IS NULL`. Selects + // only the two columns it needs so it cannot be broken by an undecoded + // JSON column elsewhere in the row. + const getThreadArchiveStateRowById = SqlSchema.findOneOption({ + Request: ThreadIdLookupInput, + Result: Schema.Struct({ + archivedAt: Schema.NullOr(Schema.String), + deletedAt: Schema.NullOr(Schema.String), + }), + execute: ({ threadId }) => + sql` + SELECT + archived_at AS "archivedAt", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE thread_id = ${threadId} + LIMIT 1 + `, + }); + + // Dedupe, not display: deliberately no join to `projection_threads` and so + // no archive/delete filter. A delivery written into a thread the user later + // archived is still written, and treating it as absent would re-append it on + // every retry. `json_extract` rather than a LIKE over the blob, so a + // deliveryId occurring inside some other field cannot false-positive. + const findThreadNotificationDeliveryRow = SqlSchema.findOneOption({ + Request: Schema.Struct({ threadId: ThreadId, deliveryId: Schema.String }), + Result: Schema.Struct({ messageId: Schema.String }), + execute: ({ threadId, deliveryId }) => + sql` + SELECT message_id AS "messageId" + FROM projection_thread_messages + WHERE thread_id = ${threadId} + AND json_extract(notification_json, '$.deliveryId') = ${deliveryId} + LIMIT 1 + `, + }); + const getActiveThreadRowById = SqlSchema.findOneOption({ Request: ThreadIdLookupInput, Result: ProjectionThreadDbRowSchema, @@ -774,6 +863,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pending_approval_count AS "pendingApprovalCount", pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", + latest_notification_json AS "latestNotification", deleted_at AS "deletedAt" FROM projection_threads WHERE thread_id = ${threadId} @@ -795,6 +885,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { role, text, attachments_json AS "attachments", + notification_json AS "notification", is_streaming AS "isStreaming", created_at AS "createdAt", updated_at AS "updatedAt" @@ -1066,6 +1157,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { role: row.role, text: row.text, ...(row.attachments !== null ? { attachments: row.attachments } : {}), + ...(row.notification !== null ? { notification: row.notification } : {}), turnId: row.turnId, streaming: row.isStreaming === 1, createdAt: row.createdAt, @@ -1184,6 +1276,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { repositoryIdentity: repositoryIdentities.get(row.projectId) ?? null, defaultModelSelection: row.defaultModelSelection, scripts: row.scripts, + agentInstanceId: row.agentInstanceId, createdAt: row.createdAt, updatedAt: row.updatedAt, deletedAt: row.deletedAt, @@ -1310,6 +1403,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { workspaceRoot: row.workspaceRoot, defaultModelSelection: row.defaultModelSelection, scripts: row.scripts, + agentInstanceId: row.agentInstanceId, createdAt: row.createdAt, updatedAt: row.updatedAt, deletedAt: row.deletedAt, @@ -1546,6 +1640,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { hasPendingApprovals: row.pendingApprovalCount > 0, hasPendingUserInput: row.pendingUserInputCount > 0, hasActionableProposedPlan: row.hasActionableProposedPlan > 0, + latestNotification: row.latestNotification, } satisfies OrchestrationThreadShell) : Result.failVoid, ), @@ -1684,6 +1779,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { hasPendingApprovals: row.pendingApprovalCount > 0, hasPendingUserInput: row.pendingUserInputCount > 0, hasActionableProposedPlan: row.hasActionableProposedPlan > 0, + latestNotification: row.latestNotification, }), ), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", @@ -1758,6 +1854,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { repositoryIdentity, defaultModelSelection: option.value.defaultModelSelection, scripts: option.value.scripts, + agentInstanceId: option.value.agentInstanceId, createdAt: option.value.createdAt, updatedAt: option.value.updatedAt, deletedAt: option.value.deletedAt, @@ -1767,6 +1864,38 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ), ); + const getActiveAgentProject: ProjectionSnapshotQueryShape["getActiveAgentProject"] = ( + agentInstanceId, + ) => + getActiveAgentProjectRow({ agentInstanceId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getActiveAgentProject:query", + "ProjectionSnapshotQuery.getActiveAgentProject:decodeRow", + ), + ), + Effect.map((option) => + Option.map( + option, + (row): OrchestrationProject => ({ + id: row.projectId, + title: row.title, + workspaceRoot: row.workspaceRoot, + // Agent projects are not checkouts, so there is no remote to + // resolve — skipping the resolver keeps this off the git path + // entirely rather than paying for a lookup that returns null. + repositoryIdentity: null, + defaultModelSelection: row.defaultModelSelection, + scripts: row.scripts, + agentInstanceId: row.agentInstanceId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + deletedAt: row.deletedAt, + }), + ), + ), + ); + const getProjectShellById: ProjectionSnapshotQueryShape["getProjectShellById"] = (projectId) => getActiveProjectRowById({ projectId }).pipe( Effect.mapError( @@ -1873,6 +2002,26 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); }); + const getThreadArchiveStateById: ProjectionSnapshotQueryShape["getThreadArchiveStateById"] = ( + threadId, + ) => + getThreadArchiveStateRowById({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadArchiveStateById:query", + "ProjectionSnapshotQuery.getThreadArchiveStateById:decodeRow", + ), + ), + Effect.map( + Option.flatMap((row) => + // A soft-deleted row is gone; an archived one is merely parked. + row.deletedAt !== null + ? Option.none<{ readonly archivedAt: string | null }>() + : Option.some({ archivedAt: row.archivedAt }), + ), + ), + ); + const getThreadShellById: ProjectionSnapshotQueryShape["getThreadShellById"] = (threadId) => Effect.gen(function* () { const [threadRow, latestTurnRow, sessionRow] = yield* Effect.all([ @@ -1928,9 +2077,22 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { hasPendingApprovals: threadRow.value.pendingApprovalCount > 0, hasPendingUserInput: threadRow.value.pendingUserInputCount > 0, hasActionableProposedPlan: threadRow.value.hasActionableProposedPlan > 0, + latestNotification: threadRow.value.latestNotification, } satisfies OrchestrationThreadShell); }); + const hasThreadNotificationDelivery: ProjectionSnapshotQueryShape["hasThreadNotificationDelivery"] = + (input) => + findThreadNotificationDeliveryRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.hasThreadNotificationDelivery:query", + "ProjectionSnapshotQuery.hasThreadNotificationDelivery:decodeRow", + ), + ), + Effect.map(Option.isSome), + ); + const getThreadDetailById: ProjectionSnapshotQueryShape["getThreadDetailById"] = (threadId) => Effect.gen(function* () { const [ @@ -2023,7 +2185,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { snoozedAt: threadRow.value.snoozedAt, deletedAt: null, messages: messageRows.map((row) => { - const message = { + const message: OrchestrationMessage = { id: row.messageId, role: row.role, text: row.text, @@ -2031,10 +2193,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { streaming: row.isStreaming === 1, createdAt: row.createdAt, updatedAt: row.updatedAt, + ...(row.attachments !== null ? { attachments: row.attachments } : {}), + ...(row.notification !== null ? { notification: row.notification } : {}), }; - if (row.attachments !== null) { - return Object.assign(message, { attachments: row.attachments }); - } return message; }), proposedPlans: proposedPlanRows.map(mapProposedPlanRow), @@ -2111,11 +2272,14 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { getSnapshotSequence, getCounts, getActiveProjectByWorkspaceRoot, + getActiveAgentProject, getProjectShellById, getFirstActiveThreadIdByProjectId, getThreadCheckpointContext, getFullThreadDiffContext, getThreadShellById, + getThreadArchiveStateById, + hasThreadNotificationDelivery, getThreadDetailById, getThreadDetailSnapshot, } satisfies ProjectionSnapshotQueryShape; diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index c49646b7a4b..98e17675798 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -477,6 +477,49 @@ describe("ProviderCommandReactor", () => { expect(thread?.session?.runtimeMode).toBe("approval-required"); }); + effectIt.effect("adopts a recovered active provider turn while binding a pending start", () => + Effect.gen(function* () { + const recoveredTurnId = asTurnId("turn-recovered-during-start"); + const harness = yield* Effect.promise(() => + createHarness({ + startSessionEffect: (session) => + Effect.succeed({ + ...session, + status: "running", + activeTurnId: recoveredTurnId, + }), + }), + ); + const now = "2026-01-01T00:00:00.000Z"; + + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-turn-start-recovered-provider-turn"), + threadId: ThreadId.make("thread-1"), + message: { + messageId: asMessageId("user-message-recovered-provider-turn"), + role: "user", + text: "steer the recovered turn", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: now, + }); + + yield* Effect.promise(() => waitFor(() => harness.sendTurn.mock.calls.length === 1)); + const readModel = yield* Effect.promise(() => harness.readModel()); + const thread = readModel.threads.find((entry) => entry.id === ThreadId.make("thread-1")); + + expect(thread?.session?.status).toBe("running"); + expect(thread?.session?.activeTurnId).toBe(recoveredTurnId); + expect(thread?.latestTurn).toMatchObject({ + turnId: recoveredTurnId, + state: "running", + }); + }), + ); + effectIt.effect("projects starting before a slow provider session finishes", () => Effect.gen(function* () { const releaseStart = yield* Deferred.make(); @@ -1181,6 +1224,62 @@ describe("ProviderCommandReactor", () => { expect(harness.stopSession.mock.calls.length).toBe(0); }); + it("reuses a cwd-bookkept Hermes session for active follow-up guidance", async () => { + const harness = await createHarness({ + threadModelSelection: { + instanceId: ProviderInstanceId.make("hermes"), + model: "hermes", + }, + }); + const now = "2026-01-01T00:00:00.000Z"; + + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-hermes-guidance-1"), + threadId: ThreadId.make("thread-1"), + message: { + messageId: asMessageId("user-message-hermes-guidance-1"), + role: "user", + text: "first", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: now, + }), + ); + + await waitFor(() => harness.startSession.mock.calls.length === 1); + await waitFor(() => harness.sendTurn.mock.calls.length === 1); + expect(harness.startSession.mock.calls[0]?.[1]).toMatchObject({ + cwd: "/tmp/provider-project", + provider: ProviderDriverKind.make("hermes"), + providerInstanceId: ProviderInstanceId.make("hermes"), + }); + + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-hermes-guidance-2"), + threadId: ThreadId.make("thread-1"), + message: { + messageId: asMessageId("user-message-hermes-guidance-2"), + role: "user", + text: "follow up", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: now, + }), + ); + + await waitFor(() => harness.sendTurn.mock.calls.length === 2); + expect(harness.startSession).toHaveBeenCalledTimes(1); + expect(harness.stopSession).not.toHaveBeenCalled(); + }); + it("restarts an existing Codex thread on a compatible requested instance", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; @@ -2174,45 +2273,45 @@ describe("ProviderCommandReactor", () => { expect(resolvedActivity).toBeUndefined(); }); - it("reacts to thread.session.stop by stopping provider session and clearing thread session state", async () => { - const harness = await createHarness(); - const now = "2026-01-01T00:00:00.000Z"; + effectIt.effect( + "reacts to thread.session.stop by stopping provider session and clearing thread session state", + () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => createHarness()); + const now = "2026-01-01T00:00:00.000Z"; - await Effect.runPromise( - harness.engine.dispatch({ - type: "thread.session.set", - commandId: CommandId.make("cmd-session-set-for-stop"), - threadId: ThreadId.make("thread-1"), - session: { + yield* harness.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make("cmd-session-set-for-stop"), threadId: ThreadId.make("thread-1"), - status: "ready", - providerName: "codex", - providerInstanceId: ProviderInstanceId.make("codex_work"), - runtimeMode: "approval-required", - activeTurnId: null, - lastError: null, - updatedAt: now, - }, - createdAt: now, - }), - ); + session: { + threadId: ThreadId.make("thread-1"), + status: "ready", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex_work"), + runtimeMode: "approval-required", + activeTurnId: null, + lastError: null, + updatedAt: now, + }, + createdAt: now, + }); - await Effect.runPromise( - harness.engine.dispatch({ - type: "thread.session.stop", - commandId: CommandId.make("cmd-session-stop"), - threadId: ThreadId.make("thread-1"), - createdAt: now, - }), - ); + yield* harness.engine.dispatch({ + type: "thread.session.stop", + commandId: CommandId.make("cmd-session-stop"), + threadId: ThreadId.make("thread-1"), + createdAt: now, + }); - await waitFor(() => harness.stopSession.mock.calls.length === 1); - const readModel = await harness.readModel(); - const thread = readModel.threads.find((entry) => entry.id === ThreadId.make("thread-1")); - expect(thread?.session).not.toBeNull(); - expect(thread?.session?.status).toBe("stopped"); - expect(thread?.session?.threadId).toBe("thread-1"); - expect(thread?.session?.providerInstanceId).toBe(ProviderInstanceId.make("codex_work")); - expect(thread?.session?.activeTurnId).toBeNull(); - }); + yield* Effect.promise(() => waitFor(() => harness.stopSession.mock.calls.length === 1)); + const readModel = yield* Effect.promise(() => harness.readModel()); + const thread = readModel.threads.find((entry) => entry.id === ThreadId.make("thread-1")); + expect(thread?.session).not.toBeNull(); + expect(thread?.session?.status).toBe("stopped"); + expect(thread?.session?.threadId).toBe("thread-1"); + expect(thread?.session?.providerInstanceId).toBe(ProviderInstanceId.make("codex_work")); + expect(thread?.session?.activeTurnId).toBeNull(); + }), + ); }); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 3044cc6029d..e26f1930444 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -530,8 +530,7 @@ const make = Effect.gen(function* () { providerName: session.provider, providerInstanceId: session.providerInstanceId, runtimeMode: desiredRuntimeMode, - // Provider turn ids are not orchestration turn ids. - activeTurnId: null, + activeTurnId: session.activeTurnId ?? null, lastError: session.lastError ?? null, updatedAt: session.updatedAt, }, diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 74ece50cd31..9ed1d8611ef 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -27,6 +27,7 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -35,7 +36,9 @@ import { afterEach, describe, expect, it } from "vite-plus/test"; import { OrchestrationEventStoreLive } from "../../persistence/Layers/OrchestrationEventStore.ts"; import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts"; +import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts"; +import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; import { ProviderService, type ProviderServiceShape, @@ -192,7 +195,10 @@ async function waitForThread( describe("ProviderRuntimeIngestion", () => { let runtime: ManagedRuntime.ManagedRuntime< - OrchestrationEngineService | ProviderRuntimeIngestionService | ProjectionSnapshotQuery, + | OrchestrationEngineService + | ProviderRuntimeIngestionService + | ProjectionSnapshotQuery + | ProjectionTurnRepository, unknown > | null = null; let scope: Scope.Closeable | null = null; @@ -234,9 +240,13 @@ describe("ProviderRuntimeIngestion", () => { Layer.provide(RepositoryIdentityResolver.layer), Layer.provide(SqlitePersistenceMemory), ); + const projectionTurnLayer = ProjectionTurnRepositoryLive.pipe( + Layer.provide(SqlitePersistenceMemory), + ); const layer = ProviderRuntimeIngestionLive.pipe( Layer.provideMerge(orchestrationLayer), Layer.provideMerge(projectionSnapshotLayer), + Layer.provideMerge(projectionTurnLayer), Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(Layer.succeed(ProviderService, provider.service)), Layer.provideMerge(makeTestServerSettingsLayer(options?.serverSettings)), @@ -247,6 +257,7 @@ describe("ProviderRuntimeIngestion", () => { const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService)); const snapshotQuery = await runtime.runPromise(Effect.service(ProjectionSnapshotQuery)); const ingestion = await runtime.runPromise(Effect.service(ProviderRuntimeIngestionService)); + const turnRepository = await runtime.runPromise(Effect.service(ProjectionTurnRepository)); scope = await Effect.runPromise(Scope.make("sequential")); await Effect.runPromise(ingestion.start().pipe(Scope.provide(scope))); const drain = () => Effect.runPromise(ingestion.drain); @@ -315,6 +326,7 @@ describe("ProviderRuntimeIngestion", () => { readModel: () => Effect.runPromise(snapshotQuery.getSnapshot()), emit: provider.emit, setProviderSession: provider.setSession, + turnRepository, drain, }; } @@ -946,6 +958,209 @@ describe("ProviderRuntimeIngestion", () => { expect(message?.streaming).toBe(false); }); + it("replaces streaming assistant text from authoritative snapshots without changing identity", async () => { + const harness = await createHarness({ serverSettings: { enableAssistantStreaming: true } }); + const now = "2026-01-01T00:00:00.000Z"; + const itemId = asItemId("item-streaming-snapshot"); + const turnId = asTurnId("turn-streaming-snapshot"); + + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-streaming-snapshot-delta"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: "draft", + }, + }); + harness.emit({ + type: "content.snapshot", + eventId: asEventId("evt-streaming-snapshot-replace"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + text: "authoritative", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-streaming-snapshot-tail"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: " answer", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-streaming-snapshot-completed"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "assistant_message", + status: "completed", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.messages.some( + (message: ProviderRuntimeTestMessage) => + message.id === `assistant:${itemId}` && + message.text === "authoritative answer" && + !message.streaming, + ), + ); + expect(thread.messages.filter((message) => message.id === `assistant:${itemId}`)).toHaveLength( + 1, + ); + }); + + it("applies repeated empty snapshots idempotently to a streaming assistant message", async () => { + const harness = await createHarness({ serverSettings: { enableAssistantStreaming: true } }); + const now = "2026-01-01T00:00:00.000Z"; + const itemId = asItemId("item-empty-snapshot"); + const turnId = asTurnId("turn-empty-snapshot"); + + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-empty-snapshot-delta"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: "remove me", + }, + }); + for (const eventId of ["evt-empty-snapshot-one", "evt-empty-snapshot-two"]) { + harness.emit({ + type: "content.snapshot", + eventId: asEventId(eventId), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + text: "", + }, + }); + } + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-empty-snapshot-completed"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "assistant_message", + status: "completed", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.messages.some( + (message: ProviderRuntimeTestMessage) => + message.id === `assistant:${itemId}` && message.text === "" && !message.streaming, + ), + ); + expect(thread.messages.filter((message) => message.id === `assistant:${itemId}`)).toHaveLength( + 1, + ); + }); + + it("authoritatively replaces buffered assistant text before completion", async () => { + const harness = await createHarness(); + const now = "2026-01-01T00:00:00.000Z"; + const itemId = asItemId("item-buffered-snapshot"); + const turnId = asTurnId("turn-buffered-snapshot"); + + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-buffered-snapshot-delta"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: "draft", + }, + }); + harness.emit({ + type: "content.snapshot", + eventId: asEventId("evt-buffered-snapshot-replace"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + text: "final", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-buffered-snapshot-tail"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: " answer", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-buffered-snapshot-completed"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "assistant_message", + status: "completed", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.messages.some( + (message: ProviderRuntimeTestMessage) => + message.id === `assistant:${itemId}` && + message.text === "final answer" && + !message.streaming, + ), + ); + expect(thread.messages.filter((message) => message.id === `assistant:${itemId}`)).toHaveLength( + 1, + ); + }); + it("uses assistant item completion detail when no assistant deltas were streamed", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; @@ -1573,6 +1788,85 @@ describe("ProviderRuntimeIngestion", () => { expect(threadAfterSteer.latestTurn?.state).toBe("running"); }); + effectIt.effect( + "consumes a follow-up pending start when Hermes acknowledges the active turn", + () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => createHarness()); + const threadId = asThreadId("thread-1"); + const activeTurnId = asTurnId("turn-hermes-active"); + const followUpMessageId = asMessageId("msg-hermes-follow-up"); + const createdAt = "2026-01-01T00:00:00.000Z"; + + harness.setProviderSession({ + provider: ProviderDriverKind.make("hermes"), + status: "running", + runtimeMode: "approval-required", + threadId, + createdAt, + updatedAt: createdAt, + activeTurnId, + }); + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-hermes-active-turn-started"), + provider: ProviderDriverKind.make("hermes"), + createdAt, + threadId, + turnId: activeTurnId, + }); + yield* Effect.promise(() => + waitForThread( + harness.readModel, + (thread) => + thread.session?.status === "running" && thread.session.activeTurnId === activeTurnId, + 2_000, + threadId, + ), + ); + + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-hermes-follow-up"), + threadId, + message: { + messageId: followUpMessageId, + role: "user", + text: "steer the active Hermes turn", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: "2026-01-01T00:00:01.000Z", + }); + + const pendingBeforeAck = yield* harness.turnRepository.getPendingTurnStartByThreadId({ + threadId, + }); + expect(Option.getOrUndefined(pendingBeforeAck)?.messageId).toBe(followUpMessageId); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-hermes-follow-up-ack"), + provider: ProviderDriverKind.make("hermes"), + createdAt: "2026-01-01T00:00:02.000Z", + threadId, + turnId: activeTurnId, + }); + yield* Effect.promise(() => harness.drain()); + + const pendingAfterAck = yield* harness.turnRepository.getPendingTurnStartByThreadId({ + threadId, + }); + const activeTurn = yield* harness.turnRepository.getByTurnId({ + threadId, + turnId: activeTurnId, + }); + expect(Option.isNone(pendingAfterAck)).toBe(true); + expect(Option.getOrUndefined(activeTurn)?.pendingMessageId).toBe(followUpMessageId); + }), + ); + it("does not mark the source proposed plan implemented for an unrelated turn.started when no thread active turn is tracked", async () => { const harness = await createHarness(); const sourceThreadId = asThreadId("thread-plan"); @@ -1873,6 +2167,97 @@ describe("ProviderRuntimeIngestion", () => { expect(message?.streaming).toBe(false); }); + it("treats turn.aborted as terminal and finalizes buffered assistant text", async () => { + const harness = await createHarness(); + const now = "2026-01-01T00:00:00.000Z"; + const turnId = asTurnId("turn-aborted-buffered"); + const itemId = asItemId("item-aborted-buffered"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-turn-started-before-abort"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + }); + await waitForThread( + harness.readModel, + (thread) => thread.session?.status === "running" && thread.session.activeTurnId === turnId, + ); + + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-content-before-abort"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: "partial response", + }, + }); + harness.emit({ + type: "turn.aborted", + eventId: asEventId("evt-turn-aborted"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + payload: { + reason: "Interrupted by user.", + }, + }); + + const thread = await waitForThread( + harness.readModel, + (entry) => + entry.session?.status === "ready" && + entry.session.activeTurnId === null && + entry.messages.some( + (message: ProviderRuntimeTestMessage) => + message.id === `assistant:${itemId}` && + message.text === "partial response" && + !message.streaming, + ), + ); + expect(thread.session?.lastError).toBeNull(); + + const nextTurnId = asTurnId("turn-after-abort"); + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-turn-started-after-abort"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId: nextTurnId, + }); + await waitForThread( + harness.readModel, + (entry) => entry.session?.status === "running" && entry.session.activeTurnId === nextTurnId, + ); + harness.emit({ + type: "turn.aborted", + eventId: asEventId("evt-stale-turn-aborted"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + payload: { + reason: "Late abort for the previous turn.", + }, + }); + await harness.drain(); + const afterStaleAbort = await harness.readModel(); + const activeThread = afterStaleAbort.threads.find( + (entry) => entry.id === ThreadId.make("thread-1"), + ); + expect(activeThread?.session?.status).toBe("running"); + expect(activeThread?.session?.activeTurnId).toBe(nextTurnId); + }); + it("flushes and completes buffered assistant text when an approval request opens", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; @@ -2452,6 +2837,65 @@ describe("ProviderRuntimeIngestion", () => { expect(message?.streaming).toBe(false); }); + it("replaces a spilled buffered projection when an authoritative snapshot arrives", async () => { + const harness = await createHarness(); + const now = "2026-01-01T00:00:00.000Z"; + const itemId = asItemId("item-buffer-spill-snapshot"); + const turnId = asTurnId("turn-buffer-spill-snapshot"); + + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-buffer-spill-before-snapshot"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + delta: "x".repeat(40_000), + }, + }); + harness.emit({ + type: "content.snapshot", + eventId: asEventId("evt-buffer-spill-snapshot"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + streamKind: "assistant_text", + text: "authoritative after spill", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-buffer-spill-snapshot-completed"), + provider: ProviderDriverKind.make("hermes"), + createdAt: now, + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "assistant_message", + status: "completed", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.messages.some( + (message: ProviderRuntimeTestMessage) => + message.id === `assistant:${itemId}` && + message.text === "authoritative after spill" && + !message.streaming, + ), + ); + expect(thread.messages.filter((message) => message.id === `assistant:${itemId}`)).toHaveLength( + 1, + ); + }); + it("does not duplicate assistant completion when item.completed is followed by turn.completed", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; @@ -2758,6 +3202,278 @@ describe("ProviderRuntimeIngestion", () => { ).toBe(true); }); + it("upserts status_text provider item lifecycle into one meaningful activity", async () => { + const harness = await createHarness(); + const itemId = asItemId("item-generic-activity"); + const turnId = asTurnId("turn-generic-activity"); + + harness.emit({ + type: "item.started", + eventId: asEventId("evt-generic-activity-started"), + provider: ProviderDriverKind.make("hermes"), + createdAt: "2026-01-01T00:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "status_text", + status: "inProgress", + title: "Reading skill hermes-agent", + detail: "Opening the skill instructions.", + data: { unsafeRawPayload: "not projected" }, + }, + }); + harness.emit({ + type: "item.updated", + eventId: asEventId("evt-generic-activity-updated"), + provider: ProviderDriverKind.make("hermes"), + createdAt: "2026-01-01T00:00:01.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "status_text", + status: "inProgress", + title: "Reading skill hermes-agent", + detail: "Parsing the workflow.", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-generic-activity-completed"), + provider: ProviderDriverKind.make("hermes"), + createdAt: "2026-01-01T00:00:02.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "status_text", + status: "completed", + title: "Read skill hermes-agent", + detail: "Skill instructions loaded.", + }, + }); + + const activityId = `provider:hermes:item:${itemId}`; + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => + activity.id === activityId && + activity.payload && + typeof activity.payload === "object" && + (activity.payload as Record).status === "completed", + ), + ); + const activities = thread.activities.filter((activity) => activity.id === activityId); + const activity = activities[0]; + const payload = + activity?.payload && typeof activity.payload === "object" + ? (activity.payload as Record) + : undefined; + + expect(activities).toHaveLength(1); + expect(activity?.kind).toBe("provider.activity"); + expect(activity?.tone).toBe("info"); + expect(activity?.summary).toBe("Skill instructions loaded."); + expect(payload).toMatchObject({ + itemType: "status_text", + status: "completed", + providerItemId: itemId, + title: "Read skill hermes-agent", + detail: "Skill instructions loaded.", + }); + expect(payload?.data).toBeUndefined(); + }); + + // Regression: `unknown` is the canonical "could not classify this item" + // sentinel and adapters rely on it being INERT. CodexAdapter deliberately + // emits it for item.updated; routing generic activity through it put rows + // summarized as the literal string "Provider activity" into Codex threads. + // Renderable status text has its own item type (`status_text`) instead. + it("does not create activities for the unknown item sentinel", async () => { + const harness = await createHarness(); + const itemId = asItemId("item-codex-unknown"); + const turnId = asTurnId("turn-codex-unknown"); + + harness.emit({ + type: "item.updated", + eventId: asEventId("evt-codex-unknown-updated"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId, + payload: { + itemType: "unknown", + status: "inProgress", + }, + }); + // A tool item after it gives the poll something real to settle on, so the + // assertion is "the unknown item produced nothing" rather than a race. + harness.emit({ + type: "item.started", + eventId: asEventId("evt-codex-unknown-tool"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:01.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-codex-tool"), + payload: { + itemType: "command_execution", + status: "inProgress", + title: "ls", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.kind === "tool.started", + ), + ); + + expect( + thread.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.id === `provider:codex:item:${itemId}`, + ), + ).toBe(false); + expect( + thread.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.kind === "provider.activity", + ), + ).toBe(false); + }); + + it("attributes a late turn-less item frame to the turn that just completed", async () => { + // Plugin tool hooks run on worker threads, so a post_tool_call → + // item.completed frame can land after turn.completed and carry no turn id. + // Without a fallback the activity projects with turnId null, which the + // timeline can never fold behind the turn's "Worked for …" row. + const harness = await createHarness(); + const turnId = asTurnId("turn-late-frame"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-late-frame-turn-started"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:00.000Z", + turnId, + }); + await waitForThread( + harness.readModel, + (entry) => entry.session?.activeTurnId === "turn-late-frame", + ); + + harness.emit({ + type: "turn.completed", + eventId: asEventId("evt-late-frame-turn-completed"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:10.000Z", + turnId, + payload: { state: "completed" }, + }); + await waitForThread(harness.readModel, (entry) => entry.session?.activeTurnId === null); + + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-late-frame-item"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:10.002Z", + itemId: asItemId("item-late-frame"), + payload: { + itemType: "command_execution", + status: "completed", + title: "Ran command", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-late-frame-item", + ), + ); + const activity = thread.activities.find( + (entry: ProviderRuntimeTestActivity) => entry.id === "evt-late-frame-item", + ); + + expect(activity?.kind).toBe("tool.completed"); + expect(activity?.turnId).toBe("turn-late-frame"); + }); + + it("leaves an item frame unattributed when no turn has ever run on the thread", async () => { + const harness = await createHarness(); + + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-orphan-frame-item"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:00.000Z", + itemId: asItemId("item-orphan-frame"), + payload: { + itemType: "command_execution", + status: "completed", + title: "Ran command", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-orphan-frame-item", + ), + ); + const activity = thread.activities.find( + (entry: ProviderRuntimeTestActivity) => entry.id === "evt-orphan-frame-item", + ); + + expect(activity?.turnId).toBeNull(); + }); + + it("keeps an explicit turn id on a frame naming a turn other than the active one", async () => { + const harness = await createHarness(); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-conflict-turn-started"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:00.000Z", + turnId: asTurnId("turn-active"), + }); + await waitForThread( + harness.readModel, + (entry) => entry.session?.activeTurnId === "turn-active", + ); + + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-conflict-item"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:01.000Z", + turnId: asTurnId("turn-other"), + itemId: asItemId("item-conflict"), + payload: { + itemType: "command_execution", + status: "completed", + title: "Ran command", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-conflict-item", + ), + ); + const activity = thread.activities.find( + (entry: ProviderRuntimeTestActivity) => entry.id === "evt-conflict-item", + ); + + expect(activity?.turnId).toBe("turn-other"); + }); + it("consumes P1 runtime events into thread metadata, diff checkpoints, and activities", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index a8a51b30260..2f0cc2ea945 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -2,6 +2,7 @@ import { ApprovalRequestId, type AssistantDeliveryMode, CommandId, + EventId, MessageId, type OrchestrationEvent, type OrchestrationMessage, @@ -113,6 +114,47 @@ function toTurnId(value: TurnId | string | undefined): TurnId | undefined { return value === undefined ? undefined : TurnId.make(String(value)); } +/** + * How long after a turn settles a turn-less provider frame is still treated as + * belonging to that turn. Plugin tool hooks run on worker threads, so a + * `post_tool_call` → `item.completed` frame can land after `turn.completed`; + * without attribution such an activity can never be folded behind the turn's + * "Worked for …" row and renders below the final answer instead. + */ +const LATE_ACTIVITY_TURN_ATTRIBUTION_GRACE_MS = 30_000; + +export interface RecentlySettledTurn { + readonly turnId: TurnId; + readonly settledAt: string; +} + +/** + * Turn id to stamp on activities produced by a frame that carries none. The + * session's active turn wins; once the turn has closed, the turn that just + * settled still claims frames landing inside a short trailing window. + */ +export function resolveActivityFallbackTurnId(input: { + readonly activeTurnId: TurnId | null; + readonly recentlySettledTurn: RecentlySettledTurn | null; + readonly at: string; +}): TurnId | null { + if (input.activeTurnId !== null) { + return input.activeTurnId; + } + const settled = input.recentlySettledTurn; + if (!settled) { + return null; + } + const atMs = Date.parse(input.at); + const settledAtMs = Date.parse(settled.settledAt); + if (!Number.isFinite(atMs) || !Number.isFinite(settledAtMs)) { + return null; + } + // Frames stamped before the turn closed always belong to it; frames stamped + // after it only within the grace window. + return atMs - settledAtMs <= LATE_ACTIVITY_TURN_ATTRIBUTION_GRACE_MS ? settled.turnId : null; +} + function toApprovalRequestId(value: string | undefined): ApprovalRequestId | undefined { return value === undefined ? undefined : ApprovalRequestId.make(value); } @@ -306,6 +348,47 @@ function requestKindFromCanonicalRequestType( } } +function genericProviderActivityId(event: ProviderRuntimeEvent): EventId { + return EventId.make( + event.itemId + ? `provider:${event.provider}:item:${event.itemId}` + : `provider:${event.provider}:event:${event.eventId}`, + ); +} + +function genericProviderActivity( + event: Extract< + ProviderRuntimeEvent, + { type: "item.started" | "item.updated" | "item.completed" } + >, + maybeSequence: { readonly sequence?: number }, +): OrchestrationThreadActivity { + const title = event.payload.title ? truncateDetail(event.payload.title, 120) : undefined; + const detail = event.payload.detail ? truncateDetail(event.payload.detail) : undefined; + const status = + event.payload.status ?? (event.type === "item.completed" ? "completed" : "inProgress"); + + return { + id: genericProviderActivityId(event), + createdAt: event.createdAt, + tone: "info", + kind: "provider.activity", + // Status text is the interesting part here; the title is a generic label + // ("Hermes activity"). Prefer the detail so the row reads as the status the + // provider is actually reporting. + summary: detail ? truncateDetail(detail, 120) : (title ?? "Working…"), + payload: { + itemType: event.payload.itemType, + status, + ...(event.itemId ? { providerItemId: event.itemId } : {}), + ...(title ? { title } : {}), + ...(detail ? { detail } : {}), + }, + turnId: toTurnId(event.turnId) ?? null, + ...maybeSequence, + }; +} + export function runtimeEventToActivities( event: ProviderRuntimeEvent, taskTitle?: string, @@ -614,6 +697,9 @@ export function runtimeEventToActivities( } case "item.updated": { + if (event.payload.itemType === "status_text") { + return [genericProviderActivity(event, maybeSequence)]; + } if (!isToolLifecycleItemType(event.payload.itemType)) { return []; } @@ -637,6 +723,9 @@ export function runtimeEventToActivities( } case "item.completed": { + if (event.payload.itemType === "status_text") { + return [genericProviderActivity(event, maybeSequence)]; + } if (!isToolLifecycleItemType(event.payload.itemType)) { return []; } @@ -659,6 +748,9 @@ export function runtimeEventToActivities( } case "item.started": { + if (event.payload.itemType === "status_text") { + return [genericProviderActivity(event, maybeSequence)]; + } if (!isToolLifecycleItemType(event.payload.itemType)) { return []; } @@ -733,6 +825,16 @@ const make = Effect.gen(function* () { lookup: () => Effect.succeed(""), }); + // The projection nulls the thread's latest turn id the moment the session + // stops running, so a frame arriving right after turn.completed can no + // longer see the turn it belongs to. Remember the turn that just settled + // per thread so late frames can still be attributed to it. + const recentlySettledTurnByThreadId = yield* Cache.make({ + capacity: TURN_MESSAGE_IDS_BY_TURN_CACHE_CAPACITY, + timeToLive: TURN_MESSAGE_IDS_BY_TURN_TTL, + lookup: () => Effect.succeed(null), + }); + const rememberTaskDescription = (threadId: ThreadId, taskId: string, description: string) => Cache.set(taskDescriptionByTaskKey, providerTaskKey(threadId, taskId), description); @@ -899,6 +1001,9 @@ const make = Effect.gen(function* () { ), ); + const replaceBufferedAssistantText = (messageId: MessageId, text: string) => + Cache.set(bufferedAssistantTextByMessageId, messageId, text); + const takeBufferedAssistantText = (messageId: MessageId) => Cache.getOption(bufferedAssistantTextByMessageId, messageId).pipe( Effect.flatMap((existingText) => @@ -1212,6 +1317,7 @@ const make = Effect.gen(function* () { key.startsWith(prefix) ? Cache.invalidate(taskDescriptionByTaskKey, key) : Effect.void, { concurrency: 1 }, ).pipe(Effect.asVoid); + yield* Cache.invalidate(recentlySettledTurnByThreadId, threadId); }); const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fn( @@ -1318,6 +1424,11 @@ const make = Effect.gen(function* () { activeTurnId !== null && eventTurnId !== undefined && !sameId(activeTurnId, eventTurnId); const missingTurnForActiveTurn = activeTurnId !== null && eventTurnId === undefined; + const getRecentlySettledTurn = () => + Cache.getOption(recentlySettledTurnByThreadId, thread.id).pipe( + Effect.map(Option.getOrNull), + ); + // A turn.started that conflicts with the active turn is legitimate when // the server itself has a turn start pending for this thread AND the // provider session already tracks the event's turn as its active turn: @@ -1343,6 +1454,7 @@ const make = Effect.gen(function* () { case "turn.started": return !conflictsWithActiveTurn || conflictingTurnStartIsPendingTurnStart; case "turn.completed": + case "turn.aborted": if (conflictsWithActiveTurn || missingTurnForActiveTurn) { return false; } @@ -1367,7 +1479,8 @@ const make = Effect.gen(function* () { event.type === "session.exited" || event.type === "thread.started" || event.type === "turn.started" || - event.type === "turn.completed" + event.type === "turn.completed" || + event.type === "turn.aborted" ) { const status = (() => { switch (event.type) { @@ -1383,6 +1496,8 @@ const make = Effect.gen(function* () { return normalizeRuntimeTurnState(event.payload.state) === "failed" ? "error" : "ready"; + case "turn.aborted": + return "ready"; case "session.started": case "thread.started": // Provider thread/session start notifications can arrive during an @@ -1393,7 +1508,9 @@ const make = Effect.gen(function* () { const nextActiveTurnId = event.type === "turn.started" ? (eventTurnId ?? null) - : event.type === "turn.completed" || event.type === "session.exited" + : event.type === "turn.completed" || + event.type === "turn.aborted" || + event.type === "session.exited" ? null : event.type === "session.state.changed" && !sessionStatusAllowsActiveTurn( @@ -1450,6 +1567,19 @@ const make = Effect.gen(function* () { }, createdAt: now, }); + + // Record the turn the session is leaving so frames that land after + // it (tool-hook completions on provider worker threads) can still be + // attributed to it. A new turn opening clears the memo. + const settlingTurnId = activeTurnId ?? eventTurnId ?? null; + if (nextActiveTurnId !== null) { + yield* Cache.set(recentlySettledTurnByThreadId, thread.id, null); + } else if (settlingTurnId !== null) { + yield* Cache.set(recentlySettledTurnByThreadId, thread.id, { + turnId: settlingTurnId, + settledAt: now, + }); + } } } @@ -1457,6 +1587,10 @@ const make = Effect.gen(function* () { event.type === "content.delta" && event.payload.streamKind === "assistant_text" ? event.payload.delta : undefined; + const assistantSnapshot = + event.type === "content.snapshot" && event.payload.streamKind === "assistant_text" + ? event.payload.text + : undefined; const proposedPlanDelta = event.type === "turn.proposed.delta" ? event.payload.delta : undefined; @@ -1501,6 +1635,53 @@ const make = Effect.gen(function* () { } } + if (assistantSnapshot !== undefined) { + const turnId = toTurnId(event.turnId); + const assistantMessageId = yield* getOrCreateAssistantMessageId({ + threadId: thread.id, + event, + ...(turnId ? { turnId } : {}), + }); + if (turnId) { + yield* rememberAssistantMessageId(thread.id, turnId, assistantMessageId); + } + + const assistantDeliveryMode: AssistantDeliveryMode = yield* Effect.map( + serverSettingsService.getSettings, + (settings) => (settings.enableAssistantStreaming ? "streaming" : "buffered"), + ); + if (assistantDeliveryMode === "buffered") { + const detailedThread = yield* getLoadedThreadDetail(); + const hasProjectedMessage = + findMessageById(detailedThread?.messages ?? [], assistantMessageId) !== undefined; + if (hasProjectedMessage) { + yield* orchestrationEngine.dispatch({ + type: "thread.message.assistant.replace", + commandId: yield* providerCommandId(event, "assistant-snapshot-buffer-spill"), + threadId: thread.id, + messageId: assistantMessageId, + text: assistantSnapshot, + ...(turnId ? { turnId } : {}), + createdAt: now, + }); + yield* clearBufferedAssistantText(assistantMessageId); + } else { + yield* replaceBufferedAssistantText(assistantMessageId, assistantSnapshot); + } + } else { + yield* orchestrationEngine.dispatch({ + type: "thread.message.assistant.replace", + commandId: yield* providerCommandId(event, "assistant-snapshot"), + threadId: thread.id, + messageId: assistantMessageId, + text: assistantSnapshot, + ...(turnId ? { turnId } : {}), + createdAt: now, + }); + yield* clearBufferedAssistantText(assistantMessageId); + } + } + const pauseForUserTurnId = event.type === "request.opened" || event.type === "user-input.requested" ? toTurnId(event.turnId) @@ -1634,7 +1815,7 @@ const make = Effect.gen(function* () { }); } - if (event.type === "turn.completed") { + if (event.type === "turn.completed" || event.type === "turn.aborted") { const detailedThread = yield* getLoadedThreadDetail(); const messages = detailedThread?.messages ?? []; const proposedPlans = detailedThread?.proposedPlans ?? []; @@ -1764,7 +1945,25 @@ const make = Effect.gen(function* () { } } - const activities = runtimeEventToActivities(event, taskTitle); + // Frames that carry no turn id (late tool-hook completions, generic + // status items emitted around turn boundaries) would otherwise project + // an unattributed activity, which the timeline can never fold behind the + // turn's "Worked for …" row. Attribute them to the thread's active turn, + // or to the turn that just settled. Frames that do name a turn keep it — + // the turn-conflict gate above governs lifecycle, not attribution. + const fallbackActivityTurnId = resolveActivityFallbackTurnId({ + activeTurnId, + // With a turn start pending, an unattributed frame is at least as + // likely to belong to the turn about to open as to the one that just + // closed — leave it unattributed rather than folding it into history. + recentlySettledTurn: hasPendingTurnStart ? null : yield* getRecentlySettledTurn(), + at: now, + }); + const activities = runtimeEventToActivities(event, taskTitle).map((activity) => + activity.turnId === null && fallbackActivityTurnId !== null + ? { ...activity, turnId: fallbackActivityTurnId } + : activity, + ); yield* Effect.forEach(activities, (activity) => providerCommandId(event, "thread-activity-append").pipe( Effect.flatMap((commandId) => diff --git a/apps/server/src/orchestration/Normalizer.ts b/apps/server/src/orchestration/Normalizer.ts index 24c65900b29..673a6ca1fce 100644 --- a/apps/server/src/orchestration/Normalizer.ts +++ b/apps/server/src/orchestration/Normalizer.ts @@ -7,6 +7,7 @@ import { type IsoDateTime, type OrchestrationCommand, OrchestrationDispatchCommandError, + PROVIDER_SEND_TURN_MAX_FILE_BYTES, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, } from "@t3tools/contracts"; @@ -109,16 +110,24 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => (attachment) => Effect.gen(function* () { const parsed = parseBase64DataUrl(attachment.dataUrl); - if (!parsed || !parsed.mimeType.startsWith("image/")) { + // The upload's declared type is trusted only after the payload's + // real mime agrees with it: an image must actually be image/*, and + // a "file" that turns out to be an image is normalized to the image + // variant so it renders inline instead of as a download card. + if (!parsed || (attachment.type === "image" && !parsed.mimeType.startsWith("image/"))) { return yield* new OrchestrationDispatchCommandError({ - message: `Invalid image attachment payload for '${attachment.name}'.`, + message: `Invalid ${attachment.type} attachment payload for '${attachment.name}'.`, }); } + const maxBytes = + attachment.type === "image" + ? PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + : PROVIDER_SEND_TURN_MAX_FILE_BYTES; const bytes = Buffer.from(parsed.base64, "base64"); - if (bytes.byteLength === 0 || bytes.byteLength > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { + if (bytes.byteLength === 0 || bytes.byteLength > maxBytes) { return yield* new OrchestrationDispatchCommandError({ - message: `Image attachment '${attachment.name}' is empty or too large.`, + message: `Attachment '${attachment.name}' is empty or too large.`, }); } @@ -130,7 +139,9 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => } const persistedAttachment = { - type: "image" as const, + type: parsed.mimeType.toLowerCase().startsWith("image/") + ? ("image" as const) + : ("file" as const), id: attachmentId, name: attachment.name, mimeType: parsed.mimeType.toLowerCase(), diff --git a/apps/server/src/orchestration/Schemas.ts b/apps/server/src/orchestration/Schemas.ts index 3b558d24739..d189d8bf6ff 100644 --- a/apps/server/src/orchestration/Schemas.ts +++ b/apps/server/src/orchestration/Schemas.ts @@ -19,6 +19,7 @@ import { ThreadTurnDiffCompletedPayload as ContractsThreadTurnDiffCompletedPayloadSchema, ThreadRevertedPayload as ContractsThreadRevertedPayloadSchema, ThreadActivityAppendedPayload as ContractsThreadActivityAppendedPayloadSchema, + ThreadNotificationDeliveredPayload as ContractsThreadNotificationDeliveredPayloadSchema, ThreadTurnStartRequestedPayload as ContractsThreadTurnStartRequestedPayloadSchema, ThreadTurnInterruptRequestedPayload as ContractsThreadTurnInterruptRequestedPayloadSchema, ThreadApprovalResponseRequestedPayload as ContractsThreadApprovalResponseRequestedPayloadSchema, @@ -49,6 +50,7 @@ export const ThreadSessionSetPayload = ContractsThreadSessionSetPayloadSchema; export const ThreadTurnDiffCompletedPayload = ContractsThreadTurnDiffCompletedPayloadSchema; export const ThreadRevertedPayload = ContractsThreadRevertedPayloadSchema; export const ThreadActivityAppendedPayload = ContractsThreadActivityAppendedPayloadSchema; +export const ThreadNotificationDeliveredPayload = ContractsThreadNotificationDeliveredPayloadSchema; export const ThreadTurnStartRequestedPayload = ContractsThreadTurnStartRequestedPayloadSchema; export const ThreadTurnInterruptRequestedPayload = diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index 23b291d8778..70fe3891eca 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -17,6 +17,7 @@ import type { OrchestrationThreadDetailSnapshot, OrchestrationThreadShell, ProjectId, + ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -115,6 +116,18 @@ export interface ProjectionSnapshotQueryShape { workspaceRoot: string, ) => Effect.Effect, ProjectionRepositoryError>; + /** + * Read the active *agent project* for one provider instance — the synthetic + * row that lets a directoryless agent thread satisfy `project_id NOT NULL`. + * + * `Option.none` means no such project exists yet (or it was soft-deleted), + * which callers treat as "create one", not as an error. See + * `getOrCreateAgentProject`. + */ + readonly getActiveAgentProject: ( + agentInstanceId: ProviderInstanceId, + ) => Effect.Effect, ProjectionRepositoryError>; + /** * Read a single active project shell row by id. */ @@ -152,6 +165,39 @@ export interface ProjectionSnapshotQueryShape { threadId: ThreadId, ) => Effect.Effect, ProjectionRepositoryError>; + /** + * Answer "does this thread row exist", independent of archive state. + * + * Every other thread read here filters `archived_at IS NULL`, which is right + * for "what should the user see" and wrong for existence: an archived thread + * is parked, not gone. Callers holding a durable reference to a specific + * thread — the Hermes home-thread designation — need this distinction, or + * archiving reads as deletion and they replace a thread the user still has. + * + * A soft-deleted row still reports absent. + */ + readonly getThreadArchiveStateById: ( + threadId: ThreadId, + ) => Effect.Effect< + Option.Option<{ readonly archivedAt: string | null }>, + ProjectionRepositoryError + >; + + /** + * Answer "has this proactive delivery already been written to this thread", + * independent of archive state. + * + * The idempotency check behind `deliveryId` dedupe, kept off the detail read + * for the same reason `getThreadArchiveStateById` exists: `archived_at IS + * NULL` would make an archived thread report every delivery as absent, and + * each retry of one delivery would then append another copy. Reads only the + * provenance column, so it costs nothing near the message bodies. + */ + readonly hasThreadNotificationDelivery: (input: { + readonly threadId: ThreadId; + readonly deliveryId: string; + }) => Effect.Effect; + /** * Read a single active thread detail snapshot by id. */ diff --git a/apps/server/src/orchestration/agentProjects.test.ts b/apps/server/src/orchestration/agentProjects.test.ts new file mode 100644 index 00000000000..9889c0533c4 --- /dev/null +++ b/apps/server/src/orchestration/agentProjects.test.ts @@ -0,0 +1,328 @@ +import { assert, it } from "@effect/vitest"; +import { + type OrchestrationCommand, + type OrchestrationProject, + ProjectId, + ProviderInstanceId, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../config.ts"; +import { OrchestrationCommandInvariantError } from "./Errors.ts"; +import { agentWorkspaceRoot, getOrCreateAgentProject } from "./agentProjects.ts"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; + +const INSTANCE_ID = ProviderInstanceId.make("hermes-siva-local-abc123"); +const BASE_DIR = "/tmp/t3-agent-projects-test"; + +const makeAgentProject = (overrides: Partial = {}): OrchestrationProject => ({ + id: ProjectId.make("agent-project-1"), + title: "Hermes Siva Local", + workspaceRoot: `${BASE_DIR}/agents/${INSTANCE_ID}`, + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + agentInstanceId: INSTANCE_ID, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + deletedAt: null, + ...overrides, +}); + +/** + * Query stub whose `getActiveAgentProject` answers from a mutable queue, so a + * test can model "absent, then present" — the shape every self-healing path + * here depends on. + */ +const makeQueryLayer = ( + responses: Ref.Ref>>, + reads: Ref.Ref, +) => + Layer.succeed(ProjectionSnapshotQuery, { + getCommandReadModel: () => Effect.die("unused"), + getSnapshot: () => Effect.die("unused"), + getShellSnapshot: () => Effect.die("unused"), + getArchivedShellSnapshot: () => Effect.die("unused"), + getSnapshotSequence: () => Effect.die("unused"), + getCounts: () => Effect.die("unused"), + getActiveProjectByWorkspaceRoot: () => Effect.die("unused"), + getActiveAgentProject: () => + Effect.gen(function* () { + yield* Ref.update(reads, (count) => count + 1); + const queue = yield* Ref.get(responses); + const [head, ...rest] = queue; + if (head === undefined) { + return Option.none(); + } + yield* Ref.set(responses, rest.length > 0 ? rest : [head]); + return head; + }), + getProjectShellById: () => Effect.die("unused"), + getFirstActiveThreadIdByProjectId: () => Effect.die("unused"), + getThreadCheckpointContext: () => Effect.die("unused"), + getFullThreadDiffContext: () => Effect.die("unused"), + getThreadShellById: () => Effect.die("unused"), + getThreadArchiveStateById: () => Effect.die("unused"), + hasThreadNotificationDelivery: () => Effect.die("unused"), + getThreadDetailById: () => Effect.die("unused"), + getThreadDetailSnapshot: () => Effect.die("unused"), + }); + +const makeEngineLayer = ( + dispatched: Ref.Ref>, + options: { readonly fail?: boolean } = {}, +) => + Layer.succeed(OrchestrationEngine.OrchestrationEngineService, { + readEvents: () => Stream.empty, + dispatch: (command) => + Ref.update(dispatched, (calls) => [...calls, command]).pipe( + Effect.andThen( + options.fail + ? Effect.fail( + new OrchestrationCommandInvariantError({ + commandType: "project.create", + detail: "A project already exists for this workspace root.", + }), + ) + : Effect.succeed({ sequence: 1 }), + ), + ), + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(0), + } as OrchestrationEngine.OrchestrationEngineService["Service"]); + +const configLayer = Layer.succeed(ServerConfig, { + baseDir: BASE_DIR, +} as unknown as ServerConfig["Service"]); + +it("derives the workspace root under the T3 home agents directory", () => { + assert.equal( + agentWorkspaceRoot({ + baseDir: "/home/user/.t3", + instanceId: INSTANCE_ID, + join: (...parts) => parts.join("/"), + }), + `/home/user/.t3/agents/${INSTANCE_ID}`, + ); +}); + +it.effect("returns the existing project without dispatching anything", () => + Effect.gen(function* () { + const existing = makeAgentProject(); + const responses = yield* Ref.make>>([ + Option.some(existing), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + const project = yield* getOrCreateAgentProject({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched), + configLayer, + NodeServices.layer, + ), + ), + ); + + assert.equal(project.id, existing.id); + // The common path must stay a single indexed read — this is called as a + // precondition on every agent thread start. + assert.equal(yield* Ref.get(reads), 1); + assert.deepEqual(yield* Ref.get(dispatched), []); + }), +); + +it.effect("creates the project when none exists yet", () => + Effect.gen(function* () { + const created = makeAgentProject(); + const responses = yield* Ref.make>>([ + Option.none(), + Option.some(created), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + const project = yield* getOrCreateAgentProject({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched), + configLayer, + NodeServices.layer, + ), + ), + ); + + assert.equal(project.id, created.id); + const commands = yield* Ref.get(dispatched); + assert.equal(commands.length, 1); + const command = commands[0]; + assert.equal(command?.type, "project.create"); + if (command?.type === "project.create") { + assert.equal(command.agentInstanceId, INSTANCE_ID); + assert.equal(command.createWorkspaceRootIfMissing, true); + assert.equal(command.workspaceRoot, `${BASE_DIR}/agents/${INSTANCE_ID}`); + assert.equal(command.title, "Hermes Siva Local"); + } + }), +); + +it.effect("self-heals a soft-deleted project by creating a fresh one", () => + Effect.gen(function* () { + // A soft-deleted row is invisible to `getActiveAgentProject`, so this is + // indistinguishable from "never created" — which is the point: the + // instance recovers instead of being permanently unusable. + const replacement = makeAgentProject({ id: ProjectId.make("agent-project-2") }); + const responses = yield* Ref.make>>([ + Option.none(), + Option.some(replacement), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + const project = yield* getOrCreateAgentProject({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched), + configLayer, + NodeServices.layer, + ), + ), + ); + + assert.equal(project.id, replacement.id); + }), +); + +it.effect("resolves to the winner's project when a concurrent caller wins the race", () => + Effect.gen(function* () { + // The decider rejects a second project on the same workspace root, so the + // loser sees a dispatch failure. It must re-read and use the winner's row + // rather than surfacing an error to the user. + const winner = makeAgentProject({ id: ProjectId.make("agent-project-winner") }); + const responses = yield* Ref.make>>([ + Option.none(), + Option.some(winner), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + const project = yield* getOrCreateAgentProject({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched, { fail: true }), + configLayer, + NodeServices.layer, + ), + ), + ); + + assert.equal(project.id, winner.id); + }), +); + +it.effect("surfaces the dispatch failure when no project appears after it", () => + Effect.gen(function* () { + const responses = yield* Ref.make>>([ + Option.none(), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + const result = yield* Effect.result( + getOrCreateAgentProject({ instanceId: INSTANCE_ID, title: "Hermes Siva Local" }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched, { fail: true }), + configLayer, + NodeServices.layer, + ), + ), + ), + ); + + // A genuine failure must not be swallowed by the race-recovery read. + assert.equal(result._tag, "Failure"); + }), +); + +it.effect("falls back to the dispatched row when the projection has not caught up", () => + Effect.gen(function* () { + const responses = yield* Ref.make>>([ + Option.none(), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + const project = yield* getOrCreateAgentProject({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched), + configLayer, + NodeServices.layer, + ), + ), + ); + + // Dispatch succeeded but the read-back saw nothing yet: the caller still + // gets a usable project rather than an error. + assert.equal(project.agentInstanceId, INSTANCE_ID); + assert.equal(project.workspaceRoot, `${BASE_DIR}/agents/${INSTANCE_ID}`); + assert.equal(project.deletedAt, null); + }), +); + +it.effect("falls back to the instance id when the nickname is blank", () => + Effect.gen(function* () { + const responses = yield* Ref.make>>([ + Option.none(), + ]); + const reads = yield* Ref.make(0); + const dispatched = yield* Ref.make>([]); + + yield* getOrCreateAgentProject({ instanceId: INSTANCE_ID, title: " " }).pipe( + Effect.provide( + Layer.mergeAll( + makeQueryLayer(responses, reads), + makeEngineLayer(dispatched), + configLayer, + NodeServices.layer, + ), + ), + ); + + // `title` is a TrimmedNonEmptyString on the command — a blank nickname + // would be a decode failure, not a cosmetic problem. + const command = (yield* Ref.get(dispatched))[0]; + if (command?.type === "project.create") { + assert.equal(command.title, INSTANCE_ID); + } + }), +); diff --git a/apps/server/src/orchestration/agentProjects.ts b/apps/server/src/orchestration/agentProjects.ts new file mode 100644 index 00000000000..44e138ce326 --- /dev/null +++ b/apps/server/src/orchestration/agentProjects.ts @@ -0,0 +1,147 @@ +/** + * Agent projects — the synthetic project rows that back directoryless threads. + * + * `projection_threads.project_id` is `NOT NULL`, so a thread with a provider + * that owns its own machine (Hermes) still needs a project row to point at. + * That row is *synthetic*: `agentInstanceId` is set, its workspace root is a + * real but empty directory under `/agents//`, and every + * "pick a project" surface filters it out. + * + * ## Why converge-on-read instead of create-on-enrollment + * + * The obvious design is to create the project when an instance enrolls and + * delete it when the instance is removed. That leaves the system able to reach + * states it cannot recover from on its own: + * + * - enrollment succeeds but the project dispatch fails → the instance has no + * project, forever, and every thread start fails + * - instances enrolled before this code shipped have no project at all + * - a project soft-deleted by hand (or by a removal that later gets undone) + * leaves live threads unopenable + * + * `getOrCreateAgentProject` is instead a **precondition every caller runs**, + * not a lifecycle event one caller owns. It reads, and creates only what is + * missing. Any of the states above self-heals on the next call, and callers + * never have to reason about ordering. Enrollment still calls it, but only to + * warm the row — nothing depends on that call having happened. + * + * @module orchestration/agentProjects + */ +import { + CommandId, + ProjectId, + type OrchestrationProject, + type ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; + +import { ServerConfig } from "../config.ts"; +import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; + +/** Directory name under `` holding one directory per agent instance. */ +export const AGENT_WORKSPACE_DIR = "agents"; + +/** + * Workspace root for one agent instance. + * + * A real directory, not a marker string: `OrchestrationProject.workspaceRoot` + * is a non-empty string that plenty of code will happily `stat`, and handing + * it a path that cannot exist would turn a cosmetic problem into a crash. + */ +export function agentWorkspaceRoot(input: { + readonly baseDir: string; + readonly instanceId: ProviderInstanceId; + readonly join: (...parts: ReadonlyArray) => string; +}): string { + return input.join(input.baseDir, AGENT_WORKSPACE_DIR, input.instanceId); +} + +/** + * Resolve the agent project for one provider instance, creating it if absent. + * + * Safe to call on every request: the common path is a single indexed read. + * Callers should treat this as a precondition rather than a mutation — it is + * idempotent, and concurrent callers converge on one row (see the recovery + * read below). + */ +export const getOrCreateAgentProject = Effect.fn("getOrCreateAgentProject")(function* (input: { + readonly instanceId: ProviderInstanceId; + /** Instance nickname, used as the project title on first creation. */ + readonly title: string; +}) { + const query = yield* ProjectionSnapshotQuery; + const existing = yield* query.getActiveAgentProject(input.instanceId); + if (Option.isSome(existing)) { + return existing.value; + } + + const engine = yield* OrchestrationEngineService; + const serverConfig = yield* ServerConfig; + const crypto = yield* Crypto.Crypto; + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + + const workspaceRoot = agentWorkspaceRoot({ + baseDir: serverConfig.baseDir, + instanceId: input.instanceId, + join: path.join, + }); + yield* fs.makeDirectory(workspaceRoot, { recursive: true }); + + const projectId = ProjectId.make(yield* crypto.randomUUIDv4); + const createdAt = DateTime.formatIso(yield* DateTime.now); + + const dispatched = yield* Effect.result( + engine.dispatch({ + type: "project.create", + commandId: CommandId.make(yield* crypto.randomUUIDv4), + projectId, + title: input.title.trim() || input.instanceId, + workspaceRoot, + createWorkspaceRootIfMissing: true, + agentInstanceId: input.instanceId, + createdAt, + }), + ); + + if (dispatched._tag === "Failure") { + // The decider rejects a second project on the same workspace root, so a + // racing caller that won shows up here as a dispatch failure rather than a + // duplicate row. Re-read before surfacing the error: if the row now + // exists, the race resolved correctly and this caller can use it. + const raced = yield* query.getActiveAgentProject(input.instanceId); + if (Option.isSome(raced)) { + return raced.value; + } + return yield* Effect.fail(dispatched.failure); + } + + // Read back rather than synthesizing the row locally, so the caller always + // sees exactly what the projection committed. + const created = yield* query.getActiveAgentProject(input.instanceId); + if (Option.isSome(created)) { + return created.value; + } + + // The dispatch succeeded but the projection has not caught up. Returning a + // locally-built row keeps the caller moving; the next call reads the real + // one. Every field matches what the decider emitted. + return { + id: projectId, + title: input.title.trim() || input.instanceId, + workspaceRoot, + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + agentInstanceId: input.instanceId, + createdAt, + updatedAt: createdAt, + deletedAt: null, + } satisfies OrchestrationProject; +}); diff --git a/apps/server/src/orchestration/commandInvariants.test.ts b/apps/server/src/orchestration/commandInvariants.test.ts index 9531cd5c3af..0ed50af42e2 100644 --- a/apps/server/src/orchestration/commandInvariants.test.ts +++ b/apps/server/src/orchestration/commandInvariants.test.ts @@ -34,6 +34,7 @@ const readModel: OrchestrationReadModel = { model: "gpt-5-codex", }, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, deletedAt: null, @@ -47,6 +48,7 @@ const readModel: OrchestrationReadModel = { model: "gpt-5-codex", }, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, deletedAt: null, diff --git a/apps/server/src/orchestration/decider.assistantMessageReplacement.test.ts b/apps/server/src/orchestration/decider.assistantMessageReplacement.test.ts new file mode 100644 index 00000000000..2246812a294 --- /dev/null +++ b/apps/server/src/orchestration/decider.assistantMessageReplacement.test.ts @@ -0,0 +1,73 @@ +import { + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + EventId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + TurnId, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import { expect, it } from "@effect/vitest"; + +import { decideOrchestrationCommand } from "./decider.ts"; +import { createEmptyReadModel, projectEvent } from "./projector.ts"; + +it.layer(NodeServices.layer)("assistant message replacement", (it) => { + it.effect("emits a streaming replacement message event", () => + Effect.gen(function* () { + const now = "2026-07-24T12:00:00.000Z"; + const threadId = ThreadId.make("thread-replacement"); + const readModel = yield* projectEvent(createEmptyReadModel(now), { + sequence: 1, + eventId: EventId.make("event-thread-created"), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.created", + occurredAt: now, + commandId: CommandId.make("command-thread-created"), + causationEventId: null, + correlationId: CommandId.make("command-thread-created"), + metadata: {}, + payload: { + threadId, + projectId: ProjectId.make("project-replacement"), + title: "Replacement", + modelSelection: { + instanceId: ProviderInstanceId.make("hermes"), + model: "hermes", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: now, + updatedAt: now, + }, + }); + + const decided = yield* decideOrchestrationCommand({ + command: { + type: "thread.message.assistant.replace", + commandId: CommandId.make("command-message-replace"), + threadId, + messageId: MessageId.make("assistant:message-1"), + text: "Hello there", + turnId: TurnId.make("turn-1"), + createdAt: now, + }, + readModel, + }); + + const event = Array.isArray(decided) ? decided[0] : decided; + if (event?.type !== "thread.message-sent") { + throw new Error("expected thread.message-sent"); + } + expect(event.payload.text).toBe("Hello there"); + expect(event.payload.streaming).toBe(true); + expect(event.payload.textOperation).toBe("replace"); + }), + ); +}); diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index fea36b5717f..22e98ee5474 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -40,6 +40,7 @@ const seedReadModel = Effect.gen(function* () { workspaceRoot: "/tmp/project-delete", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, diff --git a/apps/server/src/orchestration/decider.notification.test.ts b/apps/server/src/orchestration/decider.notification.test.ts new file mode 100644 index 00000000000..032cb51e018 --- /dev/null +++ b/apps/server/src/orchestration/decider.notification.test.ts @@ -0,0 +1,317 @@ +import { + CommandId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + TurnId, + type ChatAttachment, + type OrchestrationReadModel, + type OrchestrationThread, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; + +import { decideOrchestrationCommand } from "./decider.ts"; + +const NOW = "2026-07-01T00:00:00.000Z"; +const DELIVERED_AT = "2026-06-30T09:00:00.000Z"; +const SETTLED_AT = "2026-06-29T00:00:00.000Z"; +const SNOOZED_UNTIL = "2026-07-02T00:00:00.000Z"; + +function makeReadModel(input: { + readonly settledOverride?: OrchestrationThread["settledOverride"]; + readonly snoozedUntil?: string | null; + readonly messages?: OrchestrationThread["messages"]; +}): OrchestrationReadModel { + const settledOverride = input.settledOverride ?? null; + return { + snapshotSequence: 0, + projects: [], + threads: [ + { + id: ThreadId.make("thread-1"), + projectId: ProjectId.make("project-1"), + title: "Home", + modelSelection: { instanceId: ProviderInstanceId.make("hermes"), model: "hermes" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + latestTurn: null, + createdAt: NOW, + updatedAt: NOW, + archivedAt: null, + settledOverride, + settledAt: settledOverride === "settled" ? SETTLED_AT : null, + snoozedUntil: input.snoozedUntil ?? null, + snoozedAt: input.snoozedUntil != null ? SETTLED_AT : null, + deletedAt: null, + messages: input.messages ?? [], + proposedPlans: [], + activities: [], + checkpoints: [], + session: null, + }, + ], + updatedAt: NOW, + }; +} + +function deliverCommand(overrides: { + readonly commandId: string; + readonly kind?: "cron" | "message" | "lifecycle" | "handoff" | "other"; + readonly deliveryId?: string; + readonly messageId?: string; + readonly text?: string; + readonly attachments?: ReadonlyArray; + readonly turnId?: TurnId; +}) { + return { + type: "thread.notification.deliver", + commandId: CommandId.make(overrides.commandId), + threadId: ThreadId.make("thread-1"), + messageId: MessageId.make(overrides.messageId ?? "message-1"), + deliveryId: overrides.deliveryId ?? "delivery-1", + kind: overrides.kind ?? "cron", + label: "Cron: daily-digest", + text: overrides.text ?? "Digest ready.", + ...(overrides.attachments !== undefined ? { attachments: overrides.attachments } : {}), + ...(overrides.turnId !== undefined ? { turnId: overrides.turnId } : {}), + createdAt: NOW, + } as const; +} + +const mediaAttachment: ChatAttachment = { + type: "file", + id: "thread-1-2c8b3f1e-0000-4000-8000-000000000001", + name: "chart.mp4", + mimeType: "video/mp4", + sizeBytes: 2_048, +}; + +const deliveredMessage: OrchestrationThread["messages"][number] = { + id: MessageId.make("message-1"), + role: "assistant", + text: "Digest ready.", + turnId: null, + streaming: false, + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }, + createdAt: DELIVERED_AT, + updatedAt: DELIVERED_AT, +}; + +it.layer(NodeServices.layer)("notification delivery decider", (it) => { + it.effect("appends a turnless notification message and un-settles the thread", () => + Effect.gen(function* () { + const result = yield* decideOrchestrationCommand({ + command: deliverCommand({ commandId: "cmd-deliver" }), + readModel: makeReadModel({ settledOverride: "settled" }), + }); + const events = Array.isArray(result) ? result : [result]; + expect(events.map((event) => event.type)).toEqual([ + "thread.unsettled", + "thread.notification-delivered", + ]); + + const unsettled = events[0]; + if (unsettled?.type === "thread.unsettled") { + expect(unsettled.payload.reason).toBe("activity"); + } + + const delivered = events[1]; + if (delivered?.type === "thread.notification-delivered") { + expect(delivered.payload.messageId).toBe("message-1"); + expect(delivered.payload.text).toBe("Digest ready."); + expect(delivered.payload.notification).toEqual({ + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }); + // Nothing about a delivery belongs to a turn; attributing it to the + // thread's live turn would fold it behind that turn's summary. + expect("turnId" in delivered.payload).toBe(false); + } + }), + ); + + it.effect("un-snoozes as well as un-settles, and does neither when neither applies", () => + Effect.gen(function* () { + const both = yield* decideOrchestrationCommand({ + command: deliverCommand({ commandId: "cmd-deliver-both" }), + readModel: makeReadModel({ settledOverride: "settled", snoozedUntil: SNOOZED_UNTIL }), + }); + expect((Array.isArray(both) ? both : [both]).map((event) => event.type)).toEqual([ + "thread.unsettled", + "thread.unsnoozed", + "thread.notification-delivered", + ]); + + const neither = yield* decideOrchestrationCommand({ + command: deliverCommand({ commandId: "cmd-deliver-neither" }), + readModel: makeReadModel({}), + }); + expect((Array.isArray(neither) ? neither : [neither]).map((event) => event.type)).toEqual([ + "thread.notification-delivered", + ]); + }), + ); + + it.effect("appends lifecycle notices quietly, leaving settled and snoozed state alone", () => + Effect.gen(function* () { + const result = yield* decideOrchestrationCommand({ + command: deliverCommand({ commandId: "cmd-lifecycle", kind: "lifecycle" }), + readModel: makeReadModel({ settledOverride: "settled", snoozedUntil: SNOOZED_UNTIL }), + }); + const events = Array.isArray(result) ? result : [result]; + // Unread indicator only: a restart notice must not drag a thread the + // user has parked back into the inbox. + expect(events.map((event) => event.type)).toEqual(["thread.notification-delivered"]); + }), + ); + + it.effect("dedupes a replayed deliveryId without appending or waking the thread", () => + Effect.gen(function* () { + const result = yield* decideOrchestrationCommand({ + // The plugin's retry queue resends with a fresh command id, a new + // message id, and possibly edited text — only deliveryId is stable. + command: deliverCommand({ + commandId: "cmd-deliver-retry", + messageId: "message-retry", + text: "Digest ready (retry).", + }), + readModel: makeReadModel({ + settledOverride: "settled", + snoozedUntil: SNOOZED_UNTIL, + messages: [deliveredMessage], + }), + }); + const events = Array.isArray(result) ? result : [result]; + + // No un-settle/un-snooze: a replay is not new activity. + expect(events.map((event) => event.type)).toEqual(["thread.notification-delivered"]); + + const delivered = events[0]; + if (delivered?.type === "thread.notification-delivered") { + // Re-emitted from the persisted row, so reducing it a second time is + // a byte-identical no-op rather than a duplicate or an overwrite. + expect(delivered.payload.messageId).toBe("message-1"); + expect(delivered.payload.text).toBe("Digest ready."); + expect(delivered.payload.createdAt).toBe(DELIVERED_AT); + expect(delivered.payload.updatedAt).toBe(DELIVERED_AT); + expect(delivered.occurredAt).toBe(DELIVERED_AT); + } + }), + ); + + it.effect("treats a different deliveryId on the same thread as a new delivery", () => + Effect.gen(function* () { + const result = yield* decideOrchestrationCommand({ + command: deliverCommand({ + commandId: "cmd-deliver-second", + deliveryId: "delivery-2", + messageId: "message-2", + text: "Second digest.", + }), + readModel: makeReadModel({ + settledOverride: "settled", + messages: [deliveredMessage], + }), + }); + const events = Array.isArray(result) ? result : [result]; + expect(events.map((event) => event.type)).toEqual([ + "thread.unsettled", + "thread.notification-delivered", + ]); + const delivered = events[1]; + if (delivered?.type === "thread.notification-delivered") { + expect(delivered.payload.messageId).toBe("message-2"); + expect(delivered.payload.notification.deliveryId).toBe("delivery-2"); + } + }), + ); + + it.effect("threads media attachments and turnId through the delivered payload", () => + Effect.gen(function* () { + const turnId = TurnId.make("turn-live"); + const result = yield* decideOrchestrationCommand({ + command: deliverCommand({ + commandId: "cmd-deliver-media", + deliveryId: "delivery-media", + messageId: "message-media", + text: "A caption", + attachments: [mediaAttachment], + turnId, + }), + readModel: makeReadModel({}), + }); + const events = Array.isArray(result) ? result : [result]; + const delivered = events.find((event) => event.type === "thread.notification-delivered"); + if (delivered?.type === "thread.notification-delivered") { + expect(delivered.payload.attachments).toEqual([mediaAttachment]); + expect(delivered.payload.turnId).toBe(turnId); + } else { + expect.unreachable("expected a thread.notification-delivered event"); + } + }), + ); + + it.effect("rebuilds media replays from the persisted row, not the retry's fields", () => + Effect.gen(function* () { + const persisted: OrchestrationThread["messages"][number] = { + ...deliveredMessage, + id: MessageId.make("message-media"), + text: "A caption", + turnId: TurnId.make("turn-live"), + attachments: [mediaAttachment], + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-media", + }, + }; + const result = yield* decideOrchestrationCommand({ + // The retry names a different messageId and attachment id — its bytes + // were written under the original id, so adopting the retry's fields + // would re-point the row at files that do not exist. + command: deliverCommand({ + commandId: "cmd-deliver-media-retry", + deliveryId: "delivery-media", + messageId: "message-media-retry", + text: "A caption (retry)", + attachments: [ + { ...mediaAttachment, id: "thread-1-2c8b3f1e-0000-4000-8000-00000000dead" }, + ], + }), + readModel: makeReadModel({ messages: [persisted] }), + }); + const events = Array.isArray(result) ? result : [result]; + expect(events.map((event) => event.type)).toEqual(["thread.notification-delivered"]); + const delivered = events[0]; + if (delivered?.type === "thread.notification-delivered") { + expect(delivered.payload.messageId).toBe("message-media"); + expect(delivered.payload.text).toBe("A caption"); + expect(delivered.payload.attachments).toEqual([mediaAttachment]); + expect(delivered.payload.turnId).toBe("turn-live"); + } + }), + ); + + it.effect("rejects a delivery for a thread that does not exist", () => + Effect.gen(function* () { + const error = yield* decideOrchestrationCommand({ + command: { + ...deliverCommand({ commandId: "cmd-deliver-missing" }), + threadId: ThreadId.make("thread-missing"), + }, + readModel: makeReadModel({}), + }).pipe(Effect.flip); + expect(error._tag).toBe("OrchestrationCommandInvariantError"); + }), + ); +}); diff --git a/apps/server/src/orchestration/decider.projectScripts.test.ts b/apps/server/src/orchestration/decider.projectScripts.test.ts index a0c06840733..be806230f0f 100644 --- a/apps/server/src/orchestration/decider.projectScripts.test.ts +++ b/apps/server/src/orchestration/decider.projectScripts.test.ts @@ -63,6 +63,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/scripts", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, @@ -115,6 +116,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/project", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, @@ -161,6 +163,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/project-first", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, @@ -182,6 +185,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/project-second", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, @@ -226,6 +230,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/project", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, @@ -323,6 +328,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/project", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, @@ -401,6 +407,7 @@ it.layer(NodeServices.layer)("decider project scripts", (it) => { workspaceRoot: "/tmp/project", defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, }, diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 100369ae6e3..cb68f1df3ec 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -251,6 +251,7 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" workspaceRoot: command.workspaceRoot, defaultModelSelection: command.defaultModelSelection ?? null, scripts: [], + agentInstanceId: command.agentInstanceId ?? null, createdAt: command.createdAt, updatedAt: command.createdAt, }, @@ -1018,6 +1019,34 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" }; } + case "thread.message.assistant.replace": { + yield* requireThread({ + readModel, + command, + threadId: command.threadId, + }); + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt: command.createdAt, + commandId: command.commandId, + })), + type: "thread.message-sent", + payload: { + threadId: command.threadId, + messageId: command.messageId, + role: "assistant", + text: command.text, + turnId: command.turnId ?? null, + streaming: true, + textOperation: "replace", + createdAt: command.createdAt, + updatedAt: command.createdAt, + }, + }; + } + case "thread.message.assistant.complete": { yield* requireThread({ readModel, @@ -1168,6 +1197,109 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" return [unsettledEvent, activityAppendedEvent]; } + case "thread.notification.deliver": { + const thread = yield* requireThread({ + readModel, + command, + threadId: command.threadId, + }); + // Dedupe on deliveryId: the plugin's durable queue retries anything it + // has not seen acked, so the same delivery legitimately arrives more + // than once. Idempotency is by re-emission (see thread.settle) because + // the engine rejects a command that produces no events — but unlike + // settle, the payload must be rebuilt from the ALREADY-WRITTEN message + // rather than from the command, so reducing it a second time lands on + // byte-identical state instead of rewriting the row from a retry whose + // messageId or text may differ. + // + // NOT SUFFICIENT ON ITS OWN. This scans the in-memory read model, which + // `getCommandReadModel` builds with `messages: []` on every thread — it + // deliberately does not hydrate message bodies at boot. So this catches + // only replays of deliveries appended since the process started, and + // sees nothing after a restart. The delivery path re-checks against + // `projection_thread_messages` before dispatching + // (`hermesGatewayHttp.ts`), and that read is what actually holds across + // a restart. Keep both: this one keeps the aggregate self-consistent for + // any future caller, the other one is load-bearing. + const duplicate = thread.messages.find( + (message) => message.notification?.deliveryId === command.deliveryId, + ); + // On replay these too come from the persisted row, for the same reason + // as the text: the retry's attachment ids point at files written for a + // different messageId, and adopting them would re-point the row. + const attachments = duplicate ? duplicate.attachments : command.attachments; + const turnId = duplicate ? (duplicate.turnId ?? undefined) : command.turnId; + const notificationEvent: Omit = { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt: duplicate?.createdAt ?? command.createdAt, + commandId: command.commandId, + })), + type: "thread.notification-delivered", + payload: { + threadId: command.threadId, + messageId: duplicate?.id ?? command.messageId, + text: duplicate?.text ?? command.text, + notification: duplicate?.notification ?? { + kind: command.kind, + label: command.label, + deliveryId: command.deliveryId, + }, + ...(attachments !== undefined ? { attachments } : {}), + ...(turnId !== undefined ? { turnId } : {}), + createdAt: duplicate?.createdAt ?? command.createdAt, + updatedAt: duplicate?.updatedAt ?? command.createdAt, + }, + }; + // A replay is not new activity: it must never wake a thread the user + // has since settled or snoozed. + if (duplicate !== undefined) { + return notificationEvent; + } + // A lifecycle notice ("agent restarted") is bookkeeping, not a claim on + // the user's attention: it appends quietly and leaves a settled or + // snoozed thread where it is. Every other kind is the agent raising its + // hand and resets the lifecycle exactly like a user message does. + if (command.kind === "lifecycle") { + return notificationEvent; + } + const lifecycleResetEvents: Array> = []; + if (thread.settledOverride !== null) { + lifecycleResetEvents.push({ + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt: command.createdAt, + commandId: command.commandId, + })), + type: "thread.unsettled", + payload: { + threadId: command.threadId, + reason: "activity", + updatedAt: command.createdAt, + }, + }); + } + if (thread.snoozedUntil != null) { + lifecycleResetEvents.push({ + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt: command.createdAt, + commandId: command.commandId, + })), + type: "thread.unsnoozed", + payload: { + threadId: command.threadId, + reason: "activity", + updatedAt: command.createdAt, + }, + }); + } + return [...lifecycleResetEvents, notificationEvent]; + } + default: { command satisfies never; const fallback = command as never as { type: string }; diff --git a/apps/server/src/orchestration/homeThreads.test.ts b/apps/server/src/orchestration/homeThreads.test.ts new file mode 100644 index 00000000000..d7f6d30188d --- /dev/null +++ b/apps/server/src/orchestration/homeThreads.test.ts @@ -0,0 +1,480 @@ +import { assert, it } from "@effect/vitest"; +import { + DEFAULT_HERMES_MODEL, + HERMES_DRIVER_KIND, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationCommand, + type OrchestrationProject, + type OrchestrationThreadShell, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { OrchestrationCommandInvariantError } from "./Errors.ts"; +import { + HOME_THREAD_TITLE, + getDesignatedHomeThreadId, + getOrCreateHomeThread, + isHomeThread, + readHomeThreadId, +} from "./homeThreads.ts"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; + +const INSTANCE_ID = ProviderInstanceId.make("hermes-siva-local-abc123"); +const OTHER_INSTANCE_ID = ProviderInstanceId.make("hermes-other-def456"); +const HOME_THREAD_ID = ThreadId.make("thread-home-1"); +const BASE_DIR = "/tmp/t3-home-threads-test"; + +const agentProject: OrchestrationProject = { + id: ProjectId.make("agent-project-1"), + title: "Hermes Siva Local", + workspaceRoot: `${BASE_DIR}/agents/${INSTANCE_ID}`, + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + agentInstanceId: INSTANCE_ID, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + deletedAt: null, +}; + +const threadShell = (id: ThreadId): OrchestrationThreadShell => + ({ + id, + projectId: agentProject.id, + title: HOME_THREAD_TITLE, + modelSelection: { instanceId: INSTANCE_ID, model: DEFAULT_HERMES_MODEL }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + latestTurn: null, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + archivedAt: null, + settledOverride: null, + settledAt: null, + session: null, + latestUserMessageAt: null, + hasPendingApprovals: false, + hasPendingUserInput: false, + hasActionableProposedPlan: false, + }) as OrchestrationThreadShell; + +const hermesInstance = (config: Record) => ({ + driver: HERMES_DRIVER_KIND, + displayName: "Hermes Siva Local", + enabled: true, + config, +}); + +/** + * Query stub answering thread reads from a mutable queue, so a test can model + * "designated but deleted" — the self-healing path this module exists for. + */ +const makeQueryLayer = ( + threads: Ref.Ref>>, + archiveStates?: ReadonlyArray>, +) => + Layer.succeed(ProjectionSnapshotQuery, { + getCommandReadModel: () => Effect.die("unused"), + getSnapshot: () => Effect.die("unused"), + getShellSnapshot: () => Effect.die("unused"), + getArchivedShellSnapshot: () => Effect.die("unused"), + getSnapshotSequence: () => Effect.die("unused"), + getCounts: () => Effect.die("unused"), + getActiveProjectByWorkspaceRoot: () => Effect.die("unused"), + getActiveAgentProject: () => Effect.succeed(Option.some(agentProject)), + getProjectShellById: () => Effect.die("unused"), + getFirstActiveThreadIdByProjectId: () => Effect.die("unused"), + getThreadCheckpointContext: () => Effect.die("unused"), + getFullThreadDiffContext: () => Effect.die("unused"), + getThreadArchiveStateById: () => + Effect.gen(function* () { + if (archiveStates?.[0] !== undefined) return archiveStates[0]; + // Default: mirror the shell queue, since a thread visible to the shell + // query is also present. Tests about archiving pass this explicitly. + const queue = yield* Ref.get(threads); + return Option.map(queue[0] ?? Option.none(), () => ({ archivedAt: null })); + }), + getThreadShellById: () => + Effect.gen(function* () { + const queue = yield* Ref.get(threads); + const [head, ...rest] = queue; + if (head === undefined) return Option.none(); + yield* Ref.set(threads, rest.length > 0 ? rest : [head]); + return head; + }), + getThreadDetailById: () => Effect.die("unused"), + getThreadDetailSnapshot: () => Effect.die("unused"), + } as unknown as ProjectionSnapshotQuery["Service"]); + +const makeEngineLayer = ( + dispatched: Ref.Ref>, + options: { + readonly fail?: boolean; + /** + * Runs after each command is recorded. The only hook that fires *inside* + * `getOrCreateHomeThread`, which is what lets a test land a competing + * settings write between this caller's read and its persist. + */ + readonly onDispatch?: (command: OrchestrationCommand) => Effect.Effect; + } = {}, +) => + Layer.succeed(OrchestrationEngine.OrchestrationEngineService, { + readEvents: () => Stream.empty, + dispatch: (command: OrchestrationCommand) => + Ref.update(dispatched, (calls) => [...calls, command]).pipe( + Effect.andThen(options.onDispatch ? options.onDispatch(command) : Effect.void), + Effect.andThen( + options.fail + ? Effect.fail( + new OrchestrationCommandInvariantError({ + commandType: "thread.create", + detail: "Simulated dispatch failure.", + }), + ) + : Effect.succeed({ sequence: 1 }), + ), + ), + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(0), + } as OrchestrationEngine.OrchestrationEngineService["Service"]); + +const configLayer = Layer.succeed(ServerConfig, { + baseDir: BASE_DIR, +} as unknown as ServerConfig["Service"]); + +const testLayer = (input: { + readonly threads: Ref.Ref>>; + readonly dispatched: Ref.Ref>; + readonly providerInstances: Record; + readonly failDispatch?: boolean; + /** Fires inside `getOrCreateHomeThread`, right after `thread.create`. */ + readonly onDispatch?: (command: OrchestrationCommand) => Effect.Effect; + /** + * Settings layer to use instead of a fresh one. A test that needs to write + * settings from `onDispatch` builds the layer itself so both sides share one + * store. + */ + readonly settingsLayer?: ReturnType; + /** + * Archive-state rows. Unlike the shell queue these ignore archive state, so + * a test can model "archived, therefore invisible to the shell query but + * still very much present" — the case that used to mint a second Home. + */ + readonly archiveStates?: ReadonlyArray>; +}) => + Layer.mergeAll( + makeQueryLayer(input.threads, input.archiveStates), + makeEngineLayer(input.dispatched, { + ...(input.failDispatch ? { fail: true } : {}), + ...(input.onDispatch ? { onDispatch: input.onDispatch } : {}), + }), + configLayer, + input.settingsLayer ?? + ServerSettings.layerTest({ + providerInstances: input.providerInstances, + } as never), + NodeServices.layer, + ); + +it("reads a designation only from a Hermes envelope", () => { + assert.equal( + readHomeThreadId(hermesInstance({ homeThreadId: HOME_THREAD_ID }) as never), + HOME_THREAD_ID, + ); + assert.equal(readHomeThreadId(hermesInstance({}) as never), undefined); + // An empty string is "not designated", not a thread whose id is "". + assert.equal(readHomeThreadId(hermesInstance({ homeThreadId: "" }) as never), undefined); + // A designation on a non-Hermes envelope is not ours to honour. + assert.equal( + readHomeThreadId({ driver: "codex", homeThreadId: HOME_THREAD_ID } as never), + undefined, + ); +}); + +it.effect("returns the designated thread without dispatching anything", () => + Effect.gen(function* () { + const threads = yield* Ref.make>>([ + Option.some(threadShell(HOME_THREAD_ID)), + ]); + const dispatched = yield* Ref.make>([]); + + const resolved = yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + providerInstances: { + [INSTANCE_ID]: hermesInstance({ homeThreadId: HOME_THREAD_ID }), + }, + }), + ), + ); + + assert.equal(resolved, HOME_THREAD_ID); + // This runs on every handshake; it must not create anything on the happy path. + assert.deepEqual(yield* Ref.get(dispatched), []); + }), +); + +it.effect("creates and persists a Home thread when none is designated", () => + Effect.gen(function* () { + const threads = yield* Ref.make>>([ + Option.none(), + ]); + const dispatched = yield* Ref.make>([]); + + yield* Effect.gen(function* () { + const resolved = yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }); + + const commands = yield* Ref.get(dispatched); + const created = commands.find((command) => command.type === "thread.create"); + assert.isDefined(created); + if (created?.type === "thread.create") { + assert.equal(created.threadId, resolved); + assert.equal(created.title, HOME_THREAD_TITLE); + assert.equal(created.projectId, agentProject.id); + // Binds to the fixed Hermes slug, not to whatever model the plugin + // currently reports — otherwise the thread orphans on a model change. + assert.equal(created.modelSelection.model, DEFAULT_HERMES_MODEL); + } + + // The designation must be durable, or the next handshake mints a second + // Home and the first one's history is stranded. + assert.equal(yield* getDesignatedHomeThreadId(INSTANCE_ID), resolved); + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + providerInstances: { [INSTANCE_ID]: hermesInstance({}) }, + }), + ), + ); + }), +); + +it.effect("self-heals a designation whose thread no longer exists", () => + Effect.gen(function* () { + // A deleted home thread must not strand the instance: it re-designates + // rather than failing every future delivery. + const threads = yield* Ref.make>>([ + Option.none(), + ]); + const dispatched = yield* Ref.make>([]); + + const resolved = yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + providerInstances: { + [INSTANCE_ID]: hermesInstance({ homeThreadId: "thread-deleted-long-ago" }), + }, + }), + ), + ); + + assert.notEqual(resolved, "thread-deleted-long-ago"); + const commands = yield* Ref.get(dispatched); + assert.isDefined(commands.find((command) => command.type === "thread.create")); + }), +); + +it.effect("adopts a racing caller's designation when its own dispatch fails", () => + Effect.gen(function* () { + const threads = yield* Ref.make>>([ + Option.none(), + ]); + const dispatched = yield* Ref.make>([]); + + // The loser of the race sees a dispatch failure but must still return the + // winner's thread rather than surfacing an error to the handshake. + const resolved = yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + failDispatch: true, + providerInstances: { + // Designated, but the thread read says gone — so it falls through + // to create, fails, and re-reads the designation. + [INSTANCE_ID]: hermesInstance({ homeThreadId: HOME_THREAD_ID }), + }, + }), + ), + ); + + assert.equal(resolved, HOME_THREAD_ID); + }), +); + +it.effect("does not clobber a designation that landed while it was creating", () => + Effect.gen(function* () { + // Two handshakes race: both read "no designation", both create a thread. + // The one that persists *second* must stand down rather than overwrite — + // otherwise the first caller re-read the winner's id and returned it while + // this one overwrites with its own, so the two disagree about Home and + // concurrent proactive deliveries land in two different threads. + const threads = yield* Ref.make>>([ + Option.none(), + ]); + const dispatched = yield* Ref.make>([]); + const settingsLayer = ServerSettings.layerTest({ + providerInstances: { [INSTANCE_ID]: hermesInstance({}) }, + } as never); + + yield* Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + + const resolved = yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + providerInstances: {}, + settingsLayer, + // The racing caller wins the settings write while this one is + // still between `thread.create` and its own persist. + onDispatch: () => + settings + .updateSettingsWith((latest) => ({ + providerInstances: { + ...latest.providerInstances, + [INSTANCE_ID]: hermesInstance({ homeThreadId: HOME_THREAD_ID }), + }, + })) + .pipe(Effect.ignore), + }), + ), + ); + + // Both callers agree on the winner's thread, and the loser's own thread + // stays an empty thread in the agent project rather than a second Home. + assert.equal(resolved, HOME_THREAD_ID); + assert.equal(yield* getDesignatedHomeThreadId(INSTANCE_ID), HOME_THREAD_ID); + }).pipe(Effect.provide(settingsLayer)); + }), +); + +it.effect("recognizes any instance's home thread as undeletable", () => + Effect.gen(function* () { + const threads = yield* Ref.make>>([]); + const dispatched = yield* Ref.make>([]); + + yield* Effect.gen(function* () { + assert.isTrue(yield* isHomeThread(HOME_THREAD_ID)); + // Scans every instance, not just one — the delete path knows only a thread. + assert.isTrue(yield* isHomeThread(ThreadId.make("thread-home-2"))); + assert.isFalse(yield* isHomeThread(ThreadId.make("thread-ordinary"))); + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + providerInstances: { + [INSTANCE_ID]: hermesInstance({ homeThreadId: HOME_THREAD_ID }), + [OTHER_INSTANCE_ID]: hermesInstance({ homeThreadId: "thread-home-2" }), + }, + }), + ), + ); + }), +); + +it.effect("keeps an archived Home rather than minting a replacement", () => + Effect.gen(function* () { + // The bug this pins: `getThreadShellById` filters `archived_at IS NULL`, so + // an archived Home read as *deleted* and the self-healing path created a + // second one — stranding the real history and silently re-pointing the + // designation at an empty thread. + const threads = yield* Ref.make>>([ + Option.none(), + ]); + const dispatched = yield* Ref.make>([]); + + const resolved = yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + archiveStates: [Option.some({ archivedAt: "2026-01-02T00:00:00.000Z" })], + providerInstances: { + [INSTANCE_ID]: hermesInstance({ homeThreadId: HOME_THREAD_ID }), + }, + }), + ), + ); + + assert.equal(resolved, HOME_THREAD_ID); + const commands = yield* Ref.get(dispatched); + assert.isUndefined( + commands.find((command) => command.type === "thread.create"), + "an archived Home must never be replaced by a new thread", + ); + // A delivery is the agent raising its hand, so it brings the thread back + // rather than landing somewhere the user cannot see. + const unarchive = commands.find((command) => command.type === "thread.unarchive"); + assert.isDefined(unarchive); + if (unarchive?.type === "thread.unarchive") { + assert.equal(unarchive.threadId, HOME_THREAD_ID); + } + }), +); + +it.effect("does not un-archive a Home that was never archived", () => + Effect.gen(function* () { + const threads = yield* Ref.make>>([ + Option.some(threadShell(HOME_THREAD_ID)), + ]); + const dispatched = yield* Ref.make>([]); + + yield* getOrCreateHomeThread({ + instanceId: INSTANCE_ID, + title: "Hermes Siva Local", + }).pipe( + Effect.provide( + testLayer({ + threads, + dispatched, + archiveStates: [Option.some({ archivedAt: null })], + providerInstances: { + [INSTANCE_ID]: hermesInstance({ homeThreadId: HOME_THREAD_ID }), + }, + }), + ), + ); + + // The common path stays free of writes; this runs on every handshake. + assert.deepEqual(yield* Ref.get(dispatched), []); + }), +); diff --git a/apps/server/src/orchestration/homeThreads.ts b/apps/server/src/orchestration/homeThreads.ts new file mode 100644 index 00000000000..e4ac0b7cd49 --- /dev/null +++ b/apps/server/src/orchestration/homeThreads.ts @@ -0,0 +1,261 @@ +/** + * Home threads — the one thread per agent instance that receives its agent's + * proactive output. + * + * Hermes has a first-class "home channel" concept: a per-platform default + * destination for output nobody asked for at a specific address — a cron job's + * result, an agent-initiated `send_message`, a gateway online notice, a + * `/handoff`. Every other Hermes surface designates one interactively via + * `/sethome`. T3 designates one automatically instead, because there is no + * "current channel" here for a user to point at: threads are created on + * demand, and asking someone to pick one before their first cron job can fire + * is setup for a decision they have no basis to make. + * + * ## Converge-on-read, like agent projects + * + * `getOrCreateHomeThread` is a **precondition every caller runs**, not a + * lifecycle event one caller owns — the same reasoning as + * `getOrCreateAgentProject`, and for the same failure modes: an instance + * enrolled before this shipped has no designation, a dispatch that failed once + * would otherwise leave the instance permanently unable to receive, and a + * thread deleted out from under the designation would strand every future + * delivery. Each of those self-heals on the next handshake. + * + * ## Why settings and not a table + * + * The designation is per-instance enrollment metadata, exactly like + * `connectorUrl` and `revoked`, and it lives beside them in the Hermes + * instance's config blob. That keeps one source of truth for "facts about this + * enrollment" and avoids a migration for a single nullable id. + * + * The plugin's `T3_HOME_CHANNEL` env var is a *cache* of this value, reconciled + * on every `connection.accepted`. This module is authoritative; drift is + * bounded by one reconnect. + * + * @module orchestration/homeThreads + */ +import { + CommandId, + DEFAULT_HERMES_MODEL, + DEFAULT_PROVIDER_INTERACTION_MODE, + HERMES_DRIVER_KIND, + ThreadId, + type ProviderInstanceConfig, + type ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { ServerSettingsService } from "../serverSettings.ts"; +import { getOrCreateAgentProject } from "./agentProjects.ts"; +import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; + +/** + * Title of the home thread. Plain, no emoji: it sits in the same list as the + * user's own threads and should not shout. + */ +export const HOME_THREAD_TITLE = "Home"; + +/** Read the designated home thread id out of a Hermes instance envelope. */ +export const readHomeThreadId = ( + config: ProviderInstanceConfig | undefined, +): ThreadId | undefined => { + if (!config || config.driver !== HERMES_DRIVER_KIND) return undefined; + const blob = config.config && typeof config.config === "object" ? config.config : {}; + const raw = (blob as Record).homeThreadId; + return typeof raw === "string" && raw !== "" ? ThreadId.make(raw) : undefined; +}; + +/** + * Read one instance's designation straight from settings. + * + * Callers that only need to *know* the designation — the snapshot builder, the + * delete guard — use this rather than `getOrCreateHomeThread`, so a read never + * has the side effect of creating a thread. + */ +export const getDesignatedHomeThreadId = Effect.fn("getDesignatedHomeThreadId")(function* ( + instanceId: ProviderInstanceId, +) { + const settings = yield* ServerSettingsService; + const current = yield* settings.getSettings; + return readHomeThreadId(current.providerInstances[instanceId]); +}); + +/** + * Whether a thread is some instance's home thread, and therefore undeletable. + * + * Scans every configured instance rather than taking an instance id, because + * the delete path knows only the thread. The map is small — one entry per + * configured provider instance — and this runs once per delete. + */ +export const isHomeThread = Effect.fn("isHomeThread")(function* (threadId: ThreadId) { + const settings = yield* ServerSettingsService; + const current = yield* settings.getSettings; + for (const config of Object.values(current.providerInstances)) { + if (readHomeThreadId(config) === threadId) return true; + } + return false; +}); + +/** + * Read the designated thread's row regardless of archive state. + * + * The snapshot query's thread reads filter `archived_at IS NULL`, which is + * correct for "what should the user see" and wrong for "does this row exist". + * Going to the repository directly is what keeps an archived Home from looking + * deleted — and thereby keeps this module from minting a second one. A + * soft-deleted row still counts as gone. + */ +const readHomeThreadRow = Effect.fn("readHomeThreadRow")(function* (threadId: ThreadId) { + const query = yield* ProjectionSnapshotQuery; + const row = yield* query.getThreadArchiveStateById(threadId); + return Option.getOrUndefined(row); +}); + +/** + * Persist a designation into the instance's Hermes config blob. + * + * A compare-and-set against `expected` — the designation this caller read + * before deciding to create — rather than a blind write. Two concurrent + * callers that both mint a thread would otherwise each persist their own id, + * and the one that wrote first re-reads before the other's overwrite lands, so + * the two return *different* threads and one is orphaned from the final + * designation. Standing down when the designation moved out from under us is + * what makes the caller's re-read below an actual tiebreaker. + * + * `expected` still has to be honoured rather than "never overwrite": the + * self-healing path arrives here with a stale designation pointing at a thread + * that is really gone, and replacing exactly that value is the whole point. + */ +const persistHomeThreadId = (input: { + readonly instanceId: ProviderInstanceId; + readonly threadId: ThreadId; + /** Designation observed before creating, or `undefined` if there was none. */ + readonly expected: ThreadId | undefined; +}) => + Effect.gen(function* () { + const settings = yield* ServerSettingsService; + yield* settings.updateSettingsWith((latest) => { + // Re-read under the settings write lock: the envelope may have been + // replaced (or the driver changed) between our read and this write, and + // writing a Hermes designation onto a non-Hermes envelope would corrupt + // it. Returning an empty patch is the established no-op here. + const existing = latest.providerInstances[input.instanceId]; + if (!existing || existing.driver !== HERMES_DRIVER_KIND) return {}; + // Someone else designated (or re-designated) while we were creating. + // Adopt theirs; ours stays an empty thread in the agent project. + if (readHomeThreadId(existing) !== input.expected) return {}; + const currentConfig = + existing.config && typeof existing.config === "object" + ? (existing.config as Record) + : {}; + return { + providerInstances: { + ...latest.providerInstances, + [input.instanceId]: { + ...existing, + config: { ...currentConfig, homeThreadId: input.threadId }, + }, + }, + }; + }); + }); + +/** + * Resolve the home thread for one provider instance, creating it if absent. + * + * Safe to call on every handshake: the common path is one settings read plus + * one indexed thread read. Idempotent, and concurrent callers converge — + * the loser of a race re-reads and adopts the winner's thread. + */ +export const getOrCreateHomeThread = Effect.fn("getOrCreateHomeThread")(function* (input: { + readonly instanceId: ProviderInstanceId; + /** Instance nickname, used as the agent project's title on first creation. */ + readonly title: string; +}) { + const engine = yield* OrchestrationEngineService; + const crypto = yield* Crypto.Crypto; + + const designated = yield* getDesignatedHomeThreadId(input.instanceId); + if (designated !== undefined) { + // Deliberately NOT `getThreadShellById`: that query filters + // `archived_at IS NULL`, so an archived Home reads as absent and this + // function would "self-heal" by minting a second Home — stranding the + // history in the archived one and silently re-pointing the designation. + // Archiving is a user parking a thread, not destroying it; only a row + // that is really gone justifies a replacement. + const existing = yield* readHomeThreadRow(designated); + if (existing !== undefined) { + // Un-archive rather than deliver into a hidden thread. A delivery is + // the agent raising its hand, and the same rule the decider applies to + // settled/snoozed threads applies here: incoming activity brings the + // thread back. Best-effort — a failure here must not cost the delivery. + if (existing.archivedAt !== null) { + yield* engine + .dispatch({ + type: "thread.unarchive", + commandId: CommandId.make(yield* crypto.randomUUIDv4), + threadId: designated, + }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("could not un-archive the home thread", { + instanceId: input.instanceId, + threadId: designated, + cause: Cause.pretty(cause), + }), + ), + ); + } + return designated; + } + } + + const project = yield* getOrCreateAgentProject({ + instanceId: input.instanceId, + title: input.title, + }); + + const threadId = ThreadId.make(yield* crypto.randomUUIDv4); + const createdAt = DateTime.formatIso(yield* DateTime.now); + + const dispatched = yield* Effect.result( + engine.dispatch({ + type: "thread.create", + commandId: CommandId.make(yield* crypto.randomUUIDv4), + threadId, + projectId: project.id, + title: HOME_THREAD_TITLE, + // The same fixed slug every Hermes thread binds to — the reported model + // name is display-only, and following it here would orphan the thread + // whenever Hermes' own config changed. + modelSelection: { instanceId: input.instanceId, model: DEFAULT_HERMES_MODEL }, + runtimeMode: "full-access", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + branch: null, + worktreePath: null, + createdAt, + }), + ); + + if (dispatched._tag === "Failure") { + // A concurrent handshake may have won and already persisted its thread. + // Re-read before surfacing the error, matching getOrCreateAgentProject. + const raced = yield* getDesignatedHomeThreadId(input.instanceId); + if (raced !== undefined) return raced; + return yield* Effect.fail(dispatched.failure); + } + + yield* persistHomeThreadId({ instanceId: input.instanceId, threadId, expected: designated }); + + // Re-read rather than trusting our own write: if a racing caller persisted + // first, both callers must agree on one thread, and settings is the + // tiebreaker. Our orphaned thread stays as an empty thread in the project + // rather than becoming a second home. + const settled = yield* getDesignatedHomeThreadId(input.instanceId); + return settled ?? threadId; +}); diff --git a/apps/server/src/orchestration/projector.notification.test.ts b/apps/server/src/orchestration/projector.notification.test.ts new file mode 100644 index 00000000000..511128c0a98 --- /dev/null +++ b/apps/server/src/orchestration/projector.notification.test.ts @@ -0,0 +1,189 @@ +import { + CommandId, + EventId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationEvent, +} from "@t3tools/contracts"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; + +import { createEmptyReadModel, projectEvent } from "./projector.ts"; + +const CREATED_AT = "2026-07-01T09:00:00.000Z"; +const DELIVERED_AT = "2026-07-01T09:05:00.000Z"; + +function makeEvent(input: { + readonly sequence: number; + readonly type: OrchestrationEvent["type"]; + readonly occurredAt: string; + readonly payload: unknown; +}): OrchestrationEvent { + return { + sequence: input.sequence, + eventId: EventId.make(`event-${input.sequence}`), + type: input.type, + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-home"), + occurredAt: input.occurredAt, + commandId: CommandId.make(`command-${input.sequence}`), + causationEventId: null, + correlationId: null, + metadata: {}, + payload: input.payload as never, + } as OrchestrationEvent; +} + +const threadCreatedEvent = makeEvent({ + sequence: 1, + type: "thread.created", + occurredAt: CREATED_AT, + payload: { + threadId: ThreadId.make("thread-home"), + projectId: ProjectId.make("project-1"), + title: "Home", + modelSelection: { instanceId: ProviderInstanceId.make("hermes"), model: "hermes" }, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + }, +}); + +const deliveryEvent = makeEvent({ + sequence: 2, + type: "thread.notification-delivered", + occurredAt: DELIVERED_AT, + payload: { + threadId: ThreadId.make("thread-home"), + messageId: MessageId.make("notification:msg-1"), + text: "Digest ready.", + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }, + createdAt: DELIVERED_AT, + updatedAt: DELIVERED_AT, + }, +}); + +const expectedMessage = { + id: "notification:msg-1", + role: "assistant", + text: "Digest ready.", + // A delivery belongs to no turn and is never streamed. + turnId: null, + streaming: false, + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }, + createdAt: DELIVERED_AT, + updatedAt: DELIVERED_AT, +}; + +it.effect("appends a notification delivery as a turnless assistant message", () => + Effect.gen(function* () { + const created = yield* projectEvent(createEmptyReadModel(CREATED_AT), threadCreatedEvent); + const delivered = yield* projectEvent(created, deliveryEvent); + + const thread = delivered.threads[0]; + expect(thread?.messages).toHaveLength(1); + expect(thread?.messages[0]).toEqual(expectedMessage); + expect(thread?.updatedAt).toBe(DELIVERED_AT); + // Nothing about a delivery touches turn state — attributing it to the + // thread's live turn would fold it behind that turn's summary. + expect(thread?.latestTurn).toBeNull(); + }), +); + +it.effect("replays a delivery onto the same message instead of duplicating it", () => + Effect.gen(function* () { + const created = yield* projectEvent(createEmptyReadModel(CREATED_AT), threadCreatedEvent); + const delivered = yield* projectEvent(created, deliveryEvent); + // The decider re-emits the persisted row for a replayed deliveryId, so + // reducing the event twice must be a no-op rather than an append. + const replayed = yield* projectEvent(delivered, { ...deliveryEvent, sequence: 3 }); + + expect(replayed.threads[0]?.messages).toHaveLength(1); + expect(replayed.threads[0]?.messages[0]).toEqual(expectedMessage); + }), +); + +it.effect("projects media deliveries with attachments and an optional turnId", () => + Effect.gen(function* () { + const created = yield* projectEvent(createEmptyReadModel(CREATED_AT), threadCreatedEvent); + const attachment = { + type: "file", + id: "thread-home-2c8b3f1e-0000-4000-8000-000000000001", + name: "chart.mp4", + mimeType: "video/mp4", + sizeBytes: 2_048, + } as const; + const delivered = yield* projectEvent( + created, + makeEvent({ + sequence: 2, + type: "thread.notification-delivered", + occurredAt: DELIVERED_AT, + payload: { + threadId: ThreadId.make("thread-home"), + messageId: MessageId.make("notification:media-1"), + text: "A caption", + notification: { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-media", + }, + attachments: [attachment], + turnId: "turn-live", + createdAt: DELIVERED_AT, + updatedAt: DELIVERED_AT, + }, + }), + ); + + const message = delivered.threads[0]?.messages[0]; + expect(message?.attachments).toEqual([attachment]); + // Turn-scoped media names its turn so it sequences beside the turn's text. + expect(message?.turnId).toBe("turn-live"); + }), +); + +it.effect("keeps deliveries alongside ordinary turn messages", () => + Effect.gen(function* () { + const created = yield* projectEvent(createEmptyReadModel(CREATED_AT), threadCreatedEvent); + const withUserMessage = yield* projectEvent( + created, + makeEvent({ + sequence: 2, + type: "thread.message-sent", + occurredAt: "2026-07-01T09:01:00.000Z", + payload: { + threadId: ThreadId.make("thread-home"), + messageId: MessageId.make("msg-user"), + role: "user", + text: "hi", + turnId: null, + streaming: false, + createdAt: "2026-07-01T09:01:00.000Z", + updatedAt: "2026-07-01T09:01:00.000Z", + }, + }), + ); + const delivered = yield* projectEvent(withUserMessage, { ...deliveryEvent, sequence: 3 }); + + expect(delivered.threads[0]?.messages.map((message) => message.id)).toEqual([ + "msg-user", + "notification:msg-1", + ]); + // Absence of `notification` is what marks an ordinary message; it must + // never be synthesized onto one. + expect(delivered.threads[0]?.messages[0]?.notification).toBeUndefined(); + }), +); diff --git a/apps/server/src/orchestration/projector.test.ts b/apps/server/src/orchestration/projector.test.ts index 9c07a312023..cf6094d031f 100644 --- a/apps/server/src/orchestration/projector.test.ts +++ b/apps/server/src/orchestration/projector.test.ts @@ -7,6 +7,7 @@ import { type OrchestrationEvent, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import { it as effectIt } from "@effect/vitest"; import { describe, expect, it } from "vite-plus/test"; import { createEmptyReadModel, projectEvent } from "./projector.ts"; @@ -488,6 +489,83 @@ describe("orchestration projector", () => { expect(message?.updatedAt).toBe(completeAt); }); + effectIt.effect("replaces an assistant snapshot between legacy appended deltas", () => + Effect.gen(function* () { + const createdAt = "2026-02-23T09:00:00.000Z"; + const model = createEmptyReadModel(createdAt); + const afterCreate = yield* projectEvent( + model, + makeEvent({ + sequence: 1, + type: "thread.created", + aggregateKind: "thread", + aggregateId: "thread-1", + occurredAt: createdAt, + commandId: "cmd-create", + payload: { + threadId: "thread-1", + projectId: "project-1", + title: "demo", + modelSelection: { + provider: ProviderDriverKind.make("hermes"), + model: "hermes", + }, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt, + updatedAt: createdAt, + }, + }), + ); + + const messageEvent = (input: { + sequence: number; + text: string; + streaming: boolean; + textOperation?: "replace"; + }) => + makeEvent({ + sequence: input.sequence, + type: "thread.message-sent", + aggregateKind: "thread", + aggregateId: "thread-1", + occurredAt: createdAt, + commandId: `cmd-message-${input.sequence}`, + payload: { + threadId: "thread-1", + messageId: "assistant:msg-replace", + role: "assistant", + text: input.text, + turnId: "turn-1", + streaming: input.streaming, + ...(input.textOperation !== undefined ? { textOperation: input.textOperation } : {}), + createdAt, + updatedAt: createdAt, + }, + }); + + let projected = afterCreate; + for (const event of [ + messageEvent({ sequence: 2, text: "Hello wor", streaming: true }), + messageEvent({ + sequence: 3, + text: "Hello there", + streaming: true, + textOperation: "replace", + }), + messageEvent({ sequence: 4, text: "!", streaming: true }), + messageEvent({ sequence: 5, text: "", streaming: false }), + ]) { + projected = yield* projectEvent(projected, event); + } + + const message = projected.threads[0]?.messages[0]; + expect(message?.text).toBe("Hello there!"); + expect(message?.streaming).toBe(false); + }), + ); + it("prunes reverted turn messages from in-memory thread snapshot", async () => { const createdAt = "2026-02-23T10:00:00.000Z"; const model = createEmptyReadModel(createdAt); diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts index 0504cb36f9a..4bfa412d3f7 100644 --- a/apps/server/src/orchestration/projector.ts +++ b/apps/server/src/orchestration/projector.ts @@ -1,5 +1,6 @@ import type { OrchestrationEvent, OrchestrationReadModel, ThreadId } from "@t3tools/contracts"; import { + mergeAssistantMessageText, OrchestrationCheckpointSummary, OrchestrationMessage, OrchestrationSession, @@ -20,6 +21,7 @@ import { ThreadDeletedPayload, ThreadInteractionModeSetPayload, ThreadMetaUpdatedPayload, + ThreadNotificationDeliveredPayload, ThreadProposedPlanUpsertedPayload, ThreadRuntimeModeSetPayload, ThreadSettledPayload, @@ -212,6 +214,7 @@ export function projectEvent( workspaceRoot: payload.workspaceRoot, defaultModelSelection: payload.defaultModelSelection, scripts: payload.scripts, + agentInstanceId: payload.agentInstanceId, createdAt: payload.createdAt, updatedAt: payload.updatedAt, deletedAt: null, @@ -471,11 +474,13 @@ export function projectEvent( entry.id === message.id ? { ...entry, - text: message.streaming - ? `${entry.text}${message.text}` - : message.text.length > 0 - ? message.text - : entry.text, + text: mergeAssistantMessageText(entry.text, { + text: message.text, + streaming: message.streaming, + ...(payload.textOperation !== undefined + ? { textOperation: payload.textOperation } + : {}), + }), streaming: message.streaming, updatedAt: message.updatedAt, turnId: message.turnId, @@ -718,6 +723,57 @@ export function projectEvent( }), ); + case "thread.notification-delivered": + return Effect.gen(function* () { + const payload = yield* decodeForEvent( + ThreadNotificationDeliveredPayload, + event.payload, + event.type, + "payload", + ); + const thread = nextBase.threads.find((entry) => entry.id === payload.threadId); + if (!thread) { + return nextBase; + } + + const message: OrchestrationMessage = yield* decodeForEvent( + OrchestrationMessage, + { + id: payload.messageId, + role: "assistant", + text: payload.text, + // A proactive delivery belongs to no turn: it arrives outside the + // turn machinery and may land while an unrelated turn is live. + // Turn-scoped media is the exception — the payload names its turn. + turnId: payload.turnId ?? null, + streaming: false, + notification: payload.notification, + ...(payload.attachments !== undefined ? { attachments: payload.attachments } : {}), + createdAt: payload.createdAt, + updatedAt: payload.updatedAt, + }, + event.type, + "message", + ); + + // Replays of an already-written delivery re-emit the persisted row, so + // replacing in place is a byte-identical no-op rather than a duplicate. + const existingMessage = thread.messages.find((entry) => entry.id === message.id); + const messages = ( + existingMessage + ? thread.messages.map((entry) => (entry.id === message.id ? message : entry)) + : [...thread.messages, message] + ).slice(-MAX_THREAD_MESSAGES); + + return { + ...nextBase, + threads: updateThread(nextBase.threads, payload.threadId, { + messages, + updatedAt: event.occurredAt, + }), + }; + }); + case "thread.activity-appended": return decodeForEvent( ThreadActivityAppendedPayload, diff --git a/apps/server/src/persistence/Layers/ProjectionProjects.ts b/apps/server/src/persistence/Layers/ProjectionProjects.ts index c1ca6d3104e..0248f0159dd 100644 --- a/apps/server/src/persistence/Layers/ProjectionProjects.ts +++ b/apps/server/src/persistence/Layers/ProjectionProjects.ts @@ -36,6 +36,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { workspace_root, default_model_selection_json, scripts_json, + agent_instance_id, created_at, updated_at, deleted_at @@ -46,6 +47,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { ${row.workspaceRoot}, ${row.defaultModelSelection !== null ? JSON.stringify(row.defaultModelSelection) : null}, ${JSON.stringify(row.scripts)}, + ${row.agentInstanceId}, ${row.createdAt}, ${row.updatedAt}, ${row.deletedAt} @@ -56,6 +58,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { workspace_root = excluded.workspace_root, default_model_selection_json = excluded.default_model_selection_json, scripts_json = excluded.scripts_json, + agent_instance_id = excluded.agent_instance_id, created_at = excluded.created_at, updated_at = excluded.updated_at, deleted_at = excluded.deleted_at @@ -73,6 +76,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { workspace_root AS "workspaceRoot", default_model_selection_json AS "defaultModelSelection", scripts_json AS "scripts", + agent_instance_id AS "agentInstanceId", created_at AS "createdAt", updated_at AS "updatedAt", deleted_at AS "deletedAt" @@ -92,6 +96,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { workspace_root AS "workspaceRoot", default_model_selection_json AS "defaultModelSelection", scripts_json AS "scripts", + agent_instance_id AS "agentInstanceId", created_at AS "createdAt", updated_at AS "updatedAt", deleted_at AS "deletedAt" diff --git a/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts b/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts index 4763f565653..32744eb3043 100644 --- a/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts +++ b/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts @@ -34,6 +34,7 @@ projectionRepositoriesLayer("Projection repositories", (it) => { model: "gpt-5.4", }, scripts: [], + agentInstanceId: null, createdAt: "2026-03-24T00:00:00.000Z", updatedAt: "2026-03-24T00:00:00.000Z", deletedAt: null, @@ -99,6 +100,7 @@ projectionRepositoriesLayer("Projection repositories", (it) => { pendingApprovalCount: 0, pendingUserInputCount: 0, hasActionableProposedPlan: 0, + latestNotification: null, deletedAt: null, }); @@ -161,6 +163,7 @@ projectionRepositoriesLayer("Projection repositories", (it) => { pendingApprovalCount: 0, pendingUserInputCount: 0, hasActionableProposedPlan: 0, + latestNotification: null, deletedAt: null, }); diff --git a/apps/server/src/persistence/Layers/ProjectionThreadMessages.test.ts b/apps/server/src/persistence/Layers/ProjectionThreadMessages.test.ts index b1f394a9e57..d901c20b136 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadMessages.test.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadMessages.test.ts @@ -66,6 +66,80 @@ layer("ProjectionThreadMessageRepository", (it) => { }), ); + it.effect("round-trips notification provenance and preserves it across later upserts", () => + Effect.gen(function* () { + const repository = yield* ProjectionThreadMessageRepository; + const threadId = ThreadId.make("thread-notification"); + const messageId = MessageId.make("message-notification"); + const createdAt = "2026-07-01T09:00:00.000Z"; + const notification = { + kind: "cron" as const, + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }; + + yield* repository.upsert({ + messageId, + threadId, + turnId: null, + role: "assistant", + text: "Digest ready.", + notification, + isStreaming: false, + createdAt, + updatedAt: createdAt, + }); + + const rows = yield* repository.listByThreadId({ threadId }); + assert.equal(rows.length, 1); + assert.deepEqual(rows[0]?.notification, notification); + + // Same COALESCE rule as attachments: an unrelated update that omits the + // field must not erase a delivery's provenance. + yield* repository.upsert({ + messageId, + threadId, + turnId: null, + role: "assistant", + text: "Digest ready (edited).", + isStreaming: false, + createdAt, + updatedAt: "2026-07-01T09:00:02.000Z", + }); + + const rowById = yield* repository.getByMessageId({ messageId }); + assert.equal(rowById._tag, "Some"); + if (rowById._tag === "Some") { + assert.equal(rowById.value.text, "Digest ready (edited)."); + assert.deepEqual(rowById.value.notification, notification); + } + }), + ); + + it.effect("leaves notification absent on ordinary messages", () => + Effect.gen(function* () { + const repository = yield* ProjectionThreadMessageRepository; + const threadId = ThreadId.make("thread-ordinary-message"); + const messageId = MessageId.make("message-ordinary"); + + yield* repository.upsert({ + messageId, + threadId, + turnId: null, + role: "user", + text: "hello", + isStreaming: false, + createdAt: "2026-07-01T09:10:00.000Z", + updatedAt: "2026-07-01T09:10:00.000Z", + }); + + const rows = yield* repository.listByThreadId({ threadId }); + // Absent, not a synthesized default — consumers key on its absence to + // tell an ordinary message from a proactive delivery. + assert.equal(rows[0]?.notification, undefined); + }), + ); + it.effect("allows explicit attachment clearing with an empty array", () => Effect.gen(function* () { const repository = yield* ProjectionThreadMessageRepository; diff --git a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts index 71919166886..0b3ab859b0a 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts @@ -5,7 +5,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Struct from "effect/Struct"; -import { ChatAttachment } from "@t3tools/contracts"; +import { ChatAttachment, OrchestrationMessageNotification } from "@t3tools/contracts"; import { toPersistenceSqlError } from "../Errors.ts"; import { @@ -21,6 +21,7 @@ const ProjectionThreadMessageDbRowSchema = ProjectionThreadMessage.mapFields( Struct.assign({ isStreaming: Schema.Number, attachments: Schema.NullOr(Schema.fromJsonString(Schema.Array(ChatAttachment))), + notification: Schema.NullOr(Schema.fromJsonString(OrchestrationMessageNotification)), }), ); @@ -37,6 +38,7 @@ function toProjectionThreadMessage( createdAt: row.createdAt, updatedAt: row.updatedAt, ...(row.attachments !== null ? { attachments: row.attachments } : {}), + ...(row.notification !== null ? { notification: row.notification } : {}), }; } @@ -48,6 +50,8 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { execute: (row) => { const nextAttachmentsJson = row.attachments !== undefined ? JSON.stringify(row.attachments) : null; + const nextNotificationJson = + row.notification !== undefined ? JSON.stringify(row.notification) : null; return sql` INSERT INTO projection_thread_messages ( message_id, @@ -56,6 +60,7 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { role, text, attachments_json, + notification_json, is_streaming, created_at, updated_at @@ -74,6 +79,14 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { WHERE message_id = ${row.messageId} ) ), + COALESCE( + ${nextNotificationJson}, + ( + SELECT notification_json + FROM projection_thread_messages + WHERE message_id = ${row.messageId} + ) + ), ${row.isStreaming ? 1 : 0}, ${row.createdAt}, ${row.updatedAt} @@ -88,6 +101,10 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { excluded.attachments_json, projection_thread_messages.attachments_json ), + notification_json = COALESCE( + excluded.notification_json, + projection_thread_messages.notification_json + ), is_streaming = excluded.is_streaming, created_at = excluded.created_at, updated_at = excluded.updated_at @@ -107,6 +124,7 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { role, text, attachments_json AS "attachments", + notification_json AS "notification", is_streaming AS "isStreaming", created_at AS "createdAt", updated_at AS "updatedAt" @@ -128,6 +146,7 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { role, text, attachments_json AS "attachments", + notification_json AS "notification", is_streaming AS "isStreaming", created_at AS "createdAt", updated_at AS "updatedAt" diff --git a/apps/server/src/persistence/Layers/ProjectionThreads.ts b/apps/server/src/persistence/Layers/ProjectionThreads.ts index 7e86d49eac3..253634ead62 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreads.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreads.ts @@ -14,11 +14,14 @@ import { ProjectionThreadRepository, type ProjectionThreadRepositoryShape, } from "../Services/ProjectionThreads.ts"; -import { ModelSelection } from "@t3tools/contracts"; +import { ModelSelection, OrchestrationThreadNotificationSummary } from "@t3tools/contracts"; const ProjectionThreadDbRow = ProjectionThread.mapFields( Struct.assign({ modelSelection: Schema.fromJsonString(ModelSelection), + latestNotification: Schema.NullOr( + Schema.fromJsonString(OrchestrationThreadNotificationSummary), + ), }), ); type ProjectionThreadDbRow = typeof ProjectionThreadDbRow.Type; @@ -28,8 +31,10 @@ const makeProjectionThreadRepository = Effect.gen(function* () { const upsertProjectionThreadRow = SqlSchema.void({ Request: ProjectionThread, - execute: (row) => - sql` + execute: (row) => { + const latestNotificationJson = + row.latestNotification === null ? null : JSON.stringify(row.latestNotification); + return sql` INSERT INTO projection_threads ( thread_id, project_id, @@ -51,6 +56,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { pending_approval_count, pending_user_input_count, has_actionable_proposed_plan, + latest_notification_json, deleted_at ) VALUES ( @@ -74,6 +80,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { ${row.pendingApprovalCount}, ${row.pendingUserInputCount}, ${row.hasActionableProposedPlan}, + ${latestNotificationJson}, ${row.deletedAt} ) ON CONFLICT (thread_id) @@ -97,8 +104,10 @@ const makeProjectionThreadRepository = Effect.gen(function* () { pending_approval_count = excluded.pending_approval_count, pending_user_input_count = excluded.pending_user_input_count, has_actionable_proposed_plan = excluded.has_actionable_proposed_plan, + latest_notification_json = excluded.latest_notification_json, deleted_at = excluded.deleted_at - `, + `; + }, }); const getProjectionThreadRow = SqlSchema.findOneOption({ @@ -127,6 +136,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { pending_approval_count AS "pendingApprovalCount", pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", + latest_notification_json AS "latestNotification", deleted_at AS "deletedAt" FROM projection_threads WHERE thread_id = ${threadId} @@ -159,6 +169,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { pending_approval_count AS "pendingApprovalCount", pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", + latest_notification_json AS "latestNotification", deleted_at AS "deletedAt" FROM projection_threads WHERE project_id = ${projectId} diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index d25895671a9..6e2a7efff4d 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -47,6 +47,10 @@ import Migration0031 from "./Migrations/031_AuthAuthorizationScopes.ts"; import Migration0032 from "./Migrations/032_AuthPairingProofKeyThumbprint.ts"; import Migration0033 from "./Migrations/033_ProjectionThreadsSettled.ts"; import Migration0034 from "./Migrations/034_ProjectionThreadsSnoozed.ts"; +import Migration0035 from "./Migrations/035_ProjectionProjectsAgentInstance.ts"; +import Migration0036 from "./Migrations/036_ProjectionThreadMessageNotification.ts"; +import Migration0037 from "./Migrations/037_ProjectionThreadsLatestNotification.ts"; +import Migration0038 from "./Migrations/038_BackfillProjectionThreadsLatestTurn.ts"; /** * Migration loader with all migrations defined inline. @@ -93,6 +97,10 @@ export const migrationEntries = [ [32, "AuthPairingProofKeyThumbprint", Migration0032], [33, "ProjectionThreadsSettled", Migration0033], [34, "ProjectionThreadsSnoozed", Migration0034], + [35, "ProjectionProjectsAgentInstance", Migration0035], + [36, "ProjectionThreadMessageNotification", Migration0036], + [37, "ProjectionThreadsLatestNotification", Migration0037], + [38, "BackfillProjectionThreadsLatestTurn", Migration0038], ] as const; export const makeMigrationLoader = (throughId?: number) => diff --git a/apps/server/src/persistence/Migrations/035_ProjectionProjectsAgentInstance.test.ts b/apps/server/src/persistence/Migrations/035_ProjectionProjectsAgentInstance.test.ts new file mode 100644 index 00000000000..30fe3c59708 --- /dev/null +++ b/apps/server/src/persistence/Migrations/035_ProjectionProjectsAgentInstance.test.ts @@ -0,0 +1,65 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("035_ProjectionProjectsAgentInstance", (it) => { + it.effect("adds a nullable agent_instance_id column and its index", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* runMigrations({ toMigrationInclusive: 34 }); + + const before = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_projects) + `; + assert.ok(!before.some((column) => column.name === "agent_instance_id")); + + yield* runMigrations({ toMigrationInclusive: 35 }); + + const after = yield* sql<{ + readonly name: string; + readonly type: string; + readonly notnull: number; + readonly dflt_value: string | null; + }>` + PRAGMA table_info(projection_projects) + `; + const agentInstanceColumn = after.find((column) => column.name === "agent_instance_id"); + assert.ok(agentInstanceColumn); + assert.strictEqual(agentInstanceColumn.type, "TEXT"); + assert.strictEqual(agentInstanceColumn.notnull, 0); + assert.strictEqual(agentInstanceColumn.dflt_value, null); + + const indexes = yield* sql<{ + readonly seq: number; + readonly name: string; + readonly unique: number; + readonly origin: string; + readonly partial: number; + }>` + PRAGMA index_list(projection_projects) + `; + assert.ok( + indexes.some((index) => index.name === "idx_projection_projects_agent_instance_id"), + ); + + const indexColumns = yield* sql<{ + readonly seqno: number; + readonly cid: number; + readonly name: string; + }>` + PRAGMA index_info('idx_projection_projects_agent_instance_id') + `; + assert.deepStrictEqual( + indexColumns.map((column) => column.name), + ["agent_instance_id"], + ); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/035_ProjectionProjectsAgentInstance.ts b/apps/server/src/persistence/Migrations/035_ProjectionProjectsAgentInstance.ts new file mode 100644 index 00000000000..e9f1be0ae39 --- /dev/null +++ b/apps/server/src/persistence/Migrations/035_ProjectionProjectsAgentInstance.ts @@ -0,0 +1,21 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_projects) + `; + + if (!columns.some((column) => column.name === "agent_instance_id")) { + yield* sql` + ALTER TABLE projection_projects + ADD COLUMN agent_instance_id TEXT + `; + } + + yield* sql` + CREATE INDEX IF NOT EXISTS idx_projection_projects_agent_instance_id + ON projection_projects (agent_instance_id) + `; +}); diff --git a/apps/server/src/persistence/Migrations/036_ProjectionThreadMessageNotification.test.ts b/apps/server/src/persistence/Migrations/036_ProjectionThreadMessageNotification.test.ts new file mode 100644 index 00000000000..211c903ca08 --- /dev/null +++ b/apps/server/src/persistence/Migrations/036_ProjectionThreadMessageNotification.test.ts @@ -0,0 +1,129 @@ +import { assert, it } from "@effect/vitest"; +import { OrchestrationMessageNotification } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; +import Migration036 from "./036_ProjectionThreadMessageNotification.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("036_ProjectionThreadMessageNotification", (it) => { + it.effect("adds the notification column without disturbing existing messages", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* runMigrations({ toMigrationInclusive: 35 }); + + // An ordinary message written before the column existed. Proactive + // deliveries are the exception, so every pre-existing row must survive + // the migration with no notification rather than a synthesized default. + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + is_streaming, + created_at, + updated_at + ) VALUES ( + 'message-1', + 'thread-1', + 'turn-1', + 'assistant', + 'an ordinary answer', + 0, + '2026-01-01T00:00:00.000Z', + '2026-01-01T00:00:00.000Z' + ) + `; + + yield* runMigrations({ toMigrationInclusive: 36 }); + + const rows = yield* sql<{ + readonly message_id: string; + readonly text: string; + readonly notification_json: string | null; + }>` + SELECT message_id, text, notification_json + FROM projection_thread_messages + WHERE message_id = 'message-1' + `; + + assert.equal(rows.length, 1); + assert.equal(rows[0]?.text, "an ordinary answer"); + assert.equal(rows[0]?.notification_json, null); + + // And the new column accepts a delivery's provenance. + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + is_streaming, + created_at, + updated_at, + notification_json + ) VALUES ( + 'message-2', + 'thread-1', + NULL, + 'assistant', + 'Your digest is ready.', + 0, + '2026-01-01T00:01:00.000Z', + '2026-01-01T00:01:00.000Z', + '{"kind":"cron","label":"Cron: daily-digest","deliveryId":"delivery-1"}' + ) + `; + + const delivered = yield* sql<{ + readonly notification_json: string | null; + readonly turn_id: string | null; + }>` + SELECT notification_json, turn_id + FROM projection_thread_messages + WHERE message_id = 'message-2' + `; + + assert.equal(delivered.length, 1); + // A delivery belongs to no turn — attributing it to the thread's live + // turn would fold it away behind that turn's "Worked for …" row. + assert.equal(delivered[0]?.turn_id, null); + const decodedNotification = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(OrchestrationMessageNotification), + )(delivered[0]?.notification_json ?? ""); + assert.deepEqual(decodedNotification, { + kind: "cron", + label: "Cron: daily-digest", + deliveryId: "delivery-1", + }); + }), + ); + + it.effect("is idempotent: re-running against a migrated database is a no-op", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // The layer shares one in-memory database across this file's tests, so + // the schema may already be at 36 here; make sure it is, then apply the + // migration effect again directly — the migrator's tracking table would + // otherwise skip it, which is exactly the protection a restored backup + // or divergent branch does not have. + yield* runMigrations({ toMigrationInclusive: 36 }); + yield* Migration036; + + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_thread_messages) + `; + assert.equal(columns.filter((column) => column.name === "notification_json").length, 1); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/036_ProjectionThreadMessageNotification.ts b/apps/server/src/persistence/Migrations/036_ProjectionThreadMessageNotification.ts new file mode 100644 index 00000000000..ae7372a4129 --- /dev/null +++ b/apps/server/src/persistence/Migrations/036_ProjectionThreadMessageNotification.ts @@ -0,0 +1,16 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as Effect from "effect/Effect"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_thread_messages) + `; + + if (!columns.some((column) => column.name === "notification_json")) { + yield* sql` + ALTER TABLE projection_thread_messages + ADD COLUMN notification_json TEXT + `; + } +}); diff --git a/apps/server/src/persistence/Migrations/037_ProjectionThreadsLatestNotification.test.ts b/apps/server/src/persistence/Migrations/037_ProjectionThreadsLatestNotification.test.ts new file mode 100644 index 00000000000..4a191d6df6a --- /dev/null +++ b/apps/server/src/persistence/Migrations/037_ProjectionThreadsLatestNotification.test.ts @@ -0,0 +1,103 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; +import Migration037 from "./037_ProjectionThreadsLatestNotification.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("037_ProjectionThreadsLatestNotification", (it) => { + it.effect("adds a nullable latest_notification_json column, leaving existing rows null", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* runMigrations({ toMigrationInclusive: 36 }); + + const before = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + assert.ok(!before.some((column) => column.name === "latest_notification_json")); + + yield* sql` + INSERT INTO projection_threads ( + thread_id, + project_id, + title, + model_selection_json, + runtime_mode, + interaction_mode, + branch, + worktree_path, + latest_turn_id, + created_at, + updated_at, + deleted_at + ) + VALUES ( + 'thread-pre-037', + 'project-pre-037', + 'Existing thread', + '{"instanceId":"codex","model":"gpt-5.4"}', + 'full-access', + 'default', + NULL, + NULL, + NULL, + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + NULL + ) + `; + + yield* runMigrations({ toMigrationInclusive: 37 }); + + const after = yield* sql<{ + readonly name: string; + readonly type: string; + readonly notnull: number; + readonly dflt_value: string | null; + }>` + PRAGMA table_info(projection_threads) + `; + const column = after.find((entry) => entry.name === "latest_notification_json"); + assert.ok(column); + assert.strictEqual(column.type, "TEXT"); + assert.strictEqual(column.notnull, 0); + assert.strictEqual(column.dflt_value, null); + + // No backfill is needed or possible: notification_json (migration 036) + // is newer than every pre-existing row, so nothing to summarize. + const rows = yield* sql<{ readonly latestNotification: string | null }>` + SELECT latest_notification_json AS "latestNotification" + FROM projection_threads + WHERE thread_id = 'thread-pre-037' + `; + assert.deepStrictEqual(rows, [{ latestNotification: null }]); + }), + ); + + it.effect("is idempotent: re-running against a migrated database is a no-op", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // The layer shares one in-memory database across this file's tests, so + // the schema may already be at 37 here; make sure it is, then apply the + // migration effect again directly — the migrator's tracking table would + // otherwise skip it, which is exactly the protection a restored backup + // or divergent branch does not have. + yield* runMigrations({ toMigrationInclusive: 37 }); + yield* Migration037; + + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + assert.equal( + columns.filter((column) => column.name === "latest_notification_json").length, + 1, + ); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/037_ProjectionThreadsLatestNotification.ts b/apps/server/src/persistence/Migrations/037_ProjectionThreadsLatestNotification.ts new file mode 100644 index 00000000000..44bbc50ccbe --- /dev/null +++ b/apps/server/src/persistence/Migrations/037_ProjectionThreadsLatestNotification.ts @@ -0,0 +1,27 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as Effect from "effect/Effect"; + +/** + * Materializes the thread's newest proactive delivery onto the shell row. + * + * Shell consumers (sidebar, agent-awareness/push) never read a thread's + * messages, so notification provenance that lives on the message row is + * invisible to them unless it is summarized here — the same reason + * `latest_user_message_at` exists. + * + * No backfill: `projection_thread_messages.notification_json` is introduced by + * migration 036, so no delivery predates this column. + */ +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + + if (!columns.some((column) => column.name === "latest_notification_json")) { + yield* sql` + ALTER TABLE projection_threads + ADD COLUMN latest_notification_json TEXT + `; + } +}); diff --git a/apps/server/src/persistence/Migrations/038_BackfillProjectionThreadsLatestTurn.test.ts b/apps/server/src/persistence/Migrations/038_BackfillProjectionThreadsLatestTurn.test.ts new file mode 100644 index 00000000000..c86b204b7ac --- /dev/null +++ b/apps/server/src/persistence/Migrations/038_BackfillProjectionThreadsLatestTurn.test.ts @@ -0,0 +1,164 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +const insertThread = (sql: SqlClient.SqlClient, threadId: string, latestTurnId: string | null) => + sql` + INSERT INTO projection_threads ( + thread_id, + project_id, + title, + model_selection_json, + runtime_mode, + interaction_mode, + branch, + worktree_path, + latest_turn_id, + created_at, + updated_at, + deleted_at + ) + VALUES ( + ${threadId}, + 'project-038', + 'Thread', + '{"instanceId":"hermes-local","model":"hermes"}', + 'full-access', + 'default', + NULL, + NULL, + ${latestTurnId}, + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + NULL + ) + `; + +const insertTurn = ( + sql: SqlClient.SqlClient, + threadId: string, + turnId: string, + requestedAt: string, +) => + sql` + INSERT INTO projection_turns ( + thread_id, + turn_id, + pending_message_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_files_json + ) + VALUES ( + ${threadId}, + ${turnId}, + NULL, + NULL, + 'completed', + ${requestedAt}, + ${requestedAt}, + ${requestedAt}, + '[]' + ) + `; + +const insertPendingTurnStart = (sql: SqlClient.SqlClient, threadId: string, requestedAt: string) => + sql` + INSERT INTO projection_turns ( + thread_id, + turn_id, + pending_message_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_files_json + ) + VALUES ( + ${threadId}, + NULL, + 'pending-message-038', + NULL, + 'pending', + ${requestedAt}, + NULL, + NULL, + '[]' + ) + `; + +const readLatestTurnId = (sql: SqlClient.SqlClient, threadId: string) => + sql<{ readonly latestTurnId: string | null }>` + SELECT latest_turn_id AS "latestTurnId" + FROM projection_threads + WHERE thread_id = ${threadId} + `; + +layer("038_BackfillProjectionThreadsLatestTurn", (it) => { + // A single test on purpose: `it.layer` shares one in-memory database across + // the block, so a second `it.effect` would find the migration already run + // and assert against a no-op. All fixtures are written between migration 37 + // and 38 so every scenario exercises the actual backfill. + it.effect("backfills nulled pointers without touching set or turnless ones", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 37 }); + + // Pointer nulled at turn end (the Hermes case): restore the newest turn. + yield* insertThread(sql, "thread-hermes", null); + yield* insertTurn(sql, "thread-hermes", "hermes-turn-old", "2026-07-01T00:00:00.000Z"); + yield* insertTurn(sql, "thread-hermes", "hermes-turn-new", "2026-07-01T00:05:00.000Z"); + + // A correct pointer must survive: the backfill is a heuristic, and the + // projector's own value is authoritative wherever it exists. + yield* insertThread(sql, "thread-codex", "turn-kept"); + yield* insertTurn(sql, "thread-codex", "turn-kept", "2026-07-01T00:00:00.000Z"); + yield* insertTurn(sql, "thread-codex", "turn-newer", "2026-07-01T00:09:00.000Z"); + + // A requested-but-never-adopted turn leaves a `turn_id IS NULL` row, + // and on the stranded threads this migration targets it is often the + // NEWEST row. Selecting it would write NULL over NULL — silently + // skipping the repair of the completed turn underneath. + yield* insertThread(sql, "thread-dangling", null); + yield* insertTurn(sql, "thread-dangling", "turn-completed", "2026-07-01T00:00:00.000Z"); + yield* insertPendingTurnStart(sql, "thread-dangling", "2026-07-01T00:05:00.000Z"); + + // Inventing a turn for a thread that never ran one would make a fresh + // thread look like it had already run one. + yield* insertThread(sql, "thread-empty", null); + + // A thread whose ONLY row is a dangling pending start has no turn to + // restore — it must stay null rather than "repair" to null noisily. + yield* insertThread(sql, "thread-pending-only", null); + yield* insertPendingTurnStart(sql, "thread-pending-only", "2026-07-01T00:05:00.000Z"); + + yield* runMigrations({ toMigrationInclusive: 38 }); + + assert.deepStrictEqual(yield* readLatestTurnId(sql, "thread-hermes"), [ + { latestTurnId: "hermes-turn-new" }, + ]); + assert.deepStrictEqual(yield* readLatestTurnId(sql, "thread-codex"), [ + { latestTurnId: "turn-kept" }, + ]); + assert.deepStrictEqual(yield* readLatestTurnId(sql, "thread-dangling"), [ + { latestTurnId: "turn-completed" }, + ]); + assert.deepStrictEqual(yield* readLatestTurnId(sql, "thread-empty"), [ + { latestTurnId: null }, + ]); + assert.deepStrictEqual(yield* readLatestTurnId(sql, "thread-pending-only"), [ + { latestTurnId: null }, + ]); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/038_BackfillProjectionThreadsLatestTurn.ts b/apps/server/src/persistence/Migrations/038_BackfillProjectionThreadsLatestTurn.ts new file mode 100644 index 00000000000..c4aff9d8c55 --- /dev/null +++ b/apps/server/src/persistence/Migrations/038_BackfillProjectionThreadsLatestTurn.ts @@ -0,0 +1,58 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as Effect from "effect/Effect"; + +/** + * Repairs `projection_threads.latest_turn_id` rows that were nulled at turn end. + * + * The `thread.session-set` projector wrote `session.activeTurnId` straight + * through, so the session going back to `ready` (activeTurnId `null`) at the + * end of a turn erased the pointer to the turn that had just finished. For + * workspace providers that damage was invisible: `thread.turn-diff-completed` + * fires moments later and re-writes the id. A provider with no workspace + * (Hermes) emits no diff, so the id stayed null permanently — `latestTurn` + * never resolved, and the settle guard read every completed turn as a + * queued-but-unadopted turn start, refusing to settle the thread for the + * length of the adoption grace window. + * + * The projector no longer nulls the field, but rows already written keep the + * stale null: projections resume from their stored sequence rather than + * replaying, so nothing re-derives them. + * + * Restores the newest turn per thread by `requested_at`, matching what the + * projector would have retained. Deliberately conservative: + * + * - Only threads whose `latest_turn_id` is currently null are touched, so a + * correct pointer is never overwritten by this heuristic. + * - Only threads that actually have a turn row are updated; the `WHERE + * EXISTS` keeps a thread that has never run a turn at null rather than + * inventing one. + * - Ties on `requested_at` (possible when a turn is steered within the same + * millisecond) break on `row_id`, which is monotonic per insert. + * - Rows with a null `turn_id` are skipped: those are pending turn-starts + * (requested but never adopted), and a dangling one is often the *newest* + * row on exactly the threads this bug stranded. Without the filter the + * subquery would select it and write NULL over NULL — silently skipping + * the repair of the completed turn underneath. + */ +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* sql` + UPDATE projection_threads + SET latest_turn_id = ( + SELECT turns.turn_id + FROM projection_turns turns + WHERE turns.thread_id = projection_threads.thread_id + AND turns.turn_id IS NOT NULL + ORDER BY turns.requested_at DESC, turns.row_id DESC + LIMIT 1 + ) + WHERE projection_threads.latest_turn_id IS NULL + AND EXISTS ( + SELECT 1 + FROM projection_turns turns + WHERE turns.thread_id = projection_threads.thread_id + AND turns.turn_id IS NOT NULL + ) + `; +}); diff --git a/apps/server/src/persistence/Services/ProjectionProjects.ts b/apps/server/src/persistence/Services/ProjectionProjects.ts index 5632205a269..55c161dec7c 100644 --- a/apps/server/src/persistence/Services/ProjectionProjects.ts +++ b/apps/server/src/persistence/Services/ProjectionProjects.ts @@ -6,7 +6,13 @@ * * @module ProjectionProjectRepository */ -import { IsoDateTime, ModelSelection, ProjectId, ProjectScript } from "@t3tools/contracts"; +import { + IsoDateTime, + ModelSelection, + ProjectId, + ProjectScript, + ProviderInstanceId, +} from "@t3tools/contracts"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Context from "effect/Context"; @@ -20,6 +26,12 @@ export const ProjectionProject = Schema.Struct({ workspaceRoot: Schema.String, defaultModelSelection: Schema.NullOr(ModelSelection), scripts: Schema.Array(ProjectScript), + /** + * Set for *synthetic agent projects* standing in for a provider instance + * with no workspace of its own; `null` for every real project. See + * `OrchestrationProject.agentInstanceId`. + */ + agentInstanceId: Schema.NullOr(ProviderInstanceId), createdAt: IsoDateTime, updatedAt: IsoDateTime, deletedAt: Schema.NullOr(IsoDateTime), diff --git a/apps/server/src/persistence/Services/ProjectionThreadMessages.ts b/apps/server/src/persistence/Services/ProjectionThreadMessages.ts index d50ff320256..970eda3f495 100644 --- a/apps/server/src/persistence/Services/ProjectionThreadMessages.ts +++ b/apps/server/src/persistence/Services/ProjectionThreadMessages.ts @@ -9,6 +9,7 @@ import { ChatAttachment, MessageId, + OrchestrationMessageNotification, OrchestrationMessageRole, ThreadId, TurnId, @@ -28,6 +29,14 @@ export const ProjectionThreadMessage = Schema.Struct({ role: OrchestrationMessageRole, text: Schema.String, attachments: Schema.optional(Schema.Array(ChatAttachment)), + /** + * Proactive-delivery provenance, present only on notification rows. + * + * Optional like `attachments`, and persisted with the same + * `COALESCE(excluded, existing)` upsert rule: an ordinary update that omits + * it must not erase the provenance of a row that has it. + */ + notification: Schema.optional(OrchestrationMessageNotification), isStreaming: Schema.Boolean, createdAt: IsoDateTime, updatedAt: IsoDateTime, diff --git a/apps/server/src/persistence/Services/ProjectionThreads.ts b/apps/server/src/persistence/Services/ProjectionThreads.ts index 056425ae886..b98c4271a0f 100644 --- a/apps/server/src/persistence/Services/ProjectionThreads.ts +++ b/apps/server/src/persistence/Services/ProjectionThreads.ts @@ -10,6 +10,7 @@ import { IsoDateTime, ModelSelection, NonNegativeInt, + OrchestrationThreadNotificationSummary, ProjectId, ProviderInteractionMode, RuntimeMode, @@ -44,6 +45,12 @@ export const ProjectionThread = Schema.Struct({ pendingApprovalCount: NonNegativeInt, pendingUserInputCount: NonNegativeInt, hasActionableProposedPlan: NonNegativeInt, + /** + * Newest proactive delivery on this thread, summarized so shell readers do + * not have to load messages. Refreshed alongside the other shell-summary + * columns; `null` on every thread that has never received a delivery. + */ + latestNotification: Schema.NullOr(OrchestrationThreadNotificationSummary), deletedAt: Schema.NullOr(IsoDateTime), }); export type ProjectionThread = typeof ProjectionThread.Type; diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts index 15612908079..fe692059041 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts @@ -19,6 +19,7 @@ const makeProject = (scripts: OrchestrationProject["scripts"]): OrchestrationPro workspaceRoot: "/repo/project", defaultModelSelection: null, scripts, + agentInstanceId: null, createdAt: "2026-01-01T00:00:00.000Z", updatedAt: "2026-01-01T00:00:00.000Z", deletedAt: null, @@ -36,12 +37,15 @@ const makeProjectionSnapshotQueryLayer = (project: OrchestrationProject) => Effect.succeed( workspaceRoot === project.workspaceRoot ? Option.some(project) : Option.none(), ), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: (projectId) => Effect.succeed(projectId === project.id ? Option.some(project) : Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.die("unused"), getThreadCheckpointContext: () => Effect.die("unused"), getFullThreadDiffContext: () => Effect.die("unused"), getThreadShellById: () => Effect.die("unused"), + getThreadArchiveStateById: () => Effect.die("unused"), + hasThreadNotificationDelivery: () => Effect.die("unused"), getThreadDetailById: () => Effect.die("unused"), getThreadDetailSnapshot: () => Effect.die("unused"), }); diff --git a/apps/server/src/provider/Drivers/HermesDriver.ts b/apps/server/src/provider/Drivers/HermesDriver.ts new file mode 100644 index 00000000000..3dce0f83276 --- /dev/null +++ b/apps/server/src/provider/Drivers/HermesDriver.ts @@ -0,0 +1,166 @@ +import { + DEFAULT_HERMES_MODEL, + HERMES_DRIVER_KIND, + HermesSettings, + TextGenerationError, + type ServerProvider, + type ServerSettings, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../../config.ts"; +import { readHomeThreadId } from "../../orchestration/homeThreads.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; +import type { TextGenerationShape } from "../../textGeneration/TextGeneration.ts"; +import { makeHermesAdapter } from "../Layers/HermesAdapter.ts"; +import { + defaultProviderContinuationIdentity, + type ProviderDriver, + type ProviderInstance, +} from "../ProviderDriver.ts"; +import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; +import { HermesGatewayBroker } from "../Services/HermesGatewayBroker.ts"; + +const decodeHermesSettings = Schema.decodeSync(HermesSettings); + +export type HermesDriverEnv = + | Crypto.Crypto + | FileSystem.FileSystem + | ServerConfig + | ServerSettingsService; + +const unsupportedTextGeneration = ( + operation: + | "generateCommitMessage" + | "generatePrContent" + | "generateBranchName" + | "generateThreadTitle", +) => + Effect.fail( + new TextGenerationError({ + operation, + detail: "Hermes gateway instances do not support utility text generation.", + }), + ); + +const makeTextGeneration = (): TextGenerationShape => ({ + generateCommitMessage: () => unsupportedTextGeneration("generateCommitMessage"), + generatePrContent: () => unsupportedTextGeneration("generatePrContent"), + generateBranchName: () => unsupportedTextGeneration("generateBranchName"), + generateThreadTitle: () => unsupportedTextGeneration("generateThreadTitle"), +}); + +export const HermesDriver: ProviderDriver = { + driverKind: HERMES_DRIVER_KIND, + metadata: { + displayName: "Hermes", + supportsMultipleInstances: true, + }, + configSchema: HermesSettings, + defaultConfig: () => decodeHermesSettings({}), + create: ({ instanceId, displayName, accentColor, enabled }) => + Effect.gen(function* () { + const broker = yield* HermesGatewayBroker; + // Captured once at construction: `getSnapshot` must be context-free + // (`R = never`) because the registry calls it outside this scope. + const settings = yield* ServerSettingsService; + const adapter = yield* makeHermesAdapter({ instanceId }); + const continuationIdentity = defaultProviderContinuationIdentity({ + driverKind: HERMES_DRIVER_KIND, + instanceId, + }); + const maintenanceCapabilities = makeManualOnlyProviderMaintenanceCapabilities({ + provider: HERMES_DRIVER_KIND, + packageName: null, + }); + const getSnapshot = Effect.gen(function* () { + const connected = yield* broker.isConnected(instanceId); + const status = yield* broker + .getInstanceStatus(instanceId) + .pipe( + Effect.catchTags({ HermesGatewayManagementError: () => Effect.succeed(undefined) }), + ); + // Read-only: a snapshot must never create the thread as a side effect + // (this runs on every status tick). The handshake owns creation. + // A settings read that fails degrades to "no designation" rather than + // failing the whole snapshot — the pin is cosmetic, the status is not. + const currentSettings = yield* settings.getSettings.pipe( + Effect.map(Option.some), + Effect.orElseSucceed(() => Option.none()), + ); + const homeThreadId = Option.isSome(currentSettings) + ? readHomeThreadId(currentSettings.value.providerInstances[instanceId]) + : undefined; + return { + ...(homeThreadId !== undefined ? { homeThreadId } : {}), + instanceId, + driver: HERMES_DRIVER_KIND, + ...(displayName ? { displayName } : {}), + ...(accentColor ? { accentColor } : {}), + continuation: { groupKey: continuationIdentity.continuationKey }, + showInteractionModeToggle: false, + requiresNewThreadForModelChange: true, + requiresWorkspace: false, + enabled, + installed: true, + version: status?.hermesVersion ?? null, + status: !enabled ? "disabled" : connected ? "ready" : "warning", + auth: { + status: connected ? "authenticated" : "unauthenticated", + type: "gateway", + label: status?.nickname ?? displayName ?? "Hermes", + }, + checkedAt: DateTime.formatIso(DateTime.nowUnsafe()), + ...(!connected && enabled + ? { message: "Hermes is offline. Reconnect its T3 Code gateway plugin." } + : {}), + availability: "available", + // The slug stays `DEFAULT_HERMES_MODEL` no matter what the plugin + // reports: threads bind to it, and letting it follow Hermes' current + // config would orphan every thread whose model changed on the Hermes + // side. The reported model is used only as the human-facing name, so + // the picker says "gpt-5.6-terra" instead of a generic placeholder. + // Falls back to "Hermes" when no plugin has connected yet or the + // plugin predates the `model` field on `connection.hello`. + models: [ + { + slug: DEFAULT_HERMES_MODEL, + name: status?.model ?? "Hermes", + isCustom: false, + isDefault: true, + capabilities: null, + }, + ], + slashCommands: [], + skills: [], + } satisfies ServerProvider; + }); + const snapshot = { + maintenanceCapabilities, + getSnapshot, + refresh: getSnapshot, + streamChanges: broker.streamStatuses.pipe( + Stream.filter((status) => status.instanceId === instanceId), + Stream.mapEffect(() => getSnapshot), + ), + }; + + return { + instanceId, + driverKind: HERMES_DRIVER_KIND, + continuationIdentity, + displayName, + accentColor, + enabled, + snapshot, + adapter, + textGeneration: makeTextGeneration(), + } satisfies ProviderInstance; + }), +}; diff --git a/apps/server/src/provider/Layers/HermesAdapter.test.ts b/apps/server/src/provider/Layers/HermesAdapter.test.ts new file mode 100644 index 00000000000..a7956e4d6b7 --- /dev/null +++ b/apps/server/src/provider/Layers/HermesAdapter.test.ts @@ -0,0 +1,1614 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + ApprovalRequestId, + HERMES_GATEWAY_PROTOCOL_VERSION, + HERMES_MEDIA_MAX_BYTES, + HermesGatewayItemId, + HermesGatewayRequestId, + HermesGatewaySessionId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + TurnId, + type ChatAttachment, + type HermesGatewayInstanceStatus, + type HermesGatewayT3ToPluginMessage, + type ProviderInstanceDescription, + type ProviderRuntimeEvent, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; + +import { createAttachmentId, resolveAttachmentPath } from "../../attachmentStore.ts"; +import * as ServerConfig from "../../config.ts"; +import { ProviderAdapterRequestError } from "../Errors.ts"; +import { + HermesGatewayBroker, + type HermesGatewayBrokerShape, + type HermesGatewayEnvelope, +} from "../Services/HermesGatewayBroker.ts"; +import { + makeHermesAdapter, + sanitizeHermesItemData, + sanitizeHermesRequestArgs, +} from "./HermesAdapter.ts"; + +// The adapter reads attachment bytes from `attachmentsDir`, so every test +// needs a ServerConfig alongside the platform services. +const testEnvLayer = ServerConfig.layerTest("/tmp/hermes-adapter-test", "/tmp").pipe( + Layer.provideMerge(NodeServices.layer), +); + +/** + * The reconnect trigger is a broker status transition, so these tests drive the + * broker's two streams directly and never touch a real socket or a clock. + */ +let generationCounter = 0; +const nextGeneration = () => { + generationCounter += 1; + return generationCounter; +}; +const statusFor = ( + instanceId: ProviderInstanceId, + status: HermesGatewayInstanceStatus["status"], + // Defaults to a fresh generation per connected status so a test that just + // wants "it reconnected" gets one; pass an explicit value to model a + // republish of the SAME connection (session counts, pings), which must not + // re-trigger the resume. + connectionGeneration: number | null = status === "connected" ? nextGeneration() : null, +): HermesGatewayInstanceStatus => ({ + instanceId, + nickname: "Hermes", + status, + connectorUrl: "wss://hermes.example/gateway", + lastConnectedAt: null, + pluginVersion: null, + hermesVersion: null, + model: null, + activeSessionCount: 0, + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + capabilities: null, + connectionGeneration, +}); + +const collectEvents = (adapter: { readonly streamEvents: Stream.Stream }) => + Effect.gen(function* () { + const seen: Array = []; + const fiber = yield* Stream.runForEach(adapter.streamEvents, (event) => + Effect.sync(() => { + seen.push(event); + }), + ).pipe(Effect.forkChild({ startImmediately: true })); + return { seen, fiber } as const; + }); + +/** + * Fibers settle by explicit rescheduling, never by sleeping: a sleep would make + * these tests time-dependent and flaky under load. + */ +const drain = Effect.gen(function* () { + for (let attempt = 0; attempt < 12; attempt += 1) { + yield* Effect.yieldNow; + } +}); + +it("sanitizes persisted gateway payloads", () => { + assert.deepEqual( + sanitizeHermesItemData("command_execution", { + command: "git status", + cwd: "/workspace", + arbitrarySecret: "drop-me", + result: { unbounded: true }, + }), + { command: "git status", cwd: "/workspace" }, + ); + assert.deepEqual( + sanitizeHermesRequestArgs("command_execution_approval", { + command: "git push", + arbitrarySecret: "drop-me", + }), + { command: "git push" }, + ); +}); + +it.effect("keeps cwd local while forwarding turn text byte-for-byte and steering follow-ups", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_remote"); + const threadId = ThreadId.make("thread-1"); + const sessionId = HermesGatewaySessionId.make("session-1"); + const sent: Array = []; + const brokerEvents = yield* PubSub.unbounded(); + const broker: HermesGatewayBrokerShape = { + createEnrollment: () => Effect.die(new Error("unused")), + getInstanceStatus: () => Effect.die(new Error("unused")), + listInstances: Effect.succeed([]), + renameInstance: () => Effect.die(new Error("unused")), + revokeInstance: () => Effect.die(new Error("unused")), + removeInstance: () => Effect.die(new Error("unused")), + registerConnection: () => Effect.die(new Error("unused")), + receive: () => Effect.void, + disconnect: () => Effect.void, + request: (_instanceId, message) => { + sent.push(message); + if (message.type === "session.ensure") { + return Effect.succeed({ + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId, + resumed: false, + }); + } + if (message.type === "turn.start" || message.type === "turn.steer") { + return Effect.succeed({ + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: message.sessionId, + turnId: message.turnId, + }); + } + return Effect.die(new Error(`unexpected request ${message.type}`)); + }, + send: (_instanceId, message) => Effect.sync(() => sent.push(message)).pipe(Effect.asVoid), + isConnected: () => Effect.succeed(true), + stream: Stream.fromPubSub(brokerEvents), + streamStatuses: Stream.empty, + }; + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + yield* Effect.yieldNow; + const firstEventAfterOrphanExitFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "session.exited", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId: HermesGatewaySessionId.make("session-before-context"), + recoverable: false, + }, + }); + const session = yield* adapter.startSession({ + threadId, + providerInstanceId: instanceId, + cwd: "/must/not/leak", + runtimeMode: "full-access", + }); + assert.equal(session.cwd, "/must/not/leak"); + const sessionEnsure = sent.find((message) => message.type === "session.ensure"); + assert.isFalse(sessionEnsure !== undefined && "cwd" in sessionEnsure); + + const original = " /help keep all whitespace \n"; + yield* adapter.sendTurn({ threadId, input: original }); + const turnStart = sent.find((message) => message.type === "turn.start"); + assert.equal(turnStart?.type === "turn.start" ? turnStart.text : undefined, original); + + if (!turnStart || turnStart.type !== "turn.start") { + return yield* Effect.die(new Error("turn.start was not sent")); + } + const startEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: turnStart.requestId, + threadId, + sessionId, + turnId: turnStart.turnId, + }, + }); + const startEvent = Option.getOrUndefined(yield* Fiber.join(startEventFiber)); + assert.equal(startEvent?.type, "turn.started"); + const firstEventAfterOrphanExit = Option.getOrUndefined( + yield* Fiber.join(firstEventAfterOrphanExitFiber), + ); + assert.equal(firstEventAfterOrphanExit?.type, "session.started"); + + yield* adapter.sendTurn({ threadId, input: "follow up" }); + const turnSteer = sent.find((message) => message.type === "turn.steer"); + if (!turnSteer || turnSteer.type !== "turn.steer") { + return yield* Effect.die(new Error("turn.steer was not sent")); + } + const steerEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: turnSteer.requestId, + threadId, + sessionId, + turnId: turnSteer.turnId, + }, + }); + const steerEvent = Option.getOrUndefined(yield* Fiber.join(steerEventFiber)); + assert.equal(steerEvent?.type, "turn.started"); + + const nextEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "content.delta", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: turnSteer.turnId, + streamKind: "assistant_text", + delta: "continued", + }, + }); + const nextEvent = Option.getOrUndefined(yield* Fiber.join(nextEventFiber)); + assert.equal(nextEvent?.type, "content.delta"); + + const newerTurnId = TurnId.make("newer-turn"); + const newerTurnFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: HermesGatewayRequestId.make("newer-turn-request"), + threadId, + sessionId, + turnId: newerTurnId, + }, + }); + yield* Fiber.join(newerTurnFiber); + + const afterStaleExitFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "session.exited", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId: HermesGatewaySessionId.make("session-stale"), + recoverable: false, + }, + }); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "content.delta", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: newerTurnId, + streamKind: "assistant_text", + delta: "still active", + }, + }); + const afterStaleExit = Option.getOrUndefined(yield* Fiber.join(afterStaleExitFiber)); + assert.equal(afterStaleExit?.type, "content.delta"); + const afterStaleExitSession = (yield* adapter.listSessions())[0]; + assert.equal(afterStaleExitSession?.status, "running"); + assert.equal(afterStaleExitSession?.activeTurnId, newerTurnId); + + const staleCompletionFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "turn.completed", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: turnStart.turnId, + state: "completed", + }, + }); + yield* Fiber.join(staleCompletionFiber); + + const afterStaleCompletion = (yield* adapter.listSessions())[0]; + assert.equal(afterStaleCompletion?.status, "running"); + assert.equal(afterStaleCompletion?.activeTurnId, newerTurnId); + + const activeCompletionFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "turn.completed", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: newerTurnId, + state: "completed", + }, + }); + yield* Fiber.join(activeCompletionFiber); + + const afterActiveCompletion = (yield* adapter.listSessions())[0]; + assert.equal(afterActiveCompletion?.status, "ready"); + assert.equal(afterActiveCompletion?.activeTurnId, undefined); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("resumes active Hermes turns for steering and projects authoritative snapshots", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_reconnected"); + const threadId = ThreadId.make("thread-reconnected"); + const sessionId = HermesGatewaySessionId.make("session-reconnected"); + const activeTurnId = TurnId.make("turn-already-running"); + const sent: Array = []; + const brokerEvents = yield* PubSub.unbounded(); + const broker: HermesGatewayBrokerShape = { + createEnrollment: () => Effect.die(new Error("unused")), + getInstanceStatus: () => Effect.die(new Error("unused")), + listInstances: Effect.succeed([]), + renameInstance: () => Effect.die(new Error("unused")), + revokeInstance: () => Effect.die(new Error("unused")), + removeInstance: () => Effect.die(new Error("unused")), + registerConnection: () => Effect.die(new Error("unused")), + receive: () => Effect.void, + disconnect: () => Effect.void, + request: (_instanceId, message) => { + sent.push(message); + if (message.type === "session.ensure") { + return Effect.succeed({ + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId, + resumed: true, + activeTurnId, + }); + } + if (message.type === "turn.steer") { + return Effect.succeed({ + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: message.sessionId, + turnId: message.turnId, + }); + } + return Effect.die(new Error(`unexpected request ${message.type}`)); + }, + send: (_instanceId, message) => Effect.sync(() => sent.push(message)).pipe(Effect.asVoid), + isConnected: () => Effect.succeed(true), + stream: Stream.fromPubSub(brokerEvents), + streamStatuses: Stream.empty, + }; + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + yield* Effect.yieldNow; + + const session = yield* adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + resumeCursor: { + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + sessionId, + }, + }); + assert.equal(session.status, "running"); + assert.equal(session.activeTurnId, activeTurnId); + + yield* adapter.sendTurn({ threadId, input: "steer the restored turn" }); + const steer = sent.find((message) => message.type === "turn.steer"); + assert.equal(steer?.type === "turn.steer" ? steer.turnId : undefined, activeTurnId); + + const snapshotEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* PubSub.publish(brokerEvents, { + instanceId, + message: { + type: "content.snapshot", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: activeTurnId, + streamKind: "assistant_text", + text: "the complete restored answer", + contentIndex: 0, + }, + }); + const snapshotEvent = Option.getOrUndefined(yield* Fiber.join(snapshotEventFiber)); + assert.equal(snapshotEvent?.type, "content.snapshot"); + if (snapshotEvent?.type === "content.snapshot") { + assert.deepEqual(snapshotEvent.payload, { + streamKind: "assistant_text", + text: "the complete restored answer", + contentIndex: 0, + }); + } + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +/** + * Builds an adapter whose broker connectivity and session.ensure response are + * driven per-test, so a reconnect is expressed as: flip `connected`, decide what + * the plugin reports on resume, publish a status event. + */ +const makeReconnectHarness = (options: { + readonly instanceId: ProviderInstanceId; + readonly threadId: ThreadId; + readonly initialSessionId: HermesGatewaySessionId; + /** + * Runs before each one-way send is recorded, so a test can land a broker + * event in the middle of an adapter operation. + */ + readonly onSend?: (message: HermesGatewayT3ToPluginMessage) => Effect.Effect; +}) => + Effect.gen(function* () { + const sent: Array = []; + const brokerEvents = yield* PubSub.unbounded(); + const brokerStatuses = yield* PubSub.unbounded(); + const state = { + connected: true, + // What the plugin reports on the *next* session.ensure. `undefined` + // activeTurnId means the plugin no longer knows about any running turn. + resumeSessionId: options.initialSessionId, + resumeActiveTurnId: undefined as TurnId | undefined, + resumeFails: false, + // When set, session.ensure never completes — a plugin that has accepted + // the socket but has not answered yet. The handler must not be wedged + // waiting for it. + stallEnsure: false, + }; + const broker: HermesGatewayBrokerShape = { + createEnrollment: () => Effect.die(new Error("unused")), + getInstanceStatus: () => Effect.die(new Error("unused")), + listInstances: Effect.succeed([]), + renameInstance: () => Effect.die(new Error("unused")), + revokeInstance: () => Effect.die(new Error("unused")), + removeInstance: () => Effect.die(new Error("unused")), + registerConnection: () => Effect.die(new Error("unused")), + receive: () => Effect.void, + disconnect: () => Effect.void, + request: (_instanceId, message) => { + sent.push(message); + if (message.type === "session.ensure") { + if (state.stallEnsure) return Effect.never; + if (state.resumeFails) { + return Effect.fail( + new ProviderAdapterRequestError({ + provider: "hermes", + method: "session.ensure", + detail: "The Hermes gateway connection disconnected.", + }), + ); + } + return Effect.succeed({ + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: state.resumeSessionId, + resumed: message.resumeSessionId !== undefined, + ...(state.resumeActiveTurnId !== undefined + ? { activeTurnId: state.resumeActiveTurnId } + : {}), + }); + } + if (message.type === "turn.start" || message.type === "turn.steer") { + return Effect.succeed({ + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: message.sessionId, + turnId: message.turnId, + }); + } + return Effect.die(new Error(`unexpected request ${message.type}`)); + }, + send: (_instanceId, message) => + (options.onSend ? options.onSend(message) : Effect.void).pipe( + Effect.andThen(Effect.sync(() => sent.push(message))), + Effect.asVoid, + ), + isConnected: () => Effect.sync(() => state.connected), + stream: Stream.fromPubSub(brokerEvents), + streamStatuses: Stream.fromPubSub(brokerStatuses), + }; + const adapter = yield* makeHermesAdapter({ instanceId: options.instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + yield* Effect.yieldNow; + + const disconnect = Effect.gen(function* () { + state.connected = false; + yield* PubSub.publish(brokerStatuses, statusFor(options.instanceId, "offline")); + yield* drain; + }); + const reconnect = Effect.gen(function* () { + state.connected = true; + yield* PubSub.publish(brokerStatuses, statusFor(options.instanceId, "connected")); + yield* drain; + }); + // A plugin restart as it actually happens: the broker accepts the new + // socket and retires the old one in the same step, so the instance never + // observably goes offline and only one `connected` status is published. + const replaceConnection = Effect.gen(function* () { + state.connected = true; + yield* PubSub.publish(brokerStatuses, statusFor(options.instanceId, "connected")); + yield* drain; + }); + // The same live connection republishing (session counts, pings). Must NOT + // look like a new connection. + const republishSameConnection = (generation: number) => + Effect.gen(function* () { + state.connected = true; + yield* PubSub.publish( + brokerStatuses, + statusFor(options.instanceId, "connected", generation), + ); + yield* drain; + }); + + return { + adapter, + sent, + brokerEvents, + state, + disconnect, + reconnect, + replaceConnection, + republishSameConnection, + } as const; + }); + +it.effect("re-issues session.ensure on reconnect and keeps streaming a turn that survived", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_survivor"); + const threadId = ThreadId.make("thread-survivor"); + const initialSessionId = HermesGatewaySessionId.make("session-before-drop"); + const harness = yield* makeReconnectHarness({ instanceId, threadId, initialSessionId }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "long running work" }); + const { seen, fiber } = yield* collectEvents(harness.adapter); + + // The socket drops mid-turn. Hermes keeps generating: its plugin still + // holds the turn, so session.ensure reports the same activeTurnId back. + yield* harness.disconnect; + harness.state.resumeActiveTurnId = started.turnId; + yield* harness.reconnect; + + const resumeEnsure = harness.sent.filter((message) => message.type === "session.ensure"); + assert.equal(resumeEnsure.length, 2, "reconnect must re-issue session.ensure"); + const resume = resumeEnsure[1]; + assert.equal( + resume?.type === "session.ensure" ? resume.resumeSessionId : undefined, + initialSessionId, + "resume must carry the stored cursor", + ); + + // The turn survived, so nothing disruptive is emitted and the session keeps + // pointing at the same turn. + assert.isUndefined( + seen.find((event) => event.type === "turn.completed" || event.type === "turn.aborted"), + ); + const session = (yield* harness.adapter.listSessions())[0]; + assert.equal(session?.status, "running"); + assert.equal(session?.activeTurnId, started.turnId); + + // Deltas for the surviving turn still land after the reconnect. + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "content.delta", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId: initialSessionId, + turnId: started.turnId, + streamKind: "assistant_text", + delta: "still going", + }, + }); + yield* drain; + assert.isDefined(seen.find((event) => event.type === "content.delta")); + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +// Regression: a plugin restart REPLACES the socket rather than leaving a gap. +// The broker accepts the new connection and retires the old one together, so +// the instance never observably goes offline and exactly one `connected` +// status is published. An edge detector watching connectedness therefore never +// fires, T3 keeps a session the new plugin process has never heard of, and its +// next turn.start comes back "Call session.ensure before starting a turn" — +// the thread is dead with no way to recover from the UI. +// Regression: T3 restarting must not make existing Hermes threads unusable. +// The shutdown finalizer used to `stopAll()`, which tells the plugin to drop +// the thread — but the plugin process outlives T3, so on the next turn its +// session map no longer had the thread and turn.start came back "Call +// session.ensure before starting a turn". Threads created after the restart +// were fine, which disguised a shutdown bug as a resume bug. +it.effect("does not stop remote sessions when the adapter shuts down", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_shutdown"); + const threadId = ThreadId.make("thread-across-restart"); + const sent: Array = []; + const brokerEvents = yield* PubSub.unbounded(); + + yield* Effect.gen(function* () { + const broker: HermesGatewayBrokerShape = { + createEnrollment: () => Effect.die(new Error("unused")), + getInstanceStatus: () => Effect.die(new Error("unused")), + listInstances: Effect.succeed([]), + renameInstance: () => Effect.die(new Error("unused")), + revokeInstance: () => Effect.die(new Error("unused")), + removeInstance: () => Effect.die(new Error("unused")), + registerConnection: () => Effect.die(new Error("unused")), + receive: () => Effect.void, + disconnect: () => Effect.void, + request: (_instanceId, message) => { + sent.push(message); + if (message.type === "session.ensure") { + return Effect.succeed({ + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: HermesGatewaySessionId.make("session-across-restart"), + resumed: false, + }); + } + return Effect.die(new Error(`unexpected request ${message.type}`)); + }, + send: (_instanceId, message) => Effect.sync(() => sent.push(message)).pipe(Effect.asVoid), + isConnected: () => Effect.succeed(true), + stream: Stream.fromPubSub(brokerEvents), + streamStatuses: Stream.empty, + }; + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + yield* Effect.yieldNow; + yield* adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + }).pipe(Effect.scoped); + + // The scope closed — T3 shutting down. Hermes keeps the session, so no + // session.stop may have been sent on its behalf. + assert.isUndefined( + sent.find((message) => message.type === "session.stop"), + "shutdown must not end the remote Hermes session", + ); + }).pipe(Effect.provide(testEnvLayer)), +); + +it.effect("re-issues session.ensure when the connection is replaced without going offline", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_replaced"); + const threadId = ThreadId.make("thread-replaced"); + const initialSessionId = HermesGatewaySessionId.make("session-before-restart"); + const harness = yield* makeReconnectHarness({ instanceId, threadId, initialSessionId }); + + // Production ordering: the plugin is already connected and the adapter has + // observed that connection before any thread exists. + yield* harness.republishSameConnection(7); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + + // No disconnect: the plugin restarted, so the broker retired the old socket + // and accepted the new one together. The instance never reports offline. + yield* harness.replaceConnection; + + const ensures = harness.sent.filter((message) => message.type === "session.ensure"); + assert.equal( + ensures.length, + 2, + "a replaced connection must re-ensure, or the next turn is rejected", + ); + const resume = ensures[1]; + assert.equal( + resume?.type === "session.ensure" ? resume.resumeSessionId : undefined, + initialSessionId, + ); + + // A later turn now targets a session the new plugin knows about. + yield* harness.adapter.sendTurn({ threadId, input: "after the restart" }); + const starts = harness.sent.filter((message) => message.type === "turn.start"); + assert.equal(starts.length, 1); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +// Regression: the resume must not block the status-stream handler. +// `Stream.runForEach` is serial, and `session.ensure` is a correlated request +// whose reply arrives on the broker's OTHER stream — so awaiting it inside the +// handler wedges the subscription until the 30s request timeout. Every later +// status event queues behind it, including the next reconnect, and the thread +// is left with no session while every turn fails "Call session.ensure before +// starting a turn". +it.effect("keeps handling status events while a resume awaits its reply", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_serial"); + const threadId = ThreadId.make("thread-serial"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: HermesGatewaySessionId.make("session-serial"), + }); + yield* harness.republishSameConnection(11); + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + + // From here the fake never answers session.ensure, standing in for a plugin + // that is still starting up. + harness.state.stallEnsure = true; + yield* harness.replaceConnection; + const afterFirst = harness.sent.filter((message) => message.type === "session.ensure").length; + + // A second replacement arrives while that resume is still outstanding. If + // the handler were blocked awaiting the reply this would never be seen. + yield* harness.replaceConnection; + const afterSecond = harness.sent.filter((message) => message.type === "session.ensure").length; + + assert.isAbove( + afterSecond, + afterFirst, + "a resume awaiting its reply must not stall later reconnects", + ); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("does not re-ensure when the same connection republishes its status", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_same_connection"); + const threadId = ThreadId.make("thread-same-connection"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: HermesGatewaySessionId.make("session-stable"), + }); + + // Production ordering: the plugin connects first, so its generation is + // already known by the time a thread creates a session. + yield* harness.republishSameConnection(4242); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + + // Session-count reports and ping results republish the SAME connection. + // Re-ensuring on each would cancel pending approvals under a running turn. + yield* harness.republishSameConnection(4242); + yield* harness.republishSameConnection(4242); + + assert.equal(harness.sent.filter((message) => message.type === "session.ensure").length, 1); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("settles the thread when a turn does not survive the reconnect", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_dead_turn"); + const threadId = ThreadId.make("thread-dead-turn"); + const initialSessionId = HermesGatewaySessionId.make("session-dead-turn"); + const harness = yield* makeReconnectHarness({ instanceId, threadId, initialSessionId }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "work that dies" }); + const { seen, fiber } = yield* collectEvents(harness.adapter); + + // The plugin restarted: it comes back with a fresh session and no active + // turn. Hermes will never send a terminal frame for the old turn, so the + // adapter must produce one or the thread hangs forever. + yield* harness.disconnect; + harness.state.resumeSessionId = HermesGatewaySessionId.make("session-after-restart"); + harness.state.resumeActiveTurnId = undefined; + yield* harness.reconnect; + + const terminal = seen.find((event) => event.type === "turn.completed"); + assert.isDefined(terminal, "a dead turn must be settled with a terminal event"); + if (terminal?.type === "turn.completed") { + assert.equal(terminal.turnId, started.turnId); + assert.equal(terminal.payload.state, "failed"); + assert.include(terminal.payload.errorMessage ?? "", "did not survive"); + } + const session = (yield* harness.adapter.listSessions())[0]; + assert.isUndefined(session?.activeTurnId); + assert.equal(session?.status, "error"); + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("settles the thread when session.ensure fails on reconnect", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_resume_failed"); + const threadId = ThreadId.make("thread-resume-failed"); + const initialSessionId = HermesGatewaySessionId.make("session-resume-failed"); + const harness = yield* makeReconnectHarness({ instanceId, threadId, initialSessionId }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "work" }); + const { seen, fiber } = yield* collectEvents(harness.adapter); + + yield* harness.disconnect; + harness.state.resumeFails = true; + yield* harness.reconnect; + + const terminal = seen.find((event) => event.type === "turn.completed"); + assert.isDefined(terminal, "a failed resume must settle rather than hang"); + if (terminal?.type === "turn.completed") { + assert.equal(terminal.turnId, started.turnId); + assert.equal(terminal.payload.state, "failed"); + assert.include(terminal.payload.errorMessage ?? "", "could not resume"); + } + assert.isUndefined((yield* harness.adapter.listSessions())[0]?.activeTurnId); + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("cancels pending approvals and user input on reconnect without deciding", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_pending"); + const threadId = ThreadId.make("thread-pending"); + const sessionId = HermesGatewaySessionId.make("session-pending"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: sessionId, + }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "needs approval" }); + const { seen, fiber } = yield* collectEvents(harness.adapter); + + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "request.opened", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: started.turnId, + requestId: HermesGatewayRequestId.make("approval-in-flight"), + requestType: "command_execution_approval", + detail: "Hermes wants to run a command", + }, + }); + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "user-input.requested", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: started.turnId, + requestId: HermesGatewayRequestId.make("clarify-in-flight"), + questions: [ + { + id: "clarify-in-flight", + header: "Hermes", + question: "Which branch?", + options: [{ label: "main", description: "main" }], + multiSelect: false, + }, + ], + }, + }); + yield* drain; + + // The plugin restarts and forgets both requests; answering them afterwards + // would come back `request-not-found` and park the turn forever. A + // restarted plugin reports no active turn — that absence is exactly how + // the adapter knows the requests died with the process. + yield* harness.disconnect; + harness.state.resumeActiveTurnId = undefined; + yield* harness.reconnect; + + const resolvedApproval = seen.find( + (event) => event.type === "request.resolved" && event.requestId === "approval-in-flight", + ); + assert.isDefined(resolvedApproval, "the pending approval must be closed out"); + if (resolvedApproval?.type === "request.resolved") { + assert.equal(resolvedApproval.payload.requestType, "command_execution_approval"); + // Explicit product decision: never approve or deny on the user's behalf. + assert.isUndefined(resolvedApproval.payload.decision); + } + const resolvedUserInput = seen.find( + (event) => event.type === "user-input.resolved" && event.requestId === "clarify-in-flight", + ); + assert.isDefined(resolvedUserInput, "the pending user-input request must be closed out"); + if (resolvedUserInput?.type === "user-input.resolved") { + assert.deepEqual(resolvedUserInput.payload.answers, {}); + } + const warning = seen.find((event) => event.type === "runtime.warning"); + assert.isDefined(warning, "the cancellation must be visible in the thread"); + if (warning?.type === "runtime.warning") { + assert.include(warning.payload.message, "cancelled"); + } + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("keeps pending interactions when the turn survives the reconnect", () => + Effect.gen(function* () { + // A bare socket drop the plugin process survived: its request dicts are + // intact and the turn is genuinely blocked on the user's answer. Cancelling + // here is the hang Macroscope flagged — the UI loses the request while + // Hermes keeps waiting for the response forever. + const instanceId = ProviderInstanceId.make("hermes_pending_alive"); + const threadId = ThreadId.make("thread-pending-alive"); + const sessionId = HermesGatewaySessionId.make("session-pending-alive"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: sessionId, + }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "needs approval" }); + const { seen, fiber } = yield* collectEvents(harness.adapter); + + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "request.opened", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: started.turnId, + requestId: HermesGatewayRequestId.make("approval-still-live"), + requestType: "command_execution_approval", + detail: "Hermes wants to run a command", + }, + }); + yield* drain; + + yield* harness.disconnect; + harness.state.resumeActiveTurnId = started.turnId; + yield* harness.reconnect; + + assert.isUndefined( + seen.find((event) => event.type === "request.resolved"), + "a request the plugin still holds must stay open", + ); + assert.isUndefined( + seen.find((event) => event.type === "runtime.warning"), + "no cancellation warning may appear for a surviving turn", + ); + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("fails sends immediately and actionably while the gateway is offline", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_offline"); + const threadId = ThreadId.make("thread-offline"); + const sessionId = HermesGatewaySessionId.make("session-offline"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: sessionId, + }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + yield* harness.disconnect; + let sentBefore = harness.sent.length; + + // turn.start path. + const startError = yield* Effect.flip( + harness.adapter.sendTurn({ threadId, input: "queue me" }), + ); + assert.equal(startError._tag, "ProviderAdapterRequestError"); + if (startError._tag === "ProviderAdapterRequestError") { + assert.include(startError.detail, "Hermes is offline"); + assert.include(startError.detail, instanceId); + assert.include(startError.detail, "hermes gateway restart"); + } + // Nothing was queued or dispatched — the failure is immediate, not a + // deferred send that lands minutes later against a stale thread. + assert.equal(harness.sent.length, sentBefore); + + // With a turn actually running, the steer and interactive paths must fail + // just as clearly rather than parking on the broker's request timeout. + yield* harness.reconnect; + yield* harness.adapter.sendTurn({ threadId, input: "real work" }); + yield* harness.disconnect; + sentBefore = harness.sent.length; + + const steerError = yield* Effect.flip( + harness.adapter.sendTurn({ threadId, input: "steer me" }), + ); + assert.equal(steerError._tag, "ProviderAdapterRequestError"); + if (steerError._tag === "ProviderAdapterRequestError") { + assert.include(steerError.detail, "Hermes is offline"); + } + + const interruptError = yield* Effect.flip(harness.adapter.interruptTurn(threadId)); + assert.equal(interruptError._tag, "ProviderAdapterRequestError"); + + const approvalError = yield* Effect.flip( + harness.adapter.respondToRequest( + threadId, + ApprovalRequestId.make("approval-while-offline"), + "accept", + ), + ); + assert.equal(approvalError._tag, "ProviderAdapterRequestError"); + + const userInputError = yield* Effect.flip( + harness.adapter.respondToUserInput( + threadId, + ApprovalRequestId.make("clarify-while-offline"), + {}, + ), + ); + assert.equal(userInputError._tag, "ProviderAdapterRequestError"); + + assert.equal(harness.sent.length, sentBefore); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("keeps the session on an undeliverable stop so Hermes is not orphaned", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_stop_offline"); + const threadId = ThreadId.make("thread-stop-offline"); + const sessionId = HermesGatewaySessionId.make("session-stop-offline"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: sessionId, + }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "work" }); + const { seen, fiber } = yield* collectEvents(harness.adapter); + yield* harness.disconnect; + + const stopError = yield* Effect.flip(harness.adapter.stopSession(threadId)); + assert.equal(stopError._tag, "ProviderAdapterRequestError"); + if (stopError._tag === "ProviderAdapterRequestError") { + assert.include(stopError.detail, "Hermes is offline"); + } + // The stop never reached Hermes, so dropping local state would orphan a + // thread that is still running on the far side. + assert.isTrue(yield* harness.adapter.hasSession(threadId)); + + // stopAll (the shutdown finalizer) must not throw away the session either. + yield* harness.adapter.stopAll(); + assert.isTrue(yield* harness.adapter.hasSession(threadId)); + + // Frames that arrive for the still-live session are forwarded with full + // context rather than dropped or forwarded contextless. + yield* harness.reconnect; + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "content.delta", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: started.turnId, + streamKind: "assistant_text", + delta: "still running on Hermes", + }, + }); + yield* drain; + assert.isDefined(seen.find((event) => event.type === "content.delta")); + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +// `stopAll` snapshots `sessions.keys()` up front, and the broker event fiber +// deletes a session the moment `session.exited` lands — so a session can be +// gone by the time its own stop runs. That raises SessionNotFound, which the +// sweep used to let escape: every session after it in the snapshot was never +// stopped, and on shutdown the whole finalizer logged a failure. +it.effect("finishes the stopAll sweep when a session exits mid-sweep", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_stop_all_race"); + const firstThreadId = ThreadId.make("thread-stop-all-first"); + const secondThreadId = ThreadId.make("thread-stop-all-second"); + const sessionId = HermesGatewaySessionId.make("session-stop-all"); + const events = yield* PubSub.unbounded(); + + const harness = yield* makeReconnectHarness({ + instanceId, + threadId: firstThreadId, + initialSessionId: sessionId, + // The second session exits while the first one's stop is being written. + onSend: (message) => + message.type === "session.stop" && message.threadId === firstThreadId + ? PubSub.publish(events, { + instanceId, + message: { + type: "session.exited", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId: secondThreadId, + sessionId, + recoverable: false, + }, + }).pipe(Effect.andThen(drain)) + : Effect.void, + }); + // Route the harness's broker stream through our own pubsub so `onSend` can + // publish into the same stream the adapter is subscribed to. + yield* Stream.runForEach(Stream.fromPubSub(events), (envelope) => + PubSub.publish(harness.brokerEvents, envelope), + ).pipe(Effect.forkChild({ startImmediately: true })); + + for (const threadId of [firstThreadId, secondThreadId]) { + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + } + + // The sweep must run to completion rather than dying on the vanished one. + yield* harness.adapter.stopAll(); + assert.isFalse(yield* harness.adapter.hasSession(firstThreadId)); + assert.isFalse(yield* harness.adapter.hasSession(secondThreadId)); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("drops frames for a session it no longer tracks", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_forgotten"); + const threadId = ThreadId.make("thread-forgotten"); + const sessionId = HermesGatewaySessionId.make("session-forgotten"); + const harness = yield* makeReconnectHarness({ + instanceId, + threadId, + initialSessionId: sessionId, + }); + + yield* harness.adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const started = yield* harness.adapter.sendTurn({ threadId, input: "work" }); + yield* harness.adapter.stopSession(threadId); + const { seen, fiber } = yield* collectEvents(harness.adapter); + + // Hermes had frames in flight when the stop landed. With no session to + // match, forwarding them would inject content into a forgotten thread. + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "content.delta", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: started.turnId, + streamKind: "assistant_text", + delta: "orphaned text", + }, + }); + yield* PubSub.publish(harness.brokerEvents, { + instanceId, + message: { + type: "item.completed", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + threadId, + sessionId, + turnId: started.turnId, + itemId: HermesGatewayItemId.make("orphan-item"), + itemType: "command_execution", + }, + }); + yield* drain; + + assert.equal(seen.length, 0, "frames for an untracked session must not be forwarded"); + + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +/** + * Base description the registry hands to `describe` — the snapshot-derived + * default every driver starts from. Deliberately sparse: Hermes runs on + * another machine, so a snapshot alone knows almost nothing about it. + */ +const makeBaseDescription = (instanceId: ProviderInstanceId): ProviderInstanceDescription => ({ + identity: { + instanceId, + driver: ProviderDriverKind.make("hermes"), + displayName: "Hermes", + agentVersion: null, + pluginVersion: null, + protocolVersion: null, + host: null, + }, + connection: { + status: "ready", + detail: null, + lastConnectedAt: null, + activeSessionCount: null, + connectionGeneration: null, + }, + model: null, + skills: [], + capabilities: null, + describedAt: "2024-01-01T00:00:00.000Z", +}); + +const makeDescribeBroker = ( + respond: ( + message: HermesGatewayT3ToPluginMessage, + ) => ReturnType | undefined, + options: { readonly connected?: boolean } = {}, +): HermesGatewayBrokerShape => ({ + createEnrollment: () => Effect.die(new Error("unused")), + getInstanceStatus: () => Effect.die(new Error("unused")), + listInstances: Effect.succeed([]), + renameInstance: () => Effect.die(new Error("unused")), + revokeInstance: () => Effect.die(new Error("unused")), + removeInstance: () => Effect.die(new Error("unused")), + registerConnection: () => Effect.die(new Error("unused")), + receive: () => Effect.void, + disconnect: () => Effect.void, + request: (_instanceId, message) => + respond(message) ?? Effect.die(new Error(`unexpected request ${message.type}`)), + send: () => Effect.void, + isConnected: () => Effect.succeed(options.connected ?? true), + stream: Stream.empty, + streamStatuses: Stream.empty, +}); + +it.effect("enriches the base description with what the plugin reports", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_remote"); + const broker = makeDescribeBroker((message) => + message.type === "describe.request" + ? Effect.succeed({ + type: "describe.response", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + pluginVersion: "0.4.1", + hermesVersion: "0.19.0", + capabilities: { + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + streaming: true, + activity: true, + approvals: true, + userInput: true, + attachments: true, + }, + model: "claude-opus-5", + reasoningEffort: "high", + skills: [ + { name: "deploy", description: "Ship it", source: "workflow", enabled: true }, + { name: "bare", enabled: true }, + ], + describedAt: "2024-06-01T10:00:00.000Z", + } as const) + : undefined, + ); + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + + const description = yield* adapter.describe!(makeBaseDescription(instanceId)); + + assert.equal(description.identity.agentVersion, "0.19.0"); + assert.equal(description.identity.pluginVersion, "0.4.1"); + assert.equal(description.identity.protocolVersion, HERMES_GATEWAY_PROTOCOL_VERSION); + assert.equal(description.model?.displayName, "claude-opus-5"); + assert.equal(description.model?.reasoningEffortLabel, "high"); + assert.equal(description.describedAt, "2024-06-01T10:00:00.000Z"); + assert.equal(description.skills.length, 2); + assert.equal(description.skills[0]?.scope, "workflow"); + // No on-disk path in Hermes' skills surface: the category stands in, and a + // skill without one falls back to its own name rather than an empty path. + assert.equal(description.skills[1]?.path, "bare"); + // The wire capability struct carries a version alongside the flags; only + // the booleans belong in the description's capability map. + assert.deepEqual(description.capabilities, { + streaming: true, + activity: true, + approvals: true, + userInput: true, + attachments: true, + }); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("keeps base values for fields the plugin omitted", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_remote"); + const broker = makeDescribeBroker((message) => + message.type === "describe.request" + ? Effect.succeed({ + type: "describe.response", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + pluginVersion: "0.4.1", + hermesVersion: "0.19.0", + capabilities: { + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + streaming: true, + activity: true, + approvals: true, + userInput: true, + attachments: true, + }, + skills: [], + describedAt: "2024-06-01T10:00:00.000Z", + } as const) + : undefined, + ); + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + + const base = makeBaseDescription(instanceId); + const description = yield* adapter.describe!({ + ...base, + model: { + id: "snapshot-model", + displayName: "Snapshot Model", + vendor: "Anthropic", + contextWindow: null, + reasoningEffortLabel: "medium", + }, + }); + + // Omitted model/effort must not blank out what the snapshot already knew. + assert.equal(description.model?.displayName, "Snapshot Model"); + assert.equal(description.model?.reasoningEffortLabel, "medium"); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("fails describe rather than reporting stale data when the gateway is offline", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_remote"); + const broker = makeDescribeBroker(() => undefined, { connected: false }); + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + + const result = yield* Effect.result(adapter.describe!(makeBaseDescription(instanceId))); + + // The caller (ProviderRegistry) turns this failure into "render the + // snapshot default", which is why failing here is correct. + assert.equal(result._tag, "Failure"); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("reads a skill body and passes a null body through unchanged", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_remote"); + const broker = makeDescribeBroker((message) => + message.type === "skill.body.request" + ? Effect.succeed({ + type: "skill.body.response", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + skillName: message.skillName, + markdown: message.skillName === "empty" ? null : "# Deploy\n", + } as const) + : undefined, + ); + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + + const body = yield* adapter.getSkillBody!("deploy"); + assert.equal(body.skillName, "deploy"); + assert.equal(body.markdown, "# Deploy\n"); + + // "Asked, and there is nothing to show" is distinct from a dropped reply. + const empty = yield* adapter.getSkillBody!("empty"); + assert.equal(empty.markdown, null); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +/** + * A broker fake that answers session.ensure and turn.start/steer, recording + * everything sent — the minimum surface `sendTurn` touches. + */ +const makeTurnBroker = () => { + const sent: Array = []; + const broker: HermesGatewayBrokerShape = { + createEnrollment: () => Effect.die(new Error("unused")), + getInstanceStatus: () => Effect.die(new Error("unused")), + listInstances: Effect.succeed([]), + renameInstance: () => Effect.die(new Error("unused")), + revokeInstance: () => Effect.die(new Error("unused")), + removeInstance: () => Effect.die(new Error("unused")), + registerConnection: () => Effect.die(new Error("unused")), + receive: () => Effect.void, + disconnect: () => Effect.void, + request: (_instanceId, message) => { + sent.push(message); + if (message.type === "session.ensure") { + return Effect.succeed({ + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: HermesGatewaySessionId.make("session-attachments"), + resumed: false, + }); + } + if (message.type === "turn.start" || message.type === "turn.steer") { + return Effect.succeed({ + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + threadId: message.threadId, + sessionId: message.sessionId, + turnId: message.turnId, + }); + } + return Effect.die(new Error(`unexpected request ${message.type}`)); + }, + send: (_instanceId, message) => Effect.sync(() => sent.push(message)).pipe(Effect.asVoid), + isConnected: () => Effect.succeed(true), + stream: Stream.empty, + streamStatuses: Stream.empty, + }; + return { sent, broker } as const; +}; + +/** Write attachment bytes where the adapter will look for them. */ +const writeAttachmentFixture = (attachment: ChatAttachment, bytes: Uint8Array) => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const attachmentPath = resolveAttachmentPath({ + attachmentsDir: config.attachmentsDir, + attachment, + }); + if (!attachmentPath) return yield* Effect.die(new Error("unresolvable attachment path")); + yield* fileSystem.makeDirectory(path.dirname(attachmentPath), { recursive: true }); + yield* fileSystem.writeFile(attachmentPath, bytes); + }); + +it.effect("inlines attachment bytes as base64 on the turn frame", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_attachments"); + const threadId = ThreadId.make("thread-attachments"); + const { sent, broker } = makeTurnBroker(); + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + + const attachment: ChatAttachment = { + type: "file", + id: createAttachmentId(threadId)!, + name: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 9, + }; + yield* writeAttachmentFixture(attachment, new TextEncoder().encode("PDF BYTES")); + + yield* adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ threadId, input: "here is a file", attachments: [attachment] }); + + const turnStart = sent.find((message) => message.type === "turn.start"); + if (!turnStart || turnStart.type !== "turn.start") { + return yield* Effect.die(new Error("turn.start was not sent")); + } + assert.equal(turnStart.attachments?.length, 1); + const framed = turnStart.attachments![0]!; + assert.equal(framed.name, "notes.pdf"); + assert.equal(framed.mimeType, "application/pdf"); + // sizeBytes reflects the bytes actually read, not the caller's claim. + assert.equal(framed.sizeBytes, 9); + assert.equal(Buffer.from(framed.data, "base64").toString("utf8"), "PDF BYTES"); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); + +it.effect("fails a turn whose attachments total more than the per-turn ceiling", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("hermes_attachments_oversized"); + const threadId = ThreadId.make("thread-attachments-oversized"); + const { sent, broker } = makeTurnBroker(); + const adapter = yield* makeHermesAdapter({ instanceId }).pipe( + Effect.provideService(HermesGatewayBroker, broker), + ); + + // Two files under the per-file cap whose sum crosses the 25MB per-turn + // total — the case only the adapter can see. + const half = Math.floor(HERMES_MEDIA_MAX_BYTES / 2) + 1024; + const attachments: Array = []; + for (const name of ["first.bin", "second.bin"]) { + const attachment: ChatAttachment = { + type: "file", + id: createAttachmentId(threadId)!, + name, + mimeType: "application/octet-stream", + sizeBytes: half, + }; + yield* writeAttachmentFixture(attachment, new Uint8Array(half)); + attachments.push(attachment); + } + + yield* adapter.startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }); + const error = yield* Effect.flip( + adapter.sendTurn({ threadId, input: "too much", attachments }), + ); + assert.equal(error._tag, "ProviderAdapterValidationError"); + if (error._tag === "ProviderAdapterValidationError") { + assert.include(error.issue, "25MB"); + assert.include(error.issue, "second.bin"); + } + // The turn never reached the gateway. + assert.isUndefined(sent.find((message) => message.type === "turn.start")); + }).pipe(Effect.scoped, Effect.provide(testEnvLayer)), +); diff --git a/apps/server/src/provider/Layers/HermesAdapter.ts b/apps/server/src/provider/Layers/HermesAdapter.ts new file mode 100644 index 00000000000..d6742854f10 --- /dev/null +++ b/apps/server/src/provider/Layers/HermesAdapter.ts @@ -0,0 +1,1094 @@ +import { + EventId, + HERMES_GATEWAY_PROTOCOL_VERSION, + HERMES_MEDIA_MAX_BYTES, + HermesGatewayRequestId, + HermesGatewayResumeCursor, + HermesGatewaySessionId, + ProviderDriverKind, + RuntimeItemId, + RuntimeRequestId, + ThreadId, + TurnId, + type CanonicalRequestType, + type HermesGatewayPluginToT3Message, + type HermesGatewayTurnAttachment, + type ProviderInstanceId, + type ProviderRuntimeEvent, + type ProviderSession, + type ServerProviderSkill, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { resolveAttachmentPath } from "../../attachmentStore.ts"; +import { ServerConfig } from "../../config.ts"; +import { + ProviderAdapterRequestError, + ProviderAdapterSessionNotFoundError, + ProviderAdapterValidationError, +} from "../Errors.ts"; +import type { ProviderAdapterShape } from "../Services/ProviderAdapter.ts"; +import { HermesGatewayBroker } from "../Services/HermesGatewayBroker.ts"; + +const PROVIDER = ProviderDriverKind.make("hermes"); +const isResumeCursor = Schema.is(HermesGatewayResumeCursor); + +/** + * Hermes never queues. When the gateway socket is down there is nothing on the + * far side to accept work, and a queued turn would silently start minutes later + * against a thread the user has moved on from. Every send path fails + * immediately with this text so the failure names the fix rather than showing + * up as a generic protocol error or a 30s request timeout. + */ +const offlineDetail = (instanceId: ProviderInstanceId) => + `Hermes is offline — the '${instanceId}' gateway plugin is not connected. Start the gateway on the Hermes host (\`hermes gateway restart\`), then retry.`; + +/** + * `turns` exists only to answer `readThread`, which no Hermes call site reaches + * today. Retaining every completed item for the life of the process is an + * unbounded leak in service of that dead path, so the accumulator keeps a + * hard-capped tail of the most recent turns/items and `readThread` reports what + * it still holds. Raising these bounds is a deliberate decision, not something + * that should drift with thread length. + */ +const MAX_RETAINED_TURNS = 8; +const MAX_RETAINED_ITEMS_PER_TURN = 32; + +type HermesAdapterShape = ProviderAdapterShape< + ProviderAdapterRequestError | ProviderAdapterSessionNotFoundError | ProviderAdapterValidationError +>; + +interface PendingInteraction { + readonly requestId: string; + readonly turnId: TurnId; + readonly requestType: CanonicalRequestType; +} + +interface SessionContext { + hermesSessionId: HermesGatewaySessionId; + readonly turns: Array<{ readonly id: TurnId; readonly items: Array }>; + /** + * Approval and user-input requests T3 has shown but Hermes has not resolved. + * The plugin holds its side in plain dicts, so a plugin restart forgets them + * and any decision sent afterwards comes back `request-not-found`, parking + * the turn forever. Tracking them here lets a reconnect close them out. + */ + readonly pendingApprovals: Map; + readonly pendingUserInputs: Map; + session: ProviderSession; +} + +type PluginMessage = Exclude; + +const nowIso = () => DateTime.formatIso(DateTime.nowUnsafe()); +const MAX_PERSISTED_GATEWAY_FIELD_CHARS = 4_096; + +const boundedString = (value: unknown) => + typeof value === "string" && value.length <= MAX_PERSISTED_GATEWAY_FIELD_CHARS + ? value + : undefined; + +const asRecord = (value: unknown) => + typeof value === "object" && value !== null + ? (value as Readonly>) + : undefined; + +export const sanitizeHermesItemData = ( + itemType: Extract["itemType"], + value: unknown, +) => { + const record = asRecord(value); + if (!record) return undefined; + const fields = + itemType === "command_execution" + ? ["command", "cwd"] + : itemType === "file_change" + ? ["path"] + : itemType === "web_search" + ? ["query"] + : itemType === "image_view" + ? ["path"] + : itemType === "mcp_tool_call" + ? ["server", "operation"] + : []; + const sanitized = Object.fromEntries( + fields.flatMap((field) => { + const value = boundedString(record[field]); + return value === undefined ? [] : [[field, value]]; + }), + ); + return Object.keys(sanitized).length > 0 ? sanitized : undefined; +}; + +export const sanitizeHermesRequestArgs = ( + requestType: Extract["requestType"], + value: unknown, +) => { + const record = asRecord(value); + if (!record) return undefined; + const fields = + requestType === "command_execution_approval" || requestType === "exec_command_approval" + ? ["command", "cwd"] + : requestType === "file_read_approval" || + requestType === "file_change_approval" || + requestType === "apply_patch_approval" + ? ["path"] + : []; + const sanitized = Object.fromEntries( + fields.flatMap((field) => { + const value = boundedString(record[field]); + return value === undefined ? [] : [[field, value]]; + }), + ); + return Object.keys(sanitized).length > 0 ? sanitized : undefined; +}; + +export const makeHermesAdapter = Effect.fn("makeHermesAdapter")(function* (input: { + readonly instanceId: ProviderInstanceId; +}) { + const crypto = yield* Crypto.Crypto; + const broker = yield* HermesGatewayBroker; + const fileSystem = yield* FileSystem.FileSystem; + const serverConfig = yield* ServerConfig; + const events = yield* PubSub.unbounded(); + const sessions = new Map(); + + const randomId = crypto.randomUUIDv4.pipe( + Effect.mapError( + (cause) => + new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "crypto/randomUUIDv4", + detail: "Failed to generate a Hermes runtime identifier.", + cause, + }), + ), + ); + const requestId = randomId.pipe( + Effect.map((value) => HermesGatewayRequestId.make(`t3-${value}`)), + ); + const eventBase = (message: { + readonly threadId: ThreadId; + readonly turnId?: string | undefined; + readonly itemId?: string | undefined; + readonly requestId?: string | undefined; + }) => + randomId.pipe( + Effect.map((value) => ({ + eventId: EventId.make(value), + provider: PROVIDER, + providerInstanceId: input.instanceId, + threadId: message.threadId, + createdAt: nowIso(), + ...(message.turnId ? { turnId: TurnId.make(message.turnId) } : {}), + ...(message.itemId ? { itemId: RuntimeItemId.make(message.itemId) } : {}), + ...(message.requestId ? { requestId: RuntimeRequestId.make(message.requestId) } : {}), + })), + ); + + const emit = (event: ProviderRuntimeEvent) => PubSub.publish(events, event).pipe(Effect.asVoid); + + const findContext = (threadId: ThreadId) => + Effect.gen(function* (): Effect.fn.Return { + const context = sessions.get(threadId); + if (!context) { + return yield* new ProviderAdapterSessionNotFoundError({ + provider: PROVIDER, + threadId, + }); + } + return context; + }); + + const requireConnected = (method: string) => + Effect.gen(function* (): Effect.fn.Return { + const connected = yield* broker.isConnected(input.instanceId); + if (!connected) { + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method, + detail: offlineDetail(input.instanceId), + }); + } + }); + + const updateSession = (context: SessionContext, patch: Partial) => { + context.session = { + ...context.session, + ...patch, + updatedAt: nowIso(), + }; + }; + + const clearActiveTurn = ( + context: SessionContext, + patch: Omit, "activeTurnId">, + ) => { + const { activeTurnId: _activeTurnId, ...session } = context.session; + context.session = { ...session, ...patch, updatedAt: nowIso() }; + }; + + const trackTurn = (context: SessionContext, turnId: TurnId) => { + if (context.turns.some((turn) => turn.id === turnId)) return; + context.turns.push({ id: turnId, items: [] }); + while (context.turns.length > MAX_RETAINED_TURNS) context.turns.shift(); + }; + + const recordCompletedItem = (context: SessionContext, turnId: string, payload: unknown) => { + const turn = context.turns.find((entry) => entry.id === turnId); + if (!turn) return; + turn.items.push(payload); + while (turn.items.length > MAX_RETAINED_ITEMS_PER_TURN) turn.items.shift(); + }; + + /** + * Every session-scoped plugin frame names the session it belongs to. If we do + * not hold that session — because the thread was stopped, or the plugin is + * still emitting for a session we already tore down — the frame has no thread + * context on this side, and forwarding it would inject deltas and item rows + * into a thread T3 no longer tracks. `session.exited` always guarded this; + * the guard belongs on every session-scoped frame. + */ + const contextMatchesSession = ( + context: SessionContext | undefined, + message: { readonly sessionId: HermesGatewaySessionId }, + ): context is SessionContext => context?.hermesSessionId === message.sessionId; + + const toRuntimeEvent = (message: PluginMessage) => + Effect.gen(function* (): Effect.fn.Return< + ProviderRuntimeEvent | undefined, + ProviderAdapterRequestError + > { + if (!("threadId" in message)) return undefined; + const context = sessions.get(message.threadId); + switch (message.type) { + case "session.ready": + return undefined; + case "turn.started": { + if (!contextMatchesSession(context, message)) return undefined; + updateSession(context, { + status: "running", + activeTurnId: TurnId.make(message.turnId), + }); + trackTurn(context, TurnId.make(message.turnId)); + return { + ...(yield* eventBase(message)), + type: "turn.started", + payload: {}, + }; + } + case "content.delta": { + if (!contextMatchesSession(context, message)) return undefined; + return { + ...(yield* eventBase(message)), + type: "content.delta", + payload: { + streamKind: message.streamKind, + delta: message.delta, + ...(message.contentIndex !== undefined ? { contentIndex: message.contentIndex } : {}), + }, + }; + } + case "content.snapshot": { + if (!contextMatchesSession(context, message)) return undefined; + return { + ...(yield* eventBase(message)), + type: "content.snapshot", + payload: { + streamKind: message.streamKind, + text: message.text, + ...(message.contentIndex !== undefined ? { contentIndex: message.contentIndex } : {}), + }, + }; + } + case "item.started": + case "item.updated": + case "item.completed": { + if (!contextMatchesSession(context, message)) return undefined; + const data = sanitizeHermesItemData(message.itemType, message.data); + const payload = { + itemType: message.itemType, + ...(message.status ? { status: message.status } : {}), + ...(message.title ? { title: message.title } : {}), + ...(message.detail ? { detail: message.detail } : {}), + ...(data ? { data } : {}), + }; + if (message.type === "item.completed") { + recordCompletedItem(context, message.turnId, payload); + } + return { ...(yield* eventBase(message)), type: message.type, payload }; + } + case "request.opened": { + if (!contextMatchesSession(context, message)) return undefined; + context.pendingApprovals.set(message.requestId, { + requestId: message.requestId, + turnId: TurnId.make(message.turnId), + requestType: message.requestType, + }); + const args = sanitizeHermesRequestArgs(message.requestType, message.args); + return { + ...(yield* eventBase(message)), + type: "request.opened", + payload: { + requestType: message.requestType, + ...(message.detail ? { detail: message.detail } : {}), + ...(args ? { args } : {}), + }, + }; + } + case "request.resolved": { + if (!contextMatchesSession(context, message)) return undefined; + context.pendingApprovals.delete(message.requestId); + return { + ...(yield* eventBase(message)), + type: "request.resolved", + payload: { + requestType: message.requestType, + ...(message.decision ? { decision: message.decision } : {}), + }, + }; + } + case "user-input.requested": { + if (!contextMatchesSession(context, message)) return undefined; + context.pendingUserInputs.set(message.requestId, { + requestId: message.requestId, + turnId: TurnId.make(message.turnId), + requestType: "tool_user_input", + }); + return { + ...(yield* eventBase(message)), + type: "user-input.requested", + payload: { questions: message.questions }, + }; + } + case "user-input.resolved": { + if (!contextMatchesSession(context, message)) return undefined; + context.pendingUserInputs.delete(message.requestId); + return { + ...(yield* eventBase(message)), + type: "user-input.resolved", + payload: { answers: message.answers }, + }; + } + case "turn.completed": { + if (!contextMatchesSession(context, message)) return undefined; + if (context.session.activeTurnId === message.turnId) { + clearActiveTurn(context, { status: message.state === "failed" ? "error" : "ready" }); + } + return { + ...(yield* eventBase(message)), + type: "turn.completed", + payload: { + state: message.state, + ...(message.stopReason !== undefined ? { stopReason: message.stopReason } : {}), + ...(message.errorMessage ? { errorMessage: message.errorMessage } : {}), + }, + }; + } + case "turn.aborted": { + if (!contextMatchesSession(context, message)) return undefined; + if (context.session.activeTurnId === message.turnId) { + clearActiveTurn(context, { status: "ready" }); + } + return { + ...(yield* eventBase(message)), + type: "turn.aborted", + payload: { reason: message.reason }, + }; + } + case "session.exited": { + if (!contextMatchesSession(context, message)) return undefined; + clearActiveTurn(context, { status: message.recoverable ? "error" : "closed" }); + const base = yield* eventBase(message); + // A session Hermes has exited is gone on both sides. Dropping it here + // is what keeps the sessions map from growing without bound across a + // long-lived process's reconnect cycles. + sessions.delete(message.threadId); + return { + ...base, + type: "session.exited", + payload: { + ...(message.reason ? { reason: message.reason } : {}), + recoverable: message.recoverable, + exitKind: message.recoverable ? "error" : "graceful", + }, + }; + } + default: + return undefined; + } + }); + + yield* broker.stream.pipe( + Stream.filter((envelope) => envelope.instanceId === input.instanceId), + Stream.runForEach((envelope) => + toRuntimeEvent(envelope.message).pipe( + Effect.flatMap((event) => (event ? emit(event) : Effect.void)), + ), + ), + Effect.forkScoped, + ); + + /** + * Close out every interaction the user could still be looking at. + * + * The plugin keeps `_approval_requests` / `_user_input_requests` in plain + * dicts, so a plugin restart forgets them: any decision the user makes + * afterwards is answered `request-not-found` and the turn waits on a reply + * that can never arrive. We deliberately do not decide on the user's behalf — + * each request is marked resolved with no decision, which clears the pending + * badge, and a warning row explains why it disappeared. + */ + const cancelPendingInteractions = (context: SessionContext, reason: string) => + Effect.gen(function* () { + const approvals = Array.from(context.pendingApprovals.values()); + const userInputs = Array.from(context.pendingUserInputs.values()); + context.pendingApprovals.clear(); + context.pendingUserInputs.clear(); + if (approvals.length === 0 && userInputs.length === 0) return; + for (const pending of approvals) { + yield* emit({ + ...(yield* eventBase({ + threadId: context.session.threadId, + turnId: pending.turnId, + requestId: pending.requestId, + })), + type: "request.resolved", + payload: { requestType: pending.requestType }, + }); + } + for (const pending of userInputs) { + yield* emit({ + ...(yield* eventBase({ + threadId: context.session.threadId, + turnId: pending.turnId, + requestId: pending.requestId, + })), + type: "user-input.resolved", + payload: { answers: {} }, + }); + } + yield* emit({ + ...(yield* eventBase({ threadId: context.session.threadId })), + type: "runtime.warning", + payload: { + message: `${approvals.length + userInputs.length} pending Hermes request(s) were cancelled without a decision: ${reason} Nothing was approved or denied on your behalf — ask again to re-run the step.`, + }, + }); + }); + + /** + * Settle a thread whose turn did not survive the reconnect. + * + * Without this the turn stays "running" forever: Hermes never sends a + * terminal frame for a turn its restarted plugin no longer knows about, and + * orchestration's conflict gate rejects any later frame naming it. A failed + * `turn.completed` is the terminal event orchestration already folds into + * thread state. + */ + const settleDeadTurn = (context: SessionContext, turnId: TurnId, message: string) => + Effect.gen(function* () { + clearActiveTurn(context, { status: "error", lastError: message }); + yield* emit({ + ...(yield* eventBase({ threadId: context.session.threadId, turnId })), + type: "turn.completed", + payload: { state: "failed", errorMessage: message }, + }); + }); + + const resumeSessionAfterReconnect = (context: SessionContext) => + Effect.gen(function* () { + const threadId = context.session.threadId; + const previousTurnId = context.session.activeTurnId; + // Cancellation is deliberately NOT issued here. A connection-generation + // change covers two very different events: a plugin restart (its + // request dicts are gone, pending interactions are unanswerable) and a + // bare socket drop the plugin process survived (it still holds the + // requests and the turn is genuinely blocked on an answer). Which one + // happened is only knowable from the resume outcome below — cancelling + // up front wiped the approval UI while Hermes kept waiting for the + // response, hanging the thread with nothing left to click. + const outcome = yield* broker + .request(input.instanceId, { + type: "session.ensure", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: yield* requestId, + threadId, + resumeSessionId: context.hermesSessionId, + }) + .pipe(Effect.result); + + // Anything other than a clean session.ready leaves us unable to tell + // whether Hermes is still generating, with no channel to ask. Leaving the + // thread "running" would hang it, so it settles with the reason spelled + // out instead — and its pending interactions are unanswerable, so they + // are cancelled too. + const failure = + outcome._tag === "Failure" + ? `Hermes could not resume this thread after reconnecting: ${outcome.failure.detail}` + : outcome.success.type === "protocol.error" + ? `Hermes rejected the resume for this thread: ${outcome.success.message}` + : outcome.success.type !== "session.ready" || outcome.success.threadId !== threadId + ? `Hermes returned '${outcome.success.type}' instead of session.ready while resuming this thread.` + : undefined; + if (failure !== undefined) { + yield* cancelPendingInteractions( + context, + "the Hermes gateway reconnected but this thread could not be resumed.", + ); + if (previousTurnId) return yield* settleDeadTurn(context, previousTurnId, failure); + updateSession(context, { status: "error", lastError: failure }); + return; + } + if (outcome._tag !== "Success" || outcome.success.type !== "session.ready") return; + const ready = outcome.success; + + // The session may have been stopped or exited while the request was in + // flight; do not resurrect it. Its pending maps go with it — nothing to + // cancel on a context no longer routing events. + if (sessions.get(threadId) !== context) return; + + // Hermes owns the session id and a restarted plugin mints a new one. + // Adopting it is what keeps the inbound context guard matching. + context.hermesSessionId = ready.sessionId; + updateSession(context, { + resumeCursor: { + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + sessionId: ready.sessionId, + } satisfies HermesGatewayResumeCursor, + }); + + const activeTurnId = ready.activeTurnId; + if (activeTurnId !== undefined) { + // The turn outlived the socket, and with it the plugin process — its + // request dicts are intact and any pending approval/user-input is + // still answerable over the replacement connection. Keep streaming + // into the turn, keep the requests on screen, emit nothing disruptive. + updateSession(context, { status: "running", activeTurnId }); + trackTurn(context, activeTurnId); + return; + } + + // No active turn on the Hermes side: the plugin restarted (or finished + // and forgot). Its request dicts are per-process, so whatever the user + // is still looking at can only ever be answered `request-not-found`. + yield* cancelPendingInteractions( + context, + "the Hermes gateway reconnected and the plugin no longer holds them.", + ); + if (previousTurnId) { + yield* settleDeadTurn( + context, + previousTurnId, + "The Hermes gateway reconnected without this turn — it did not survive the disconnect.", + ); + return; + } + updateSession(context, { status: "ready" }); + }); + + /** + * The reconnect trigger. Protocol v2 already carries `activeTurnId` on + * `session.ready` for exactly this case, but nothing re-issued + * `session.ensure` after a drop — so a surviving turn's eventual + * `turn.completed` arrived for a turn orchestration had written off, was + * rejected by its conflict gate, and the thread never settled. + */ + // Track the connection identity, not merely whether we are connected. A + // plugin restart usually REPLACES the socket: the old one dies as the new + // one is accepted, so exactly one `connected` status is published and a + // connectedness edge never fires. The plugin's session map is per-process, + // so skipping the re-ensure there leaves T3 holding a session the new + // process has never heard of, and its next turn.start is rejected with + // "Call session.ensure before starting a turn" — a dead thread. + // + // Starts null rather than reading the current generation: sessions are only + // created after this point, so the first status we observe has nothing to + // resume, and treating it as new is harmless. + let seenGeneration: number | null = null; + yield* broker.streamStatuses.pipe( + Stream.filter((status) => status.instanceId === input.instanceId), + Stream.runForEach((status) => + Effect.gen(function* () { + const generation = status.connectionGeneration; + const reconnected = + status.status === "connected" && generation !== null && generation !== seenGeneration; + // Only advance on a live connection: holding the last generation + // through an offline gap means the reconnect still registers as new. + if (generation !== null) seenGeneration = generation; + if (!reconnected) return; + // Forked, NOT awaited. `Stream.runForEach` processes status events + // serially, and the resume issues `session.ensure` — a correlated + // request whose reply is delivered by the broker's message stream. + // Awaiting it here holds up this handler, and the deadlock is only + // broken by the 30s request timeout, after which the thread is left + // with no session and every turn fails "Call session.ensure before + // starting a turn". Resuming off the handler lets the reply land. + yield* Effect.forEach( + Array.from(sessions.values()), + (context) => + resumeSessionAfterReconnect(context).pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider.hermes.resume-after-reconnect-failed", { + instanceId: input.instanceId, + threadId: context.session.threadId, + cause, + }), + ), + ), + { discard: true }, + ).pipe(Effect.forkScoped); + }), + ), + Effect.forkScoped, + ); + + const startSession: HermesAdapterShape["startSession"] = Effect.fn("startSession")( + function* (sessionInput) { + yield* requireConnected("session.ensure"); + if (sessionInput.providerInstanceId && sessionInput.providerInstanceId !== input.instanceId) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "startSession", + issue: `Session targets instance '${sessionInput.providerInstanceId}', expected '${input.instanceId}'.`, + }); + } + const resume = + sessionInput.resumeCursor === undefined + ? undefined + : isResumeCursor(sessionInput.resumeCursor) + ? sessionInput.resumeCursor + : undefined; + if (sessionInput.resumeCursor !== undefined && !resume) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "startSession", + issue: "The Hermes resume cursor is invalid or from an unsupported protocol version.", + }); + } + const response = yield* broker.request(input.instanceId, { + type: "session.ensure", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: yield* requestId, + threadId: sessionInput.threadId, + ...(resume ? { resumeSessionId: resume.sessionId } : {}), + }); + if (response.type !== "session.ready" || response.threadId !== sessionInput.threadId) { + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "session.ensure", + detail: + response.type === "protocol.error" + ? response.message + : `Expected session.ready, received '${response.type}'.`, + }); + } + const createdAt = nowIso(); + const activeTurnId = response.activeTurnId; + const session = { + provider: PROVIDER, + providerInstanceId: input.instanceId, + status: activeTurnId === undefined ? "ready" : "running", + runtimeMode: sessionInput.runtimeMode, + ...(sessionInput.cwd !== undefined ? { cwd: sessionInput.cwd } : {}), + ...(sessionInput.modelSelection?.model ? { model: sessionInput.modelSelection.model } : {}), + ...(activeTurnId !== undefined ? { activeTurnId } : {}), + threadId: sessionInput.threadId, + resumeCursor: { + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + sessionId: response.sessionId, + } satisfies HermesGatewayResumeCursor, + createdAt, + updatedAt: createdAt, + } satisfies ProviderSession; + sessions.set(sessionInput.threadId, { + session, + hermesSessionId: response.sessionId, + turns: activeTurnId === undefined ? [] : [{ id: activeTurnId, items: [] }], + pendingApprovals: new Map(), + pendingUserInputs: new Map(), + }); + yield* emit({ + ...(yield* eventBase({ threadId: sessionInput.threadId })), + type: "session.started", + payload: { + message: response.resumed ? "Hermes session resumed" : "Hermes session started", + resume: session.resumeCursor, + }, + }); + yield* emit({ + ...(yield* eventBase({ threadId: sessionInput.threadId })), + type: "thread.started", + payload: { providerThreadId: response.sessionId }, + }); + return session; + }, + ); + + /** + * Read a turn's attachments off disk and inline them onto the frame. + * + * Inline base64 on purpose: the plugin may run on another machine with no + * authenticated route back to T3's asset endpoint, so a URL would be a + * side channel that sometimes works. The per-turn total is enforced here — + * the schema bounds each file, but only the adapter sees the whole turn. + */ + const readTurnAttachments = Effect.fn("readTurnAttachments")(function* ( + attachments: NonNullable[0]["attachments"]>, + ): Effect.fn.Return< + Array, + ProviderAdapterRequestError | ProviderAdapterValidationError + > { + const frameAttachments: Array = []; + let totalBytes = 0; + for (const attachment of attachments) { + const attachmentPath = resolveAttachmentPath({ + attachmentsDir: serverConfig.attachmentsDir, + attachment, + }); + if (!attachmentPath) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "sendTurn", + issue: `Invalid attachment id '${attachment.id}'.`, + }); + } + const bytes = yield* fileSystem.readFile(attachmentPath).pipe( + Effect.mapError( + (cause) => + new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "turn.start", + detail: `Failed to read attachment '${attachment.name}'.`, + cause, + }), + ), + ); + totalBytes += bytes.byteLength; + if (totalBytes > HERMES_MEDIA_MAX_BYTES) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "sendTurn", + issue: `Attachments total more than ${Math.floor(HERMES_MEDIA_MAX_BYTES / (1024 * 1024))}MB for one turn — remove '${attachment.name}' or send it separately.`, + }); + } + frameAttachments.push({ + name: attachment.name, + mimeType: attachment.mimeType, + sizeBytes: bytes.byteLength, + data: Buffer.from(bytes).toString("base64"), + }); + } + return frameAttachments; + }); + + const sendTurn: HermesAdapterShape["sendTurn"] = Effect.fn("sendTurn")(function* (turnInput) { + const context = yield* findContext(turnInput.threadId); + const text = turnInput.input; + if (!text || text.trim().length === 0) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "sendTurn", + issue: "Hermes turns require text input.", + }); + } + const attachments = + turnInput.attachments && turnInput.attachments.length > 0 + ? yield* readTurnAttachments(turnInput.attachments) + : undefined; + const activeTurnId = context.session.activeTurnId; + const method = activeTurnId ? "turn.steer" : "turn.start"; + // Checked before the request so an offline gateway fails immediately with + // an actionable message instead of parking on the broker's request timeout. + yield* requireConnected(method); + const turnId = activeTurnId ?? TurnId.make(`hermes-turn-${yield* randomId}`); + const outboundRequestId = yield* requestId; + const response = yield* broker.request(input.instanceId, { + type: activeTurnId ? "turn.steer" : "turn.start", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: outboundRequestId, + threadId: turnInput.threadId, + sessionId: context.hermesSessionId, + turnId, + text, + ...(attachments !== undefined ? { attachments } : {}), + }); + if (response.type !== "turn.started") { + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method, + detail: + response.type === "protocol.error" + ? response.message + : `Expected turn.started, received '${response.type}'.`, + }); + } + updateSession(context, { status: "running", activeTurnId: turnId }); + trackTurn(context, turnId); + return { + threadId: turnInput.threadId, + turnId, + resumeCursor: context.session.resumeCursor, + }; + }); + + /** + * Enrich the snapshot-derived description with what the live plugin knows. + * + * Hermes runs on its own machine, so almost everything worth showing on the + * Agent page — skills, reasoning effort, the agent's own version — is only + * knowable by asking. Fields the plugin omits (because it could not read + * them from Hermes) deliberately fall through to the base description rather + * than overwriting it with a null. + */ + const describe: NonNullable = (base) => + Effect.gen(function* () { + yield* requireConnected("describe.request"); + const response = yield* broker.request(input.instanceId, { + type: "describe.request", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: yield* requestId, + }); + if (response.type !== "describe.response") { + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "describe.request", + detail: + response.type === "protocol.error" + ? response.message + : `Expected describe.response, received '${response.type}'.`, + }); + } + return { + ...base, + identity: { + ...base.identity, + agentVersion: response.hermesVersion, + pluginVersion: response.pluginVersion, + protocolVersion: response.protocolVersion, + }, + model: + response.model === undefined && response.reasoningEffort === undefined + ? base.model + : { + id: response.model ?? base.model?.id ?? null, + displayName: response.model ?? base.model?.displayName ?? null, + vendor: base.model?.vendor ?? null, + contextWindow: base.model?.contextWindow ?? null, + reasoningEffortLabel: + response.reasoningEffort ?? base.model?.reasoningEffortLabel ?? null, + }, + skills: response.skills.map( + (skill): ServerProviderSkill => ({ + name: skill.name, + ...(skill.description !== undefined ? { description: skill.description } : {}), + // Hermes' public skills surface publishes no on-disk path; its + // category is the closest analogue and doubles as the scope chip. + path: skill.source ?? skill.name, + ...(skill.source !== undefined ? { scope: skill.source } : {}), + enabled: skill.enabled, + }), + ), + // The wire struct mixes a version number in with the boolean flags; + // the description's capability map is booleans only, and the version + // is already carried on `identity.protocolVersion`. + capabilities: { + streaming: response.capabilities.streaming, + activity: response.capabilities.activity, + approvals: response.capabilities.approvals, + userInput: response.capabilities.userInput, + attachments: response.capabilities.attachments, + }, + describedAt: response.describedAt, + }; + }); + + const getSkillBody: NonNullable = (skillName) => + Effect.gen(function* () { + yield* requireConnected("skill.body.request"); + const response = yield* broker.request(input.instanceId, { + type: "skill.body.request", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: yield* requestId, + skillName, + }); + if (response.type !== "skill.body.response") { + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "skill.body.request", + detail: + response.type === "protocol.error" + ? response.message + : `Expected skill.body.response, received '${response.type}'.`, + }); + } + return { skillName: response.skillName, markdown: response.markdown }; + }); + + const adapter: HermesAdapterShape = { + provider: PROVIDER, + capabilities: { sessionModelSwitch: "unsupported" }, + describe, + getSkillBody, + startSession, + sendTurn, + interruptTurn: (threadId, selectedTurnId) => + Effect.gen(function* () { + const context = yield* findContext(threadId); + const turnId = selectedTurnId ?? context.session.activeTurnId; + if (!turnId) return; + yield* requireConnected("turn.interrupt"); + yield* broker.send(input.instanceId, { + type: "turn.interrupt", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: yield* requestId, + threadId, + sessionId: context.hermesSessionId, + turnId, + }); + }), + respondToRequest: (threadId, selectedRequestId, decision) => + Effect.gen(function* () { + const context = yield* findContext(threadId); + if (!context.session.activeTurnId) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "respondToRequest", + issue: "The Hermes session has no active turn.", + }); + } + yield* requireConnected("approval.respond"); + yield* broker.send(input.instanceId, { + type: "approval.respond", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: HermesGatewayRequestId.make(selectedRequestId), + threadId, + sessionId: context.hermesSessionId, + turnId: context.session.activeTurnId, + decision, + }); + context.pendingApprovals.delete(selectedRequestId); + }), + respondToUserInput: (threadId, selectedRequestId, answers) => + Effect.gen(function* () { + const context = yield* findContext(threadId); + if (!context.session.activeTurnId) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "respondToUserInput", + issue: "The Hermes session has no active turn.", + }); + } + yield* requireConnected("user-input.respond"); + yield* broker.send(input.instanceId, { + type: "user-input.respond", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: HermesGatewayRequestId.make(selectedRequestId), + threadId, + sessionId: context.hermesSessionId, + turnId: context.session.activeTurnId, + answers, + }); + context.pendingUserInputs.delete(selectedRequestId); + }), + stopSession: (threadId) => + Effect.gen(function* () { + const context = yield* findContext(threadId); + if (!(yield* broker.isConnected(input.instanceId))) { + // The stop was never delivered: Hermes still owns this thread and + // keeps generating. Deleting our state here would orphan it — frames + // arriving after a reconnect would have no session to match, and the + // user would have no way to stop it. Keep the session so the reconnect + // path can resume it and a later stop can actually land. + updateSession(context, { status: "error", lastError: offlineDetail(input.instanceId) }); + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "session.stop", + detail: offlineDetail(input.instanceId), + }); + } + yield* broker.send(input.instanceId, { + type: "session.stop", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: yield* requestId, + threadId, + sessionId: context.hermesSessionId, + }); + updateSession(context, { status: "closed" }); + sessions.delete(threadId); + }), + listSessions: () => Effect.sync(() => Array.from(sessions.values(), ({ session }) => session)), + hasSession: (threadId) => Effect.sync(() => sessions.has(threadId)), + readThread: (threadId) => + findContext(threadId).pipe( + Effect.map((context) => ({ + threadId, + // Bounded to the most recent turns/items — older history lives in the + // orchestration projections, not in adapter memory. + turns: context.turns, + })), + ), + rollbackThread: (threadId) => + findContext(threadId).pipe( + Effect.flatMap(() => + Effect.fail( + new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "rollbackThread", + issue: "Hermes gateway thread rollback is not supported.", + }), + ), + ), + ), + stopAll: () => + Effect.forEach( + Array.from(sessions.keys()), + (threadId) => + // A stop that could not be delivered (typically: the gateway is + // offline) must not abort the sweep or fail shutdown. The session is + // deliberately left in place rather than orphaned. + adapter.stopSession(threadId).pipe( + Effect.catchTags({ + ProviderAdapterRequestError: (error) => + Effect.logWarning("provider.hermes.stop-session-undeliverable", { + instanceId: input.instanceId, + threadId, + detail: error.detail, + }), + // The keys snapshot is taken before the first stop runs, and the + // broker event fiber deletes sessions on `session.exited` — so a + // session can disappear mid-sweep. That is the outcome this sweep + // wants anyway; aborting the remaining threads over it is not. + ProviderAdapterSessionNotFoundError: () => Effect.void, + }), + ), + { discard: true }, + ), + streamEvents: Stream.fromPubSub(events), + }; + + // Deliberately NOT `stopAll()`. Hermes sessions live in a separate process + // that outlives this one, and `session.stop` makes the plugin drop the + // thread from its active set — so tearing down on shutdown meant every + // thread open when T3 restarted came back unusable: the next turn.start was + // rejected with "Call session.ensure before starting a turn", with no way to + // recover from the UI. Threads created after the restart worked, which is + // what made it look like a resume bug rather than a shutdown bug. + // + // Dropping local state without telling the plugin is the correct shutdown: + // the resume cursor is persisted, and the next turn re-issues session.ensure + // against the session Hermes still holds. Explicit user-initiated stops + // still go through `stopSession` and do end the remote session. + yield* Effect.addFinalizer(() => + Effect.sync(() => { + sessions.clear(); + }), + ); + return adapter; +}); diff --git a/apps/server/src/provider/Layers/HermesConnectionRegistry.ts b/apps/server/src/provider/Layers/HermesConnectionRegistry.ts new file mode 100644 index 00000000000..7e343a4a74d --- /dev/null +++ b/apps/server/src/provider/Layers/HermesConnectionRegistry.ts @@ -0,0 +1,213 @@ +/** + * Live {@link HermesConnectionRegistry}. See the service module for why this + * state is memory-only and how generation fencing works. + * + * @module Layers/HermesConnectionRegistry + */ +import type { ProviderInstanceId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Ref from "effect/Ref"; +import * as Scope from "effect/Scope"; + +import type { + HermesActiveConnection, + HermesConnectionRegistry, + HermesInstanceLiveness, + HermesLivenessStatusFields, + HermesObservedConnection, +} from "../Services/HermesConnectionRegistry.ts"; + +/** Strip the transport/scope/generation from a connection to keep as `lastSeen`. */ +const toObserved = (connection: HermesActiveConnection): HermesObservedConnection => ({ + pluginVersion: connection.pluginVersion, + hermesVersion: connection.hermesVersion, + capabilities: connection.capabilities, + model: connection.model, + connectedAt: connection.connectedAt, + activeSessionCount: connection.activeSessionCount, +}); + +/** + * Project liveness into the status fields it owns. Prefers the live connection, + * falls back to `lastSeen`, and finally to whatever the incompatible plugin + * advertised. + */ +export const livenessStatusFields = ( + liveness: HermesInstanceLiveness | undefined, +): HermesLivenessStatusFields => { + const observed = liveness?.connection ?? liveness?.lastSeen; + return { + lastConnectedAt: observed?.connectedAt ?? null, + pluginVersion: observed?.pluginVersion ?? liveness?.upgradeRequired?.pluginVersion ?? null, + hermesVersion: observed?.hermesVersion ?? liveness?.upgradeRequired?.hermesVersion ?? null, + model: observed?.model ?? liveness?.upgradeRequired?.model ?? null, + activeSessionCount: liveness?.connection?.activeSessionCount ?? 0, + protocolVersion: + observed?.capabilities.protocolVersion ?? liveness?.upgradeRequired?.protocolVersion ?? null, + capabilities: observed?.capabilities ?? null, + // Exposed so consumers can tell a replacement (old socket dies as a new + // one is accepted, publishing a single `connected` status) from a + // continuous connection. Watching connectedness alone cannot. + connectionGeneration: liveness?.connection?.generation ?? null, + connected: liveness?.connection !== undefined, + upgradeRequired: liveness?.upgradeRequired !== undefined, + }; +}; + +export const makeHermesConnectionRegistry = Effect.gen(function* () { + const states = yield* Ref.make(new Map()); + const generation = yield* Ref.make(0); + + /** Write liveness for one instance, dropping the entry when it goes empty. */ + const setLiveness = ( + current: ReadonlyMap, + instanceId: ProviderInstanceId, + liveness: HermesInstanceLiveness, + ) => { + const next = new Map(current); + if ( + liveness.connection === undefined && + liveness.lastSeen === undefined && + liveness.upgradeRequired === undefined + ) { + next.delete(instanceId); + } else { + next.set(instanceId, liveness); + } + return next; + }; + + const liveness = (instanceId: ProviderInstanceId) => + Ref.get(states).pipe(Effect.map((current) => current.get(instanceId))); + + const accept: HermesConnectionRegistry["accept"] = (input) => + Effect.gen(function* () { + const nextGeneration = yield* Ref.getAndUpdate(generation, (value) => value + 1); + // The connection scope is a child of nothing in particular: it is closed + // explicitly on disconnect/replace so per-connection fibers (ping) die + // exactly with the connection they belong to. + const scope = yield* Scope.make(); + const connection: HermesActiveConnection = { + ...input.observed, + generation: nextGeneration, + transport: input.transport, + scope, + }; + + const displaced = yield* Ref.modify(states, (current) => { + const found = current.get(input.instanceId); + return [ + found?.connection, + setLiveness(current, input.instanceId, { + connection, + lastSeen: toObserved(connection), + // Accepting a compatible connection clears any stale + // upgrade-required observation for this instance. + upgradeRequired: undefined, + }), + ] as const; + }); + + return { generation: nextGeneration, displaced }; + }); + + const markUpgradeRequired: HermesConnectionRegistry["markUpgradeRequired"] = (input) => + Ref.modify(states, (current) => { + const found = current.get(input.instanceId); + return [ + found?.connection, + setLiveness(current, input.instanceId, { + connection: undefined, + lastSeen: found?.lastSeen, + upgradeRequired: input.upgradeRequired, + }), + ] as const; + }); + + const recordSessionCount: HermesConnectionRegistry["recordSessionCount"] = ( + registration, + activeSessionCount, + ) => + Ref.modify(states, (current) => { + const found = current.get(registration.instanceId); + // Fence inside the update so a replacement that lands between a read and + // this write cannot be clobbered. + if (found?.connection?.generation !== registration.generation) { + return [false, current] as const; + } + const connection: HermesActiveConnection = { ...found.connection, activeSessionCount }; + return [ + true, + setLiveness(current, registration.instanceId, { + ...found, + connection, + lastSeen: toObserved(connection), + }), + ] as const; + }); + + const disconnect: HermesConnectionRegistry["disconnect"] = (registration) => + Effect.gen(function* () { + const retired = yield* Ref.modify(states, (current) => { + const found = current.get(registration.instanceId); + if (found?.connection?.generation !== registration.generation) { + return [undefined, current] as const; + } + return [ + found.connection, + setLiveness(current, registration.instanceId, { + connection: undefined, + lastSeen: toObserved(found.connection), + upgradeRequired: found.upgradeRequired, + }), + ] as const; + }); + if (retired) yield* Scope.close(retired.scope, Exit.void).pipe(Effect.ignore); + return retired; + }); + + const clearConnection: HermesConnectionRegistry["clearConnection"] = (instanceId) => + Effect.gen(function* () { + const cleared = yield* Ref.modify(states, (current) => { + const found = current.get(instanceId); + if (!found?.connection) return [undefined, current] as const; + return [ + found.connection, + setLiveness(current, instanceId, { + connection: undefined, + lastSeen: toObserved(found.connection), + upgradeRequired: found.upgradeRequired, + }), + ] as const; + }); + if (cleared) yield* Scope.close(cleared.scope, Exit.void).pipe(Effect.ignore); + return cleared; + }); + + const forget = (instanceId: ProviderInstanceId) => + Effect.gen(function* () { + const dropped = yield* Ref.modify(states, (current) => { + const found = current.get(instanceId); + if (!found) return [undefined, current] as const; + const next = new Map(current); + next.delete(instanceId); + return [found.connection, next] as const; + }); + if (dropped) yield* Scope.close(dropped.scope, Exit.void).pipe(Effect.ignore); + }); + + return { + liveness, + accept, + markUpgradeRequired, + recordSessionCount, + disconnect, + clearConnection, + forget, + isConnected: (instanceId) => + liveness(instanceId).pipe(Effect.map((found) => found?.connection !== undefined)), + transport: (instanceId) => + liveness(instanceId).pipe(Effect.map((found) => found?.connection?.transport)), + } satisfies HermesConnectionRegistry; +}); diff --git a/apps/server/src/provider/Layers/HermesEnrollmentStore.ts b/apps/server/src/provider/Layers/HermesEnrollmentStore.ts new file mode 100644 index 00000000000..6804d45803f --- /dev/null +++ b/apps/server/src/provider/Layers/HermesEnrollmentStore.ts @@ -0,0 +1,106 @@ +/** + * Live {@link HermesEnrollmentStore}. See the service module for the token + * rules this upholds. + * + * @module Layers/HermesEnrollmentStore + */ +import { HermesGatewayEnrollmentToken, type ProviderInstanceId } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Crypto from "effect/Crypto"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as Ref from "effect/Ref"; + +import type { + HermesEnrollmentStore, + PendingEnrollment, +} from "../Services/HermesEnrollmentStore.ts"; + +const ENROLLMENT_TOKEN_BYTES = 32; + +export interface MakeHermesEnrollmentStoreOptions { + readonly ttl: Duration.Duration; +} + +export const makeHermesEnrollmentStore = ( + options: MakeHermesEnrollmentStoreOptions, +): Effect.Effect => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const enrollments = yield* Ref.make(new Map()); + + const mint: HermesEnrollmentStore["mint"] = (input) => + Effect.gen(function* () { + const bytes = yield* crypto.randomBytes(ENROLLMENT_TOKEN_BYTES).pipe(Effect.orDie); + const token = HermesGatewayEnrollmentToken.make(Encoding.encodeBase64Url(bytes)); + const expiresAtMillis = (yield* Clock.currentTimeMillis) + Duration.toMillis(options.ttl); + + yield* Ref.update(enrollments, (current) => { + const next = new Map(current); + // Minting supersedes this instance's previous tokens so only the + // newest unconsumed token is redeemable. + for (const [pendingToken, pending] of current) { + if (pending.input.instanceId === input.instanceId) next.delete(pendingToken); + } + return next.set(token, { input, expiresAtMillis }); + }); + + return { token, expiresAtMillis }; + }); + + const peek: HermesEnrollmentStore["peek"] = (token) => + Effect.gen(function* () { + const found = (yield* Ref.get(enrollments)).get(token); + if (!found) return undefined; + return found.expiresAtMillis < (yield* Clock.currentTimeMillis) ? undefined : found; + }); + + const consume: HermesEnrollmentStore["consume"] = (token, expected) => + Effect.gen(function* () { + // Compare-and-swap: delete only if the entry is still the exact object + // the caller authenticated against, so a concurrent redemption or a + // re-mint in between cannot be consumed by this caller. + const claimed = yield* Ref.modify(enrollments, (current) => { + const found = current.get(token); + if (found !== expected) return [undefined, current] as const; + const next = new Map(current); + next.delete(token); + return [found, next] as const; + }); + if (claimed === undefined) return undefined; + // Re-check expiry after winning the swap: the entry may have aged out + // between authentication and consumption. + return claimed.expiresAtMillis < (yield* Clock.currentTimeMillis) ? undefined : claimed; + }); + + const forget = (instanceId: ProviderInstanceId) => + Ref.update(enrollments, (current) => { + const next = new Map(current); + for (const [token, pending] of current) { + if (pending.input.instanceId === instanceId) next.delete(token); + } + return next; + }); + + const sweep = Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + Ref.update(enrollments, (current) => { + const next = new Map(current); + for (const [token, pending] of current) { + if (pending.expiresAtMillis < now) next.delete(token); + } + return next.size === current.size ? current : next; + }), + ), + ); + + return { + mint, + peek, + consume, + forget, + sweep, + size: Ref.get(enrollments).pipe(Effect.map((current) => current.size)), + } satisfies HermesEnrollmentStore; + }); diff --git a/apps/server/src/provider/Layers/HermesGatewayBroker.test.ts b/apps/server/src/provider/Layers/HermesGatewayBroker.test.ts new file mode 100644 index 00000000000..5a39fc86a45 --- /dev/null +++ b/apps/server/src/provider/Layers/HermesGatewayBroker.test.ts @@ -0,0 +1,1507 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + HERMES_GATEWAY_PROTOCOL_VERSION, + HermesGatewayCredential, + HermesGatewayRequestId, + HermesGatewaySessionId, + ProviderInstanceId, + ThreadId, + type HermesGatewayConnectionHello, + type HermesGatewayT3ToPluginMessage, +} from "@t3tools/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; +import * as ServerConfig from "../../config.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import { HermesDriver } from "../Drivers/HermesDriver.ts"; +import type { ProviderInstance } from "../ProviderDriver.ts"; +import { + HermesGatewayBroker, + type HermesGatewayBrokerShape, + type HermesGatewayConnectionRegistration, + type HermesGatewayTransport, +} from "../Services/HermesGatewayBroker.ts"; +import { + HermesGatewayBrokerLive, + hermesGatewayCredentialSecretName, + hermesGatewayLegacyMetadataSecretName, + makeHermesGatewayBroker, +} from "./HermesGatewayBroker.ts"; + +/** + * A version this server does not speak. Expressed relative to the supported + * version rather than as a literal, so bumping the protocol does not turn + * these "must be rejected" cases into "is the current version" cases. + */ +const UNSUPPORTED_PROTOCOL_VERSION = HERMES_GATEWAY_PROTOCOL_VERSION + 1; + +const instanceId = ProviderInstanceId.make("hermes_remote"); +const otherInstanceId = ProviderInstanceId.make("hermes_other"); +const defaultHermesInstanceId = ProviderInstanceId.make("hermes"); + +class HermesTestInstance extends Context.Service()( + "t3/provider/Layers/HermesGatewayBroker.test/HermesTestInstance", +) {} + +const capabilities = { + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + streaming: true, + activity: true, + approvals: true, + userInput: true, + // Part of the v4 contract itself: a hello advertising `false` fails the + // strict capabilities check and is rejected at the version gate. + attachments: true, +} as const; + +const makeSecretStore = () => { + const values = new Map(); + const service: ServerSecretStore.ServerSecretStore["Service"] = { + get: (name) => Effect.succeed(Option.fromUndefinedOr(values.get(name))), + set: (name, value) => Effect.sync(() => values.set(name, value)).pipe(Effect.asVoid), + create: (name, value) => Effect.sync(() => values.set(name, value)).pipe(Effect.asVoid), + getOrCreateRandom: (_name, bytes) => Effect.succeed(new Uint8Array(bytes)), + remove: (name) => Effect.sync(() => values.delete(name)).pipe(Effect.asVoid), + }; + return service; +}; + +const hermesInstances = { + [defaultHermesInstanceId]: { driver: "hermes", displayName: "Hermes", config: {} }, + [instanceId]: { driver: "hermes", displayName: "Remote", config: {} }, + [otherInstanceId]: { driver: "hermes", displayName: "Other", config: {} }, +} as const; + +/** + * Broker over the ambient settings service, so a test can assert on what the + * broker wrote and can build a second broker over the same durable state to + * model a server restart. + */ +const makeBroker = (secrets: ServerSecretStore.ServerSecretStore["Service"]) => + makeHermesGatewayBroker.pipe( + Effect.provideService(ServerSecretStore.ServerSecretStore, secrets), + Effect.provide(NodeServices.layer), + ); + +/** + * Broker over an explicitly supplied settings service, so a test can interpose + * on `getSettings` and hold a handshake at a chosen read while another + * operation runs to completion. + */ +const makeBrokerWith = ( + secrets: ServerSecretStore.ServerSecretStore["Service"], + settingsService: ServerSettings.ServerSettingsService["Service"], +) => + makeHermesGatewayBroker.pipe( + Effect.provideService(ServerSecretStore.ServerSecretStore, secrets), + Effect.provideService(ServerSettings.ServerSettingsService, settingsService), + Effect.provide(NodeServices.layer), + ); + +/** Ambient layer: one settings service shared by the test and its brokers. */ +const testLayer = Layer.mergeAll( + ServerSettings.layerTest({ providerInstances: hermesInstances }), + NodeServices.layer, +); + +const hello = ( + authentication: HermesGatewayConnectionHello["authentication"], + protocolVersion: number = HERMES_GATEWAY_PROTOCOL_VERSION, + overrides: { + readonly model?: string; + readonly role?: HermesGatewayConnectionHello["role"]; + } = {}, +): HermesGatewayConnectionHello => ({ + type: "connection.hello", + requestId: HermesGatewayRequestId.make(`hello-${protocolVersion}`), + protocolVersion, + pluginVersion: "0.2.0", + hermesVersion: "1.0.0", + capabilities: { ...capabilities, protocolVersion }, + authentication, + role: overrides.role ?? "gateway", + ...(overrides.model !== undefined ? { model: overrides.model } : {}), +}); + +/** Transport that records everything it is asked to do. */ +const recordingTransport = () => { + const sent: Array = []; + const closes: Array = []; + const transport: HermesGatewayTransport = { + send: (message) => Effect.sync(() => sent.push(message)).pipe(Effect.asVoid), + close: (code) => Effect.sync(() => closes.push(code)).pipe(Effect.asVoid), + }; + return { sent, closes, transport }; +}; + +const enroll = (broker: HermesGatewayBrokerShape, id: ProviderInstanceId, nickname: string) => + broker.createEnrollment({ + instanceId: id, + nickname, + connectorUrl: "https://t3.example.test", + }); + +it.effect("authenticates before applying incompatible connection state", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const broker = yield* makeBroker(secrets); + yield* enroll(broker, defaultHermesInstanceId, "Default Hermes"); + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + + const first = recordingTransport(); + const registered = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + first.transport, + ); + assert.isTrue(registered.accepted.credential !== undefined); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "connected"); + + const second = recordingTransport(); + const replacement = yield* broker.registerConnection( + hello({ + type: "instance-credential", + instanceId, + credential: registered.accepted.credential!, + }), + second.transport, + ); + assert.deepEqual(first.closes, [4001]); + + const threadId = ThreadId.make("pending-thread"); + const pendingRequestId = HermesGatewayRequestId.make("pending-session"); + const pendingRequest = yield* broker + .request(instanceId, { + type: "session.ensure", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: pendingRequestId, + threadId, + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + assert.equal(second.sent.at(-1)?.type, "session.ensure"); + + // A bad credential carried on an incompatible protocol version must be + // rejected for the credential, never for the version: rejecting on version + // first would let an unauthenticated caller knock the instance offline. + const malicious = yield* Effect.flip( + broker.registerConnection( + hello( + { + type: "instance-credential", + instanceId, + credential: HermesGatewayCredential.make("not-the-real-credential"), + }, + UNSUPPORTED_PROTOCOL_VERSION, + ), + second.transport, + ), + ); + assert.equal(malicious.code, "invalid-authentication"); + assert.deepEqual(second.closes, []); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "connected"); + assert.isUndefined(pendingRequest.pollUnsafe()); + + yield* broker.receive(replacement, { + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: pendingRequestId, + threadId, + sessionId: HermesGatewaySessionId.make("pending-session-id"), + resumed: false, + }); + assert.equal((yield* Fiber.join(pendingRequest)).type, "session.ready"); + + const incompatible = yield* Effect.flip( + broker.registerConnection( + hello( + { + type: "instance-credential", + instanceId, + credential: registered.accepted.credential!, + }, + UNSUPPORTED_PROTOCOL_VERSION, + ), + second.transport, + ), + ); + assert.equal(incompatible.code, "version-incompatible"); + assert.deepEqual(second.closes, [4004]); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "upgrade-required"); + + const otherEnrollment = yield* enroll(broker, otherInstanceId, "Other Hermes"); + const incompatibleEnrollment = yield* Effect.flip( + broker.registerConnection( + hello( + { type: "enrollment-token", token: otherEnrollment.oneTimeToken }, + UNSUPPORTED_PROTOCOL_VERSION, + ), + second.transport, + ), + ); + assert.equal(incompatibleEnrollment.code, "version-incompatible"); + const enrolledAfterUpgrade = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: otherEnrollment.oneTimeToken }), + second.transport, + ); + assert.equal(enrolledAfterUpgrade.instanceId, otherInstanceId); + + const revoked = yield* broker.revokeInstance(instanceId); + assert.equal(revoked.status, "revoked"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("invalidates the previous credential when replacement enrollment begins", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + const { closes, transport } = recordingTransport(); + const first = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ); + const replacement = yield* enroll(broker, instanceId, "Remote Hermes"); + assert.deepEqual(closes, [4001]); + + const staleCredential = yield* Effect.flip( + broker.registerConnection( + hello({ + type: "instance-credential", + instanceId, + credential: first.accepted.credential!, + }), + transport, + ), + ); + assert.equal(staleCredential.code, "invalid-authentication"); + const replacementConnection = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: replacement.oneTimeToken }), + transport, + ); + assert.equal(replacementConnection.instanceId, instanceId); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("accepts only the newest unconsumed enrollment token for an instance", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const older = yield* enroll(broker, instanceId, "Remote Hermes"); + const newest = yield* enroll(broker, instanceId, "Remote Hermes"); + const { transport } = recordingTransport(); + + const olderTokenError = yield* Effect.flip( + broker.registerConnection( + hello({ type: "enrollment-token", token: older.oneTimeToken }), + transport, + ), + ); + assert.equal(olderTokenError.code, "enrollment-expired"); + const newestConnection = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: newest.oneTimeToken }), + transport, + ); + assert.equal(newestConnection.instanceId, instanceId); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("redeems an enrollment token exactly once under concurrent redemption", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + + const raceFirst = yield* broker + .registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + const raceSecond = yield* broker + .registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + + const outcomes = [yield* Fiber.join(raceFirst), yield* Fiber.join(raceSecond)]; + const accepted = outcomes.filter((outcome) => outcome._tag === "Success"); + const rejected = outcomes.filter((outcome) => outcome._tag === "Failure"); + assert.equal(accepted.length, 1); + assert.equal(rejected.length, 1); + assert.equal( + rejected[0]?._tag === "Failure" ? rejected[0].failure.code : "", + "enrollment-expired", + ); + }).pipe(Effect.provide(testLayer)), +); + +/** + * Secret store that can hold the *first* `set` — the credential write at the + * heart of the handshake — open until a test releases it. That is the exact + * point a competing enrollment or revocation used to be able to slip past a + * half-finished handshake. + */ +const gatedCredentialWriteSecretStore = () => + Effect.gen(function* () { + const inner = makeSecretStore(); + const reachedWrite = yield* Deferred.make(); + const releaseWrite = yield* Deferred.make(); + let gated = false; + const service: ServerSecretStore.ServerSecretStore["Service"] = { + ...inner, + set: (name, value) => + Effect.suspend(() => { + if (gated) return inner.set(name, value); + gated = true; + return Deferred.succeed(reachedWrite, undefined).pipe( + Effect.andThen(Deferred.await(releaseWrite)), + Effect.andThen(inner.set(name, value)), + ); + }), + }; + return { secrets: service, reachedWrite, releaseWrite } as const; + }); + +// The credential write and the acceptance that follows it are the mutation +// half of the handshake. Before they were fenced by the instance lock, a +// `createEnrollment` could complete entirely inside that window: it removed the +// outgoing credential and minted a replacement token, and then the suspended +// handshake resumed and wrote a *fresh* credential for the enrollment that had +// just been superseded — leaving the old enrollee able to authenticate forever. +it.effect("does not resurrect a credential for an enrollment superseded mid-handshake", () => + Effect.gen(function* () { + const { secrets, reachedWrite, releaseWrite } = yield* gatedCredentialWriteSecretStore(); + const broker = yield* makeBrokerWith(secrets, yield* ServerSettings.ServerSettingsService); + const enrollment = yield* enroll(broker, instanceId, "Superseded Hermes"); + + const handshake = yield* broker + .registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + // The handshake is now parked with its token already consumed, holding the + // instance lock. + yield* Deferred.await(reachedWrite); + + const replacement = yield* enroll(broker, instanceId, "Superseded Hermes").pipe( + Effect.forkChild({ startImmediately: true }), + ); + // The replacement enrollment must not be able to interleave with the + // handshake's mutation section: it waits for the lock instead. + for (let i = 0; i < 20; i += 1) yield* Effect.yieldNow; + assert.isUndefined( + replacement.pollUnsafe(), + "a replacement enrollment must not run inside a handshake's mutation section", + ); + + yield* Deferred.succeed(releaseWrite, undefined); + const registered = yield* Fiber.join(handshake); + yield* Fiber.join(replacement); + + // Whichever order they serialize in, the enrollment that ran last wins: the + // superseded enrollee's credential is gone from the store... + assert.isTrue( + Option.isNone(yield* secrets.get(hermesGatewayCredentialSecretName(instanceId))), + "the superseded enrollment's credential must not survive the replacement", + ); + // ...and cannot be replayed to authenticate. + if (registered._tag === "Success" && registered.success.accepted.credential) { + const replayed = yield* Effect.flip( + broker.registerConnection( + hello({ + type: "instance-credential", + instanceId, + credential: registered.success.accepted.credential, + }), + recordingTransport().transport, + ), + ); + assert.equal(replayed.code, "invalid-authentication"); + } + }).pipe(Effect.provide(testLayer)), +); + +// The mirror of the above for revocation. `registerConnection` used to read a +// non-revoked record, pause, and then call `connections.accept` after a +// concurrent `revokeInstance` had already persisted `revoked: true` and cleared +// the connection — re-establishing a live socket for a revoked instance that +// nothing would reap until the next reconnect. +it.effect("leaves no live connection when revocation races a handshake", () => + Effect.gen(function* () { + const { secrets, reachedWrite, releaseWrite } = yield* gatedCredentialWriteSecretStore(); + const broker = yield* makeBrokerWith(secrets, yield* ServerSettings.ServerSettingsService); + const enrollment = yield* enroll(broker, instanceId, "Revoked Race Hermes"); + const { closes, transport } = recordingTransport(); + + const handshake = yield* broker + .registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(reachedWrite); + + const revoke = yield* broker + .revokeInstance(instanceId) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + for (let i = 0; i < 20; i += 1) yield* Effect.yieldNow; + assert.isUndefined( + revoke.pollUnsafe(), + "revocation must not run inside a handshake's mutation section", + ); + + yield* Deferred.succeed(releaseWrite, undefined); + yield* Fiber.join(handshake); + const revoked = yield* Fiber.join(revoke); + assert.equal(revoked._tag, "Success"); + + // Serializing the two is enough: the handshake may win the lock and be + // accepted, but revocation then runs to completion behind it and tears the + // socket down. What must never hold is a revoked instance with a live + // connection. + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "revoked"); + assert.isFalse( + yield* broker.isConnected(instanceId), + "a revoked instance must not be left connected", + ); + assert.deepEqual(closes, [4003]); + assert.isTrue(Option.isNone(yield* secrets.get(hermesGatewayCredentialSecretName(instanceId)))); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("expires enrollment tokens from memory without a redemption attempt", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + + // Past the 10 minute TTL and past a sweep tick. + yield* TestClock.adjust(Duration.minutes(11)); + + const expired = yield* Effect.flip( + broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ), + ); + assert.equal(expired.code, "enrollment-expired"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("finalizes revocation when credential deletion fails", () => + Effect.gen(function* () { + const storedSecrets = makeSecretStore(); + let failRemovals = false; + const secrets: ServerSecretStore.ServerSecretStore["Service"] = { + ...storedSecrets, + remove: (name) => + failRemovals + ? Effect.fail( + new ServerSecretStore.SecretStoreRemoveError({ + resource: `secret ${name}`, + cause: new Error("forced credential deletion failure"), + }), + ) + : storedSecrets.remove(name), + }; + const broker = yield* makeBroker(secrets); + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + const { sent, closes, transport } = recordingTransport(); + const registration = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ); + const credential = registration.accepted.credential; + if (!credential) { + return yield* Effect.die(new Error("enrollment did not issue a credential")); + } + + const threadId = ThreadId.make("revoke-pending-thread"); + const pendingRequestId = HermesGatewayRequestId.make("revoke-pending-request"); + const pendingRequest = yield* broker + .request(instanceId, { + type: "session.ensure", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: pendingRequestId, + threadId, + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + assert.equal(sent.at(-1)?.type, "session.ensure"); + const statusEvent = yield* Stream.runHead(broker.streamStatuses).pipe( + Effect.forkChild({ startImmediately: true }), + ); + + failRemovals = true; + const revokeError = yield* Effect.flip(broker.revokeInstance(instanceId)); + assert.equal(revokeError.code, "persistence-failed"); + assert.deepEqual(closes, [4003]); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "revoked"); + assert.equal(Option.getOrUndefined(yield* Fiber.join(statusEvent))?.status, "revoked"); + const pendingError = yield* Effect.flip(Fiber.join(pendingRequest)); + assert.include(pendingError.detail, "revoked"); + + const staleReconnect = yield* Effect.flip( + broker.registerConnection( + hello({ type: "instance-credential", instanceId, credential }), + recordingTransport().transport, + ), + ); + assert.equal(staleReconnect.code, "instance-revoked"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("shares one live broker between the gateway route and Hermes provider instance", () => { + const secrets = makeSecretStore(); + const providerLayer = Layer.effect( + HermesTestInstance, + HermesDriver.create({ + instanceId, + displayName: "Remote Hermes", + environment: [], + enabled: true, + config: HermesDriver.defaultConfig(), + }), + ).pipe( + Layer.provideMerge(HermesGatewayBrokerLive), + Layer.provide( + ServerSettings.layerTest({ + providerInstances: { + [instanceId]: { driver: "hermes", displayName: "Remote", config: {} }, + }, + }), + ), + Layer.provide(Layer.succeed(ServerSecretStore.ServerSecretStore, secrets)), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3code-hermes-broker-test-" })), + Layer.provide(NodeServices.layer), + ); + + return Effect.gen(function* () { + const broker = yield* HermesGatewayBroker; + const provider = yield* HermesTestInstance; + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + const { sent, transport } = recordingTransport(); + const registration = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ); + assert.equal((yield* provider.snapshot.getSnapshot).status, "ready"); + + const threadId = ThreadId.make("shared-broker-thread"); + const startSession = yield* provider.adapter + .startSession({ + threadId, + providerInstanceId: instanceId, + runtimeMode: "full-access", + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + const ensure = sent.at(-1); + if (!ensure || ensure.type !== "session.ensure") { + return yield* Effect.die(new Error("session.ensure did not reach the gateway transport")); + } + const sessionId = HermesGatewaySessionId.make("shared-broker-session"); + yield* broker.receive(registration, { + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: ensure.requestId, + threadId, + sessionId, + resumed: false, + }); + yield* Fiber.join(startSession); + + const sendTurn = yield* provider.adapter + .sendTurn({ threadId, input: "hello through the shared broker" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + const turnStart = sent.at(-1); + if (!turnStart || turnStart.type !== "turn.start") { + return yield* Effect.die(new Error("turn.start did not reach the gateway transport")); + } + yield* broker.receive(registration, { + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: turnStart.requestId, + threadId, + sessionId, + turnId: turnStart.turnId, + }); + assert.equal((yield* Fiber.join(sendTurn)).turnId, turnStart.turnId); + }).pipe(Effect.provide(providerLayer), Effect.scoped); +}); + +// Reproduces the user-reported failure end to end through the REAL broker and +// the REAL adapter (earlier adapter-only tests used a fake broker and passed, +// which is why this survived): send a turn, restart the gateway, send another. +// The second turn must not be rejected with "Call session.ensure before +// starting a turn". +it.effect("resumes an existing thread after the gateway restarts", () => { + const secrets = makeSecretStore(); + const providerLayer = Layer.effect( + HermesTestInstance, + HermesDriver.create({ + instanceId, + displayName: "Restart Hermes", + environment: [], + enabled: true, + config: HermesDriver.defaultConfig(), + }), + ).pipe( + Layer.provideMerge(HermesGatewayBrokerLive), + Layer.provide( + ServerSettings.layerTest({ + providerInstances: { + [instanceId]: { driver: "hermes", displayName: "Restart", config: {} }, + }, + }), + ), + Layer.provide(Layer.succeed(ServerSecretStore.ServerSecretStore, secrets)), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3code-hermes-broker-test-" })), + Layer.provide(NodeServices.layer), + ); + + return Effect.gen(function* () { + const broker = yield* HermesGatewayBroker; + const provider = yield* HermesTestInstance; + const enrollment = yield* enroll(broker, instanceId, "Restart Hermes"); + const threadId = ThreadId.make("gateway-restart-thread"); + const sessionId = HermesGatewaySessionId.make("gateway-restart-session"); + + // ── First connection: establish a session and complete a turn ── + const first = recordingTransport(); + const firstRegistration = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + first.transport, + ); + const startSession = yield* provider.adapter + .startSession({ threadId, providerInstanceId: instanceId, runtimeMode: "full-access" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + const ensure = first.sent.at(-1); + if (!ensure || ensure.type !== "session.ensure") { + return yield* Effect.die(new Error("session.ensure did not reach the transport")); + } + yield* broker.receive(firstRegistration, { + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: ensure.requestId, + threadId, + sessionId, + resumed: false, + }); + yield* Fiber.join(startSession); + + // ── `hermes gateway stop` + `start`. Deliberately NOT a clean + // offline-then-connected: under systemd the replacement socket can be + // accepted before the dead one's close is processed, so the broker sees a + // replacement and the instance never reports offline. + + // ── `hermes gateway start`: a NEW connection authenticates ── + const credential = yield* secrets.get(hermesGatewayCredentialSecretName(instanceId)).pipe( + Effect.map(Option.getOrThrow), + Effect.map((bytes) => new TextDecoder().decode(bytes)), + ); + const second = recordingTransport(); + const secondRegistration = yield* broker.registerConnection( + hello({ + type: "instance-credential", + instanceId, + credential: HermesGatewayCredential.make(credential), + }), + second.transport, + ); + // The dead socket's close lands late, after the replacement is live. It is + // generation-fenced, so it must not retire the new connection. + yield* broker.disconnect(firstRegistration); + // Let the adapter observe the reconnect and re-issue session.ensure. + // + // Critically the reply is NOT delivered until after this settles: the + // resume must not block the status-stream handler waiting for it. It used + // to, and since the reply is delivered by the broker's own stream the + // deadlock only broke on the 30s request timeout — leaving the thread with + // no session and every later turn rejected. + for (let i = 0; i < 40; i += 1) yield* Effect.yieldNow; + + const resumeEnsure = second.sent.find((message) => message.type === "session.ensure"); + assert.isDefined( + resumeEnsure, + "a gateway restart must re-issue session.ensure on the new connection", + ); + if (!resumeEnsure || resumeEnsure.type !== "session.ensure") { + return yield* Effect.die(new Error("unreachable")); + } + assert.equal(resumeEnsure.threadId, threadId); + assert.equal(resumeEnsure.resumeSessionId, sessionId); + yield* broker.receive(secondRegistration, { + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: resumeEnsure.requestId, + threadId, + sessionId, + resumed: true, + }); + for (let i = 0; i < 20; i += 1) yield* Effect.yieldNow; + + // ── The follow-up turn must reach the plugin ── + const sendTurn = yield* provider.adapter + .sendTurn({ threadId, input: "follow up after restart" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + const turnStart = second.sent.find((message) => message.type === "turn.start"); + assert.isDefined(turnStart, "the follow-up turn must be dispatched after a gateway restart"); + if (!turnStart || turnStart.type !== "turn.start") { + return yield* Effect.die(new Error("unreachable")); + } + yield* broker.receive(secondRegistration, { + type: "turn.started", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: turnStart.requestId, + threadId, + sessionId, + turnId: turnStart.turnId, + }); + assert.equal((yield* Fiber.join(sendTurn)).turnId, turnStart.turnId); + }).pipe(Effect.provide(providerLayer), Effect.scoped); +}); + +// ── Durable config lives in settings; liveness never does ────────── + +it.effect("stores durable enrollment facts in the provider instance config", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const settings = yield* ServerSettings.ServerSettingsService; + yield* broker.createEnrollment({ + instanceId, + nickname: "Durable Hermes", + connectorUrl: "https://durable.example.test", + }); + + const configured = (yield* settings.getSettings).providerInstances[instanceId]; + assert.equal(configured?.displayName, "Durable Hermes"); + assert.deepEqual(configured?.config, { + connectorUrl: "https://durable.example.test", + revoked: false, + }); + + yield* broker.revokeInstance(instanceId); + assert.deepEqual((yield* settings.getSettings).providerInstances[instanceId]?.config, { + connectorUrl: "https://durable.example.test", + revoked: true, + }); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("never writes liveness into settings, so reconnects cannot restart instances", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const settings = yield* ServerSettings.ServerSettingsService; + const enrollment = yield* enroll(broker, instanceId, "Liveness Hermes"); + + const beforeConnect = (yield* settings.getSettings).providerInstances[instanceId]; + const registration = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ); + yield* broker.receive(registration, { + type: "connection.status", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + activeSessionCount: 3, + }); + + // Connecting and reporting sessions is observable on status... + const connected = yield* broker.getInstanceStatus(instanceId); + assert.equal(connected.status, "connected"); + assert.equal(connected.activeSessionCount, 3); + assert.isNotNull(connected.lastConnectedAt); + + // ...but the settings envelope is unchanged, so the provider instance + // registry sees no config change and never closes the instance scope — + // which would otherwise stop every live Hermes session. + assert.deepEqual((yield* settings.getSettings).providerInstances[instanceId], beforeConnect); + + yield* broker.disconnect(registration); + assert.deepEqual((yield* settings.getSettings).providerInstances[instanceId], beforeConnect); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("reports liveness as offline after a restart until the plugin reconnects", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const broker = yield* makeBroker(secrets); + const enrollment = yield* enroll(broker, instanceId, "Restart Hermes"); + yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "connected"); + + // A fresh broker over the same durable state: enrollment facts survive, + // liveness deliberately does not. + const restarted = yield* makeBroker(secrets); + const status = yield* restarted.getInstanceStatus(instanceId); + assert.equal(status.status, "offline"); + assert.equal(status.nickname, "Restart Hermes"); + assert.equal(status.connectorUrl, "https://t3.example.test"); + assert.isNull(status.lastConnectedAt); + assert.isNull(status.model); + assert.equal(status.activeSessionCount, 0); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("surfaces the model reported at handshake and clears it on restart", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const broker = yield* makeBroker(secrets); + const enrollment = yield* enroll(broker, instanceId, "Model Hermes"); + + assert.isNull((yield* broker.getInstanceStatus(instanceId)).model); + + yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }, undefined, { + model: "hermes-4-preview", + }), + recordingTransport().transport, + ); + assert.equal((yield* broker.getInstanceStatus(instanceId)).model, "hermes-4-preview"); + + const restarted = yield* makeBroker(secrets); + assert.isNull((yield* restarted.getInstanceStatus(instanceId)).model); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("reports a null model for a plugin that predates the field", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Legacy Model Hermes"); + yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ); + const status = yield* broker.getInstanceStatus(instanceId); + assert.equal(status.status, "connected"); + assert.isNull(status.model); + }).pipe(Effect.provide(testLayer)), +); + +// ── Ping loop ───────────────────────────────────────────────────── + +it.effect("pings an active connection and stays connected while pongs arrive", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Ping Hermes"); + + const sent: Array = []; + const closes: Array = []; + // The registration only exists after registerConnection returns, but the + // ping loop starts inside it — so the responder waits on this gate rather + // than racing it. + const registrationReady = yield* Deferred.make(); + const pongDelivered = yield* Deferred.make(); + const transport: HermesGatewayTransport = { + send: (message) => + Effect.gen(function* () { + sent.push(message); + if (message.type !== "ping") return; + const registration = yield* Deferred.await(registrationReady); + yield* broker.receive(registration, { + type: "pong", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: message.requestId, + sentAt: message.sentAt, + }); + yield* Deferred.succeed(pongDelivered, undefined); + }), + close: (code) => Effect.sync(() => closes.push(code)).pipe(Effect.asVoid), + }; + + const registration = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ); + yield* Deferred.succeed(registrationReady, registration); + + yield* TestClock.adjust(Duration.seconds(25)); + yield* Deferred.await(pongDelivered); + assert.isTrue(sent.some((message) => message.type === "ping")); + assert.deepEqual(closes, []); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "connected"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("marks a half-open connection offline after consecutive missed pongs", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Half Open Hermes"); + + // A half-open socket: writes succeed, nothing ever answers. + const { sent, closes, transport } = recordingTransport(); + yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "connected"); + + const pending = yield* broker + .request(instanceId, { + type: "session.ensure", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: HermesGatewayRequestId.make("half-open-request"), + threadId: ThreadId.make("half-open-thread"), + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + + // Two probe cycles, each sending a ping that times out unanswered. + yield* TestClock.adjust(Duration.seconds(90)); + + assert.isTrue(sent.filter((message) => message.type === "ping").length >= 2); + assert.deepEqual(closes, [4008]); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "offline"); + // Pending work fails immediately instead of waiting out its own timeout. + const failure = yield* Effect.flip(Fiber.join(pending)); + assert.include(failure.detail, "stopped responding"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("stops pinging once the connection is retired", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Retired Hermes"); + const { sent, transport } = recordingTransport(); + const registration = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + transport, + ); + + yield* broker.disconnect(registration); + const afterDisconnect = sent.length; + // The ping fiber lives in the connection scope, so retiring the connection + // interrupts it rather than leaving it probing a dead socket. + yield* TestClock.adjust(Duration.minutes(2)); + assert.equal(sent.length, afterDisconnect); + }).pipe(Effect.provide(testLayer)), +); + +// ── Management operations ───────────────────────────────────────── + +it.effect("renames the display label while preserving instance identity", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const settings = yield* ServerSettings.ServerSettingsService; + yield* enroll(broker, instanceId, "Remote Hermes"); + yield* enroll(broker, otherInstanceId, "Other Hermes"); + + const renamed = yield* broker.renameInstance({ + instanceId, + nickname: "Research Hermes", + }); + assert.equal(renamed.instanceId, instanceId); + assert.equal(renamed.nickname, "Research Hermes"); + assert.equal( + (yield* settings.getSettings).providerInstances[instanceId]?.displayName, + "Research Hermes", + ); + + // displayName is a free-text label, exactly like every other provider: + // reusing one is allowed, because the instance id is the identity. + const duplicate = yield* broker.renameInstance({ + instanceId: otherInstanceId, + nickname: " Research Hermes ", + }); + assert.equal(duplicate.instanceId, otherInstanceId); + assert.equal(duplicate.nickname, "Research Hermes"); + assert.equal((yield* broker.getInstanceStatus(instanceId)).nickname, "Research Hermes"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("preserves concurrent provider settings edits during rename and removal", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const settings = yield* ServerSettings.ServerSettingsService; + yield* enroll(broker, instanceId, "Concurrent Hermes"); + + yield* broker.renameInstance({ instanceId, nickname: "Concurrent Research" }); + yield* settings.updateSettingsWith((current) => ({ + providerInstances: { + ...current.providerInstances, + codex_concurrent: { driver: "codex", displayName: "Concurrent Codex", config: {} }, + }, + })); + + yield* broker.revokeInstance(instanceId); + yield* broker.removeInstance(instanceId); + yield* settings.updateSettingsWith((current) => ({ + providerInstances: { + ...current.providerInstances, + claude_concurrent: { + driver: "claudeAgent", + displayName: "Concurrent Claude", + config: {}, + }, + }, + })); + + const finalSettings = yield* settings.getSettings; + assert.equal( + finalSettings.providerInstances[ProviderInstanceId.make("codex_concurrent")]?.displayName, + "Concurrent Codex", + ); + assert.equal( + finalSettings.providerInstances[ProviderInstanceId.make("claude_concurrent")]?.displayName, + "Concurrent Claude", + ); + assert.isUndefined(finalSettings.providerInstances[instanceId]); + }).pipe(Effect.provide(testLayer)), +); + +// `createEnrollment`'s settings callback no-ops when the instance is gone (or +// changed driver) between its read and its write. Continuing past that meant +// minting a token `registerConnection` can only ever reject — the user is +// handed a `hermes t3 connect …` command that fails with no explanation. +it.effect("fails rather than minting a token when the instance vanishes mid-enrollment", () => + Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const inner = makeSecretStore(); + // The credential invalidation sits exactly between the read and the write, + // so removing the instance from there lands the race deterministically. + const secrets: ServerSecretStore.ServerSecretStore["Service"] = { + ...inner, + remove: (name) => + settings + .updateSettingsWith((current) => { + const providerInstances = { ...current.providerInstances }; + delete providerInstances[instanceId]; + return { providerInstances }; + }) + .pipe(Effect.ignore, Effect.andThen(inner.remove(name))), + }; + const broker = yield* makeBrokerWith(secrets, settings); + + const error = yield* Effect.flip(enroll(broker, instanceId, "Vanishing Hermes")); + assert.equal(error.operation, "create-enrollment"); + assert.equal(error.code, "instance-not-found"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("requires revocation before removing an enrolled instance", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const broker = yield* makeBroker(secrets); + const settings = yield* ServerSettings.ServerSettingsService; + const enrollment = yield* enroll(broker, instanceId, "Disposable Hermes"); + yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + recordingTransport().transport, + ); + + const liveError = yield* Effect.flip(broker.removeInstance(instanceId)); + assert.equal(liveError.operation, "remove-instance"); + assert.equal(liveError.code, "instance-not-revoked"); + + yield* broker.revokeInstance(instanceId); + assert.deepEqual(yield* broker.removeInstance(instanceId), { instanceId }); + assert.isUndefined((yield* settings.getSettings).providerInstances[instanceId]); + assert.isFalse( + (yield* broker.listInstances).some((status) => status.instanceId === instanceId), + ); + const missing = yield* Effect.flip(broker.getInstanceStatus(instanceId)); + assert.equal(missing.code, "instance-not-found"); + // Removal drops the credential, so a stale one cannot be replayed. + assert.isTrue(Option.isNone(yield* secrets.get(hermesGatewayCredentialSecretName(instanceId)))); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("removes an explicitly configured Hermes instance that was never enrolled", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + const settings = yield* ServerSettings.ServerSettingsService; + + assert.deepEqual(yield* broker.removeInstance(instanceId), { instanceId }); + assert.isUndefined((yield* settings.getSettings).providerInstances[instanceId]); + assert.equal( + (yield* Effect.flip(broker.getInstanceStatus(instanceId))).code, + "instance-not-found", + ); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("lists only enrolled Hermes instances", () => + Effect.gen(function* () { + const broker = yield* makeBroker(makeSecretStore()); + yield* enroll(broker, instanceId, "Listed Hermes"); + + assert.deepEqual( + (yield* broker.listInstances).map((status) => status.instanceId), + [instanceId], + ); + }).pipe(Effect.provide(testLayer)), +); + +// ── Legacy metadata migration ───────────────────────────────────── + +const legacyBlob = (metadata: Record) => + new TextEncoder().encode(JSON.stringify(metadata)); + +it.effect("migrates legacy metadata into settings and deletes the secret file", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const secretName = hermesGatewayLegacyMetadataSecretName(instanceId); + yield* secrets.set( + secretName, + legacyBlob({ + nickname: "Legacy Remote", + connectorUrl: "https://legacy.example.test/api/hermes-gateway/ws", + revoked: false, + // Liveness from a v1 plugin. Migration must drop it rather than carry + // it into the config envelope. + lastSeen: { + pluginVersion: "0.1.0", + hermesVersion: "0.19.0", + capabilities: { + protocolVersion: 1, + streaming: true, + activity: true, + approvals: true, + userInput: true, + attachments: false, + }, + connectedAt: "2026-07-24T00:00:00.000Z", + activeSessionCount: 0, + }, + }), + ); + yield* secrets.set( + hermesGatewayCredentialSecretName(instanceId), + new TextEncoder().encode("legacy-credential"), + ); + + const broker = yield* makeBroker(secrets); + const settings = yield* ServerSettings.ServerSettingsService; + + assert.deepEqual((yield* settings.getSettings).providerInstances[instanceId]?.config, { + connectorUrl: "https://legacy.example.test/api/hermes-gateway/ws", + revoked: false, + }); + + const status = yield* broker.getInstanceStatus(instanceId); + assert.equal(status.connectorUrl, "https://legacy.example.test/api/hermes-gateway/ws"); + assert.equal(status.status, "offline"); + // Liveness from the blob is not resurrected. + assert.isNull(status.lastConnectedAt); + assert.isNull(status.protocolVersion); + assert.equal(status.activeSessionCount, 0); + + // The blob is gone, but the credential of a still-configured, unrevoked + // instance is kept so its plugin can reconnect. + assert.isTrue(Option.isNone(yield* secrets.get(secretName))); + assert.isTrue(Option.isSome(yield* secrets.get(hermesGatewayCredentialSecretName(instanceId)))); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("deletes tombstoned legacy metadata and its orphaned credential", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const secretName = hermesGatewayLegacyMetadataSecretName(instanceId); + // Shape taken from a real orphan found on disk: revoked + removed. The old + // code only ever passed this name to get/set, never remove, so tombstones + // leaked for the lifetime of the install. + yield* secrets.set( + secretName, + legacyBlob({ + nickname: "Research Renamed", + connectorUrl: "https://siva.example.ts.net:7446/api/hermes-gateway/ws", + revoked: true, + removed: true, + }), + ); + yield* secrets.set( + hermesGatewayCredentialSecretName(instanceId), + new TextEncoder().encode("orphaned-credential"), + ); + + const broker = yield* makeBroker(secrets); + const settings = yield* ServerSettings.ServerSettingsService; + + assert.isTrue(Option.isNone(yield* secrets.get(secretName))); + assert.isTrue(Option.isNone(yield* secrets.get(hermesGatewayCredentialSecretName(instanceId)))); + // A tombstone contributes nothing to settings, and the id stays reusable. + assert.deepEqual((yield* settings.getSettings).providerInstances[instanceId]?.config, {}); + assert.isFalse( + (yield* broker.listInstances).some((status) => status.instanceId === instanceId), + ); + }).pipe(Effect.provide(testLayer)), +); + +/** + * Orphan cleanup can only be exercised against a real secrets directory: the + * whole point is that these files belong to instances settings no longer + * mentions, so they are unreachable by name and must be discovered by scanning. + * The two blobs below are verbatim copies of orphans found on a real install. + */ +it.effect("discovers and deletes orphaned legacy metadata left on disk", () => { + const removedInstanceId = ProviderInstanceId.make("hermes-first-hermes-demo-7f63c3052018"); + const renamedInstanceId = ProviderInstanceId.make("hermes-research-e4fcf4b39fa1"); + const configLayer = ServerConfig.layerTest(process.cwd(), { + prefix: "t3code-hermes-orphan-migration-test-", + }).pipe(Layer.provide(NodeServices.layer)); + + return Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const secrets = yield* ServerSecretStore.ServerSecretStore; + + const writeOrphan = (id: ProviderInstanceId, metadata: Record) => + Effect.gen(function* () { + yield* secrets.set(hermesGatewayLegacyMetadataSecretName(id), legacyBlob(metadata)); + yield* secrets.set( + hermesGatewayCredentialSecretName(id), + new TextEncoder().encode(`credential-for-${id}`), + ); + }); + + yield* writeOrphan(removedInstanceId, { + nickname: "First Hermes Demo", + connectorUrl: "https://siva.otter-hawksbill.ts.net:7446/api/hermes-gateway/ws", + revoked: true, + lastSeen: { + pluginVersion: "0.1.0", + hermesVersion: "0.19.0", + capabilities: { + protocolVersion: 1, + streaming: true, + activity: true, + approvals: true, + userInput: true, + attachments: false, + }, + connectedAt: "2026-07-25T01:11:26.467Z", + activeSessionCount: 0, + }, + removed: true, + }); + yield* writeOrphan(renamedInstanceId, { + nickname: "Research Renamed", + connectorUrl: "https://siva.otter-hawksbill.ts.net:7446/api/hermes-gateway/ws", + revoked: true, + removed: true, + }); + + // Neither id appears in settings, so name-based lookup alone would miss + // both files entirely. + yield* makeHermesGatewayBroker.pipe( + Effect.provide(ServerSettings.layerTest({ providerInstances: {} })), + ); + + for (const id of [removedInstanceId, renamedInstanceId]) { + assert.isTrue(Option.isNone(yield* secrets.get(hermesGatewayLegacyMetadataSecretName(id)))); + assert.isTrue(Option.isNone(yield* secrets.get(hermesGatewayCredentialSecretName(id)))); + } + const remaining = yield* fs.readDirectory(config.secretsDir); + assert.deepEqual( + remaining.filter((entry) => entry.includes("hermes-gateway")), + [], + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + ServerSecretStore.layer.pipe(Layer.provide(configLayer), Layer.provide(NodeServices.layer)), + configLayer, + NodeServices.layer, + ), + ), + ); +}); + +it.effect("keeps legacy metadata when the settings write fails", () => + Effect.gen(function* () { + const storedSecrets = makeSecretStore(); + const secretName = hermesGatewayLegacyMetadataSecretName(instanceId); + yield* storedSecrets.set( + secretName, + legacyBlob({ + nickname: "Deferred Hermes", + connectorUrl: "https://deferred.example.test", + revoked: false, + }), + ); + + // Deleting the blob is conditional on committing its durable equivalent, + // so a settings failure must leave the input intact for the next boot. + let metadataRemovals = 0; + const secrets: ServerSecretStore.ServerSecretStore["Service"] = { + ...storedSecrets, + remove: (name) => + name === secretName + ? Effect.sync(() => { + metadataRemovals += 1; + }).pipe(Effect.andThen(storedSecrets.remove(name))) + : storedSecrets.remove(name), + }; + + const failingSettings = Layer.effect( + ServerSettings.ServerSettingsService, + ServerSettings.layerTest({ providerInstances: hermesInstances }).pipe( + Layer.build, + Effect.map((context) => { + const inner = Context.get(context, ServerSettings.ServerSettingsService); + return { + ...inner, + updateSettingsWith: () => Effect.die(new Error("settings write failed")), + } satisfies ServerSettings.ServerSettingsService["Service"]; + }), + ), + ); + + yield* makeHermesGatewayBroker.pipe( + Effect.provide(failingSettings), + Effect.provideService(ServerSecretStore.ServerSecretStore, secrets), + Effect.provide(NodeServices.layer), + Effect.scoped, + ); + + assert.equal(metadataRemovals, 0); + assert.isTrue(Option.isSome(yield* storedSecrets.get(secretName))); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("discards unreadable legacy metadata instead of failing boot", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + const secretName = hermesGatewayLegacyMetadataSecretName(instanceId); + yield* secrets.set(secretName, new TextEncoder().encode("{not-valid-json")); + + const broker = yield* makeBroker(secrets); + assert.isTrue(Option.isNone(yield* secrets.get(secretName))); + // Boot survives; the instance is simply un-enrolled. + assert.equal( + (yield* Effect.flip(broker.getInstanceStatus(instanceId))).code, + "instance-not-found", + ); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("migrates a revoked legacy instance and drops its credential", () => + Effect.gen(function* () { + const secrets = makeSecretStore(); + yield* secrets.set( + hermesGatewayLegacyMetadataSecretName(instanceId), + legacyBlob({ + nickname: "Revoked Legacy", + connectorUrl: "https://revoked.example.test", + revoked: true, + }), + ); + yield* secrets.set( + hermesGatewayCredentialSecretName(instanceId), + new TextEncoder().encode("revoked-credential"), + ); + + const broker = yield* makeBroker(secrets); + assert.equal((yield* broker.getInstanceStatus(instanceId)).status, "revoked"); + assert.isTrue(Option.isNone(yield* secrets.get(hermesGatewayCredentialSecretName(instanceId)))); + + const reconnect = yield* Effect.flip( + broker.registerConnection( + hello({ + type: "instance-credential", + instanceId, + credential: HermesGatewayCredential.make("revoked-credential"), + }), + recordingTransport().transport, + ), + ); + assert.equal(reconnect.code, "invalid-authentication"); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("a delivery connection never displaces the live gateway connection", () => + Effect.gen(function* () { + // The failure this guards: an out-of-process cron run dials in to hand + // over one delivery, `registerConnection` treats it as a replacement, and + // the real plugin gets closed with 4001 — knocking Hermes offline for the + // duration of a cron job. + const broker = yield* makeBroker(makeSecretStore()); + const enrollment = yield* enroll(broker, instanceId, "Remote Hermes"); + const live = recordingTransport(); + const gateway = yield* broker.registerConnection( + hello({ type: "enrollment-token", token: enrollment.oneTimeToken }), + live.transport, + ); + assert.equal(gateway.role, "gateway"); + assert.isTrue(yield* broker.isConnected(instanceId)); + + const credential = gateway.accepted.credential!; + const cron = recordingTransport(); + const delivery = yield* broker.registerConnection( + hello( + { type: "instance-credential", instanceId, credential }, + HERMES_GATEWAY_PROTOCOL_VERSION, + { role: "delivery" }, + ), + cron.transport, + ); + + assert.equal(delivery.role, "delivery"); + // No generation: it was never registered, so there is nothing for it to + // be stale against — and a generation would let it be compared with, and + // mistaken for, the live connection. + assert.equal(delivery.generation, null); + assert.deepEqual(live.closes, [], "the live gateway socket must not be closed"); + assert.isTrue(yield* broker.isConnected(instanceId)); + + // A delivery socket holds no session, so it must not be able to answer + // the live connection's outstanding requests — otherwise a throwaway cron + // socket could speak for the real plugin. + const threadId = ThreadId.make("delivery-fencing-thread"); + const pendingRequestId = HermesGatewayRequestId.make("delivery-fencing-session"); + const pending = yield* broker + .request(instanceId, { + type: "session.ensure", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: pendingRequestId, + threadId, + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + yield* broker.receive(delivery, { + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: pendingRequestId, + threadId, + sessionId: HermesGatewaySessionId.make("delivery-forged-session"), + resumed: false, + }); + yield* Effect.yieldNow; + assert.isUndefined( + pending.pollUnsafe(), + "a delivery connection must not complete the live connection's requests", + ); + + // The live connection still can. + yield* broker.receive(gateway, { + type: "session.ready", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: pendingRequestId, + threadId, + sessionId: HermesGatewaySessionId.make("real-session"), + resumed: false, + }); + assert.equal((yield* Fiber.join(pending)).type, "session.ready"); + + // Closing the delivery socket is its normal lifecycle and must leave the + // instance's liveness untouched. + yield* broker.disconnect(delivery); + assert.isTrue(yield* broker.isConnected(instanceId)); + assert.deepEqual(live.closes, []); + }).pipe(Effect.provide(testLayer)), +); diff --git a/apps/server/src/provider/Layers/HermesGatewayBroker.ts b/apps/server/src/provider/Layers/HermesGatewayBroker.ts new file mode 100644 index 00000000000..f44e8a0fc6d --- /dev/null +++ b/apps/server/src/provider/Layers/HermesGatewayBroker.ts @@ -0,0 +1,1360 @@ +/** + * HermesGatewayBroker — enrollment, authentication, and message routing for + * Hermes gateway plugin connections. + * + * State ownership + * --------------- + * The broker splits gateway state three ways, by durability: + * + * - **Credential** → `ServerSecretStore`. A 0600 file is exactly the right + * home for a secret, and nothing else needs to enumerate it. + * + * - **Durable config** (display name, connector URL, revoked) → the Hermes + * provider instance config in `ServerSettings`, written through + * `updateSettingsWith` so the read-modify-write happens under the settings + * write lock and concurrent edits to unrelated instances are preserved. + * This replaced a JSON blob in the secret store, which had no listing API, + * no index, and no transactions — and therefore forced O(N) secret-file + * scans, whole-file rewrites on every connect, and hand-rolled + * compensating writes. + * + * - **Volatile liveness** (connection, transport, lastSeen, session count, + * reported model, upgrade-required) → `HermesConnectionRegistry`, in + * memory only. See that module for why persisting it would stop every live + * Hermes session on every reconnect. + * + * After a T3 restart the UI reports "never connected" until the plugin dials + * back in. That is intended, and is the correct reading of the facts: T3 has + * no live connection until one is re-established. + * + * Locking + * ------- + * There is no global broker lock. Everything that mutates one instance's + * enrollment — `createEnrollment`, `revokeInstance`, `removeInstance`, + * `renameInstance`, and the state-changing half of `registerConnection` — + * serializes on that instance's lock, so a slow credential write during one + * instance's enrollment still cannot block another instance's WebSocket + * handshake. `registerConnection` authenticates *before* taking the lock, so an + * unauthenticated caller cannot queue on it, and then re-validates the durable + * record and the credential once inside — the pre-lock read is only a filter. + * Where two writers could still race within a locked section — token + * redemption, connection replacement — correctness comes from compare-and-swap + * and generation fencing rather than from the mutex. + * + * @module HermesGatewayBroker + */ +import * as NodeCrypto from "node:crypto"; +import { + HERMES_GATEWAY_PROTOCOL_VERSION, + HERMES_DRIVER_KIND, + defaultInstanceIdForDriver, + HermesGatewayCredential, + HermesGatewayCapabilities, + HermesGatewayManagementError, + ProviderInstanceId, + type HermesGatewayConnectionHello, + type HermesGatewayCreateEnrollmentInput, + type HermesGatewayEnrollmentResult, + type HermesGatewayInstanceStatus, + type HermesGatewayPluginToT3Message, + type HermesGatewayRemoveInstanceResult, + type HermesGatewayRenameInstanceInput, + type HermesGatewayRenameInstanceResult, + type HermesGatewayRevokeInstanceResult, + type HermesGatewayT3ToPluginMessage, + type ProviderInstanceConfig, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; + +import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; +import * as ServerConfig from "../../config.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; +import { ProviderAdapterRequestError } from "../Errors.ts"; +import { + HermesGatewayBroker, + type HermesGatewayConnectionRegistration, + type HermesGatewayEnvelope, + type HermesGatewayTransport, + type HermesGatewayBrokerShape, +} from "../Services/HermesGatewayBroker.ts"; +import type { HermesObservedConnection } from "../Services/HermesConnectionRegistry.ts"; +import type { PendingEnrollment } from "../Services/HermesEnrollmentStore.ts"; +import { livenessStatusFields, makeHermesConnectionRegistry } from "./HermesConnectionRegistry.ts"; +import { makeHermesEnrollmentStore } from "./HermesEnrollmentStore.ts"; +import { makeRequestCorrelator } from "./RequestCorrelator.ts"; + +const ENROLLMENT_TTL = Duration.minutes(10); +const REQUEST_TIMEOUT = Duration.seconds(30); +/** Safety net for pending requests whose awaiting fiber died abnormally. */ +const REQUEST_MAX_AGE = Duration.seconds(90); +/** How often expired enrollment tokens and abandoned requests are reaped. */ +const SWEEP_INTERVAL = Duration.minutes(1); +/** + * Server→plugin liveness probe. Without it a half-open socket reads as + * "connected" indefinitely: `send` succeeds into the void and the failure only + * surfaces when some unrelated request times out. + * + * These are sized so worst-case detection (first probe immediately, then + * `PING_MAX_MISSED` failures) lands inside `REQUEST_TIMEOUT`. That ordering is + * the whole point: an in-flight request over a dead socket should fail with + * "the connection stopped responding" rather than sit out a generic timeout. + */ +const PING_INTERVAL = Duration.seconds(10); +/** + * How long a single ping may go unanswered before it counts as missed. + * + * Generous on purpose. The plugin answers pings from its socket read loop + * without touching Hermes, so a slow reply means the process is genuinely + * starved rather than merely busy — but a Python event loop under a heavy + * turn can still be slow to schedule, and killing a working connection is far + * worse than noticing a dead one a few seconds later. + */ +const PING_TIMEOUT = Duration.seconds(6); +/** + * Consecutive missed pings tolerated before the connection is torn down. + * + * Worst case detection is `PING_INTERVAL` (the wait before the first probe) + * plus `PING_TIMEOUT * PING_MAX_MISSED`, since a miss re-probes immediately. + * That total must stay under `REQUEST_TIMEOUT` so an in-flight request over a + * dead socket fails with the liveness reason rather than a generic timeout: + * 10 + 6*3 = 28s against a 30s request timeout. + */ +const PING_MAX_MISSED = 3; + +const CREDENTIAL_BYTES = 32; + +const textEncoder = new TextEncoder(); +const isStrictCapabilities = Schema.is(HermesGatewayCapabilities); +const isProviderInstanceId = Schema.is(ProviderInstanceId); + +type PluginMessage = Exclude; + +/** + * Durable enrollment facts, read from and written to the Hermes provider + * instance config in settings. + */ +interface InstanceRecord { + readonly nickname: string; + readonly connectorUrl: string; + readonly revoked: boolean; +} + +/** + * A Hermes instance can be *configured* (someone added the envelope) without + * being *enrolled* (no plugin has ever been paired). Only enrollment produces + * a connector URL, so its presence is what distinguishes the two. Status, + * rename, and revoke all require an enrolled instance; create-enrollment and + * remove deliberately accept a merely-configured one. + */ +const isEnrolled = (record: InstanceRecord) => record.connectorUrl !== ""; + +const credentialSecretName = (instanceId: ProviderInstanceId) => + `hermes-gateway-credential-${Buffer.from(instanceId, "utf8").toString("base64url")}`; + +/** Legacy metadata blob location, read once at boot then deleted. */ +const legacyMetadataSecretName = (instanceId: ProviderInstanceId) => + `hermes-gateway-metadata-${Buffer.from(instanceId, "utf8").toString("base64url")}`; + +/** + * Schema for the legacy secret-store metadata blob. Retained only so boot can + * migrate pre-existing files into settings and delete them; nothing writes it. + */ +const LegacyInstanceMetadata = Schema.Struct({ + nickname: Schema.String, + connectorUrl: Schema.String, + revoked: Schema.Boolean, + removed: Schema.optionalKey(Schema.Boolean), +}); +type LegacyInstanceMetadata = typeof LegacyInstanceMetadata.Type; +const decodeLegacyMetadata = Schema.decodeUnknownEffect( + // The blob also carried a `lastSeen` liveness object. It is intentionally + // dropped: liveness is no longer persisted at all. + Schema.fromJsonString(LegacyInstanceMetadata), +); + +const managementError = ( + operation: HermesGatewayManagementError["operation"], + code: HermesGatewayManagementError["code"], + message: string, + instanceId?: ProviderInstanceId, +) => + new HermesGatewayManagementError({ + operation, + code, + message, + ...(instanceId ? { instanceId } : {}), + }); + +const rejection = ( + requestId: HermesGatewayConnectionHello["requestId"], + code: Extract["code"], + message: string, +): Extract => ({ + type: "connection.rejected", + requestId, + code, + message, + expectedProtocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, +}); + +const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** + * Constant-time credential comparison. The length pre-check is required: + * `timingSafeEqual` throws on mismatched lengths, and comparing lengths first + * leaks only the length, which is not secret. + */ +const credentialsEqual = (left: Uint8Array, right: string) => { + const rightBytes = textEncoder.encode(right); + return left.byteLength === rightBytes.byteLength && NodeCrypto.timingSafeEqual(left, rightBytes); +}; + +const normalizeNickname = (nickname: string) => nickname.trim(); + +/** Read a Hermes instance's durable config out of a settings envelope. */ +const readHermesConfig = (config: ProviderInstanceConfig | undefined) => { + if (!config || config.driver !== HERMES_DRIVER_KIND) return undefined; + const raw = config.config; + const blob = raw && typeof raw === "object" ? (raw as Record) : {}; + return { + displayName: config.displayName, + connectorUrl: typeof blob.connectorUrl === "string" ? blob.connectorUrl : undefined, + revoked: blob.revoked === true, + }; +}; + +const recordFrom = ( + instanceId: ProviderInstanceId, + config: ProviderInstanceConfig | undefined, +): InstanceRecord | undefined => { + const hermes = readHermesConfig(config); + if (!hermes) return undefined; + return { + nickname: hermes.displayName ?? instanceId, + connectorUrl: hermes.connectorUrl ?? "", + revoked: hermes.revoked, + }; +}; + +/** Merge durable config edits into an existing Hermes envelope. */ +const withHermesConfig = ( + existing: ProviderInstanceConfig, + patch: { + readonly nickname?: string | undefined; + readonly connectorUrl?: string | undefined; + readonly revoked?: boolean | undefined; + }, +): ProviderInstanceConfig => { + const currentConfig = + existing.config && typeof existing.config === "object" + ? (existing.config as Record) + : {}; + return { + ...existing, + ...(patch.nickname !== undefined ? { displayName: patch.nickname } : {}), + config: { + ...currentConfig, + ...(patch.connectorUrl !== undefined ? { connectorUrl: patch.connectorUrl } : {}), + ...(patch.revoked !== undefined ? { revoked: patch.revoked } : {}), + }, + }; +}; + +export const makeHermesGatewayBroker = Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const secretStore = yield* ServerSecretStore.ServerSecretStore; + const settings = yield* ServerSettingsService; + + const connections = yield* makeHermesConnectionRegistry; + const enrollmentStore = yield* makeHermesEnrollmentStore({ ttl: ENROLLMENT_TTL }); + const correlator = yield* makeRequestCorrelator({ + provider: HERMES_DRIVER_KIND, + timeout: REQUEST_TIMEOUT, + maxAge: REQUEST_MAX_AGE, + }); + + const events = yield* PubSub.unbounded(); + const statusEvents = yield* PubSub.unbounded(); + + /** + * Per-instance enrollment lock. Serializes the read-modify-write of one + * instance's durable config against itself without coupling unrelated + * instances — and, critically, without gating the WebSocket handshake. + */ + const enrollmentLocks = yield* Ref.make(new Map()); + const withInstanceLock = ( + instanceId: ProviderInstanceId, + effect: Effect.Effect, + ) => + Effect.gen(function* () { + const existing = (yield* Ref.get(enrollmentLocks)).get(instanceId); + const semaphore = existing ?? (yield* Semaphore.make(1)); + if (!existing) { + // Two fibers can reach here concurrently for a new instance; the + // modify below keeps whichever landed first so both share one lock. + const winner = yield* Ref.modify(enrollmentLocks, (current) => { + const found = current.get(instanceId); + if (found) return [found, current] as const; + return [semaphore, new Map(current).set(instanceId, semaphore)] as const; + }); + return yield* winner.withPermits(1)(effect); + } + return yield* semaphore.withPermits(1)(effect); + }); + + const readSettings = (operation: HermesGatewayManagementError["operation"]) => + settings.getSettings.pipe( + Effect.mapError(() => + managementError(operation, "internal-error", "Failed to read server settings."), + ), + ); + + /** Durable record for one instance, or `undefined` if not a Hermes instance. */ + const readRecord = ( + instanceId: ProviderInstanceId, + operation: HermesGatewayManagementError["operation"], + ) => + readSettings(operation).pipe( + Effect.map((current) => recordFrom(instanceId, current.providerInstances[instanceId])), + ); + + const requireRecord = ( + instanceId: ProviderInstanceId, + operation: HermesGatewayManagementError["operation"], + ) => + readRecord(instanceId, operation).pipe( + Effect.flatMap((record) => + record && isEnrolled(record) + ? Effect.succeed(record) + : Effect.fail( + managementError( + operation, + "instance-not-found", + record + ? `Hermes gateway instance '${instanceId}' has not been enrolled.` + : `Hermes gateway instance '${instanceId}' is not configured.`, + instanceId, + ), + ), + ), + ); + + /** + * Compose a public status from the durable record plus volatile liveness. + * `revoked` wins over everything: a revoked instance is revoked even if a + * stale connection object has not been reaped yet. + */ + const statusOf = (instanceId: ProviderInstanceId, record: InstanceRecord) => + connections.liveness(instanceId).pipe( + Effect.map((liveness) => { + const fields = livenessStatusFields(liveness); + const { connected, upgradeRequired, capabilities, ...rest } = fields; + return { + ...rest, + // A capability shape from a newer protocol is reported as null + // rather than passed through as if T3 understood it. + capabilities: + capabilities !== null && isStrictCapabilities(capabilities) ? capabilities : null, + instanceId, + nickname: record.nickname, + connectorUrl: record.connectorUrl, + status: record.revoked + ? "revoked" + : upgradeRequired + ? "upgrade-required" + : connected + ? "connected" + : "offline", + } satisfies HermesGatewayInstanceStatus; + }), + ); + + const publishStatus = (instanceId: ProviderInstanceId, record: InstanceRecord) => + statusOf(instanceId, record).pipe( + Effect.flatMap((status) => PubSub.publish(statusEvents, status)), + Effect.asVoid, + ); + + /** Publish using whatever durable record settings currently holds. */ + const publishCurrentStatus = ( + instanceId: ProviderInstanceId, + operation: HermesGatewayManagementError["operation"], + ) => + readRecord(instanceId, operation).pipe( + Effect.flatMap((record) => (record ? publishStatus(instanceId, record) : Effect.void)), + Effect.ignore, + ); + + const failPendingRequests = (instanceId: ProviderInstanceId, detail: string) => + correlator.failOwner(instanceId, detail); + + /** + * Tear down a connection: fail its in-flight requests, then close the + * socket. Ordering matters — callers waiting on a request should see the + * specific reason rather than a generic disconnect. + */ + const teardownConnection = ( + instanceId: ProviderInstanceId, + transport: HermesGatewayTransport, + code: number, + reason: string, + ) => failPendingRequests(instanceId, reason).pipe(Effect.andThen(transport.close(code, reason))); + + // ── Management operations ───────────────────────────────────────── + + const createEnrollment = (input: HermesGatewayCreateEnrollmentInput) => + withInstanceLock( + input.instanceId, + Effect.gen(function* () { + const nickname = normalizeNickname(input.nickname); + const currentSettings = yield* readSettings("create-enrollment"); + const configured = currentSettings.providerInstances[input.instanceId]; + if (!configured || configured.driver !== HERMES_DRIVER_KIND) { + return yield* managementError( + "create-enrollment", + "instance-not-found", + `Provider instance '${input.instanceId}' is not configured with the Hermes driver.`, + input.instanceId, + ); + } + + // Invalidate the outgoing credential before anything else: from here + // on the old credential must not authenticate, even if a later step + // fails. + yield* secretStore + .remove(credentialSecretName(input.instanceId)) + .pipe( + Effect.mapError(() => + managementError( + "create-enrollment", + "persistence-failed", + "Failed to invalidate the previous Hermes gateway credential.", + input.instanceId, + ), + ), + ); + + const committed = yield* settings + .updateSettingsWith((latest) => { + const latestConfigured = latest.providerInstances[input.instanceId]; + if (!latestConfigured || latestConfigured.driver !== HERMES_DRIVER_KIND) return {}; + return { + providerInstances: { + ...latest.providerInstances, + [input.instanceId]: withHermesConfig(latestConfigured, { + nickname, + connectorUrl: input.connectorUrl, + revoked: false, + }), + }, + }; + }) + .pipe( + Effect.mapError(() => + managementError( + "create-enrollment", + "persistence-failed", + "Failed to persist the Hermes gateway enrollment.", + input.instanceId, + ), + ), + ); + + // Same committed-settings check `renameInstance` and `removeInstance` + // make, and for the sharper reason: the callback above no-ops when the + // instance vanished (or changed driver) between `readSettings` and the + // write, and continuing would mint a token `registerConnection` can + // only ever reject — handing the user a `hermes t3 connect` command + // that fails with no explanation. Failing here says so instead. + if (!recordFrom(input.instanceId, committed.providerInstances[input.instanceId])) { + return yield* managementError( + "create-enrollment", + "instance-not-found", + `Provider instance '${input.instanceId}' is no longer configured with the Hermes driver.`, + input.instanceId, + ); + } + + // Any live connection is authenticating with the credential we just + // invalidated, so it can no longer be trusted. + const displaced = yield* connections.clearConnection(input.instanceId); + if (displaced) { + yield* teardownConnection( + input.instanceId, + displaced.transport, + 4001, + "A new enrollment was created for this instance", + ); + } + + const { token, expiresAtMillis } = yield* enrollmentStore.mint({ ...input, nickname }); + + const record: InstanceRecord = { + nickname, + connectorUrl: input.connectorUrl, + revoked: false, + }; + yield* publishStatus(input.instanceId, record); + + return { + instanceId: input.instanceId, + expiresAt: DateTime.formatIso(DateTime.makeUnsafe(expiresAtMillis)), + connectorUrl: input.connectorUrl, + command: `hermes t3 connect --url ${shellQuote(input.connectorUrl)} --token ${shellQuote(token)}`, + // The long-lived credential is structurally absent here: it only + // ever exists on the server→plugin `connection.accepted` frame. + oneTimeToken: token, + } satisfies HermesGatewayEnrollmentResult; + }), + ); + + const getInstanceStatus = (instanceId: ProviderInstanceId) => + requireRecord(instanceId, "get-status").pipe( + Effect.flatMap((record) => statusOf(instanceId, record)), + ); + + const listInstances = readSettings("list-instances").pipe( + Effect.flatMap((currentSettings) => + Effect.forEach( + Object.entries(currentSettings.providerInstances).filter( + ([, config]) => config.driver === HERMES_DRIVER_KIND, + ), + ([rawId, config]) => { + const instanceId = rawId as ProviderInstanceId; + const record = recordFrom(instanceId, config); + // Configured-but-never-enrolled instances have no gateway status to + // report, so they are omitted rather than listed as offline. + return record && isEnrolled(record) + ? statusOf(instanceId, record) + : Effect.succeed(undefined); + }, + { concurrency: "unbounded" }, + ).pipe(Effect.map((values) => values.filter((value) => value !== undefined))), + ), + ); + + /** + * Rename is a pure display-name edit. Instance id is identity; the label is + * free text like every other provider, so there is no uniqueness scan. + */ + const renameInstance = (input: HermesGatewayRenameInstanceInput) => + withInstanceLock( + input.instanceId, + Effect.gen(function* () { + const nickname = normalizeNickname(input.nickname); + yield* requireRecord(input.instanceId, "rename-instance").pipe( + Effect.mapError((error) => + managementError("rename-instance", error.code, error.message, input.instanceId), + ), + ); + + const updated = yield* settings + .updateSettingsWith((latest) => { + const latestConfigured = latest.providerInstances[input.instanceId]; + if (!latestConfigured || latestConfigured.driver !== HERMES_DRIVER_KIND) return {}; + return { + providerInstances: { + ...latest.providerInstances, + [input.instanceId]: withHermesConfig(latestConfigured, { nickname }), + }, + }; + }) + .pipe( + Effect.mapError(() => + managementError( + "rename-instance", + "persistence-failed", + "Failed to persist the Hermes instance display name.", + input.instanceId, + ), + ), + ); + + // The update callback is a pure function of `latest`, so the committed + // settings are the single source of truth for whether it applied — + // no side-channel flag needed. + const record = recordFrom(input.instanceId, updated.providerInstances[input.instanceId]); + if (!record) { + return yield* managementError( + "rename-instance", + "instance-not-found", + `Provider instance '${input.instanceId}' is no longer configured with the Hermes driver.`, + input.instanceId, + ); + } + + yield* publishStatus(input.instanceId, record); + return (yield* statusOf( + input.instanceId, + record, + )) satisfies HermesGatewayRenameInstanceResult; + }), + ); + + const revokeInstance = (instanceId: ProviderInstanceId) => + withInstanceLock( + instanceId, + Effect.gen(function* () { + const existing = yield* requireRecord(instanceId, "revoke-instance").pipe( + Effect.mapError((error) => + managementError("revoke-instance", error.code, error.message, instanceId), + ), + ); + + yield* settings + .updateSettingsWith((latest) => { + const latestConfigured = latest.providerInstances[instanceId]; + if (!latestConfigured || latestConfigured.driver !== HERMES_DRIVER_KIND) return {}; + return { + providerInstances: { + ...latest.providerInstances, + [instanceId]: withHermesConfig(latestConfigured, { revoked: true }), + }, + }; + }) + .pipe( + Effect.mapError(() => + managementError( + "revoke-instance", + "persistence-failed", + "Failed to persist the Hermes gateway revocation.", + instanceId, + ), + ), + ); + + const record: InstanceRecord = { ...existing, revoked: true }; + yield* enrollmentStore.forget(instanceId); + + const cleared = yield* connections.clearConnection(instanceId); + if (cleared) { + yield* teardownConnection( + instanceId, + cleared.transport, + 4003, + "The Hermes gateway credential was revoked.", + ); + } + yield* publishStatus(instanceId, record); + + // Credential removal is last and is allowed to fail loudly: revocation + // is already durable in settings, so a reconnect is refused either way. + yield* secretStore + .remove(credentialSecretName(instanceId)) + .pipe( + Effect.mapError(() => + managementError( + "revoke-instance", + "persistence-failed", + "Failed to remove the Hermes gateway credential.", + instanceId, + ), + ), + ); + + return (yield* statusOf(instanceId, record)) satisfies HermesGatewayRevokeInstanceResult; + }), + ); + + const removeInstance = (instanceId: ProviderInstanceId) => + withInstanceLock( + instanceId, + Effect.gen(function* () { + const record = yield* readRecord(instanceId, "remove-instance"); + if (!record) { + return yield* managementError( + "remove-instance", + "instance-not-found", + `Hermes gateway instance '${instanceId}' is not configured or enrolled.`, + instanceId, + ); + } + // An enrolled instance must be revoked first so its credential is + // invalidated before the config that describes it disappears. + if (isEnrolled(record) && !record.revoked) { + return yield* managementError( + "remove-instance", + "instance-not-revoked", + "Revoke the Hermes gateway instance before removing it.", + instanceId, + ); + } + + const updated = yield* settings + .updateSettingsWith((current) => { + if (current.providerInstances[instanceId]?.driver !== HERMES_DRIVER_KIND) return {}; + const providerInstances = { ...current.providerInstances }; + delete providerInstances[instanceId]; + return { providerInstances }; + }) + .pipe( + Effect.mapError(() => + managementError( + "remove-instance", + "persistence-failed", + "Failed to remove the Hermes provider instance from server settings.", + instanceId, + ), + ), + ); + if (updated.providerInstances[instanceId]?.driver === HERMES_DRIVER_KIND) { + return yield* managementError( + "remove-instance", + "instance-not-found", + `Hermes provider instance '${instanceId}' changed while it was being removed.`, + instanceId, + ); + } + + yield* enrollmentStore.forget(instanceId); + const cleared = yield* connections.clearConnection(instanceId); + if (cleared) { + yield* teardownConnection( + instanceId, + cleared.transport, + 4003, + "The Hermes gateway instance was removed.", + ); + } + yield* connections.forget(instanceId); + + // Removing the instance config is the authoritative act; a failure to + // delete the now-unreferenced credential is logged, not surfaced. + yield* secretStore.remove(credentialSecretName(instanceId)).pipe( + Effect.catch((error) => + Effect.logWarning("Failed to clean up a removed Hermes gateway credential", { + instanceId, + error, + }), + ), + ); + return { instanceId } satisfies HermesGatewayRemoveInstanceResult; + }), + ); + + // ── Connection lifecycle ────────────────────────────────────────── + + /** + * Ping loop for one connection. Forked into the connection's scope, so it is + * interrupted precisely when that connection is retired or replaced. + * + * A missed pong is not immediately fatal — a busy plugin can be slow — but + * `PING_MAX_MISSED` consecutive misses mean the socket is half-open, and we + * stop pretending otherwise. + */ + const pingLoop = ( + registration: HermesGatewayConnectionRegistration, + transport: HermesGatewayTransport, + ) => + Effect.gen(function* () { + const missed = yield* Ref.make(0); + + const probe = Effect.gen(function* () { + const requestId = `hermes-ping-${registration.instanceId}-${registration.generation}-${yield* Clock.currentTimeMillis}`; + const outcome = yield* correlator + .request({ + owner: registration.instanceId, + requestId, + method: "ping", + timeout: PING_TIMEOUT, + send: transport.send({ + type: "ping", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + requestId: requestId as HermesGatewayConnectionHello["requestId"], + sentAt: DateTime.formatIso(yield* DateTime.now), + }), + }) + .pipe(Effect.result); + + if (outcome._tag === "Success") { + yield* Ref.set(missed, 0); + // Healthy: wait a full interval before probing again. + return yield* Effect.sleep(PING_INTERVAL); + } + + const missedCount = yield* Ref.updateAndGet(missed, (value) => value + 1); + // Once a ping is missed, re-probe immediately rather than idling for + // another interval: the point is to confirm or clear the suspicion + // fast enough that in-flight requests get the real reason instead of + // a generic timeout. + if (missedCount < PING_MAX_MISSED) return; + + yield* Effect.logWarning("Hermes gateway connection failed liveness checks", { + instanceId: registration.instanceId, + missedCount, + }); + // Retire before closing: `disconnect` is generation-fenced, so if a + // replacement already landed this is a no-op and we leave it alone. + const retired = yield* connections.disconnect(registration); + if (retired) { + yield* publishCurrentStatus(registration.instanceId, "get-status"); + yield* teardownConnection( + registration.instanceId, + transport, + 4008, + "The Hermes gateway connection stopped responding.", + ); + } + // This connection is gone either way — retired here, or already + // replaced. Stop probing rather than interrogating a dead socket and + // logging another miss for a connection nobody is using. + return yield* Effect.interrupt; + }); + + // First probe runs after one interval — long enough that a plugin which + // just completed the handshake is not immediately re-interrogated. The + // pacing of subsequent probes lives inside `probe` itself, because a + // healthy connection and a suspected-dead one warrant different delays. + yield* Effect.sleep(PING_INTERVAL); + yield* probe.pipe(Effect.forever, Effect.ignoreCause({ log: true })); + }); + + /** + * Handshake authentication: prove the caller holds a live credential or a + * live enrollment token, and resolve the instance it is speaking for. + * + * Deliberately a pure read. Nothing here mutates broker state, which is what + * makes it safe to run twice — once before the instance lock is taken, so an + * unauthenticated caller never joins the lock queue, and again inside it, so + * the facts the mutation section acts on were re-read after every competing + * enrollment or revocation had to finish. + */ + const authenticateHello = (hello: HermesGatewayConnectionHello) => + Effect.gen(function* () { + let instanceId: ProviderInstanceId; + let enrollment: PendingEnrollment | undefined; + + if (hello.authentication.type === "enrollment-token") { + const authentication = hello.authentication; + const pending = yield* enrollmentStore.peek(authentication.token); + if (!pending) { + return yield* Effect.fail( + rejection( + hello.requestId, + "enrollment-expired", + "The enrollment token is invalid, expired, or already used.", + ), + ); + } + instanceId = pending.input.instanceId; + enrollment = pending; + } else { + const authentication = hello.authentication; + instanceId = authentication.instanceId; + const stored = yield* secretStore + .get(credentialSecretName(instanceId)) + .pipe( + Effect.mapError(() => + rejection( + hello.requestId, + "internal-error", + "Failed to read the gateway credential.", + ), + ), + ); + if (Option.isNone(stored) || !credentialsEqual(stored.value, authentication.credential)) { + return yield* Effect.fail( + rejection( + hello.requestId, + "invalid-authentication", + "The Hermes gateway credential is invalid.", + ), + ); + } + } + + const record = yield* readRecord(instanceId, "get-status").pipe( + Effect.mapError(() => + rejection( + hello.requestId, + "internal-error", + "Failed to validate the Hermes gateway provider instance.", + ), + ), + ); + if (!record) { + return yield* Effect.fail( + rejection( + hello.requestId, + "invalid-authentication", + "The Hermes gateway provider instance is no longer configured.", + ), + ); + } + if (record.revoked) { + return yield* Effect.fail( + rejection( + hello.requestId, + "instance-revoked", + "This Hermes gateway instance has been revoked.", + ), + ); + } + + return { instanceId, enrollment, record } as const; + }); + + const registerConnection = ( + hello: HermesGatewayConnectionHello, + transport: HermesGatewayTransport, + ) => + Effect.gen(function* () { + const strictCapabilities = isStrictCapabilities(hello.capabilities) + ? hello.capabilities + : undefined; + + // ── Phase 1: authenticate, unlocked. Nothing below this line mutates + // connection state, and nothing above it is allowed to. An earlier bug + // applied "incompatible version" state before checking the credential, + // letting an unauthenticated caller knock a healthy instance offline; + // the test "authenticates before applying incompatible connection state" + // guards this ordering. Running it before the lock also keeps a caller + // with a junk credential from queueing behind an in-flight enrollment. + const preliminary = yield* authenticateHello(hello); + + // ── Phase 2: the caller is authenticated. State changes may begin — + // under the instance's enrollment lock, so `createEnrollment`, + // `revokeInstance`, and `removeInstance` cannot interleave with token + // redemption, credential persistence, or acceptance. + // + // Everything inside is bounded and takes no other lock: settings reads + // are a `Ref.get`, and `teardownConnection` only fails correlated + // requests and closes the *displaced* socket — never the one currently + // handshaking, which is not in the registry yet. So the WS accept path + // cannot deadlock against a displaced connection's teardown. + return yield* withInstanceLock( + preliminary.instanceId, + Effect.gen(function* () { + // Re-read every fact behind the lock. The pre-lock pass was only a + // filter: between it and here a `createEnrollment` may have removed + // the credential and superseded the token, or a `revokeInstance` may + // have marked the instance revoked and cleared its connection. Both + // are now visible, and both reject. + const { + instanceId, + enrollment: authenticatedEnrollment, + record, + } = yield* authenticateHello(hello); + if (instanceId !== preliminary.instanceId) { + // The lock we hold is the wrong one, so this cannot proceed + // safely. Only reachable if a token were re-pointed at another + // instance, which minting never does. + return yield* Effect.fail( + rejection( + hello.requestId, + "invalid-authentication", + "The Hermes gateway enrollment changed during the handshake.", + ), + ); + } + + if ( + hello.protocolVersion !== HERMES_GATEWAY_PROTOCOL_VERSION || + strictCapabilities === undefined + ) { + const displaced = yield* connections.markUpgradeRequired({ + instanceId, + upgradeRequired: { + pluginVersion: hello.pluginVersion, + hermesVersion: hello.hermesVersion, + protocolVersion: hello.protocolVersion, + model: hello.model ?? null, + }, + }); + if (displaced) { + yield* teardownConnection( + instanceId, + displaced.transport, + 4004, + "The Hermes gateway plugin requires a protocol upgrade.", + ); + } + yield* publishStatus(instanceId, record); + // Names the fix, not just the mismatch: the rejection text is what + // a v3 (pre-media) plugin's operator sees in its logs. A + // right-version hello can still land here when its capabilities + // don't satisfy the v4 contract (e.g. `attachments: false`), so + // say which it was. + return yield* Effect.fail( + rejection( + hello.requestId, + "version-incompatible", + hello.protocolVersion !== HERMES_GATEWAY_PROTOCOL_VERSION + ? `Expected protocol version ${HERMES_GATEWAY_PROTOCOL_VERSION}, received ${hello.protocolVersion}. Upgrade the T3 Code gateway plugin on the Hermes host to reconnect.` + : `The advertised capabilities do not satisfy the version ${HERMES_GATEWAY_PROTOCOL_VERSION} contract. Upgrade the T3 Code gateway plugin on the Hermes host to reconnect.`, + ), + ); + } + + let credential: HermesGatewayCredential | undefined; + if (hello.authentication.type === "enrollment-token" && authenticatedEnrollment) { + const authentication = hello.authentication; + const consumed = yield* enrollmentStore.consume( + authentication.token, + authenticatedEnrollment, + ); + if (!consumed) { + return yield* Effect.fail( + rejection( + hello.requestId, + "enrollment-expired", + "The enrollment token is invalid, expired, or already used.", + ), + ); + } + credential = HermesGatewayCredential.make( + Encoding.encodeBase64Url( + yield* crypto + .randomBytes(CREDENTIAL_BYTES) + .pipe( + Effect.mapError(() => + rejection( + hello.requestId, + "internal-error", + "Failed to generate the Hermes gateway credential.", + ), + ), + ), + ), + ); + yield* secretStore + .set(credentialSecretName(instanceId), textEncoder.encode(credential)) + .pipe( + Effect.mapError(() => + rejection( + hello.requestId, + "internal-error", + "Failed to persist the Hermes gateway credential.", + ), + ), + ); + } + + // ── Delivery connections: authenticated, but never the primary. + // + // An out-of-process cron run dials in only to hand over a + // `home.deliver` and leave. Registering it would displace the + // instance's live gateway socket ("replaced by a newer connection") + // and bump the generation, knocking the real plugin offline for the + // duration of a cron job. So it is accepted and then deliberately + // left out of the connection registry: no generation, no liveness + // ping, no status publish, nothing to disconnect. The existing + // primary is untouched. + if (hello.role === "delivery") { + return { + instanceId, + generation: null, + role: "delivery", + accepted: { + type: "connection.accepted", + requestId: hello.requestId, + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + instanceId, + nickname: record.nickname, + ...(credential ? { credential } : {}), + }, + } as const satisfies HermesGatewayConnectionRegistration; + } + + const observed: HermesObservedConnection = { + pluginVersion: hello.pluginVersion, + hermesVersion: hello.hermesVersion, + capabilities: strictCapabilities, + model: hello.model ?? null, + connectedAt: DateTime.formatIso(yield* DateTime.now), + activeSessionCount: 0, + }; + const accepted = yield* connections.accept({ instanceId, transport, observed }); + if (accepted.displaced) { + yield* teardownConnection( + instanceId, + accepted.displaced.transport, + 4001, + "The Hermes gateway connection was replaced by a newer connection.", + ); + } + + const registration = { + instanceId, + generation: accepted.generation, + role: "gateway", + accepted: { + type: "connection.accepted", + requestId: hello.requestId, + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + instanceId, + nickname: record.nickname, + ...(credential ? { credential } : {}), + }, + } as const satisfies HermesGatewayConnectionRegistration; + + // Liveness probing belongs to this connection's scope, so it is + // interrupted the moment the connection is retired. + const liveness = yield* connections.liveness(instanceId); + if (liveness?.connection?.generation === accepted.generation) { + yield* pingLoop(registration, transport).pipe(Effect.forkIn(liveness.connection.scope)); + } + + yield* publishStatus(instanceId, record); + return registration; + }), + ); + }); + + const receive = (registration: HermesGatewayConnectionRegistration, message: PluginMessage) => + Effect.gen(function* () { + // A delivery connection's frames — `home.deliver` and `media.deliver` — + // are handled at the transport layer, because their acks have to go back + // to that specific short-lived socket. Anything else it sends is outside + // its remit and is dropped here: it holds no session, so completing a + // correlated request or publishing a status off it would let a throwaway + // cron socket speak for the live connection. + if (registration.role === "delivery") return; + + if (message.type === "connection.status") { + // Generation-fenced inside the Ref update; a stale registration is a + // no-op rather than an overwrite. + const applied = yield* connections.recordSessionCount( + registration, + message.activeSessionCount, + ); + if (!applied) return; + yield* publishCurrentStatus(registration.instanceId, "get-status"); + } else { + const liveness = yield* connections.liveness(registration.instanceId); + if (liveness?.connection?.generation !== registration.generation) return; + } + + if ("requestId" in message && message.requestId) { + yield* correlator.complete(message.requestId, message); + } + yield* PubSub.publish(events, { instanceId: registration.instanceId, message }); + }); + + const disconnect = (registration: HermesGatewayConnectionRegistration) => + Effect.gen(function* () { + // Delivery sockets close as a matter of course — that is their whole + // lifecycle. They were never in the registry, so retiring one must not + // touch the instance's liveness or fail the live connection's requests. + if (registration.role === "delivery") return; + const retired = yield* connections.disconnect(registration); + if (!retired) return; + yield* publishCurrentStatus(registration.instanceId, "get-status"); + yield* failPendingRequests( + registration.instanceId, + "The Hermes gateway connection disconnected.", + ); + }); + + // ── Messaging ───────────────────────────────────────────────────── + + const send = (instanceId: ProviderInstanceId, message: HermesGatewayT3ToPluginMessage) => + connections.transport(instanceId).pipe( + Effect.flatMap((transport) => + transport + ? transport.send(message) + : Effect.fail( + new ProviderAdapterRequestError({ + provider: HERMES_DRIVER_KIND, + method: message.type, + detail: `Hermes gateway instance '${instanceId}' is offline.`, + }), + ), + ), + ); + + const request = (instanceId: ProviderInstanceId, message: HermesGatewayT3ToPluginMessage) => { + if (!("requestId" in message) || !message.requestId) { + return Effect.fail( + new ProviderAdapterRequestError({ + provider: HERMES_DRIVER_KIND, + method: message.type, + detail: "A correlated Hermes gateway request requires a request id.", + }), + ); + } + return correlator.request({ + owner: instanceId, + requestId: message.requestId, + method: message.type, + send: send(instanceId, message), + }); + }; + + // ── Boot: migrate legacy metadata, then start sweepers ──────────── + + const LEGACY_METADATA_PREFIX = "hermes-gateway-metadata-"; + + /** + * Recover instance ids from legacy metadata filenames in the secrets + * directory. `ServerSecretStore` has no listing API, and an orphaned blob is + * by definition one that settings no longer references — so without this + * scan the exact files we most need to clean up are unreachable. + * + * Entirely best-effort: no config, no filesystem, or an unreadable directory + * all degrade to "found nothing" rather than failing boot. + */ + const discoverLegacyMetadataInstanceIds = Effect.gen(function* () { + const config = yield* Effect.serviceOption(ServerConfig.ServerConfig); + const fs = yield* Effect.serviceOption(FileSystem.FileSystem); + if (Option.isNone(config) || Option.isNone(fs)) return []; + + const entries = yield* fs.value + .readDirectory(config.value.secretsDir) + .pipe(Effect.orElseSucceed(() => [] as Array)); + + return entries.flatMap((entry) => { + const base = entry.endsWith(".bin") ? entry.slice(0, -".bin".length) : entry; + if (!base.startsWith(LEGACY_METADATA_PREFIX)) return []; + const encoded = base.slice(LEGACY_METADATA_PREFIX.length); + const decodedId = Buffer.from(encoded, "base64url").toString("utf8"); + // Round-trip guard: only accept names this scheme could have produced. + if (Buffer.from(decodedId, "utf8").toString("base64url") !== encoded) return []; + return isProviderInstanceId(decodedId) ? [decodedId] : []; + }); + }).pipe(Effect.orElseSucceed(() => [] as Array)); + + /** + * Fold pre-existing secret-store metadata blobs into settings and delete + * them. Ordering is deliberate: the settings write must succeed before the + * secret file is deleted, so a crash in between re-runs the migration rather + * than losing the enrollment facts. + */ + const migrateLegacyMetadata = Effect.gen(function* () { + const currentSettings = yield* settings.getSettings; + const defaultInstanceId = defaultInstanceIdForDriver(HERMES_DRIVER_KIND); + const candidates = new Set([ + defaultInstanceId, + ...Object.entries(currentSettings.providerInstances) + .filter(([, config]) => config.driver === HERMES_DRIVER_KIND) + .map(([rawId]) => rawId as ProviderInstanceId), + // The real orphans are precisely the ones settings no longer mentions, + // so scanning the secrets directory is the only way to find them. The + // store has no listing API; reading the directory directly is + // best-effort and never fatal. + ...(yield* discoverLegacyMetadataInstanceIds), + ]); + + for (const instanceId of candidates) { + const secretName = legacyMetadataSecretName(instanceId); + const stored = yield* secretStore.get(secretName); + if (Option.isNone(stored)) continue; + + const decoded = yield* decodeLegacyMetadata(new TextDecoder().decode(stored.value)).pipe( + Effect.result, + ); + if (decoded._tag === "Failure") { + yield* Effect.logWarning("Discarding unreadable legacy Hermes gateway metadata", { + instanceId, + }); + yield* secretStore.remove(secretName).pipe(Effect.ignore); + continue; + } + const metadata: LegacyInstanceMetadata = decoded.success; + + const configured = currentSettings.providerInstances[instanceId]; + const isHermesInstance = configured?.driver === HERMES_DRIVER_KIND; + + // A tombstone, or a blob for an instance nobody configures any more, has + // nothing to fold into. Drop the file and the orphaned credential — this + // is the leak the old code never cleaned up, because the tombstone name + // was only ever passed to get/set and never to remove. + if (metadata.removed || !isHermesInstance) { + yield* secretStore.remove(secretName).pipe(Effect.ignore); + yield* secretStore.remove(credentialSecretName(instanceId)).pipe(Effect.ignore); + continue; + } + + const committed = yield* settings + .updateSettingsWith((latest) => { + const latestConfigured = latest.providerInstances[instanceId]; + if (!latestConfigured || latestConfigured.driver !== HERMES_DRIVER_KIND) return {}; + const existing = readHermesConfig(latestConfigured); + return { + providerInstances: { + ...latest.providerInstances, + [instanceId]: withHermesConfig(latestConfigured, { + // Settings already win where they carry a value; the blob only + // fills gaps. + nickname: latestConfigured.displayName ?? metadata.nickname, + connectorUrl: existing?.connectorUrl ?? metadata.connectorUrl, + revoked: existing?.revoked === true ? true : metadata.revoked, + }), + }, + }; + }) + .pipe(Effect.result); + + if (committed._tag === "Failure") { + yield* Effect.logWarning("Deferring Hermes gateway metadata migration", { instanceId }); + continue; + } + + const migrated = recordFrom(instanceId, committed.success.providerInstances[instanceId]); + if (!migrated) { + yield* Effect.logWarning("Deferring Hermes gateway metadata migration", { instanceId }); + continue; + } + + // Only now, with the durable equivalent committed, is the blob + // redundant. The credential is kept: this instance is still configured + // and its plugin must be able to reconnect. + yield* secretStore.remove(secretName).pipe(Effect.ignore); + if (migrated.revoked) { + yield* secretStore.remove(credentialSecretName(instanceId)).pipe(Effect.ignore); + } + } + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to migrate legacy Hermes gateway metadata", { cause }), + ), + ); + + yield* migrateLegacyMetadata; + + // Bounded memory: expired tokens and abandoned request entries are reaped + // on a timer rather than only lazily at redemption. + yield* enrollmentStore.sweep.pipe( + Effect.andThen(correlator.sweep), + Effect.repeat(Schedule.spaced(SWEEP_INTERVAL)), + Effect.ignoreCause({ log: true }), + Effect.forkScoped, + ); + + return { + createEnrollment, + getInstanceStatus, + listInstances, + renameInstance, + revokeInstance, + removeInstance, + registerConnection, + receive, + disconnect, + request, + send, + isConnected: connections.isConnected, + stream: Stream.fromPubSub(events), + streamStatuses: Stream.fromPubSub(statusEvents), + } satisfies HermesGatewayBrokerShape; +}); + +export const HermesGatewayBrokerLive = Layer.effect(HermesGatewayBroker, makeHermesGatewayBroker); + +/** Exported so tests can seed and assert on migration inputs by exact name. */ +export const hermesGatewayLegacyMetadataSecretName = legacyMetadataSecretName; +export const hermesGatewayCredentialSecretName = credentialSecretName; diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.test.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.test.ts new file mode 100644 index 00000000000..6f83f9d1d5b --- /dev/null +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.test.ts @@ -0,0 +1,37 @@ +import { assert, it } from "@effect/vitest"; +import { + DEFAULT_SERVER_SETTINGS, + HERMES_DRIVER_KIND, + ProviderInstanceId, +} from "@t3tools/contracts"; + +import { deriveProviderInstanceConfigMap } from "./ProviderInstanceRegistryHydration.ts"; + +it("does not synthesize a legacy default Hermes instance", () => { + const instances = deriveProviderInstanceConfigMap(DEFAULT_SERVER_SETTINGS); + + assert.isUndefined(instances[ProviderInstanceId.make("hermes")]); + assert.equal(instances[ProviderInstanceId.make("codex")]?.driver, "codex"); +}); + +it("preserves an explicit Hermes instance whose id matches the driver default", () => { + const instanceId = ProviderInstanceId.make("hermes"); + const instances = deriveProviderInstanceConfigMap({ + ...DEFAULT_SERVER_SETTINGS, + providerInstances: { + [instanceId]: { + driver: HERMES_DRIVER_KIND, + displayName: "Research Hermes", + enabled: true, + config: {}, + }, + }, + }); + + assert.deepEqual(instances[instanceId], { + driver: HERMES_DRIVER_KIND, + displayName: "Research Hermes", + enabled: true, + config: {}, + }); +}); diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts index 0fd88b4262a..ccfc512811e 100644 --- a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts @@ -18,7 +18,8 @@ * `defaultInstanceIdForDriver(driverKind)` — literally the driver kind as a * routing slug), we synthesize an envelope from the legacy field. The * registry decodes both flavours through the same `configSchema` and ends - * up with one uniform `ProviderInstance` per entry. + * up with one uniform `ProviderInstance` per entry. Drivers such as Hermes + * that require an explicitly paired remote instance opt out of synthesis. * * Explicit `providerInstances` entries always win — users can already * override the legacy `providers.` blob by authoring a @@ -43,6 +44,7 @@ */ import { defaultInstanceIdForDriver, + HERMES_DRIVER_KIND, type ProviderInstanceConfig, type ProviderInstanceConfigMap, ServerSettings, @@ -62,9 +64,10 @@ import { ProviderInstanceRegistryMutableLayer } from "./ProviderInstanceRegistry * * Strategy: * 1. Copy all explicit `settings.providerInstances` entries verbatim. - * 2. For each built-in driver whose `defaultInstanceIdForDriver(id)` key - * is *not* already in the explicit map, synthesize an entry from the - * matching legacy `settings.providers.` blob. + * 2. For each legacy-compatible built-in driver whose + * `defaultInstanceIdForDriver(id)` key is *not* already in the explicit + * map, synthesize an entry from the matching legacy + * `settings.providers.` blob. * * The returned map is the input the registry consumes; pure & exported * separately so the hydration logic can be exercised by unit tests @@ -82,6 +85,12 @@ export const deriveProviderInstanceConfigMap = ( // config always wins over the legacy mirror. continue; } + if (driver.driverKind === HERMES_DRIVER_KIND) { + // Hermes is configured only through explicit gateway instances. A + // synthesized legacy default would resurrect a revoked-and-removed + // instance in settings and provider pickers. + continue; + } // Only built-in drivers have a legacy mirror; the registry's // `providers` struct is keyed on the same literal slug as diff --git a/apps/server/src/provider/Layers/ProviderRegistry.test.ts b/apps/server/src/provider/Layers/ProviderRegistry.test.ts index 3f00d3cc662..ae69664301d 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.test.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.test.ts @@ -280,20 +280,24 @@ function makeMutableServerSettingsService( return Effect.gen(function* () { const settingsRef = yield* Ref.make(initial); const changes = yield* PubSub.unbounded(); + const updateSettingsWith = ( + update: (current: ContractServerSettings) => Parameters[1], + ) => + Effect.gen(function* () { + const current = yield* Ref.get(settingsRef); + const next = applyServerSettingsPatch(current, update(current)); + encodeServerSettings(next); + yield* Ref.set(settingsRef, next); + yield* PubSub.publish(changes, next); + return next; + }); return { start: Effect.void, ready: Effect.void, getSettings: Ref.get(settingsRef), - updateSettings: (patch) => - Effect.gen(function* () { - const current = yield* Ref.get(settingsRef); - const next = applyServerSettingsPatch(current, patch); - encodeServerSettings(next); - yield* Ref.set(settingsRef, next); - yield* PubSub.publish(changes, next); - return next; - }), + updateSettings: (patch) => updateSettingsWith(() => patch), + updateSettingsWith, get streamChanges() { return Stream.fromPubSub(changes); }, diff --git a/apps/server/src/provider/Layers/ProviderRegistry.ts b/apps/server/src/provider/Layers/ProviderRegistry.ts index 760c8e1c59e..e96f62d74b6 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.ts @@ -24,12 +24,14 @@ */ import { defaultInstanceIdForDriver, + ProviderDescribeError, ProviderDriverKind, type ProviderInstanceId, type ServerProvider, type ServerProviderUpdateState, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Equal from "effect/Equal"; import * as FileSystem from "effect/FileSystem"; @@ -51,6 +53,7 @@ import { resolveProviderStatusCachePath, writeProviderStatusCache, } from "../providerStatusCache.ts"; +import { describeFromServerProvider } from "../providerDescribe.ts"; import type { ProviderInstance } from "../ProviderDriver.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; import type { ProviderSnapshotSource } from "../builtInProviderCatalog.ts"; @@ -704,12 +707,85 @@ export const ProviderRegistryLive = Layer.effect( return yield* Ref.get(providersRef); }); + const describeInstance = Effect.fn("describeInstance")(function* ( + instanceId: ProviderInstanceId, + ) { + const providers = yield* Ref.get(providersRef); + const provider = providers.find((candidate) => candidate.instanceId === instanceId); + if (!provider) { + return yield* new ProviderDescribeError({ + code: "instance-not-found", + message: `Provider instance '${instanceId}' is not configured.`, + instanceId, + }); + } + const describedAt = DateTime.formatIso(yield* DateTime.now); + const base = describeFromServerProvider({ provider, describedAt }); + const instance = yield* instanceRegistry.getInstance(instanceId); + const describe = instance?.adapter.describe; + if (!describe) { + return base; + } + // Enrichment is best-effort by design: an agent that is offline right + // now must still render a page from its cached snapshot rather than an + // error screen. See `ProviderAdapterShape.describe`. + return yield* describe(base).pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider describe enrichment failed; using snapshot", { + instanceId, + cause: Cause.pretty(cause), + }).pipe(Effect.as(base)), + ), + ); + }); + + const getSkillBody = Effect.fn("getSkillBody")(function* (input: { + readonly instanceId: ProviderInstanceId; + readonly skillName: string; + }) { + const instance = yield* instanceRegistry.getInstance(input.instanceId); + if (!instance) { + return yield* new ProviderDescribeError({ + code: "instance-not-found", + message: `Provider instance '${input.instanceId}' is not available.`, + instanceId: input.instanceId, + }); + } + const read = instance.adapter.getSkillBody; + if (!read) { + return yield* new ProviderDescribeError({ + code: "unsupported", + message: "This provider cannot read skill contents.", + instanceId: input.instanceId, + }); + } + return yield* read(input.skillName).pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider skill body read failed", { + instanceId: input.instanceId, + skillName: input.skillName, + cause: Cause.pretty(cause), + }).pipe( + Effect.andThen( + new ProviderDescribeError({ + code: "unavailable", + message: `Could not read skill '${input.skillName}'.`, + instanceId: input.instanceId, + }), + ), + ), + ), + ); + }); + return { getProviders: Ref.get(providersRef), refresh: (provider?: ProviderDriverKind) => refresh(provider).pipe(Effect.catchCause(recoverRefreshFailure)), refreshInstance: (instanceId: ProviderInstanceId) => refreshInstance(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), + describeInstance, + getSkillBody, getProviderMaintenanceCapabilitiesForInstance, setProviderMaintenanceActionState, get streamChanges() { diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index ccbbce1759f..6ea5c9a7560 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -1280,6 +1280,170 @@ routing.layer("ProviderServiceLive routing", (it) => { assert.equal(runtimePayload.lastRuntimeEvent, "provider.sendTurn"); } } + + yield* advanceTestClock(50); + routing.codex.emit({ + type: "turn.completed", + eventId: asEventId("evt-runtime-status-stale-completion"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:01.000Z", + threadId: session.threadId, + turnId: asTurnId("turn-from-an-older-request"), + payload: { + state: "completed", + }, + }); + yield* advanceTestClock(50); + + const afterStaleCompletion = yield* runtimeRepository.getByThreadId({ + threadId: session.threadId, + }); + assert.equal(Option.isSome(afterStaleCompletion), true); + if (Option.isSome(afterStaleCompletion)) { + assert.equal(afterStaleCompletion.value.status, "running"); + const payload = afterStaleCompletion.value.runtimePayload; + assert.equal(payload !== null && typeof payload === "object", true); + if (payload !== null && typeof payload === "object" && !Array.isArray(payload)) { + assert.equal( + "activeTurnId" in payload ? payload.activeTurnId : undefined, + `turn-${String(session.threadId)}`, + ); + assert.equal( + "lastRuntimeEvent" in payload ? payload.lastRuntimeEvent : undefined, + "turn.completed", + ); + } + } + + routing.codex.emit({ + type: "turn.completed", + eventId: asEventId("evt-runtime-status-active-completion"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:02.000Z", + threadId: session.threadId, + turnId: asTurnId(`turn-${String(session.threadId)}`), + payload: { + state: "completed", + }, + }); + yield* advanceTestClock(50); + + const afterActiveCompletion = yield* runtimeRepository.getByThreadId({ + threadId: session.threadId, + }); + assert.equal(Option.isSome(afterActiveCompletion), true); + if (Option.isSome(afterActiveCompletion)) { + assert.equal(afterActiveCompletion.value.status, "running"); + const payload = afterActiveCompletion.value.runtimePayload; + assert.equal(payload !== null && typeof payload === "object", true); + if (payload !== null && typeof payload === "object" && !Array.isArray(payload)) { + assert.equal("activeTurnId" in payload ? payload.activeTurnId : undefined, null); + assert.equal("cwd" in payload ? payload.cwd : undefined, session.cwd); + assert.equal( + "lastRuntimeEvent" in payload ? payload.lastRuntimeEvent : undefined, + "turn.completed", + ); + } + } + + // A *failed* completion of the active turn must persist "error". This + // shared a ternary branch with turn.started and wrote "running" for every + // settled turn, failed ones included. + yield* provider.sendTurn({ + threadId: session.threadId, + input: "hello again", + attachments: [], + }); + yield* advanceTestClock(50); + routing.codex.emit({ + type: "turn.completed", + eventId: asEventId("evt-runtime-status-failed-completion"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:02.500Z", + threadId: session.threadId, + turnId: asTurnId(`turn-${String(session.threadId)}`), + payload: { + state: "failed", + errorMessage: "Hermes could not resume this thread after reconnecting.", + }, + }); + yield* advanceTestClock(50); + + const afterFailedCompletion = yield* runtimeRepository.getByThreadId({ + threadId: session.threadId, + }); + assert.equal(Option.isSome(afterFailedCompletion), true); + if (Option.isSome(afterFailedCompletion)) { + assert.equal(afterFailedCompletion.value.status, "error"); + const payload = afterFailedCompletion.value.runtimePayload; + if (payload !== null && typeof payload === "object" && !Array.isArray(payload)) { + assert.equal("activeTurnId" in payload ? payload.activeTurnId : undefined, null); + } + } + + // A matching turn.aborted settles the same way, minus the error. + yield* provider.sendTurn({ + threadId: session.threadId, + input: "hello once more", + attachments: [], + }); + yield* advanceTestClock(50); + routing.codex.emit({ + type: "turn.aborted", + eventId: asEventId("evt-runtime-status-active-abort"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:02.750Z", + threadId: session.threadId, + turnId: asTurnId(`turn-${String(session.threadId)}`), + payload: { reason: "interrupted" }, + }); + yield* advanceTestClock(50); + + const afterAbort = yield* runtimeRepository.getByThreadId({ + threadId: session.threadId, + }); + assert.equal(Option.isSome(afterAbort), true); + if (Option.isSome(afterAbort)) { + assert.equal(afterAbort.value.status, "running"); + const payload = afterAbort.value.runtimePayload; + if (payload !== null && typeof payload === "object" && !Array.isArray(payload)) { + assert.equal("activeTurnId" in payload ? payload.activeTurnId : undefined, null); + assert.equal( + "lastRuntimeEvent" in payload ? payload.lastRuntimeEvent : undefined, + "turn.aborted", + ); + } + } + + routing.codex.emit({ + type: "session.exited", + eventId: asEventId("evt-runtime-status-session-exit"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:03.000Z", + threadId: session.threadId, + payload: { + exitKind: "error", + recoverable: true, + }, + }); + yield* advanceTestClock(50); + + const afterSessionExit = yield* runtimeRepository.getByThreadId({ + threadId: session.threadId, + }); + assert.equal(Option.isSome(afterSessionExit), true); + if (Option.isSome(afterSessionExit)) { + assert.equal(afterSessionExit.value.status, "error"); + const payload = afterSessionExit.value.runtimePayload; + assert.equal(payload !== null && typeof payload === "object", true); + if (payload !== null && typeof payload === "object" && !Array.isArray(payload)) { + assert.equal("activeTurnId" in payload ? payload.activeTurnId : undefined, null); + assert.equal( + "lastRuntimeEvent" in payload ? payload.lastRuntimeEvent : undefined, + "session.exited", + ); + } + } }), ); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index ecf26a914c1..076f533c72e 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -162,6 +162,19 @@ function readPersistedCwd( return trimmed.length > 0 ? trimmed : undefined; } +function readPersistedActiveTurnId( + runtimePayload: ProviderSessionDirectory.ProviderRuntimeBinding["runtimePayload"], +): string | null | undefined { + if (!runtimePayload || typeof runtimePayload !== "object" || Array.isArray(runtimePayload)) { + return undefined; + } + const rawActiveTurnId = + "activeTurnId" in runtimePayload ? runtimePayload.activeTurnId : undefined; + return typeof rawActiveTurnId === "string" || rawActiveTurnId === null + ? rawActiveTurnId + : undefined; +} + const dieOnMissingBindingInstanceId = ( operation: string, payload: { @@ -281,6 +294,84 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); }); + const persistRuntimeLifecycle = (event: ProviderRuntimeEvent) => + Effect.gen(function* () { + if ( + event.type !== "turn.started" && + event.type !== "turn.completed" && + event.type !== "turn.aborted" && + event.type !== "session.exited" + ) { + return; + } + + const binding = Option.getOrUndefined(yield* directory.getBinding(event.threadId)); + if (!binding || event.providerInstanceId === undefined) { + return; + } + if ( + binding.provider !== event.provider || + binding.providerInstanceId !== event.providerInstanceId + ) { + yield* Effect.logWarning("provider.session.runtime.lifecycle-binding-mismatch", { + threadId: event.threadId, + eventProvider: event.provider, + eventProviderInstanceId: event.providerInstanceId, + bindingProvider: binding.provider, + bindingProviderInstanceId: binding.providerInstanceId, + eventType: event.type, + }); + return; + } + + const persistedActiveTurnId = readPersistedActiveTurnId(binding.runtimePayload); + const terminalMatchesActiveTurn = + (event.type === "turn.completed" || event.type === "turn.aborted") && + event.turnId !== undefined && + persistedActiveTurnId === event.turnId; + const isSessionExit = event.type === "session.exited"; + // `turn.started` and a terminal event that closes the active turn are + // opposite transitions and must not share a branch. Folded together they + // persisted "running" for every settled turn — including a *failed* one, + // which is the lie this split removes. + // + // A non-failed terminal still writes "running": this status describes the + // session, not the turn (`toRuntimeStatus` maps both ready and running to + // it), and the settled turn is expressed by `activeTurnId: null` below. + // "stopped" is reserved for sessions that are actually gone — the reaper + // skips those, so using it for an idle session would strand it. + const status = + event.type === "turn.started" + ? "running" + : terminalMatchesActiveTurn + ? event.type === "turn.completed" && event.payload.state === "failed" + ? "error" + : "running" + : isSessionExit + ? event.payload.recoverable === true || event.payload.exitKind === "error" + ? "error" + : "stopped" + : undefined; + const activeTurnId = + event.type === "turn.started" + ? (event.turnId ?? persistedActiveTurnId) + : terminalMatchesActiveTurn || isSessionExit + ? null + : persistedActiveTurnId; + + yield* directory.upsert({ + threadId: event.threadId, + provider: event.provider, + providerInstanceId: event.providerInstanceId, + ...(status !== undefined ? { status } : {}), + runtimePayload: { + ...(activeTurnId !== undefined ? { activeTurnId } : {}), + lastRuntimeEvent: event.type, + lastRuntimeEventAt: event.createdAt, + }, + }); + }); + const processRuntimeEvent = ( source: { readonly instanceId: ProviderInstanceId; @@ -293,7 +384,22 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( increment(providerRuntimeEventsTotal, { provider: canonicalEvent.provider, eventType: canonicalEvent.type, - }).pipe(Effect.andThen(publishRuntimeEvent(canonicalEvent))), + }).pipe( + Effect.andThen( + persistRuntimeLifecycle(canonicalEvent).pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider.session.runtime.lifecycle-persist-failed", { + threadId: canonicalEvent.threadId, + provider: canonicalEvent.provider, + providerInstanceId: canonicalEvent.providerInstanceId, + eventType: canonicalEvent.type, + errorTag: causeErrorTag(cause), + }), + ), + ), + ), + Effect.andThen(publishRuntimeEvent(canonicalEvent)), + ), ), ); diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts index 3843c8acbcd..77c9cb96a00 100644 --- a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts +++ b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts @@ -200,10 +200,13 @@ describe("ProviderSessionReaper", () => { Effect.succeed({ snapshotSequence: input.readModel.snapshotSequence }), getCounts: () => Effect.die("unused"), getActiveProjectByWorkspaceRoot: () => Effect.die("unused"), + getActiveAgentProject: () => Effect.die("unused"), getProjectShellById: () => Effect.die("unused"), getFirstActiveThreadIdByProjectId: () => Effect.die("unused"), getThreadCheckpointContext: () => Effect.die("unused"), getFullThreadDiffContext: () => Effect.die("unused"), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadShellById: (threadId) => Effect.succeed( input.readModel.threads.find((thread) => thread.id === threadId) diff --git a/apps/server/src/provider/Layers/RequestCorrelator.test.ts b/apps/server/src/provider/Layers/RequestCorrelator.test.ts new file mode 100644 index 00000000000..d49f5ff7445 --- /dev/null +++ b/apps/server/src/provider/Layers/RequestCorrelator.test.ts @@ -0,0 +1,184 @@ +import { assert, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; + +import { ProviderAdapterRequestError } from "../Errors.ts"; +import { makeRequestCorrelator } from "./RequestCorrelator.ts"; + +const TIMEOUT = Duration.seconds(30); + +const makeCorrelator = () => + makeRequestCorrelator({ + provider: "test", + timeout: TIMEOUT, + maxAge: Duration.seconds(90), + }); + +const owner = "owner-a"; + +it.effect("resolves a correlated response and releases the pending entry", () => + Effect.gen(function* () { + const correlator = yield* makeCorrelator(); + const pending = yield* correlator + .request({ owner, requestId: "r1", method: "test.method", send: Effect.void }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + assert.equal(yield* correlator.pendingCount, 1); + + assert.isTrue(yield* correlator.complete("r1", { requestId: "r1" })); + assert.deepEqual(yield* Fiber.join(pending), { requestId: "r1" }); + assert.equal(yield* correlator.pendingCount, 0); + }), +); + +it.effect("releases the pending entry when the send fails", () => + Effect.gen(function* () { + const correlator = yield* makeCorrelator(); + const failure = yield* Effect.flip( + correlator.request({ + owner, + requestId: "r-send-fail", + method: "test.method", + send: Effect.fail( + new ProviderAdapterRequestError({ + provider: "test", + method: "test.method", + detail: "the transport refused the write", + }), + ), + }), + ); + assert.include(failure.detail, "refused"); + assert.equal(yield* correlator.pendingCount, 0); + }), +); + +// The send used to sit outside `restore`, so a transport write that never +// returned could not be interrupted: `Fiber.interrupt` would hang forever +// waiting for the uninterruptible region to finish. The write is now inside the +// restored region, so an interrupt lands — and the entry is still released. +it.effect("interrupts a request whose transport write never returns", () => + Effect.gen(function* () { + const correlator = yield* makeCorrelator(); + const writeStarted = yield* Deferred.make(); + const neverCompletes = yield* Deferred.make(); + + const pending = yield* correlator + .request({ + owner, + requestId: "r-stalled-write", + method: "test.method", + send: Deferred.succeed(writeStarted, undefined).pipe( + Effect.andThen(Deferred.await(neverCompletes)), + ), + }) + .pipe(Effect.forkChild({ startImmediately: true })); + + yield* Deferred.await(writeStarted); + assert.equal(yield* correlator.pendingCount, 1); + + // Would never return before the fix. + yield* Fiber.interrupt(pending); + assert.equal( + yield* correlator.pendingCount, + 0, + "an interrupted request must not leave a pending entry behind", + ); + }), +); + +// `Effect.timeout` only wrapped `Deferred.await`, so the clock did not start +// until the write returned: a request over a wedged socket outlived its own +// timeout for as long as the socket stayed wedged. The timeout now covers the +// send too. +it.effect("times out a request whose transport write stalls", () => + Effect.gen(function* () { + const correlator = yield* makeCorrelator(); + const writeStarted = yield* Deferred.make(); + const neverCompletes = yield* Deferred.make(); + + const pending = yield* correlator + .request({ + owner, + requestId: "r-stalled-timeout", + method: "test.method", + send: Deferred.succeed(writeStarted, undefined).pipe( + Effect.andThen(Deferred.await(neverCompletes)), + ), + }) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + + yield* Deferred.await(writeStarted); + yield* TestClock.adjust(Duration.seconds(31)); + + const outcome = yield* Fiber.join(pending); + assert.equal(outcome._tag, "Failure"); + if (outcome._tag === "Failure") assert.include(outcome.failure.detail, "timed out"); + assert.equal(yield* correlator.pendingCount, 0); + }), +); + +// A response can land the instant the write completes. Registration happens +// uninterruptibly before the send, so `complete` always finds an entry — +// moving the send inside `restore` must not have opened that window. +it.effect("completes a response that arrives from inside the transport write", () => + Effect.gen(function* () { + const correlator = yield* makeCorrelator(); + const pending = yield* correlator + .request({ + owner, + requestId: "r-immediate", + method: "test.method", + // The transport answers synchronously, before `send` even returns. + send: correlator.complete("r-immediate", { requestId: "r-immediate" }).pipe( + Effect.flatMap((completed) => + completed + ? Effect.void + : Effect.fail( + new ProviderAdapterRequestError({ + provider: "test", + method: "test.method", + detail: "the response found no pending entry", + }), + ), + ), + ), + }) + .pipe(Effect.forkChild({ startImmediately: true })); + + assert.deepEqual(yield* Fiber.join(pending), { requestId: "r-immediate" }); + assert.equal(yield* correlator.pendingCount, 0); + }), +); + +it.effect("fails every request routed over a dead owner", () => + Effect.gen(function* () { + const correlator = yield* makeCorrelator(); + const first = yield* correlator + .request({ owner, requestId: "r-own-1", method: "test.method", send: Effect.void }) + .pipe(Effect.forkChild({ startImmediately: true })); + const second = yield* correlator + .request({ owner, requestId: "r-own-2", method: "test.method", send: Effect.void }) + .pipe(Effect.forkChild({ startImmediately: true })); + const other = yield* correlator + .request({ + owner: "owner-b", + requestId: "r-own-3", + method: "test.method", + send: Effect.void, + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + + yield* correlator.failOwner(owner, "the connection dropped"); + assert.include((yield* Effect.flip(Fiber.join(first))).detail, "dropped"); + assert.include((yield* Effect.flip(Fiber.join(second))).detail, "dropped"); + assert.isUndefined(other.pollUnsafe(), "another owner's request must be untouched"); + + yield* correlator.complete("r-own-3", { requestId: "r-own-3" }); + assert.deepEqual(yield* Fiber.join(other), { requestId: "r-own-3" }); + }), +); diff --git a/apps/server/src/provider/Layers/RequestCorrelator.ts b/apps/server/src/provider/Layers/RequestCorrelator.ts new file mode 100644 index 00000000000..a6687e92d02 --- /dev/null +++ b/apps/server/src/provider/Layers/RequestCorrelator.ts @@ -0,0 +1,175 @@ +/** + * Live {@link RequestCorrelator} implementation. + * + * Generic over the owner key (the route a request travels over) and the + * response payload, so this is reusable by any provider with correlated + * request/response framing — see the module docs on the service for why the + * interrupt-safety and sweeping guarantees live here rather than at call sites. + * + * @module Layers/RequestCorrelator + */ +import * as Clock from "effect/Clock"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Ref from "effect/Ref"; + +import { ProviderAdapterRequestError } from "../Errors.ts"; +import type { RequestCorrelator, RequestCorrelatorSend } from "../Services/RequestCorrelator.ts"; + +interface PendingRequest { + readonly owner: Owner; + readonly method: string; + readonly registeredAtMillis: number; + readonly deferred: Deferred.Deferred; +} + +export interface MakeRequestCorrelatorOptions { + /** Provider name stamped onto `ProviderAdapterRequestError`. */ + readonly provider: string; + /** Default response timeout when a request does not override it. */ + readonly timeout: Duration.Duration; + /** + * Entries older than this are reaped by `sweep`. Comfortably larger than + * `timeout` so the sweeper only ever catches entries whose own timeout + * pipeline failed to run — a normal slow request is never stolen from + * underneath its awaiting fiber. + */ + readonly maxAge: Duration.Duration; +} + +export const makeRequestCorrelator = ( + options: MakeRequestCorrelatorOptions, +): Effect.Effect> => + Effect.gen(function* () { + const pending = yield* Ref.make(new Map>()); + + const requestError = (method: string, detail: string) => + new ProviderAdapterRequestError({ provider: options.provider, method, detail }); + + /** Remove one entry by id. Safe to call when it is already gone. */ + const release = (requestId: string) => + Ref.update(pending, (current) => { + if (!current.has(requestId)) return current; + const next = new Map(current); + next.delete(requestId); + return next; + }); + + const request = (input: RequestCorrelatorSend) => + Effect.gen(function* () { + const deferred = yield* Deferred.make(); + const registeredAtMillis = yield* Clock.currentTimeMillis; + + // Registration must not be interruptible independently of the cleanup + // that removes the entry. `uninterruptibleMask` closes that window: the + // registration is applied uninterruptibly — so a response arriving the + // instant the request is written always finds an entry to complete — + // and `ensuring` guarantees release regardless of how the rest exits. + // + // Everything after registration — the send *and* the await — lives + // inside `restore`, with the timeout inside it too. Both placements + // matter: + // + // - Send outside `restore`: a transport write that never returns is + // unkillable, and the response timeout does not start counting + // until the write returns, so a request over a wedged socket hangs + // for as long as the socket does. + // - Timeout outside `restore`: `Effect.timeout` races internally, and + // that race inherits the surrounding uninterruptible region — so an + // interrupt cannot land on the awaiting fiber even though the body + // it wraps was restored. + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* Ref.update(pending, (current) => + new Map(current).set(input.requestId, { + owner: input.owner, + method: input.method, + registeredAtMillis, + deferred, + }), + ); + + return yield* restore( + input.send.pipe( + Effect.andThen(Deferred.await(deferred)), + Effect.timeout(input.timeout ?? options.timeout), + ), + ).pipe( + Effect.mapError((error) => + error._tag === "TimeoutError" + ? requestError(input.method, `${options.provider} request timed out.`) + : error, + ), + Effect.ensuring(release(input.requestId)), + ); + }), + ); + }); + + const complete = (requestId: string, response: Response) => + Ref.modify(pending, (current) => { + const found = current.get(requestId); + if (!found) return [undefined, current] as const; + const next = new Map(current); + next.delete(requestId); + return [found, next] as const; + }).pipe( + Effect.flatMap((found) => + found === undefined + ? Effect.succeed(false) + : Deferred.succeed(found.deferred, response).pipe(Effect.as(true)), + ), + ); + + /** + * Extract every entry matching `select`, then fail their deferreds. The + * extraction is a single `Ref.modify` so two concurrent failures cannot + * both claim the same entry. + */ + const failMatching = ( + select: (entry: PendingRequest) => boolean, + detail: (entry: PendingRequest) => string, + ) => + Ref.modify(pending, (current) => { + const claimed: Array> = []; + const next = new Map(current); + for (const [requestId, entry] of current) { + if (!select(entry)) continue; + claimed.push(entry); + next.delete(requestId); + } + return [claimed, next] as const; + }).pipe( + Effect.flatMap((claimed) => + Effect.forEach( + claimed, + (entry) => Deferred.fail(entry.deferred, requestError(entry.method, detail(entry))), + { discard: true }, + ), + ), + ); + + const failOwner = (owner: Owner, detail: string) => + failMatching( + (entry) => entry.owner === owner, + () => detail, + ); + + const sweep = Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + failMatching( + (entry) => now - entry.registeredAtMillis >= Duration.toMillis(options.maxAge), + () => `${options.provider} request was abandoned before it completed.`, + ), + ), + ); + + return { + request, + complete, + failOwner, + sweep, + pendingCount: Ref.get(pending).pipe(Effect.map((current) => current.size)), + } satisfies RequestCorrelator; + }); diff --git a/apps/server/src/provider/Services/HermesConnectionRegistry.ts b/apps/server/src/provider/Services/HermesConnectionRegistry.ts new file mode 100644 index 00000000000..fca36e208ed --- /dev/null +++ b/apps/server/src/provider/Services/HermesConnectionRegistry.ts @@ -0,0 +1,167 @@ +/** + * HermesConnectionRegistry — owner of **volatile** Hermes gateway liveness. + * + * Everything in here is deliberately memory-only and is never persisted: + * the live connection, its transport, `lastSeen`, `activeSessionCount`, the + * reported model, and the `upgrade-required` observation. + * + * Why liveness must not be persisted + * ---------------------------------- + * `ProviderInstanceRegistryLive.reconcile` structurally compares each + * `ProviderInstanceConfig` envelope and closes the instance scope on **any** + * change. Closing that scope runs `HermesAdapter`'s finalizer, which calls + * `stopAll()` and sends `session.stop` to Hermes for every live thread. So + * writing something like `connectedAt` into the instance config on every + * connect would tear down every Hermes session on every reconnect. + * + * The broker (and therefore this registry) is a top-level layer that survives + * reconcile, which makes it the correct owner of liveness. The accepted + * consequence is that after a T3 server restart the UI reports "never + * connected" until the plugin dials back in. That is intended. + * + * Generation fencing + * ------------------ + * Every accepted connection gets a monotonically increasing generation. All + * post-acceptance mutations are fenced on it, and the comparison happens + * *inside* the `Ref` update rather than as a separate read, so a replacement + * landing concurrently cannot be clobbered by a stale writer (TOCTOU). + * + * @module HermesConnectionRegistry + */ +import type { + HermesGatewayCapabilities, + HermesGatewayInstanceStatus, + ProviderInstanceId, +} from "@t3tools/contracts"; +import type * as Effect from "effect/Effect"; +import type * as Scope from "effect/Scope"; + +import type { + HermesGatewayConnectionRegistration, + HermesGatewayTransport, +} from "./HermesGatewayBroker.ts"; + +/** Observed facts about a plugin connection, independent of its transport. */ +export interface HermesObservedConnection { + readonly pluginVersion: string; + readonly hermesVersion: string; + readonly capabilities: HermesGatewayCapabilities; + /** Model reported at handshake. Null when the plugin predates the field. */ + readonly model: string | null; + readonly connectedAt: string; + readonly activeSessionCount: number; +} + +export interface HermesActiveConnection extends HermesObservedConnection { + readonly generation: number; + readonly transport: HermesGatewayTransport; + /** + * Scope tied to this connection's lifetime. Per-connection fibers (the ping + * loop) are forked into it, so they die exactly when the connection does. + */ + readonly scope: Scope.Closeable; +} + +export interface HermesUpgradeRequired { + readonly pluginVersion: string; + readonly hermesVersion: string; + readonly protocolVersion: number; + readonly model: string | null; +} + +/** Volatile per-instance liveness. Absent entry means "nothing observed yet". */ +export interface HermesInstanceLiveness { + readonly connection?: HermesActiveConnection | undefined; + readonly lastSeen?: HermesObservedConnection | undefined; + readonly upgradeRequired?: HermesUpgradeRequired | undefined; +} + +export interface HermesAcceptedConnection { + readonly generation: number; + /** + * The connection this one displaced, if any. The caller is responsible for + * closing it — the registry never touches a transport itself, so callers can + * order the close against failing pending requests. + */ + readonly displaced: HermesActiveConnection | undefined; +} + +export interface HermesConnectionRegistry { + /** Current liveness for one instance, or `undefined` if nothing observed. */ + readonly liveness: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + + /** Accept a connection, assigning it the next generation. */ + readonly accept: (input: { + readonly instanceId: ProviderInstanceId; + readonly transport: HermesGatewayTransport; + readonly observed: HermesObservedConnection; + }) => Effect.Effect; + + /** + * Record an incompatible plugin. Clears any live connection for the instance + * and returns it so the caller can close it. + */ + readonly markUpgradeRequired: (input: { + readonly instanceId: ProviderInstanceId; + readonly upgradeRequired: HermesUpgradeRequired; + }) => Effect.Effect; + + /** + * Update the active session count. Generation-fenced; returns `false` when + * the registration is stale, in which case nothing was written. + */ + readonly recordSessionCount: ( + registration: HermesGatewayConnectionRegistration, + activeSessionCount: number, + ) => Effect.Effect; + + /** + * Retire a connection, demoting it to `lastSeen`. Generation-fenced; + * returns the retired connection, or `undefined` when the registration is + * stale (already replaced) and nothing was changed. + */ + readonly disconnect: ( + registration: HermesGatewayConnectionRegistration, + ) => Effect.Effect; + + /** + * Clear the live connection for an instance regardless of generation, e.g. + * on revoke. Returns it so the caller can close the transport. + */ + readonly clearConnection: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + + /** Drop all liveness for an instance (on remove). */ + readonly forget: (instanceId: ProviderInstanceId) => Effect.Effect; + + /** Whether a live connection exists. */ + readonly isConnected: (instanceId: ProviderInstanceId) => Effect.Effect; + + /** Live transport for an instance, if connected. */ + readonly transport: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; +} + +/** + * Project the status fields owned by liveness. The durable half (nickname, + * connectorUrl, revoked) is layered on by the broker, which reads it from + * settings. + */ +export type HermesLivenessStatusFields = Pick< + HermesGatewayInstanceStatus, + | "lastConnectedAt" + | "pluginVersion" + | "hermesVersion" + | "model" + | "activeSessionCount" + | "protocolVersion" + | "capabilities" + | "connectionGeneration" +> & { + readonly connected: boolean; + readonly upgradeRequired: boolean; +}; diff --git a/apps/server/src/provider/Services/HermesEnrollmentStore.ts b/apps/server/src/provider/Services/HermesEnrollmentStore.ts new file mode 100644 index 00000000000..ebcbdd134ee --- /dev/null +++ b/apps/server/src/provider/Services/HermesEnrollmentStore.ts @@ -0,0 +1,68 @@ +/** + * HermesEnrollmentStore — the mint/consume/expire lifecycle of one-time + * enrollment tokens. + * + * Separated from the broker so the token rules live in one place: + * + * - A token is redeemable exactly once. Consumption is compare-and-swap: + * the entry is deleted only if it is still byte-identical to the one the + * caller authenticated against, and expiry is re-checked afterwards, so + * two concurrent redemptions cannot both win. + * - Minting a token for an instance invalidates that instance's previous + * tokens, so only the newest unconsumed token is ever redeemable. + * - Tokens expire from memory on a sweep, not only lazily at redemption. + * Redemption-time expiry alone let any operator-scoped client grow the + * map without bound by calling createEnrollment in a loop. + * + * @module HermesEnrollmentStore + */ +import type { + HermesGatewayCreateEnrollmentInput, + HermesGatewayEnrollmentToken, + ProviderInstanceId, +} from "@t3tools/contracts"; +import type * as Effect from "effect/Effect"; + +export interface PendingEnrollment { + readonly input: HermesGatewayCreateEnrollmentInput; + readonly expiresAtMillis: number; +} + +export interface HermesEnrollmentStore { + /** + * Mint a token for `input`, invalidating any prior unconsumed token for the + * same instance. Returns the token and its absolute expiry. + */ + readonly mint: ( + input: HermesGatewayCreateEnrollmentInput, + ) => Effect.Effect< + { readonly token: HermesGatewayEnrollmentToken; readonly expiresAtMillis: number }, + never + >; + + /** + * Look up a token without consuming it. Returns `undefined` when the token + * is unknown or already expired — the caller authenticates against this + * value and later passes it back to `consume`. + */ + readonly peek: (token: string) => Effect.Effect; + + /** + * Compare-and-swap consumption. Deletes the entry only if it is still + * identical to `expected`, then re-checks expiry. Returns the consumed + * enrollment, or `undefined` if it was raced away or expired in between. + */ + readonly consume: ( + token: string, + expected: PendingEnrollment, + ) => Effect.Effect; + + /** Drop every unconsumed token belonging to `instanceId`. */ + readonly forget: (instanceId: ProviderInstanceId) => Effect.Effect; + + /** Drop every expired token. Run periodically; bounds memory. */ + readonly sweep: Effect.Effect; + + /** Number of unconsumed tokens. Exposed for tests and diagnostics. */ + readonly size: Effect.Effect; +} diff --git a/apps/server/src/provider/Services/HermesGatewayBroker.ts b/apps/server/src/provider/Services/HermesGatewayBroker.ts new file mode 100644 index 00000000000..d1e428a896a --- /dev/null +++ b/apps/server/src/provider/Services/HermesGatewayBroker.ts @@ -0,0 +1,128 @@ +import type { + HermesGatewayConnectionHello, + HermesGatewayConnectionRole, + HermesGatewayCreateEnrollmentInput, + HermesGatewayEnrollmentResult, + HermesGatewayInstanceStatus, + HermesGatewayPluginToT3Message, + HermesGatewayRemoveInstanceResult, + HermesGatewayRenameInstanceInput, + HermesGatewayRenameInstanceResult, + HermesGatewayRevokeInstanceResult, + HermesGatewayT3ToPluginMessage, + ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; + +import type { HermesGatewayManagementError } from "@t3tools/contracts"; +import type { ProviderAdapterRequestError } from "../Errors.ts"; + +export interface HermesGatewayTransport { + readonly send: ( + message: HermesGatewayT3ToPluginMessage, + ) => Effect.Effect; + readonly close: (code: number, reason: string) => Effect.Effect; +} + +export interface HermesGatewayConnectionRegistration { + readonly instanceId: ProviderInstanceId; + /** + * Fencing token for the instance's primary connection, or `null` for a + * delivery connection. + * + * A delivery connection is deliberately outside the generation scheme: it + * was never registered as the primary, so there is nothing for it to be + * stale against, and giving it a generation would let it be compared — + * and therefore mistaken for a replacement — against the live socket. + */ + readonly generation: number | null; + /** + * What this socket registered as. `delivery` sockets may only hand over + * `home.deliver` frames; they carry no session, take no ping loop, and + * their disconnect must not disturb the instance's liveness state. + */ + readonly role: HermesGatewayConnectionRole; + readonly accepted: Extract< + HermesGatewayT3ToPluginMessage, + { readonly type: "connection.accepted" } + >; +} + +export interface HermesGatewayEnvelope { + readonly instanceId: ProviderInstanceId; + readonly message: Exclude; +} + +export interface HermesGatewayBrokerShape { + readonly createEnrollment: ( + input: HermesGatewayCreateEnrollmentInput, + ) => Effect.Effect; + readonly getInstanceStatus: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + readonly listInstances: Effect.Effect< + ReadonlyArray, + HermesGatewayManagementError + >; + readonly renameInstance: ( + input: HermesGatewayRenameInstanceInput, + ) => Effect.Effect; + readonly revokeInstance: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + readonly removeInstance: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + readonly registerConnection: ( + hello: HermesGatewayConnectionHello, + transport: HermesGatewayTransport, + ) => Effect.Effect< + HermesGatewayConnectionRegistration, + Extract + >; + readonly receive: ( + registration: HermesGatewayConnectionRegistration, + message: Exclude, + ) => Effect.Effect; + readonly disconnect: (registration: HermesGatewayConnectionRegistration) => Effect.Effect; + readonly request: ( + instanceId: ProviderInstanceId, + message: HermesGatewayT3ToPluginMessage, + ) => Effect.Effect< + Exclude, + ProviderAdapterRequestError + >; + readonly send: ( + instanceId: ProviderInstanceId, + message: HermesGatewayT3ToPluginMessage, + ) => Effect.Effect; + readonly isConnected: (instanceId: ProviderInstanceId) => Effect.Effect; + readonly stream: Stream.Stream; + readonly streamStatuses: Stream.Stream; +} + +const unavailable = () => Effect.die(new Error("HermesGatewayBroker live layer is not installed")); + +export const HermesGatewayBroker = Context.Reference( + "t3/provider/Services/HermesGatewayBroker", + { + defaultValue: () => ({ + createEnrollment: unavailable, + getInstanceStatus: unavailable, + listInstances: unavailable(), + renameInstance: unavailable, + revokeInstance: unavailable, + removeInstance: unavailable, + registerConnection: unavailable, + receive: unavailable, + disconnect: unavailable, + request: unavailable, + send: unavailable, + isConnected: () => Effect.succeed(false), + stream: Stream.empty, + streamStatuses: Stream.empty, + }), + }, +); diff --git a/apps/server/src/provider/Services/ProviderAdapter.ts b/apps/server/src/provider/Services/ProviderAdapter.ts index 01eeae7b7bd..d908181fd58 100644 --- a/apps/server/src/provider/Services/ProviderAdapter.ts +++ b/apps/server/src/provider/Services/ProviderAdapter.ts @@ -11,6 +11,8 @@ import type { ApprovalRequestId, ProviderApprovalDecision, ProviderDriverKind, + ProviderInstanceDescription, + ProviderSkillBody, ProviderUserInputAnswers, ProviderRuntimeEvent, ProviderSendTurnInput, @@ -119,6 +121,29 @@ export interface ProviderAdapterShape { */ readonly stopAll: () => Effect.Effect; + /** + * Driver-specific enrichment for the Agent page. + * + * OPTIONAL by design. The caller always builds a complete description from + * the `ServerProvider` snapshot first (`describeFromServerProvider`), then + * lets an adapter that knows more overwrite parts of it. A driver that + * implements nothing here still gets a working Agent page, and one that + * cannot reach its agent right now should fail rather than return a + * half-filled description — the caller falls back to the snapshot default. + */ + readonly describe?: ( + base: ProviderInstanceDescription, + ) => Effect.Effect; + + /** + * Fetch one skill's markdown body. Fired on row expand, never eagerly — + * bodies are unbounded in size and most are never opened. + * + * Returning `markdown: null` means "this skill exists but its body is not + * retrievable"; failing means the request itself could not be made. + */ + readonly getSkillBody?: (skillName: string) => Effect.Effect; + /** * Canonical runtime event stream emitted by this adapter. */ diff --git a/apps/server/src/provider/Services/ProviderRegistry.ts b/apps/server/src/provider/Services/ProviderRegistry.ts index b7426b30338..f820fc57402 100644 --- a/apps/server/src/provider/Services/ProviderRegistry.ts +++ b/apps/server/src/provider/Services/ProviderRegistry.ts @@ -7,8 +7,11 @@ * @module ProviderRegistry */ import type { + ProviderDescribeError, + ProviderInstanceDescription, ProviderInstanceId, ProviderDriverKind, + ProviderSkillBody, ServerProvider, ServerProviderUpdateState, } from "@t3tools/contracts"; @@ -69,6 +72,28 @@ export interface ProviderRegistryShape { readonly state: ServerProviderUpdateState | null; }) => Effect.Effect>; + /** + * Provider-generic description of one configured instance, for the Agent + * page. Always answerable from the cached snapshot; drivers that can reach + * their agent enrich it via `ProviderAdapterShape.describe`. + * + * Enrichment failure is not description failure — an unreachable agent + * still yields the snapshot-derived description, so the page renders + * last-known state instead of an error. Only an unknown instance fails. + */ + readonly describeInstance: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + + /** + * Read one skill's markdown body from the instance's adapter. Fired on row + * expand — never eagerly, since bodies are unbounded and rarely opened. + */ + readonly getSkillBody: (input: { + readonly instanceId: ProviderInstanceId; + readonly skillName: string; + }) => Effect.Effect; + /** * Stream of provider snapshot updates — one emission per aggregated * change. The array contains the full current state. diff --git a/apps/server/src/provider/Services/RequestCorrelator.ts b/apps/server/src/provider/Services/RequestCorrelator.ts new file mode 100644 index 00000000000..99682062b74 --- /dev/null +++ b/apps/server/src/provider/Services/RequestCorrelator.ts @@ -0,0 +1,69 @@ +/** + * RequestCorrelator — generic request/response correlation over a one-way + * message transport. + * + * Nothing in here is Hermes-specific. Any provider that speaks a protocol + * where a request carries an id and the reply echoes it can use this: register + * a pending entry, write the request, await the correlated reply, and have the + * entry torn down on success, failure, timeout, or interrupt. + * + * The correlator owns three guarantees the call sites used to hand-roll: + * + * 1. **No interrupt window.** Registration happens uninterruptibly and the + * await pipeline is wrapped in `ensuring`, so there is no point at which + * an interrupt can leave an entry in the map with nobody to clean it up. + * 2. **Age-based sweeping.** An entry whose awaiting fiber died abnormally + * is reaped by `sweep` instead of leaking for the process lifetime. + * 3. **Owner-scoped failure.** When a connection drops, every request routed + * over it fails immediately rather than waiting out its timeout. + * + * @module RequestCorrelator + */ +import type * as Duration from "effect/Duration"; +import type * as Effect from "effect/Effect"; + +import type { ProviderAdapterRequestError } from "../Errors.ts"; + +export interface RequestCorrelatorSend { + /** + * The logical route the request travels over. Used to fail an entire group + * of in-flight requests at once when that route dies (see `failOwner`). + */ + readonly owner: Owner; + /** Correlation key. Must match the id echoed on the response. */ + readonly requestId: string; + /** Protocol method name, surfaced on errors for diagnostics. */ + readonly method: string; + /** Writes the request. Registration is already in place when this runs. */ + readonly send: Effect.Effect; + /** Overrides the correlator's default response timeout. */ + readonly timeout?: Duration.Duration | undefined; +} + +export interface RequestCorrelator { + /** + * Register, send, and await the correlated response. Cleans up the pending + * entry on every exit path, including interrupt. + */ + readonly request: ( + input: RequestCorrelatorSend, + ) => Effect.Effect; + + /** + * Resolve a pending request. Returns `false` when no entry was waiting, + * which lets callers distinguish a correlated reply from an unsolicited one. + */ + readonly complete: (requestId: string, response: Response) => Effect.Effect; + + /** Fail every pending request registered against `owner`. */ + readonly failOwner: (owner: Owner, detail: string) => Effect.Effect; + + /** + * Fail pending entries older than the configured max age. Safety net for + * entries whose awaiting fiber died without running its own cleanup. + */ + readonly sweep: Effect.Effect; + + /** Number of in-flight requests. Exposed for tests and diagnostics. */ + readonly pendingCount: Effect.Effect; +} diff --git a/apps/server/src/provider/builtInDrivers.ts b/apps/server/src/provider/builtInDrivers.ts index 791a96e1da3..58822279797 100644 --- a/apps/server/src/provider/builtInDrivers.ts +++ b/apps/server/src/provider/builtInDrivers.ts @@ -24,6 +24,7 @@ import { ClaudeDriver, type ClaudeDriverEnv } from "./Drivers/ClaudeDriver.ts"; import { CodexDriver, type CodexDriverEnv } from "./Drivers/CodexDriver.ts"; import { CursorDriver, type CursorDriverEnv } from "./Drivers/CursorDriver.ts"; import { GrokDriver, type GrokDriverEnv } from "./Drivers/GrokDriver.ts"; +import { HermesDriver, type HermesDriverEnv } from "./Drivers/HermesDriver.ts"; import { OpenCodeDriver, type OpenCodeDriverEnv } from "./Drivers/OpenCodeDriver.ts"; import type { AnyProviderDriver } from "./ProviderDriver.ts"; @@ -37,6 +38,7 @@ export type BuiltInDriversEnv = | CodexDriverEnv | CursorDriverEnv | GrokDriverEnv + | HermesDriverEnv | OpenCodeDriverEnv; /** @@ -50,4 +52,5 @@ export const BUILT_IN_DRIVERS: ReadonlyArray = {}, +): HermesGatewayMediaDeliver => + ({ + type: "media.deliver", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + deliveryId: HermesGatewayDeliveryId.make("media-delivery-1"), + threadId: HOME_THREAD_ID, + kind: "cron", + label: "Cron: daily-digest", + name: "chart.png", + mimeType: "image/png", + sizeBytes: 8, + caption: "Today's chart", + data: Buffer.from("PNGBYTES").toString("base64"), + createdAt: CREATED_AT, + ...overrides, + }) as HermesGatewayMediaDeliver; + +/** + * The handlers only reach `adapter.hasSession` on the turn-scoped path, so + * everything else can stay unimplemented. + */ +const instanceTracking = (threadId: ThreadId): ProviderInstance => + ({ + instanceId: INSTANCE_ID, + adapter: { + hasSession: (candidate: ThreadId) => Effect.succeed(candidate === threadId), + }, + }) as unknown as ProviderInstance; + +const makeHarness = (options?: { + readonly existingDeliveries?: ReadonlyArray<{ threadId: ThreadId; deliveryId: string }>; + readonly trackedThreadId?: ThreadId; + readonly failDispatch?: boolean; + /** Archive state of the designated home thread, as the projection sees it. */ + readonly homeThreadArchivedAt?: string; +}) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-hermes-media-attachments-", + }); + const dispatched = yield* Ref.make>([]); + const sent: Array = []; + const transport = { + send: (frame: HermesGatewayT3ToPluginMessage) => + Effect.sync(() => sent.push(frame)).pipe(Effect.asVoid), + }; + + const engineLayer = Layer.mock(OrchestrationEngine.OrchestrationEngineService)({ + readEvents: () => Stream.empty, + dispatch: (command) => + Ref.update(dispatched, (commands) => [...commands, command]).pipe( + Effect.andThen( + options?.failDispatch + ? Effect.fail( + new OrchestrationCommandInvariantError({ + commandType: "thread.notification.deliver", + detail: "Simulated dispatch failure.", + }), + ) + : Effect.succeed({ sequence: 1 }), + ), + ), + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(0), + }); + + const queryLayer = Layer.mock(ProjectionSnapshotQuery)({ + // Row-level, and deliberately indifferent to archive state — the whole + // point of the lookup under test. `getThreadDetailById` is left + // unimplemented so a regression back to it fails loudly here. + hasThreadNotificationDelivery: ({ threadId, deliveryId }) => + Effect.succeed( + (options?.existingDeliveries ?? []).some( + (entry) => entry.threadId === threadId && entry.deliveryId === deliveryId, + ), + ), + // The home thread is designated in settings and exists, so home-thread + // resolution takes the fast path without dispatching a thread.create. + getThreadArchiveStateById: () => + Effect.succeed(Option.some({ archivedAt: options?.homeThreadArchivedAt ?? null })), + }); + + const registryLayer = Layer.mock(ProviderInstanceRegistry)({ + getInstance: () => + Effect.succeed( + options?.trackedThreadId !== undefined + ? instanceTracking(options.trackedThreadId) + : undefined, + ), + }); + + const settingsLayer = ServerSettings.layerTest({ + providerInstances: { + [INSTANCE_ID]: { + driver: "hermes", + displayName: "Hermes Media", + config: { homeThreadId: HOME_THREAD_ID }, + }, + }, + } as never); + + // Only `attachmentsDir` is read off the config by the handlers. + const configLayer = Layer.succeed(ServerConfig, { + attachmentsDir, + } as unknown as ServerConfig["Service"]); + + // Provided at invocation too: the handlers resolve the home thread at + // delivery time, which pulls services from the runtime context. + const servicesLayer = Layer.mergeAll( + engineLayer, + queryLayer, + registryLayer, + settingsLayer, + configLayer, + ).pipe(Layer.provideMerge(NodeServices.layer)); + + const handlers = yield* makeHermesDeliveryHandlers().pipe(Effect.provide(servicesLayer)); + const deliverMedia = (...args: Parameters) => + handlers.deliverMedia(...args).pipe(Effect.provide(servicesLayer), Effect.orDie); + + return { deliverMedia, dispatched, sent, transport, attachmentsDir } as const; + }); + +const dispatchedDeliveries = (commands: ReadonlyArray) => + commands.filter( + (command): command is Extract => + command.type === "thread.notification.deliver", + ); + +it.effect("writes turnless media to the home thread with provenance and acks after dispatch", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + // The frame's threadId is deliberately NOT the home thread: turnless + // media must land in the re-resolved home thread regardless. + yield* harness.deliverMedia( + registration, + mediaFrame({ threadId: ThreadId.make("thread-attacker-named") }), + harness.transport, + ); + + const deliveries = dispatchedDeliveries(yield* Ref.get(harness.dispatched)); + assert.equal(deliveries.length, 1); + const delivery = deliveries[0]!; + assert.equal(delivery.threadId, HOME_THREAD_ID); + assert.equal(delivery.kind, "cron"); + assert.equal(delivery.label, "Cron: daily-digest"); + assert.equal(delivery.text, "Today's chart"); + assert.isUndefined(delivery.turnId); + assert.equal(delivery.attachments?.length, 1); + const attachment = delivery.attachments![0]!; + // image/* rides the image variant so the web's inline grid renders it. + assert.equal(attachment.type, "image"); + assert.equal(attachment.mimeType, "image/png"); + assert.equal(attachment.sizeBytes, 8); + + // The bytes are durably on disk under the dispatched attachment id. + const fileSystem = yield* FileSystem.FileSystem; + const entries = yield* fileSystem.readDirectory(harness.attachmentsDir); + const written = entries.find((entry) => entry.startsWith(attachment.id)); + assert.isDefined(written, "the media bytes must be written to the attachments dir"); + const bytes = yield* fileSystem.readFile(`${harness.attachmentsDir}/${written}`); + assert.equal(Buffer.from(bytes).toString("utf8"), "PNGBYTES"); + + assert.deepEqual(harness.sent, [ + { + type: "media.deliver.ack", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + deliveryId: mediaFrame().deliveryId, + }, + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("acks a duplicate deliveryId without dispatching a second message", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ + existingDeliveries: [{ threadId: HOME_THREAD_ID, deliveryId: "media-delivery-1" }], + }); + yield* harness.deliverMedia(registration, mediaFrame(), harness.transport); + + assert.equal(dispatchedDeliveries(yield* Ref.get(harness.dispatched)).length, 0); + // Still acked: the plugin's queue purges only on the ack, and the row is + // already durably written. + assert.equal(harness.sent.length, 1); + assert.equal(harness.sent[0]?.type, "media.deliver.ack"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("dedupes a delivery into an ARCHIVED home thread", () => + Effect.gen(function* () { + // The retry loop this guards against: the dedupe read used to go through + // `getThreadDetailById`, which filters `archived_at IS NULL`. A home + // thread the user archived therefore reported every delivery as new, so a + // failed un-archive (best-effort by design) turned each of the plugin's + // retries into another appended row — all of them acked. + const harness = yield* makeHarness({ + homeThreadArchivedAt: CREATED_AT, + existingDeliveries: [{ threadId: HOME_THREAD_ID, deliveryId: "media-delivery-1" }], + }); + yield* harness.deliverMedia(registration, mediaFrame(), harness.transport); + + assert.equal(dispatchedDeliveries(yield* Ref.get(harness.dispatched)).length, 0); + assert.equal(harness.sent[0]?.type, "media.deliver.ack"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("routes turn-scoped media into the tracked thread and carries the turnId", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ trackedThreadId: SESSION_THREAD_ID }); + const turnId = TurnId.make("hermes-turn-live"); + yield* harness.deliverMedia( + registration, + mediaFrame({ + threadId: SESSION_THREAD_ID, + turnId, + deliveryId: HermesGatewayDeliveryId.make("media-delivery-turn"), + mimeType: "video/mp4", + name: "clip.mp4", + }), + harness.transport, + ); + + const delivery = dispatchedDeliveries(yield* Ref.get(harness.dispatched))[0]!; + assert.equal(delivery.threadId, SESSION_THREAD_ID); + assert.equal(delivery.turnId, turnId); + // Non-image media takes the generic file variant. + assert.equal(delivery.attachments?.[0]?.type, "file"); + assert.equal(harness.sent[0]?.type, "media.deliver.ack"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("refuses turn-scoped media for a thread the adapter does not track", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ trackedThreadId: SESSION_THREAD_ID }); + yield* harness.deliverMedia( + registration, + mediaFrame({ + threadId: ThreadId.make("thread-not-ours"), + turnId: TurnId.make("turn-x"), + }), + harness.transport, + ); + + // Refused outright: nothing written, and no ack so the plugin retries + // (against a session that may exist by then). + assert.equal(dispatchedDeliveries(yield* Ref.get(harness.dispatched)).length, 0); + assert.equal(harness.sent.length, 0); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("refuses an oversized decoded payload without writing or acking", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + const oversized = Buffer.alloc(HERMES_MEDIA_MAX_BYTES + 4, 1); + yield* harness.deliverMedia( + registration, + mediaFrame({ + data: oversized.toString("base64"), + sizeBytes: oversized.byteLength, + }), + harness.transport, + ); + + assert.equal(dispatchedDeliveries(yield* Ref.get(harness.dispatched)).length, 0); + assert.equal(harness.sent.length, 0); + assert.deepEqual( + yield* (yield* FileSystem.FileSystem).readDirectory(harness.attachmentsDir), + [], + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("refuses a payload whose declared size grossly disagrees with its bytes", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.deliverMedia(registration, mediaFrame({ sizeBytes: 5_000 }), harness.transport); + + assert.equal(dispatchedDeliveries(yield* Ref.get(harness.dispatched)).length, 0); + assert.equal(harness.sent.length, 0); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("does not ack media when the dispatch fails", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ failDispatch: true }); + yield* harness.deliverMedia(registration, mediaFrame(), harness.transport); + + // The plugin keeps the delivery queued and retries on the next connect. + assert.equal(harness.sent.length, 0); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/hermesGatewayHttp.ts b/apps/server/src/provider/hermesGatewayHttp.ts new file mode 100644 index 00000000000..70f7399120d --- /dev/null +++ b/apps/server/src/provider/hermesGatewayHttp.ts @@ -0,0 +1,463 @@ +import { + CommandId, + HERMES_GATEWAY_PROTOCOL_VERSION, + HERMES_MEDIA_MAX_BYTES, + HermesGatewayConnectionHello, + HermesGatewayPluginToT3Message, + HermesGatewayT3ToPluginMessage, + MessageId, + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, + type ChatAttachment, + type ThreadId, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; +import * as Socket from "effect/unstable/socket/Socket"; + +import { createAttachmentId, resolveAttachmentPath } from "../attachmentStore.ts"; +import { ServerConfig } from "../config.ts"; +import { getOrCreateHomeThread } from "../orchestration/homeThreads.ts"; +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { + HermesGatewayBroker, + type HermesGatewayConnectionRegistration, +} from "./Services/HermesGatewayBroker.ts"; +import { ProviderInstanceRegistry } from "./Services/ProviderInstanceRegistry.ts"; +import { ProviderAdapterRequestError } from "./Errors.ts"; + +export const HERMES_GATEWAY_WEBSOCKET_PATH = "/api/hermes-gateway/ws"; + +const decodePluginFrame = Schema.decodeUnknownEffect( + Schema.fromJsonString(HermesGatewayPluginToT3Message), +); +const encodeServerFrame = Schema.encodeSync(Schema.fromJsonString(HermesGatewayT3ToPluginMessage)); +const isConnectionHello = Schema.is(HermesGatewayConnectionHello); + +interface HermesDeliveryTransport { + readonly send: ( + frame: HermesGatewayT3ToPluginMessage, + ) => Effect.Effect; +} + +/** + * Build the durable-write-then-ack handlers for plugin-initiated deliveries. + * + * A factory rather than route-inlined closures so the delivery contract — + * dedupe, pessimistic ack, thread resolution — is testable without standing + * up a WebSocket route around it. + */ +export const makeHermesDeliveryHandlers = Effect.fn("makeHermesDeliveryHandlers")(function* () { + const engine = yield* OrchestrationEngineService; + const query = yield* ProjectionSnapshotQuery; + const registry = yield* ProviderInstanceRegistry; + const crypto = yield* Crypto.Crypto; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + + // Dedupe against the projection, before dispatching. + // + // The decider also dedupes, but it can only see the in-memory command + // read model — which is built with `messages: []` on every thread + // (`getCommandReadModel`, deliberately, to avoid hydrating message + // bodies at boot). Its scan therefore matches nothing, and a replay + // after a server restart appends a second copy. This read hits + // `projection_thread_messages`, where the delivery actually lives, so + // it is the check that holds across a restart — exactly the case the + // plugin's durable queue exists to produce. + // + // Deliberately NOT `getThreadDetailById`: that read filters `archived_at + // IS NULL`, so an archived Home reported every delivery as new. Combined + // with a failed un-archive (best-effort by design in `getOrCreateHomeThread`) + // that turned the plugin's retry loop into an append loop — one row per + // reconnect, all acked. This is the only durable dedupe after a restart, so + // it has to be archive-independent. + const alreadyDelivered = (threadId: ThreadId, deliveryId: string) => + query + .hasThreadNotificationDelivery({ threadId, deliveryId }) + .pipe(Effect.orElseSucceed(() => false)); + + /** + * Write one proactive delivery into the instance's home thread and ack it. + * + * The ack is sent **only after the dispatch succeeds**. The plugin purges + * its queued copy on the ack and on nothing else, so acking optimistically + * would silently drop deliveries whenever a write failed. An unacked + * delivery is retried on the next connect and deduped there by + * `deliveryId`, which makes the pessimistic order the safe one. + * + * Accepted from either connection role, and deliberately not generation- + * fenced: a delivery carries no session and cannot conflict with the live + * connection's turns, and fencing it would reject exactly the out-of- + * process cron case this exists to serve. + */ + const deliverHomeNotification = ( + registration: HermesGatewayConnectionRegistration, + message: Extract, + transport: HermesDeliveryTransport, + ) => + Effect.gen(function* () { + // Re-resolve rather than trusting the plugin's threadId: its cached + // T3_HOME_CHANNEL may name a thread from a previous designation, and a + // delivery must never land in an arbitrary caller-named thread. + const homeThreadId = yield* getOrCreateHomeThread({ + instanceId: registration.instanceId, + title: registration.accepted.nickname, + }); + + if (!(yield* alreadyDelivered(homeThreadId, message.deliveryId))) { + const dispatched = yield* Effect.result( + engine.dispatch({ + type: "thread.notification.deliver", + commandId: CommandId.make(yield* crypto.randomUUIDv4), + threadId: homeThreadId, + messageId: MessageId.make(yield* crypto.randomUUIDv4), + deliveryId: message.deliveryId, + kind: message.kind, + label: message.label, + text: message.text, + createdAt: message.createdAt, + }), + ); + + // A dispatch failure is not proof the delivery is absent — a racing + // socket may have won between the read above and this write. Re-read + // before giving up, or that delivery goes unacked forever and the + // plugin replays it on every single reconnect. + if ( + dispatched._tag === "Failure" && + !(yield* alreadyDelivered(homeThreadId, message.deliveryId)) + ) { + return yield* Effect.fail(dispatched.failure); + } + } + + yield* transport.send({ + type: "home.deliver.ack", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + deliveryId: message.deliveryId, + }); + }).pipe( + Effect.catchCause((cause) => + // Deliberately no ack: the plugin keeps the delivery queued and + // retries it on the next connect. + Effect.logWarning("Hermes home delivery failed", { + instanceId: registration.instanceId, + deliveryId: message.deliveryId, + cause: Cause.pretty(cause), + }), + ), + ); + + /** + * Write one media delivery — bytes to the attachment store, then a + * notification-shaped message row — and ack it. + * + * The same pessimistic contract as `deliverHomeNotification`: the ack is + * sent only after the dispatch succeeds, an unacked delivery is retried + * and deduped on `deliveryId`, and both connection roles are accepted. + * + * Thread resolution is scope-dependent: + * - `turnId` present — media produced during a live turn. The frame's + * threadId is honored only if this instance's adapter actually tracks a + * session for it; anything else is refused, so a confused plugin cannot + * spray files into arbitrary threads. + * - turnless — proactive media. The threadId is advisory exactly as it is + * for `home.deliver`: the server re-resolves the instance's home thread + * and writes only there. + */ + const deliverMedia = ( + registration: HermesGatewayConnectionRegistration, + message: Extract, + transport: HermesDeliveryTransport, + ) => + Effect.gen(function* () { + const bytes = Buffer.from(message.data, "base64"); + // The schema bounds the encoded string; re-check the decoded bytes so + // a frame whose base64 hides more than the ceiling (or whose declared + // size lies) is refused before anything is written. `sizeBytes` only + // needs to be honest, not exact — base64 length is ambiguous by up to + // 2 bytes of padding. + if (bytes.byteLength === 0 || bytes.byteLength > HERMES_MEDIA_MAX_BYTES) { + return yield* Effect.fail( + new ProviderAdapterRequestError({ + provider: "hermes", + method: "media.deliver", + detail: `Media payload is empty or exceeds ${HERMES_MEDIA_MAX_BYTES} bytes after decoding.`, + }), + ); + } + if (Math.abs(bytes.byteLength - message.sizeBytes) > 2) { + return yield* Effect.fail( + new ProviderAdapterRequestError({ + provider: "hermes", + method: "media.deliver", + detail: `Media payload decoded to ${bytes.byteLength} bytes but declared ${message.sizeBytes}.`, + }), + ); + } + + const threadId = yield* Effect.gen(function* () { + if (message.turnId === undefined) { + // Proactive: re-resolve, never trust the frame (see doc above). + return yield* getOrCreateHomeThread({ + instanceId: registration.instanceId, + title: registration.accepted.nickname, + }); + } + // Turn-scoped: the named thread must be one this instance's adapter + // holds a live session for. + const instance = yield* registry.getInstance(registration.instanceId); + const tracked = + instance !== undefined && (yield* instance.adapter.hasSession(message.threadId)); + if (!tracked) { + return yield* Effect.fail( + new ProviderAdapterRequestError({ + provider: "hermes", + method: "media.deliver", + detail: `Turn-scoped media names thread '${message.threadId}', which this instance has no session for.`, + }), + ); + } + return message.threadId; + }); + + if (!(yield* alreadyDelivered(threadId, message.deliveryId))) { + const mimeType = message.mimeType.trim().toLowerCase(); + const attachmentId = createAttachmentId(threadId); + if (!attachmentId) { + return yield* Effect.fail( + new ProviderAdapterRequestError({ + provider: "hermes", + method: "media.deliver", + detail: "Failed to create a safe attachment id.", + }), + ); + } + // Images take the image variant so they ride the existing inline + // grid; the image schema caps sizeBytes at 10MB, so a larger image + // degrades to the generic file card rather than being refused. + const attachment: ChatAttachment = + mimeType.startsWith("image/") && bytes.byteLength <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + ? { + type: "image", + id: attachmentId, + name: message.name, + mimeType, + sizeBytes: bytes.byteLength, + } + : { + type: "file", + id: attachmentId, + name: message.name, + mimeType, + sizeBytes: bytes.byteLength, + }; + + const attachmentPath = resolveAttachmentPath({ + attachmentsDir: serverConfig.attachmentsDir, + attachment, + }); + if (!attachmentPath) { + return yield* Effect.fail( + new ProviderAdapterRequestError({ + provider: "hermes", + method: "media.deliver", + detail: `Failed to resolve a persisted path for '${message.name}'.`, + }), + ); + } + // Bytes before row, deliberately: a row pointing at a missing file + // renders broken forever, while an orphaned file from a failed + // dispatch is retried under a fresh id and merely leaks bytes. + yield* fileSystem.makeDirectory(path.dirname(attachmentPath), { recursive: true }); + yield* fileSystem.writeFile(attachmentPath, bytes); + + const dispatched = yield* Effect.result( + engine.dispatch({ + type: "thread.notification.deliver", + commandId: CommandId.make(yield* crypto.randomUUIDv4), + threadId, + messageId: MessageId.make(yield* crypto.randomUUIDv4), + deliveryId: message.deliveryId, + kind: message.kind, + label: message.label, + // The caption is the row's text; empty is fine — the schema + // allows it and the web renders media-only rows without a body. + text: message.caption ?? "", + attachments: [attachment], + ...(message.turnId !== undefined ? { turnId: message.turnId } : {}), + createdAt: message.createdAt, + }), + ); + + // Same racing-socket rule as home deliveries: re-read before giving + // up, or the delivery goes unacked forever. + if ( + dispatched._tag === "Failure" && + !(yield* alreadyDelivered(threadId, message.deliveryId)) + ) { + return yield* Effect.fail(dispatched.failure); + } + } + + yield* transport.send({ + type: "media.deliver.ack", + protocolVersion: HERMES_GATEWAY_PROTOCOL_VERSION, + deliveryId: message.deliveryId, + }); + }).pipe( + Effect.catchCause((cause) => + // Deliberately no ack: the plugin keeps the delivery queued and + // retries it on the next connect. + Effect.logWarning("Hermes media delivery failed", { + instanceId: registration.instanceId, + deliveryId: message.deliveryId, + cause: Cause.pretty(cause), + }), + ), + ); + + return { deliverHomeNotification, deliverMedia } as const; +}); + +export const hermesGatewayWebSocketRouteLayer = Layer.unwrap( + Effect.gen(function* () { + const broker = yield* HermesGatewayBroker; + const { deliverHomeNotification, deliverMedia } = yield* makeHermesDeliveryHandlers(); + + return HttpRouter.add( + "GET", + HERMES_GATEWAY_WEBSOCKET_PATH, + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const socket = yield* Effect.orDie(request.upgrade); + const write = yield* socket.writer; + const registration = yield* Ref.make>( + Option.none(), + ); + const transport = { + send: (message: HermesGatewayT3ToPluginMessage) => + write(encodeServerFrame(message)).pipe( + Effect.mapError( + (cause) => + new ProviderAdapterRequestError({ + provider: "hermes", + method: message.type, + detail: "Failed to write a Hermes gateway WebSocket frame.", + cause, + }), + ), + ), + close: (code: number, reason: string) => + write(new Socket.CloseEvent(code, reason)).pipe(Effect.ignore), + }; + + yield* socket + .runString((frame) => + Effect.gen(function* () { + const message = yield* decodePluginFrame(frame); + const current = yield* Ref.get(registration); + + if (Option.isNone(current)) { + if (!isConnectionHello(message)) { + yield* transport.close(4002, "First message must be connection.hello"); + return; + } + const registered = yield* broker + .registerConnection(message, transport) + .pipe( + Effect.tapError((rejected) => + transport + .send(rejected) + .pipe(Effect.andThen(transport.close(4003, rejected.message))), + ), + ); + yield* Ref.set(registration, Option.some(registered)); + + // Resolve the home thread here rather than inside + // `registerConnection`, whose error channel is exactly + // `connection.rejected`: a thread-creation failure is not a + // reason to refuse an authenticated plugin. On failure the + // handshake completes without `homeThreadId`, the plugin keeps + // whatever designation it already had, and the next connect + // converges — which is the whole point of converge-on-read. + const homeThreadId = yield* getOrCreateHomeThread({ + instanceId: registered.instanceId, + title: registered.accepted.nickname, + }).pipe( + Effect.map(Option.some), + Effect.catchCause((cause) => + Effect.logWarning("home thread resolution failed", { + instanceId: registered.instanceId, + cause: Cause.pretty(cause), + }).pipe(Effect.as(Option.none())), + ), + ); + + yield* transport.send({ + ...registered.accepted, + ...(Option.isSome(homeThreadId) ? { homeThreadId: homeThreadId.value } : {}), + }); + return; + } + + if (isConnectionHello(message)) { + yield* transport.close(4002, "connection.hello may only be sent once"); + return; + } + + // Deliveries are handled on the socket that carried them rather + // than through the broker's event stream, because the ack has to + // go back to *this* connection — which for an out-of-process cron + // run is a short-lived delivery socket that no stream subscriber + // can address. + if (message.type === "home.deliver") { + yield* deliverHomeNotification(current.value, message, transport); + return; + } + + if (message.type === "media.deliver") { + yield* deliverMedia(current.value, message, transport); + return; + } + + yield* broker.receive(current.value, message); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Rejected Hermes gateway WebSocket frame", { cause }).pipe( + Effect.andThen(transport.close(4002, "Invalid Hermes gateway message")), + ), + ), + ), + ) + .pipe( + Effect.catch((cause) => + Effect.logDebug("Hermes gateway WebSocket disconnected", { cause }), + ), + Effect.ensuring( + Ref.get(registration).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.void, + onSome: broker.disconnect, + }), + ), + ), + ), + ); + + return HttpServerResponse.empty(); + }), + ); + }), +); diff --git a/apps/server/src/provider/providerDescribe.test.ts b/apps/server/src/provider/providerDescribe.test.ts new file mode 100644 index 00000000000..426a4a1412b --- /dev/null +++ b/apps/server/src/provider/providerDescribe.test.ts @@ -0,0 +1,227 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + type ServerProvider, + type ServerProviderModel, +} from "@t3tools/contracts"; + +import { + describeFromServerProvider, + resolveConnectionStatus, + resolveCurrentSnapshotModel, + resolveReasoningEffortLabel, +} from "./providerDescribe.ts"; + +const DESCRIBED_AT = "2024-05-01T12:00:00.000Z"; + +const makeProvider = (overrides: Partial = {}): ServerProvider => ({ + instanceId: ProviderInstanceId.make("codex"), + driver: ProviderDriverKind.make("codex"), + enabled: true, + installed: true, + version: "1.2.3", + status: "ready", + auth: { status: "authenticated" }, + checkedAt: "2024-05-01T11:59:00.000Z", + models: [], + slashCommands: [], + skills: [], + ...overrides, +}); + +const makeModel = (overrides: Partial = {}): ServerProviderModel => ({ + slug: "gpt-5.6-sol", + name: "GPT-5.6 Sol", + isCustom: false, + capabilities: null, + ...overrides, +}); + +describe("resolveCurrentSnapshotModel", () => { + it("prefers the model flagged isDefault over list order", () => { + const model = resolveCurrentSnapshotModel([ + makeModel({ slug: "first" }), + makeModel({ slug: "flagged", isDefault: true }), + ]); + assert.strictEqual(model?.slug, "flagged"); + }); + + it("falls back to the first model when nothing is flagged", () => { + const model = resolveCurrentSnapshotModel([ + makeModel({ slug: "first" }), + makeModel({ slug: "second" }), + ]); + assert.strictEqual(model?.slug, "first"); + }); + + it("returns undefined rather than fabricating a model", () => { + assert.strictEqual(resolveCurrentSnapshotModel([]), undefined); + }); +}); + +describe("resolveReasoningEffortLabel", () => { + it("resolves the label for a driver's own effort option id", () => { + const label = resolveReasoningEffortLabel( + makeModel({ + capabilities: { + optionDescriptors: [ + { + id: "reasoningEffort", + label: "Reasoning effort", + type: "select", + currentValue: "high", + options: [ + { id: "low", label: "Low" }, + { id: "high", label: "High" }, + ], + }, + ], + }, + }), + ); + assert.strictEqual(label, "High"); + }); + + it("resolves Cursor's differently-named option id", () => { + const label = resolveReasoningEffortLabel( + makeModel({ + capabilities: { + optionDescriptors: [ + { + id: "reasoning", + label: "Reasoning", + type: "select", + options: [{ id: "balanced", label: "Balanced", isDefault: true }], + }, + ], + }, + }), + ); + // No currentValue — falls through to the descriptor's own default. + assert.strictEqual(label, "Balanced"); + }); + + it("matches a normalized id variant", () => { + const label = resolveReasoningEffortLabel( + makeModel({ + capabilities: { + optionDescriptors: [ + { + id: "reasoning_effort", + label: "Effort", + type: "select", + currentValue: "max", + options: [{ id: "max", label: "Max" }], + }, + ], + }, + }), + ); + assert.strictEqual(label, "Max"); + }); + + it("returns null rather than guessing at an unrelated option", () => { + const label = resolveReasoningEffortLabel( + makeModel({ + capabilities: { + optionDescriptors: [ + { + id: "webSearch", + label: "Web search", + type: "boolean", + currentValue: true, + }, + ], + }, + }), + ); + assert.strictEqual(label, null); + }); + + it("returns null when the model reports no descriptors at all", () => { + assert.strictEqual(resolveReasoningEffortLabel(makeModel()), null); + assert.strictEqual(resolveReasoningEffortLabel(undefined), null); + }); +}); + +describe("resolveConnectionStatus", () => { + it("reports disabled for a disabled instance regardless of probe result", () => { + assert.strictEqual( + resolveConnectionStatus(makeProvider({ enabled: false, status: "ready" })), + "disabled", + ); + }); + + it("does not claim ready for a warning snapshot", () => { + assert.strictEqual(resolveConnectionStatus(makeProvider({ status: "warning" })), "offline"); + }); + + it("maps ready and error through directly", () => { + assert.strictEqual(resolveConnectionStatus(makeProvider({ status: "ready" })), "ready"); + assert.strictEqual(resolveConnectionStatus(makeProvider({ status: "error" })), "error"); + }); +}); + +describe("describeFromServerProvider", () => { + it("projects identity, connection, model, and skills from the snapshot", () => { + const description = describeFromServerProvider({ + provider: makeProvider({ + displayName: "Codex Work", + models: [ + makeModel({ + isDefault: true, + subProvider: "OpenAI", + capabilities: { + optionDescriptors: [ + { + id: "reasoningEffort", + label: "Reasoning effort", + type: "select", + currentValue: "medium", + options: [{ id: "medium", label: "Medium" }], + }, + ], + }, + }), + ], + skills: [{ name: "deploy", path: "/skills/deploy", enabled: true }], + }), + describedAt: DESCRIBED_AT, + }); + + assert.strictEqual(description.identity.displayName, "Codex Work"); + assert.strictEqual(description.identity.agentVersion, "1.2.3"); + assert.strictEqual(description.connection.status, "ready"); + assert.strictEqual(description.model?.id, "gpt-5.6-sol"); + assert.strictEqual(description.model?.vendor, "OpenAI"); + assert.strictEqual(description.model?.reasoningEffortLabel, "Medium"); + assert.strictEqual(description.skills.length, 1); + assert.strictEqual(description.describedAt, DESCRIBED_AT); + }); + + it("falls back to the instance id when the snapshot has no display name", () => { + const description = describeFromServerProvider({ + provider: makeProvider(), + describedAt: DESCRIBED_AT, + }); + assert.strictEqual(description.identity.displayName, "codex"); + }); + + it("reports a null model block when the driver lists no models", () => { + const description = describeFromServerProvider({ + provider: makeProvider({ models: [] }), + describedAt: DESCRIBED_AT, + }); + assert.strictEqual(description.model, null); + }); + + it("does not report a last-connected time for a provider that is not ready", () => { + const description = describeFromServerProvider({ + provider: makeProvider({ status: "error", message: "codex CLI not found" }), + describedAt: DESCRIBED_AT, + }); + assert.strictEqual(description.connection.lastConnectedAt, null); + assert.strictEqual(description.connection.detail, "codex CLI not found"); + }); +}); diff --git a/apps/server/src/provider/providerDescribe.ts b/apps/server/src/provider/providerDescribe.ts new file mode 100644 index 00000000000..042757bfab8 --- /dev/null +++ b/apps/server/src/provider/providerDescribe.ts @@ -0,0 +1,166 @@ +/** + * Provider-generic instance description. + * + * Backs the Agent page (`/agents/$instanceId`). The default projection here + * derives everything from the `ServerProvider` snapshot every driver already + * publishes, so Claude, Codex, Cursor, Grok, and OpenCode get a populated page + * without any driver work. Drivers that can answer more precisely — Hermes, + * which can ask its connected plugin — layer their own data on top via + * `ProviderAdapterShape.describe`. + * + * Kept as pure functions over snapshots (no Effect, no services) so the + * projection is directly testable and so `ws.ts` stays a thin transport. + * + * @module provider/providerDescribe + */ +import type { + ProviderInstanceConnection, + ProviderInstanceDescription, + ProviderInstanceModelInfo, + ServerProvider, + ServerProviderModel, +} from "@t3tools/contracts"; +import { getProviderOptionCurrentLabel } from "@t3tools/shared/model"; + +/** + * Option-descriptor ids that mean "reasoning effort" across drivers. + * + * Effort is not first-class anywhere in the contracts: each driver declares it + * as a `ProviderOptionDescriptor` under its own id (`reasoningEffort` on Codex, + * `reasoning` on Cursor, `effort` on some forks). Matching on a small known set + * — plus a normalized-substring fallback — is what lets the Agent page show one + * "Reasoning effort" row without teaching the client each driver's vocabulary. + * + * A driver whose descriptor matches nothing here reports `null`, and the page + * renders "not reported" rather than guessing at an unrelated option. + */ +const REASONING_EFFORT_OPTION_IDS: ReadonlyArray = [ + "reasoningEffort", + "reasoning_effort", + "reasoning", + "effort", + "thinking", + "thinkingLevel", +]; + +const normalizeOptionId = (id: string): string => id.toLowerCase().replaceAll(/[^a-z]/g, ""); + +const NORMALIZED_REASONING_EFFORT_IDS = new Set(REASONING_EFFORT_OPTION_IDS.map(normalizeOptionId)); + +/** + * Pick the model the snapshot considers current. + * + * `isDefault` is the only "which one is live" marker on the snapshot; when no + * model carries it we fall back to the first, which is the order drivers + * publish their preferred model in. Returns `undefined` for an empty list + * rather than fabricating a model row. + */ +export function resolveCurrentSnapshotModel( + models: ReadonlyArray, +): ServerProviderModel | undefined { + return models.find((model) => model.isDefault === true) ?? models[0]; +} + +/** + * Resolve reasoning effort to a human label, or `null` when this driver does + * not expose one. Never returns a raw option id — see + * `ProviderInstanceModelInfo.reasoningEffortLabel`. + */ +export function resolveReasoningEffortLabel(model: ServerProviderModel | undefined): string | null { + const descriptors = model?.capabilities?.optionDescriptors; + if (!descriptors || descriptors.length === 0) { + return null; + } + const descriptor = + descriptors.find((candidate) => REASONING_EFFORT_OPTION_IDS.includes(candidate.id)) ?? + descriptors.find((candidate) => + NORMALIZED_REASONING_EFFORT_IDS.has(normalizeOptionId(candidate.id)), + ); + if (!descriptor) { + return null; + } + return getProviderOptionCurrentLabel(descriptor) ?? null; +} + +/** + * Map the snapshot's `status` onto the description's connection vocabulary. + * + * The two literal sets are deliberately different: `ServerProviderState` is + * about whether a provider is *usable* (`warning` covers "works but something + * is off"), while the Agent page asks whether the agent is *reachable*. A + * disabled instance reports `disabled` regardless of its probe result, because + * "we are not talking to this agent" is the honest answer. + */ +export function resolveConnectionStatus( + provider: ServerProvider, +): ProviderInstanceConnection["status"] { + if (!provider.enabled) { + return "disabled"; + } + switch (provider.status) { + case "ready": + return "ready"; + case "error": + return "error"; + case "warning": + // A reachable provider reporting a non-fatal problem. `detail` carries + // the specifics; the badge stays honest by not claiming "ready". + return "offline"; + case "disabled": + return "disabled"; + } +} + +function buildModelInfo(provider: ServerProvider): ProviderInstanceModelInfo | null { + const model = resolveCurrentSnapshotModel(provider.models); + const reasoningEffortLabel = resolveReasoningEffortLabel(model); + if (!model && reasoningEffortLabel === null) { + return null; + } + return { + id: model?.slug ?? null, + displayName: model?.name ?? null, + vendor: model?.subProvider ?? null, + // Not carried on `ServerProviderModel` today. Kept in the contract so the + // shape is stable when a driver starts reporting it. + contextWindow: null, + reasoningEffortLabel, + }; +} + +/** + * Default description, derived entirely from a `ServerProvider` snapshot. + * + * `describedAt` is passed in rather than read from the clock so this stays a + * pure function — the caller stamps it from the Effect clock. + */ +export function describeFromServerProvider(input: { + readonly provider: ServerProvider; + readonly describedAt: string; +}): ProviderInstanceDescription { + const { provider, describedAt } = input; + return { + identity: { + instanceId: provider.instanceId, + driver: provider.driver, + displayName: provider.displayName ?? provider.instanceId, + agentVersion: provider.version, + pluginVersion: null, + protocolVersion: null, + host: null, + }, + connection: { + status: resolveConnectionStatus(provider), + detail: provider.message ?? provider.unavailableReason ?? null, + // Local-process drivers have no connection timeline to report; the + // probe timestamp is the closest honest analogue. + lastConnectedAt: provider.status === "ready" ? provider.checkedAt : null, + activeSessionCount: null, + connectionGeneration: null, + }, + model: buildModelInfo(provider), + skills: provider.skills, + capabilities: null, + describedAt, + }; +} diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 641c9b52e56..10b330da3cd 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -5,7 +5,7 @@ import { type ServerProvider, type ServerProviderUpdateState, } from "@t3tools/contracts"; -import { ServerProviderUpdateError } from "@t3tools/contracts"; +import { ProviderDescribeError, ServerProviderUpdateError } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -189,6 +189,22 @@ function makeRegistry( getProviders: Ref.get(providersRef), refresh: () => Ref.get(providersRef), refreshInstance: () => Ref.get(providersRef), + describeInstance: (instanceId) => + Effect.fail( + new ProviderDescribeError({ + code: "unsupported", + message: "describeInstance is not exercised by maintenance-runner tests.", + instanceId, + }), + ), + getSkillBody: ({ instanceId }) => + Effect.fail( + new ProviderDescribeError({ + code: "unsupported", + message: "getSkillBody is not exercised by maintenance-runner tests.", + instanceId, + }), + ), getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => Effect.succeed(lifecycleFor(provider)), setProviderMaintenanceActionState, diff --git a/apps/server/src/provider/providerSnapshot.ts b/apps/server/src/provider/providerSnapshot.ts index e741a7a2c1d..ef5c4d46b96 100644 --- a/apps/server/src/provider/providerSnapshot.ts +++ b/apps/server/src/provider/providerSnapshot.ts @@ -57,6 +57,7 @@ export interface ServerProviderPresentation { readonly badgeLabel?: string; readonly showInteractionModeToggle?: boolean; readonly requiresNewThreadForModelChange?: boolean; + readonly requiresWorkspace?: boolean; } export type ServerProviderDraft = Omit; @@ -233,6 +234,9 @@ export function buildServerProvider(input: { ...(typeof input.presentation.requiresNewThreadForModelChange === "boolean" ? { requiresNewThreadForModelChange: input.presentation.requiresNewThreadForModelChange } : {}), + ...(typeof input.presentation.requiresWorkspace === "boolean" + ? { requiresWorkspace: input.presentation.requiresWorkspace } + : {}), enabled: input.enabled, installed: input.probe.installed, version: input.probe.version, diff --git a/apps/server/src/provider/testUtils/providerRegistryMock.ts b/apps/server/src/provider/testUtils/providerRegistryMock.ts index 36598b05900..94879c26e14 100644 --- a/apps/server/src/provider/testUtils/providerRegistryMock.ts +++ b/apps/server/src/provider/testUtils/providerRegistryMock.ts @@ -1,9 +1,13 @@ import { ProviderRegistry, type ProviderRegistryShape } from "../Services/ProviderRegistry.ts"; -import type { ServerProvider } from "@t3tools/contracts"; +import { ProviderDescribeError, type ServerProvider } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; +import { describeFromServerProvider } from "../providerDescribe.ts"; + +/** Fixed so mock descriptions are deterministic across test runs. */ +const MOCK_DESCRIBED_AT = "2024-01-01T00:00:00.000Z"; export const makeProviderRegistryMock = ( providers: ReadonlyArray = [], @@ -11,6 +15,26 @@ export const makeProviderRegistryMock = ( getProviders: Effect.succeed(providers), refresh: () => Effect.succeed(providers), refreshInstance: () => Effect.succeed(providers), + describeInstance: (instanceId) => { + const provider = providers.find((candidate) => candidate.instanceId === instanceId); + return provider + ? Effect.succeed(describeFromServerProvider({ provider, describedAt: MOCK_DESCRIBED_AT })) + : Effect.fail( + new ProviderDescribeError({ + code: "instance-not-found", + message: `Provider instance '${instanceId}' is not configured.`, + instanceId, + }), + ); + }, + getSkillBody: ({ instanceId }) => + Effect.fail( + new ProviderDescribeError({ + code: "unsupported", + message: "This provider cannot read skill contents.", + instanceId, + }), + ), getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => Effect.succeed(makeManualOnlyProviderMaintenanceCapabilities({ provider, packageName: null })), setProviderMaintenanceActionState: () => Effect.succeed(providers), diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 74a4de594a1..31ccf944edc 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -201,6 +201,30 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { }, } as unknown as OrchestrationEvent), ).toBe(false); + // A cron result or agent-initiated message is exactly the news push + // exists for; a lifecycle notice appends quietly and must never ring. + for (const kind of ["cron", "message", "handoff", "other"] as const) { + expect( + AgentAwarenessRelay.shouldPublishAgentAwarenessEvent({ + ...base, + type: "thread.notification-delivered", + payload: { + threadId: "thread-1" as ThreadId, + notification: { kind, label: "Cron: daily-digest", deliveryId: "delivery-1" }, + }, + } as unknown as OrchestrationEvent), + ).toBe(true); + } + expect( + AgentAwarenessRelay.shouldPublishAgentAwarenessEvent({ + ...base, + type: "thread.notification-delivered", + payload: { + threadId: "thread-1" as ThreadId, + notification: { kind: "lifecycle", label: "Agent restarted", deliveryId: "delivery-2" }, + }, + } as unknown as OrchestrationEvent), + ).toBe(false); }); it("deduplicates awareness state updates whose only change is their event timestamp", () => { @@ -429,6 +453,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { repositoryIdentity: null, defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, } satisfies OrchestrationProjectShell; @@ -587,6 +612,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { repositoryIdentity: null, defaultModelSelection: null, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, } satisfies OrchestrationProjectShell; diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 58de98f1ca1..dde9ff156c1 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -78,6 +78,11 @@ export function shouldPublishAgentAwarenessEvent(event: OrchestrationEvent): boo case "thread.runtime-mode-set": case "thread.interaction-mode-set": return false; + case "thread.notification-delivered": + // A cron result or agent-initiated message is exactly the kind of news + // push exists for. A lifecycle notice ("agent restarted") is not: it + // appends quietly and must never ring a device. + return event.payload.notification.kind !== "lifecycle"; case "thread.activity-appended": return ( event.payload.activity.kind === "approval.requested" || diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index cb68c0bba8a..b83b410cfd7 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -84,6 +84,7 @@ import { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts"; import { PersistenceSqlError } from "./persistence/Errors.ts"; +import * as ProviderInstanceRegistry from "./provider/Services/ProviderInstanceRegistry.ts"; import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "./provider/providerMaintenance.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; @@ -149,6 +150,7 @@ const makeDefaultOrchestrationReadModel = () => { workspaceRoot: "/tmp/default-project", defaultModelSelection, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, deletedAt: null, @@ -566,6 +568,7 @@ const buildAppUnderTest = (options?: { ready: Effect.void, getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS), updateSettings: () => Effect.succeed(DEFAULT_SERVER_SETTINGS), + updateSettingsWith: () => Effect.succeed(DEFAULT_SERVER_SETTINGS), streamChanges: Stream.empty, ...options?.layers?.serverSettings, }), @@ -685,13 +688,20 @@ const buildAppUnderTest = (options?: { ), ), Layer.provide( - Layer.mock(OrchestrationEngine.OrchestrationEngineService)({ - readEvents: () => Stream.empty, - dispatch: () => Effect.succeed({ sequence: 0 }), - streamDomainEvents: Stream.empty, - latestSequence: Effect.succeed(0), - ...options?.layers?.orchestrationEngine, - }), + Layer.mergeAll( + Layer.mock(OrchestrationEngine.OrchestrationEngineService)({ + readEvents: () => Stream.empty, + dispatch: () => Effect.succeed({ sequence: 0 }), + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(0), + ...options?.layers?.orchestrationEngine, + }), + // The Hermes gateway route resolves adapters through the instance + // registry for turn-scoped media; no router test exercises that path. + Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ + getInstance: () => Effect.succeed(undefined), + }), + ), ), Layer.provide( Layer.mock(ProjectionSnapshotQuery.ProjectionSnapshotQuery)({ @@ -5600,6 +5610,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { workspaceRoot: "/tmp/project-a", defaultModelSelection, scripts: [], + agentInstanceId: null, createdAt: now, updatedAt: now, deletedAt: null, diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0b7c8ccd074..d1bed970019 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -23,6 +23,8 @@ import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import { ProviderSessionDirectoryLive } from "./provider/Layers/ProviderSessionDirectory.ts"; import * as ProviderSessionRuntime from "./persistence/ProviderSessionRuntime.ts"; import { ProviderAdapterRegistryLive } from "./provider/Layers/ProviderAdapterRegistry.ts"; +import { HermesGatewayBrokerLive } from "./provider/Layers/HermesGatewayBroker.ts"; +import { hermesGatewayWebSocketRouteLayer } from "./provider/hermesGatewayHttp.ts"; import * as ProviderEventLoggers from "./provider/Layers/ProviderEventLoggers.ts"; import { ProviderServiceLive } from "./provider/Layers/ProviderService.ts"; import { ProviderSessionReaperLive } from "./provider/Layers/ProviderSessionReaper.ts"; @@ -296,6 +298,10 @@ const ProviderRuntimeLayerLive = ProviderSessionReaperLive.pipe( Layer.provideMerge(OrchestrationLayerLive), ); +const HermesProviderRuntimeLive = ProviderInstanceRegistryHydrationLive.pipe( + Layer.provideMerge(HermesGatewayBrokerLive), +); + const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // Core Services Layer.provideMerge(CheckpointingLayerLive), @@ -312,7 +318,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // through this layer. Built-in drivers come from `BUILT_IN_DRIVERS`; // `providerInstances` hydration merges `settings.providers.` // with explicit `providerInstances` entries on boot. - Layer.provideMerge(ProviderInstanceRegistryHydrationLive), + Layer.provideMerge(HermesProviderRuntimeLive), // Shared native/canonical NDJSON writers used by both the per-instance // drivers (native stream, written from inside each `Adapter`) and // `ProviderService` (canonical stream, written after event normalization). @@ -371,6 +377,7 @@ export const makeRoutesLayer = Layer.mergeAll( assetRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, + hermesGatewayWebSocketRouteLayer, ), McpHttpServer.layer.pipe(Layer.provide(McpSessionRegistry.layer)), ).pipe( diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index b8102bda9ad..320f6d4a3c2 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -90,11 +90,14 @@ it.effect("launchStartupHeartbeat does not block the caller while counts are loa }), ), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.succeed(Option.none()), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.none()), getFullThreadDiffContext: () => Effect.succeed(Option.none()), getThreadShellById: () => Effect.succeed(Option.none()), + getThreadArchiveStateById: () => Effect.succeed(Option.none()), + hasThreadNotificationDelivery: () => Effect.succeed(false), getThreadDetailById: () => Effect.succeed(Option.none()), getThreadDetailSnapshot: () => Effect.succeed(Option.none()), }), @@ -148,16 +151,20 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threa workspaceRoot: "/tmp/startup-project", defaultModelSelection: ServerRuntimeStartup.getAutoBootstrapDefaultModelSelection(), scripts: [], + agentInstanceId: null, createdAt: "2026-01-01T00:00:00.000Z", updatedAt: "2026-01-01T00:00:00.000Z", deletedAt: null, }), ), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.die("unused"), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.some(bootstrapThreadId)), getThreadCheckpointContext: () => Effect.succeed(Option.none()), getFullThreadDiffContext: () => Effect.succeed(Option.none()), getThreadShellById: () => Effect.die("unused"), + getThreadArchiveStateById: () => Effect.die("unused"), + hasThreadNotificationDelivery: () => Effect.die("unused"), getThreadDetailById: () => Effect.die("unused"), getThreadDetailSnapshot: () => Effect.die("unused"), }), @@ -197,11 +204,14 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread when getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.die("unused"), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.none()), getFullThreadDiffContext: () => Effect.succeed(Option.none()), getThreadShellById: () => Effect.die("unused"), + getThreadArchiveStateById: () => Effect.die("unused"), + hasThreadNotificationDelivery: () => Effect.die("unused"), getThreadDetailById: () => Effect.die("unused"), getThreadDetailSnapshot: () => Effect.die("unused"), }), @@ -247,11 +257,14 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation fa getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getActiveAgentProject: () => Effect.succeed(Option.none()), getProjectShellById: () => Effect.die("unused"), getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), getThreadCheckpointContext: () => Effect.succeed(Option.none()), getFullThreadDiffContext: () => Effect.succeed(Option.none()), getThreadShellById: () => Effect.die("unused"), + getThreadArchiveStateById: () => Effect.die("unused"), + hasThreadNotificationDelivery: () => Effect.die("unused"), getThreadDetailById: () => Effect.die("unused"), getThreadDetailSnapshot: () => Effect.die("unused"), }), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 50ca810a95a..d7dbfa41572 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -90,6 +90,55 @@ it.layer(NodeServices.layer)("server settings", (it) => { }).pipe(Effect.provide(settingsLayer)); }); + it.effect("does not report post-commit materialization failure as an update failure", () => { + const platformCause = PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFile", + pathOrDescriptor: "provider environment secret", + description: "Secret backend unavailable after settings commit.", + }); + const cause = new ServerSecretStore.SecretStoreReadError({ + resource: "provider environment secret", + cause: platformCause, + }); + const configLayer = Layer.fresh( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3code-server-settings-post-commit-failure-test-", + }), + ); + const settingsLayer = ServerSettingsModule.layer.pipe( + Layer.provide(makeFailingSecretStoreLayer(cause)), + Layer.provideMerge(configLayer), + ); + + return Effect.gen(function* () { + const serverConfig = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const serverSettings = yield* ServerSettingsModule.ServerSettingsService; + const hermesId = ProviderInstanceId.make("hermes_remote"); + yield* fileSystem.writeFileString( + serverConfig.settingsPath, + '{"providerInstances":{"hermes_remote":{"driver":"hermes","displayName":"Remote Hermes","config":{}},"codex_personal":{"driver":"codex","environment":[{"name":"OPENROUTER_API_KEY","value":"","sensitive":true,"valueRedacted":true}],"config":{}}}}', + ); + + const next = yield* serverSettings.updateSettingsWith((current) => { + const providerInstances = { ...current.providerInstances }; + delete providerInstances[hermesId]; + return { providerInstances }; + }); + + assert.isUndefined(next.providerInstances[hermesId]); + assert.equal( + next.providerInstances[ProviderInstanceId.make("codex_personal")]?.environment?.[0]?.value, + "", + ); + const persisted = yield* fileSystem.readFileString(serverConfig.settingsPath); + assert.notInclude(persisted, "hermes_remote"); + assert.include(persisted, "codex_personal"); + }).pipe(Effect.provide(settingsLayer)); + }); + it.effect("decodes nested settings patches", () => Effect.gen(function* () { assert.deepEqual( diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 82fd2f29f03..a1e1b75b56e 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -129,6 +129,15 @@ export class ServerSettingsService extends Context.Service< patch: ServerSettingsPatch, ) => Effect.Effect; + /** + * Compute and persist a patch while holding the settings write lock. + * Use this for read-modify-write operations on whole-map fields such as + * `providerInstances`, so concurrent settings edits are preserved. + */ + readonly updateSettingsWith: ( + update: (current: ServerSettings) => ServerSettingsPatch, + ) => Effect.Effect; + /** Stream of settings change events. */ readonly streamChanges: Stream.Stream; } @@ -148,18 +157,25 @@ const makeTest = (overrides: DeepPartial = {}) => : {}), }); const currentSettingsRef = yield* Ref.make(initialSettings); - - return { - start: Effect.void, - ready: Effect.void, - getSettings: Ref.get(currentSettingsRef).pipe(Effect.map(resolveTextGenerationProvider)), - updateSettings: (patch) => + const writeSemaphore = yield* Semaphore.make(1); + const updateSettingsWith = (update: (current: ServerSettings) => ServerSettingsPatch) => + writeSemaphore.withPermits(1)( Ref.get(currentSettingsRef).pipe( - Effect.map((currentSettings) => applyServerSettingsPatch(currentSettings, patch)), + Effect.map((currentSettings) => + applyServerSettingsPatch(currentSettings, update(currentSettings)), + ), Effect.flatMap(normalizeServerSettings), Effect.tap((nextSettings) => Ref.set(currentSettingsRef, nextSettings)), Effect.map(resolveTextGenerationProvider), ), + ); + + return { + start: Effect.void, + ready: Effect.void, + getSettings: Ref.get(currentSettingsRef).pipe(Effect.map(resolveTextGenerationProvider)), + updateSettings: (patch) => updateSettingsWith(() => patch), + updateSettingsWith, streamChanges: Stream.empty, } satisfies ServerSettingsService["Service"]; }); @@ -537,6 +553,35 @@ const make = Effect.gen(function* () { yield* Deferred.succeed(startedDeferred, undefined).pipe(Effect.orDie); }); + const updateSettingsWith = (update: (current: ServerSettings) => ServerSettingsPatch) => + writeSemaphore.withPermits(1)( + Effect.gen(function* () { + const current = yield* getSettingsFromCache; + const nextPersisted = yield* persistProviderEnvironmentSecrets( + current, + applyServerSettingsPatch(current, update(current)), + ); + const next = yield* normalizeServerSettings(nextPersisted); + yield* writeSettingsAtomically(next); + yield* Cache.set(settingsCache, cacheKey, next); + yield* emitChange(next); + const materialized = yield* materializeProviderEnvironmentSecrets(next).pipe( + Effect.catch((error: ServerSettingsError) => + Effect.logWarning( + "settings update committed but provider environment secrets could not be materialized", + { + operation: error.operation, + providerInstanceId: error.providerInstanceId, + environmentVariable: error.environmentVariable, + cause: error.cause, + }, + ).pipe(Effect.as(next)), + ), + ); + return resolveTextGenerationProvider(materialized); + }), + ); + return { start, ready: Deferred.await(startedDeferred), @@ -544,22 +589,8 @@ const make = Effect.gen(function* () { Effect.flatMap(materializeProviderEnvironmentSecrets), Effect.map(resolveTextGenerationProvider), ), - updateSettings: (patch) => - writeSemaphore.withPermits(1)( - Effect.gen(function* () { - const current = yield* getSettingsFromCache; - const nextPersisted = yield* persistProviderEnvironmentSecrets( - current, - applyServerSettingsPatch(current, patch), - ); - const next = yield* normalizeServerSettings(nextPersisted); - yield* writeSettingsAtomically(next); - yield* Cache.set(settingsCache, cacheKey, next); - yield* emitChange(next); - const materialized = yield* materializeProviderEnvironmentSecrets(next); - return resolveTextGenerationProvider(materialized); - }), - ), + updateSettings: (patch) => updateSettingsWith(() => patch), + updateSettingsWith, get streamChanges() { return Stream.fromPubSub(changesPubSub).pipe( Stream.mapEffect((settings) => diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 744e48661bf..2ee5877418f 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -46,6 +46,7 @@ import { ProjectWriteFileError, RelayClientInstallFailedError, type RelayClientInstallProgressEvent, + ProviderDescribeError, type FilesystemBrowseFailure, FilesystemBrowseError, AssetWorkspaceContextNotFoundError, @@ -73,6 +74,8 @@ import { import { normalizeDispatchCommand } from "./orchestration/Normalizer.ts"; import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; +import { getOrCreateAgentProject } from "./orchestration/agentProjects.ts"; +import { isHomeThread } from "./orchestration/homeThreads.ts"; import { observeRpcEffect as instrumentRpcEffect, observeRpcStream as instrumentRpcStream, @@ -80,6 +83,7 @@ import { } from "./observability/RpcInstrumentation.ts"; import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts"; import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; +import { HermesGatewayBroker } from "./provider/Services/HermesGatewayBroker.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; @@ -269,6 +273,7 @@ function isThreadDetailEvent(event: OrchestrationEvent): event is Extract< { type: | "thread.message-sent" + | "thread.notification-delivered" | "thread.proposed-plan-upserted" | "thread.activity-appended" | "thread.turn-diff-completed" @@ -278,6 +283,9 @@ function isThreadDetailEvent(event: OrchestrationEvent): event is Extract< > { return ( event.type === "thread.message-sent" || + // A proactive delivery appends a transcript message, so an open detail + // subscription has to receive it or the row only appears on reload. + event.type === "thread.notification-delivered" || event.type === "thread.proposed-plan-upserted" || event.type === "thread.activity-appended" || event.type === "thread.turn-diff-completed" || @@ -316,6 +324,15 @@ const RPC_REQUIRED_SCOPE = new Map([ [WS_METHODS.serverGetProcessDiagnostics, AuthOrchestrationReadScope], [WS_METHODS.serverGetProcessResourceHistory, AuthOrchestrationReadScope], [WS_METHODS.serverSignalProcess, AuthOrchestrationOperateScope], + [WS_METHODS.providerDescribeInstance, AuthOrchestrationReadScope], + [WS_METHODS.providerGetSkillBody, AuthOrchestrationReadScope], + [WS_METHODS.providerEnsureAgentProject, AuthOrchestrationOperateScope], + [WS_METHODS.hermesGatewayCreateEnrollment, AuthOrchestrationOperateScope], + [WS_METHODS.hermesGatewayGetInstanceStatus, AuthOrchestrationReadScope], + [WS_METHODS.hermesGatewayListInstances, AuthOrchestrationReadScope], + [WS_METHODS.hermesGatewayRenameInstance, AuthOrchestrationOperateScope], + [WS_METHODS.hermesGatewayRevokeInstance, AuthOrchestrationOperateScope], + [WS_METHODS.hermesGatewayRemoveInstance, AuthOrchestrationOperateScope], [WS_METHODS.cloudGetRelayClientStatus, AuthRelayWriteScope], [WS_METHODS.cloudInstallRelayClient, AuthRelayWriteScope], [WS_METHODS.sourceControlLookupRepository, AuthOrchestrationReadScope], @@ -429,6 +446,7 @@ const makeWsRpcLayer = ( const portDiscovery = yield* PortScanner.PortDiscovery; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerMaintenanceRunner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; + const hermesGatewayBroker = yield* HermesGatewayBroker; const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; const config = yield* ServerConfig.ServerConfig; const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; @@ -1073,6 +1091,23 @@ const makeWsRpcLayer = ( ORCHESTRATION_WS_METHODS.dispatchCommand, Effect.gen(function* () { const normalizedCommand = yield* normalizeDispatchCommand(command); + + // A home thread is a fixed designation, not a movable one: + // deleting it would leave its agent with nowhere to deliver, and + // the next handshake would silently mint a replacement, orphaning + // the history in the old one. This is the real guard — the UI + // only declines to offer the action. + if ( + normalizedCommand.type === "thread.delete" && + (yield* isHomeThread(normalizedCommand.threadId)) + ) { + return yield* Effect.fail( + new OrchestrationDispatchCommandError({ + message: "This agent's Home thread cannot be deleted.", + }), + ); + } + const shouldStopSessionAfterArchive = normalizedCommand.type === "thread.archive" ? yield* projectionSnapshotQuery @@ -1463,6 +1498,93 @@ const makeWsRpcLayer = ( "rpc.aggregate": "server", }, ), + [WS_METHODS.providerDescribeInstance]: ({ instanceId }) => + observeRpcEffect( + WS_METHODS.providerDescribeInstance, + providerRegistry.describeInstance(instanceId), + { "rpc.aggregate": "provider" }, + ), + [WS_METHODS.providerGetSkillBody]: (input) => + observeRpcEffect(WS_METHODS.providerGetSkillBody, providerRegistry.getSkillBody(input), { + "rpc.aggregate": "provider", + }), + [WS_METHODS.providerEnsureAgentProject]: ({ instanceId }) => + observeRpcEffect( + WS_METHODS.providerEnsureAgentProject, + Effect.gen(function* () { + const providers = yield* providerRegistry.getProviders; + const provider = providers.find((candidate) => candidate.instanceId === instanceId); + if (!provider) { + return yield* new ProviderDescribeError({ + code: "instance-not-found", + message: `Provider instance '${instanceId}' is not configured.`, + instanceId, + }); + } + const project = yield* getOrCreateAgentProject({ + instanceId, + title: provider.displayName ?? instanceId, + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("agent project resolution failed", { + instanceId, + cause: Cause.pretty(cause), + }).pipe( + Effect.andThen( + new ProviderDescribeError({ + code: "unavailable", + message: "Could not resolve this agent's project.", + instanceId, + }), + ), + ), + ), + ); + return { + projectId: project.id, + instanceId, + workspaceRoot: project.workspaceRoot, + title: project.title, + }; + }), + { "rpc.aggregate": "provider" }, + ), + [WS_METHODS.hermesGatewayCreateEnrollment]: (input) => + observeRpcEffect( + WS_METHODS.hermesGatewayCreateEnrollment, + hermesGatewayBroker.createEnrollment(input), + { "rpc.aggregate": "hermes-gateway" }, + ), + [WS_METHODS.hermesGatewayGetInstanceStatus]: ({ instanceId }) => + observeRpcEffect( + WS_METHODS.hermesGatewayGetInstanceStatus, + hermesGatewayBroker.getInstanceStatus(instanceId), + { "rpc.aggregate": "hermes-gateway" }, + ), + [WS_METHODS.hermesGatewayListInstances]: (_input) => + observeRpcEffect( + WS_METHODS.hermesGatewayListInstances, + hermesGatewayBroker.listInstances, + { "rpc.aggregate": "hermes-gateway" }, + ), + [WS_METHODS.hermesGatewayRenameInstance]: (input) => + observeRpcEffect( + WS_METHODS.hermesGatewayRenameInstance, + hermesGatewayBroker.renameInstance(input), + { "rpc.aggregate": "hermes-gateway" }, + ), + [WS_METHODS.hermesGatewayRevokeInstance]: ({ instanceId }) => + observeRpcEffect( + WS_METHODS.hermesGatewayRevokeInstance, + hermesGatewayBroker.revokeInstance(instanceId), + { "rpc.aggregate": "hermes-gateway" }, + ), + [WS_METHODS.hermesGatewayRemoveInstance]: ({ instanceId }) => + observeRpcEffect( + WS_METHODS.hermesGatewayRemoveInstance, + hermesGatewayBroker.removeInstance(instanceId), + { "rpc.aggregate": "hermes-gateway" }, + ), [WS_METHODS.serverDiscoverSourceControl]: (_input) => observeRpcEffect( WS_METHODS.serverDiscoverSourceControl, diff --git a/apps/web/public/hermes-agent-icon.png b/apps/web/public/hermes-agent-icon.png new file mode 100644 index 00000000000..003f0dc3ce2 Binary files /dev/null and b/apps/web/public/hermes-agent-icon.png differ diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index 1b2e537c35d..ddf49e163ae 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -2,6 +2,7 @@ import { EnvironmentId, MessageId, ProjectId, + ProviderDriverKind, ProviderInstanceId, ThreadId, TurnId, @@ -9,7 +10,9 @@ import { import { describe, expect, it } from "vite-plus/test"; import type { Thread } from "../types"; +import type { ComposerAttachment } from "../composerDraftStore"; import { + buildTurnUploadAttachment, MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, MAX_HIDDEN_MOUNTED_TERMINAL_THREADS, branchMismatchKey, @@ -17,6 +20,7 @@ import { buildThreadTurnInterruptInput, createLocalDispatchSnapshot, deriveComposerSendState, + deriveLockedProvider, dismissBranchMismatchForSession, getStartedThreadModelChangeBlockReason, hasServerAcknowledgedLocalDispatch, @@ -627,3 +631,144 @@ describe("hasServerAcknowledgedLocalDispatch", () => { expect(hasServerAcknowledgedLocalDispatch({ ...common, threadError: "failed" })).toBe(true); }); }); + +describe("deriveLockedProvider", () => { + const hermesInstanceId = ProviderInstanceId.make("hermes-siva-local-c9cc"); + const providers = [ + { instanceId: hermesInstanceId, driver: ProviderDriverKind.make("hermes") }, + { instanceId: ProviderInstanceId.make("codex"), driver: ProviderDriverKind.make("codex") }, + ]; + + it("leaves an unstarted thread unlocked", () => { + expect( + deriveLockedProvider({ + thread: makeThread(), + selectedProvider: null, + threadProvider: "codex", + providers, + }), + ).toBe(null); + }); + + it("follows the open session's driver kind when there is one", () => { + expect( + deriveLockedProvider({ + thread: makeThread({ session: readySession }), + selectedProvider: null, + threadProvider: null, + providers, + }), + ).toBe("codex"); + }); + + it("resolves a sessionless thread's instance id back to its driver kind", () => { + // A Hermes home thread never opens a session — deliveries bypass the turn + // machinery — so `modelSelection.instanceId` is the only binding left. + // Before this resolved, the composer fell through to the unknown-instance + // defaults and showed a runtime mode selector Hermes opts out of. + expect( + deriveLockedProvider({ + thread: makeThread({ + session: null, + messages: [ + { + id: MessageId.make("message-1"), + role: "assistant", + text: "Gateway online", + turnId: null, + streaming: false, + createdAt: now, + updatedAt: now, + }, + ], + modelSelection: { instanceId: hermesInstanceId, model: "hermes" }, + }), + selectedProvider: null, + threadProvider: hermesInstanceId, + providers, + }), + ).toBe("hermes"); + }); + + it("prefers the registry over the slug shape for an instance id", () => { + // `isProviderDriverKind` validates slug *shape*, not membership, so an + // instance id passes it. Checking shape first would hand back + // `hermes-siva-local-c9cc` as a driver kind matching no driver — which is + // exactly how a home thread ended up with another provider's composer + // controls. + expect( + deriveLockedProvider({ + thread: makeThread({ latestTurn: completedTurn }), + selectedProvider: null, + threadProvider: hermesInstanceId, + providers, + }), + ).toBe("hermes"); + }); + + it("stays unlocked when the thread carries no usable binding", () => { + expect( + deriveLockedProvider({ + thread: makeThread({ latestTurn: completedTurn }), + selectedProvider: null, + threadProvider: null, + providers, + }), + ).toBe(null); + }); + + it("degrades to slug-only matching when no snapshots are supplied", () => { + expect( + deriveLockedProvider({ + thread: makeThread({ latestTurn: completedTurn }), + selectedProvider: null, + threadProvider: "codex", + }), + ).toBe("codex"); + }); +}); + +describe("buildTurnUploadAttachment", () => { + const readDataUrl = (file: File) => Promise.resolve(`data:${file.type};base64,AAAA`); + + function makeComposerAttachment(input: { + type: "image" | "file"; + name: string; + mimeType: string; + }): ComposerAttachment { + const file = new File([new Uint8Array(4)], input.name, { type: input.mimeType }); + return { + type: input.type, + id: `att-${input.name}`, + name: input.name, + mimeType: input.mimeType, + sizeBytes: file.size, + previewUrl: `blob:${input.name}`, + file, + } as ComposerAttachment; + } + + it("sends a non-image attachment as the file upload variant", async () => { + await expect( + buildTurnUploadAttachment( + makeComposerAttachment({ type: "file", name: "spec.pdf", mimeType: "application/pdf" }), + readDataUrl, + ), + ).resolves.toEqual({ + type: "file", + name: "spec.pdf", + mimeType: "application/pdf", + sizeBytes: 4, + dataUrl: "data:application/pdf;base64,AAAA", + }); + }); + + it("leaves images on the image upload variant", async () => { + await expect( + buildTurnUploadAttachment( + makeComposerAttachment({ type: "image", name: "shot.png", mimeType: "image/png" }), + readDataUrl, + ), + ).resolves.toMatchObject({ type: "image", mimeType: "image/png" }); + }); +}); diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index 4fb007b85b5..b7bcb4c1c76 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -11,7 +11,7 @@ import { type TurnId, } from "@t3tools/contracts"; import { type ChatMessage, type SessionPhase, type Thread } from "../types"; -import { type ComposerImageAttachment, type DraftThreadState } from "../composerDraftStore"; +import { type ComposerAttachment, type DraftThreadState } from "../composerDraftStore"; import * as Schema from "effect/Schema"; import { appAtomRegistry } from "../rpc/atomRegistry"; import { environmentThreadDetails } from "../state/threads"; @@ -179,12 +179,24 @@ export function revokeBlobPreviewUrl(previewUrl: string | undefined): void { URL.revokeObjectURL(previewUrl); } -export function revokeUserMessagePreviewUrls(message: ChatMessage): void { +/** + * Releases the blob previews an optimistic user message is holding. + * + * Every attachment variant gets a blob URL at intake, so the release is + * variant-agnostic. `retainedPreviewUrls` skips the image previews that were + * handed off to cross-fade against the server's signed URL — file cards have + * no such cross-fade, so their blobs are always released here. + */ +export function revokeUserMessagePreviewUrls( + message: ChatMessage, + retainedPreviewUrls?: ReadonlyArray, +): void { if (message.role !== "user" || !message.attachments) { return; } + const retained = retainedPreviewUrls ? new Set(retainedPreviewUrls) : null; for (const attachment of message.attachments) { - if (attachment.type !== "image") { + if (attachment.previewUrl && retained?.has(attachment.previewUrl)) { continue; } revokeBlobPreviewUrl(attachment.previewUrl); @@ -226,6 +238,32 @@ export function readFileAsDataUrl(file: File): Promise { }); } +/** + * Upload shape for one composer attachment on `thread.turn.start`. + * + * The variant tag rides along unchanged: images stay `type: "image"` (mime + * constrained to `image/*` by the contract), and everything staged on a + * Hermes thread goes as `type: "file"` with its own mime. + */ +export async function buildTurnUploadAttachment( + attachment: ComposerAttachment, + readDataUrl: (file: File) => Promise, +): Promise<{ + type: "image" | "file"; + name: string; + mimeType: string; + sizeBytes: number; + dataUrl: string; +}> { + return { + type: attachment.type, + name: attachment.name, + mimeType: attachment.mimeType, + sizeBytes: attachment.sizeBytes, + dataUrl: await readDataUrl(attachment.file), + }; +} + export function resolveSendEnvMode(input: { requestedEnvMode: DraftThreadEnvMode; isGitRepo: boolean; @@ -233,9 +271,7 @@ export function resolveSendEnvMode(input: { return input.isGitRepo ? input.requestedEnvMode : "local"; } -export function cloneComposerImageForRetry( - image: ComposerImageAttachment, -): ComposerImageAttachment { +export function cloneComposerImageForRetry(image: ComposerAttachment): ComposerAttachment { if (typeof URL === "undefined" || !image.previewUrl.startsWith("blob:")) { return image; } @@ -353,15 +389,19 @@ export function threadHasStarted(thread: Thread | null | undefined): boolean { // // `selectedProvider` takes the same open-string shape because the composer // now tracks the picker selection as a `ProviderInstanceId` (e.g. -// `codex_personal`). Custom instance ids that don't directly match a -// registered driver resolve to `null` here, which matches the existing -// "unknown driver -> unlocked" semantics. Callers that want the lock to track -// a custom instance's underlying driver kind should resolve the instance id -// upstream and pass the correlated kind. +// `codex_personal`). Such an id is not itself a driver-kind slug, so +// `providers` is consulted to map it back to its owning kind; without that +// lookup a thread bound to a user-authored instance reads as unlocked. export function deriveLockedProvider(input: { thread: Thread | null | undefined; selectedProvider: string | null; threadProvider: string | null; + /** + * Instance snapshots used to resolve a `ProviderInstanceId` to its driver + * kind. Optional so callers that already hold a kind can omit it; absent (or + * not yet streamed) it degrades to the previous slug-only matching. + */ + providers?: ReadonlyArray>; }): ProviderDriverKind | null { if (!threadHasStarted(input.thread)) { return null; @@ -370,15 +410,24 @@ export function deriveLockedProvider(input: { if (sessionProvider && isProviderDriverKind(sessionProvider)) { return sessionProvider; } - const narrowedThreadProvider = - input.threadProvider && isProviderDriverKind(input.threadProvider) - ? input.threadProvider - : null; - const narrowedSelectedProvider = - input.selectedProvider && isProviderDriverKind(input.selectedProvider) - ? input.selectedProvider - : null; - return narrowedThreadProvider ?? narrowedSelectedProvider ?? null; + // A Hermes home thread never opens a session (deliveries bypass the turn + // machinery), so `providerName` is absent and the only binding left is the + // instance id on `modelSelection`. Resolving it through the snapshots is + // what keeps the composer on the thread's own instance instead of falling + // through to the unknown-instance defaults, which show a workspace runtime + // mode selector and a plan toggle that Hermes explicitly opts out of. + // + // The registry lookup must come FIRST. `isProviderDriverKind` validates slug + // *shape*, not membership, so an instance id like `hermes-siva-local-c9cc` + // satisfies it and would otherwise be returned as a driver kind that matches + // no driver — locking the thread to a provider that does not exist. + const narrow = (candidate: string | null): ProviderDriverKind | null => { + if (!candidate) return null; + const resolved = input.providers?.find((entry) => entry.instanceId === candidate)?.driver; + if (resolved) return resolved; + return isProviderDriverKind(candidate) ? candidate : null; + }; + return narrow(input.threadProvider) ?? narrow(input.selectedProvider) ?? null; } export function getStartedThreadModelChangeBlockReason(input: { diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index bdde62a3e0d..69d39ddd05f 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -1,5 +1,6 @@ import { type ApprovalRequestId, + DEFAULT_HERMES_MODEL, DEFAULT_MODEL, defaultInstanceIdForDriver, type EnvironmentId, @@ -178,8 +179,9 @@ import { } from "../logicalProject"; import { buildDraftThreadRouteParams } from "../threadRoutes"; import { - type ComposerImageAttachment, + type ComposerAttachment, type DraftThreadEnvMode, + hydrateImagesFromPersisted, useComposerDraftStore, type DraftId, } from "../composerDraftStore"; @@ -266,6 +268,7 @@ import { PullRequestDialogState, cloneComposerImageForRetry, deriveLockedProvider, + buildTurnUploadAttachment, readFileAsDataUrl, reconcileMountedTerminalThreadIds, resolveThreadMetadataUpdateForNextTurn, @@ -305,6 +308,8 @@ import { useAssetUrls } from "../assets/assetUrls"; const IMAGE_ONLY_BOOTSTRAP_PROMPT = "[User attached one or more images without additional text. Respond using the conversation context and the attached image(s).]"; +const FILE_ONLY_BOOTSTRAP_PROMPT = + "[User attached one or more files without additional text. Respond using the conversation context and the attached file(s).]"; const EMPTY_ACTIVITIES: OrchestrationThreadActivity[] = []; const EMPTY_PROVIDERS: ServerProvider[] = []; const EMPTY_PROVIDER_SKILLS: ServerProvider["skills"] = []; @@ -1139,6 +1144,9 @@ function ChatViewContent(props: ChatViewProps) { ); const routeThreadKey = useMemo(() => scopedThreadKey(routeThreadRef), [routeThreadRef]); const updateProject = useAtomCommand(projectEnvironment.update, { reportFailure: false }); + const ensureAgentProject = useAtomCommand(serverEnvironment.providerEnsureAgentProject, { + reportFailure: false, + }); const upsertKeybinding = useAtomCommand(serverEnvironment.upsertKeybinding, { reportFailure: false, }); @@ -1242,7 +1250,7 @@ function ChatViewContent(props: ChatViewProps) { (store) => store.setLogicalProjectDraftThreadId, ); const promptRef = useRef(""); - const composerImagesRef = useRef([]); + const composerImagesRef = useRef([]); const composerTerminalContextsRef = useRef([]); const composerElementContextsRef = useRef([]); const localComposerRef = useRef(null); @@ -1413,10 +1421,22 @@ function ChatViewContent(props: ChatViewProps) { ? buildLocalDraftThread( threadId, draftThread, - fallbackDraftProject?.defaultModelSelection ?? NO_PROVIDER_MODEL_SELECTION, + // A draft in an agent's synthetic project is born bound to that + // agent. The pin cannot rely on the project's + // `defaultModelSelection` — agent projects are created with it + // null — and must beat the carried-over selection from the + // previously viewed thread. + fallbackDraftProject?.agentInstanceId + ? { instanceId: fallbackDraftProject.agentInstanceId, model: DEFAULT_HERMES_MODEL } + : (fallbackDraftProject?.defaultModelSelection ?? NO_PROVIDER_MODEL_SELECTION), ) : undefined, - [draftThread, fallbackDraftProject?.defaultModelSelection, threadId], + [ + draftThread, + fallbackDraftProject?.agentInstanceId, + fallbackDraftProject?.defaultModelSelection, + threadId, + ], ); // Promotion is data-driven: the draft route keeps rendering while the // server thread (same pre-allocated ref) starts, so live state must not @@ -1586,6 +1606,10 @@ function ChatViewContent(props: ChatViewProps) { ? scopeProjectRef(activeThread.environmentId, activeThread.projectId) : null; const activeProject = useProject(activeProjectRef); + // Non-null iff the thread lives in an agent's synthetic project. Every + // thread there is bound to that one agent instance, so the composer's + // model picker locks to it and drafts can never route elsewhere. + const activeProjectAgentInstanceId = activeProject?.agentInstanceId ?? null; const handleNewThreadInActiveProject = useCallback(() => { startNewThreadForProject(activeProjectRef, handleNewThread); }, [activeProjectRef, handleNewThread]); @@ -1831,16 +1855,17 @@ function ChatViewContent(props: ChatViewProps) { activeThread?.modelSelection.instanceId ?? activeProject?.defaultModelSelection?.instanceId ?? null; - const lockedProvider = deriveLockedProvider({ - thread: activeThread, - selectedProvider: selectedProviderByThreadId, - threadProvider, - }); // Once a thread selects an environment, never substitute the primary // environment's config while the selected environment is still loading. const serverConfig = activeThread ? (activeEnvironment?.serverConfig ?? null) : (primaryEnvironment?.serverConfig ?? null); + const lockedProvider = deriveLockedProvider({ + thread: activeThread, + selectedProvider: selectedProviderByThreadId, + threadProvider, + providers: serverConfig?.providers ?? EMPTY_PROVIDERS, + }); const versionMismatch = resolveServerConfigVersionMismatch(serverConfig); const versionMismatchDismissKey = versionMismatch && activeThread @@ -2409,7 +2434,19 @@ function ChatViewContent(props: ChatViewProps) { terminalUiLaunchContext?.threadId === activeThreadId ? terminalUiLaunchContext : null; // Default true while loading to avoid toolbar flicker. const isGitRepo = gitStatusQuery.data?.isRepo ?? true; - const showComposerContextStrip = isGitRepo && activeProject !== null; + // Read the flag off the instance the thread is actually bound to, not off + // the driver kind: Hermes instance ids are user-authored slugs + // (`hermes-