From 8e7c6413dc412b41649b4cefe1a34760ee88120c Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 11:39:06 -0500 Subject: [PATCH 01/16] =?UTF-8?q?=F0=9F=A4=96=20perf:=20shard=20OpenSSH=20?= =?UTF-8?q?masters=20and=20dedupe=20SSH=20project=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add an explicit OpenSSH master pool with sharded control sockets and lease-based exec reuse - hash remote project layouts and move workspace metadata/snapshot markers to project-scoped files - dedupe per-project bundle sync work and update unit plus SSH integration coverage for the new layout --- _Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$36.24`_ --- src/node/runtime/DockerRuntime.ts | 5 +- src/node/runtime/RemoteRuntime.ts | 42 +- src/node/runtime/SSHRuntime.test.ts | 178 +++-- src/node/runtime/SSHRuntime.ts | 507 ++++++++---- src/node/runtime/openSshMasterPool.test.ts | 359 +++++++++ src/node/runtime/openSshMasterPool.ts | 756 ++++++++++++++++++ .../runtime/projectSyncCoordinator.test.ts | 78 ++ src/node/runtime/projectSyncCoordinator.ts | 128 +++ src/node/runtime/remoteProjectLayout.ts | 76 ++ src/node/runtime/runtimeFactory.ts | 1 + src/node/runtime/sshConnectionPool.ts | 4 + .../transports/OpenSSHTransport.test.ts | 60 +- .../runtime/transports/OpenSSHTransport.ts | 85 +- src/node/runtime/transports/SSH2Transport.ts | 12 +- src/node/runtime/transports/SSHTransport.ts | 2 + .../services/workspaceProjectRepos.test.ts | 21 + src/node/services/workspaceProjectRepos.ts | 2 + tests/e2e/utils/ui.ts | 7 +- tests/runtime/runtime.test.ts | 285 ++++++- 19 files changed, 2322 insertions(+), 286 deletions(-) create mode 100644 src/node/runtime/openSshMasterPool.test.ts create mode 100644 src/node/runtime/openSshMasterPool.ts create mode 100644 src/node/runtime/projectSyncCoordinator.test.ts create mode 100644 src/node/runtime/projectSyncCoordinator.ts create mode 100644 src/node/runtime/remoteProjectLayout.ts diff --git a/src/node/runtime/DockerRuntime.ts b/src/node/runtime/DockerRuntime.ts index 989c10816d..7b914e766b 100644 --- a/src/node/runtime/DockerRuntime.ts +++ b/src/node/runtime/DockerRuntime.ts @@ -353,7 +353,10 @@ export class DockerRuntime extends RemoteRuntime { return `cd ${shescape.quote(cwd)}`; } - protected spawnRemoteProcess(fullCommand: string, _options: ExecOptions): Promise { + protected spawnRemoteProcess( + fullCommand: string, + _options: ExecOptions & { deadlineMs?: number } + ): Promise { // Verify container name is available if (!this.containerName) { throw new RuntimeError( diff --git a/src/node/runtime/RemoteRuntime.ts b/src/node/runtime/RemoteRuntime.ts index a60aef5c51..20bb977e36 100644 --- a/src/node/runtime/RemoteRuntime.ts +++ b/src/node/runtime/RemoteRuntime.ts @@ -47,6 +47,10 @@ export interface SpawnResult { process: ChildProcess; /** Optional async work to do before exec (e.g., acquire connection) */ preExec?: Promise; + /** Optional transport-scoped exit handling (e.g., master-pool health accounting). */ + onExit?: (exitCode: number, stderr: string) => void; + /** Optional transport-scoped spawn error handling. */ + onError?: (error: Error) => void; } /** @@ -63,7 +67,7 @@ export abstract class RemoteRuntime implements Runtime { */ protected abstract spawnRemoteProcess( fullCommand: string, - options: ExecOptions + options: ExecOptions & { deadlineMs?: number } ): Promise; /** @@ -138,8 +142,13 @@ export abstract class RemoteRuntime implements Runtime { } // Spawn the remote process (SSH or Docker) - // For SSH, this awaits connection pool backoff before spawning - const { process: childProcess } = await this.spawnRemoteProcess(fullCommand, options); + const timeoutMs = options.timeout !== undefined ? options.timeout * 1000 : undefined; + const deadlineMs = timeoutMs !== undefined ? Date.now() + timeoutMs : undefined; + const spawnResult = await this.spawnRemoteProcess(fullCommand, { + ...options, + deadlineMs, + }); + const { process: childProcess } = spawnResult; // Short-lived commands can close stdin before writes/close complete. if (childProcess.stdin) { @@ -163,16 +172,14 @@ export abstract class RemoteRuntime implements Runtime { // Create promises for exit code and duration immediately. const exitCode = new Promise((resolve, reject) => { childProcess.on("close", (code, signal) => { - if (aborted || options.abortSignal?.aborted) { - resolve(EXIT_CODE_ABORTED); - return; - } - if (timedOut) { - resolve(EXIT_CODE_TIMEOUT); - return; - } - const finalExitCode = code ?? (signal ? -1 : 0); - + const finalExitCode = + aborted || options.abortSignal?.aborted + ? EXIT_CODE_ABORTED + : timedOut + ? EXIT_CODE_TIMEOUT + : (code ?? (signal ? -1 : 0)); + + spawnResult.onExit?.(finalExitCode, stderrForErrorReporting); // Let subclass handle exit code (e.g., SSH connection pool) this.onExitCode(finalExitCode, options, stderrForErrorReporting); @@ -180,6 +187,7 @@ export abstract class RemoteRuntime implements Runtime { }); childProcess.on("error", (err) => { + spawnResult.onError?.(err); reject( new RuntimeError( `Failed to execute ${this.commandPrefix} command: ${err.message}`, @@ -226,8 +234,10 @@ export abstract class RemoteRuntime implements Runtime { void exitCode.finally(() => abortSignal.removeEventListener("abort", onAbort)); } - // Handle timeout - if (options.timeout !== undefined) { + // Handle timeout. Include connection acquisition time in the local deadline so + // user-configured timeouts do not silently stretch while the runtime waits for SSH capacity. + if (timeoutMs !== undefined) { + const remainingTimeoutMs = Math.max(0, (deadlineMs ?? Date.now()) - Date.now()); const timeoutHandle = setTimeout(() => { timedOut = true; @@ -245,7 +255,7 @@ export abstract class RemoteRuntime implements Runtime { disposable[Symbol.dispose](); }, 1000); hardKillHandle.unref(); - }, options.timeout * 1000); + }, remainingTimeoutMs); void exitCode.finally(() => clearTimeout(timeoutHandle)); } diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index 60abe4d2d0..ea35d4f6e3 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -3,6 +3,11 @@ import * as runtimeHelpers from "@/node/utils/runtime/helpers"; import * as disposableExec from "@/node/utils/disposableExec"; import * as submoduleSync from "./submoduleSync"; import { SSHRuntime, computeBaseRepoPath } from "./SSHRuntime"; +import { + buildLegacyRemoteProjectLayout, + buildRemoteProjectLayout, + getRemoteWorkspacePath, +} from "./remoteProjectLayout"; import { createSSHTransport } from "./transports"; /** @@ -576,10 +581,8 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { }); describe("SSHRuntime.createWorkspace", () => { - it("uses directoryName for the workspace path while preparing the remote parent directory", async () => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const runtime = new SSHRuntime(config, createSSHTransport(config, false)); - const execSpy = spyOn(runtime, "exec").mockResolvedValue({ + function createExecStream(exitCode = 0) { + return { stdout: new ReadableStream({ start(controller) { controller.close(); @@ -591,18 +594,29 @@ describe("SSHRuntime.createWorkspace", () => { }, }), stdin: new WritableStream(), - exitCode: Promise.resolve(0), + exitCode: Promise.resolve(exitCode), duration: Promise.resolve(0), - }); - const readFileSpy = spyOn(runtime, "readFile").mockReturnValue( - new ReadableStream({ - start(controller) { - controller.error(new Error("missing branch map")); - }, - }) + }; + } + + it("uses directoryName for the workspace path while preparing the remote parent directory", async () => { + const config = { host: "example.com", srcBaseDir: "/home/user/src" }; + const runtime = new SSHRuntime(config, createSSHTransport(config, false)); + const expectedLayout = buildRemoteProjectLayout(config.srcBaseDir, "/projects/demo"); + const expectedWorkspacePath = getRemoteWorkspacePath(expectedLayout, "review-slot"); + const execSpy = spyOn(runtime, "exec").mockImplementation(() => + Promise.resolve(createExecStream()) + ); + const readFileSpy = spyOn(runtime, "readFile").mockImplementation( + () => + new ReadableStream({ + start(controller) { + controller.error(new Error("missing branch metadata")); + }, + }) ); - const writeFileSpy = spyOn(runtime, "writeFile").mockReturnValue( - new WritableStream() + const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation( + () => new WritableStream() ); try { @@ -621,13 +635,16 @@ describe("SSHRuntime.createWorkspace", () => { expect(result).toEqual({ success: true, - workspacePath: "/home/user/src/demo/review-slot", - }); - expect(execSpy).toHaveBeenCalledWith('mkdir -p "/home/user/src/demo"', { - cwd: "/tmp", - timeout: 10, - abortSignal: undefined, + workspacePath: expectedWorkspacePath, }); + expect(execSpy).toHaveBeenCalledWith( + `mkdir -p ${JSON.stringify(expectedLayout.projectRoot)}`, + { + cwd: "/tmp", + timeout: 10, + abortSignal: undefined, + } + ); } finally { execSpy.mockRestore(); readFileSpy.mockRestore(); @@ -650,6 +667,8 @@ describe("SSHRuntime.deleteWorkspace", () => { it("deletes the mapped workspace branch instead of the current remote checkout", async () => { const config = { host: "example.com", srcBaseDir: "/home/user/src" }; const runtime = new SSHRuntime(config, createSSHTransport(config, false)); + const expectedLayout = buildRemoteProjectLayout(config.srcBaseDir, "/projects/demo"); + const expectedDeletedPath = getRemoteWorkspacePath(expectedLayout, "review-slot"); const execSpy = spyOn(runtime, "exec").mockImplementation((command) => { if (command.includes("git diff --quiet") || command.includes("test -d")) { return Promise.resolve(createExecStream(0)); @@ -659,13 +678,14 @@ describe("SSHRuntime.deleteWorkspace", () => { } throw new Error(`Unexpected exec command: ${command}`); }); - const readFileSpy = spyOn(runtime, "readFile").mockReturnValue( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode('{"review-slot":"feature-branch"}\n')); - controller.close(); - }, - }) + const readFileSpy = spyOn(runtime, "readFile").mockImplementation( + () => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"review-slot":"feature-branch"}\n')); + controller.close(); + }, + }) ); const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( (_runtime, command) => { @@ -688,7 +708,7 @@ describe("SSHRuntime.deleteWorkspace", () => { const result = await runtime.deleteWorkspace("/projects/demo", "review-slot", true); expect(result).toEqual({ success: true, - deletedPath: "/home/user/src/demo/review-slot", + deletedPath: expectedDeletedPath, }); } finally { execSpy.mockRestore(); @@ -717,25 +737,27 @@ describe("SSHRuntime.ensureReady repository checks", () => { it("accepts worktrees where .git is a file", async () => { execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") + .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: ".git", stderr: "", exitCode: 0, duration: 0 }) + .mockResolvedValueOnce({ stdout: ".git\n", stderr: "", exitCode: 0, duration: 0 }) .mockResolvedValueOnce({ stdout: "true\n", stderr: "", exitCode: 0, duration: 0 }); const result = await runtime.ensureReady(); - expect(execBufferedSpy).toHaveBeenCalledTimes(3); - const firstCommand = execBufferedSpy?.mock.calls[0]?.[1]; - expect(firstCommand).toContain("test -d"); - expect(firstCommand).toContain("test -f"); - const thirdCommand = execBufferedSpy?.mock.calls[2]?.[1]; - expect(thirdCommand).toContain("rev-parse --is-inside-work-tree"); + expect(execBufferedSpy).toHaveBeenCalledTimes(4); + const secondCommand = execBufferedSpy?.mock.calls[1]?.[1]; + expect(secondCommand).toContain("test -d"); + expect(secondCommand).toContain("test -f"); + const fourthCommand = execBufferedSpy?.mock.calls[3]?.[1]; + expect(fourthCommand).toContain("rev-parse --is-inside-work-tree"); expect(result).toEqual({ ready: true }); }); it("returns runtime_not_ready when git reports the workspace is not inside a work tree", async () => { execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") + .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: ".git", stderr: "", exitCode: 0, duration: 0 }) + .mockResolvedValueOnce({ stdout: ".git\n", stderr: "", exitCode: 0, duration: 0 }) .mockResolvedValueOnce({ stdout: "false\n", stderr: "", exitCode: 0, duration: 0 }); const result = await runtime.ensureReady(); @@ -747,12 +769,14 @@ describe("SSHRuntime.ensureReady repository checks", () => { }); it("returns runtime_not_ready when the repo is missing", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "", - stderr: "", - exitCode: 1, - duration: 0, - }); + execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") + .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) + .mockResolvedValue({ + stdout: "", + stderr: "", + exitCode: 1, + duration: 0, + }); const result = await runtime.ensureReady(); @@ -764,6 +788,7 @@ describe("SSHRuntime.ensureReady repository checks", () => { it("returns runtime_start_failed when git is unavailable", async () => { execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") + .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 0, duration: 0 }) .mockResolvedValueOnce({ stdout: "", @@ -821,16 +846,77 @@ describe("SSHRuntime.resolvePath", () => { }); }); }); +describe("SSHRuntime project sync coordination", () => { + it("uses srcBaseDir in the per-project sync key", () => { + const projectId = "demo-project-123456789abc"; + const configA = { host: "example.com", srcBaseDir: "/home/user/src-a" }; + const configB = { host: "example.com", srcBaseDir: "/home/user/src-b" }; + const runtimeA = new SSHRuntime(configA, createSSHTransport(configA, false)); + const runtimeB = new SSHRuntime(configB, createSSHTransport(configB, false)); + + const getProjectSyncKey = (runtime: SSHRuntime): ((projectIdArg: string) => string) => { + const maybeMethod: unknown = Reflect.get(runtime, "getProjectSyncKey"); + if (typeof maybeMethod !== "function") { + throw new Error("getProjectSyncKey is unavailable"); + } + return maybeMethod as (projectIdArg: string) => string; + }; + + expect(getProjectSyncKey(runtimeA).call(runtimeA, projectId)).not.toBe( + getProjectSyncKey(runtimeB).call(runtimeB, projectId) + ); + }); +}); + +describe("SSHRuntime layout detection", () => { + let execBufferedSpy: ReturnType> | null = + null; + + afterEach(() => { + execBufferedSpy?.mockRestore(); + execBufferedSpy = null; + }); + + it("does not treat legacy root existence alone as evidence of a legacy layout", async () => { + const config = { host: "example.com", srcBaseDir: "/home/user/src" }; + const projectPath = "/projects/demo"; + const workspaceName = "fresh-workspace"; + const runtime = new SSHRuntime(config, createSSHTransport(config, false)); + const preferredLayout = buildRemoteProjectLayout(config.srcBaseDir, projectPath); + const legacyLayout = buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath); + + execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ + stdout: "preferred\n", + stderr: "", + exitCode: 0, + duration: 0, + }); + + const resolveProjectLayout = Reflect.get(runtime, "resolveProjectLayout") as ( + projectPathArg: string, + workspaceNameArg?: string + ) => Promise<{ projectRoot: string }>; + const layout = await resolveProjectLayout.call(runtime, projectPath, workspaceName); + + expect(layout.projectRoot).toBe(preferredLayout.projectRoot); + const detectionCommand = execBufferedSpy.mock.calls[0]?.[1]; + expect(detectionCommand).toContain(`test -e "${legacyLayout.projectRoot}/${workspaceName}"`); + expect(detectionCommand).not.toContain(`test -d "${legacyLayout.projectRoot}"`); + }); +}); + describe("computeBaseRepoPath", () => { it("computes the correct bare repo path", () => { - // computeBaseRepoPath uses getProjectName (basename) to compute: - // //.mux-base.git + const layout = buildRemoteProjectLayout("~/mux", "/Users/me/code/my-project"); const result = computeBaseRepoPath("~/mux", "/Users/me/code/my-project"); - expect(result).toBe("~/mux/my-project/.mux-base.git"); + expect(result).toBe(layout.baseRepoPath); + expect(result).toMatch(/^~\/mux\/my-project-[a-f0-9]{12}\/\.mux-base\.git$/); }); it("handles absolute srcBaseDir", () => { + const layout = buildRemoteProjectLayout("/home/user/src", "/code/repo"); const result = computeBaseRepoPath("/home/user/src", "/code/repo"); - expect(result).toBe("/home/user/src/repo/.mux-base.git"); + expect(result).toBe(layout.baseRepoPath); + expect(result).toMatch(/^\/home\/user\/src\/repo-[a-f0-9]{12}\/\.mux-base\.git$/); }); }); diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index 76e23d96b1..876f92ae93 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -16,6 +16,7 @@ */ import { spawn, type ChildProcess } from "child_process"; +import * as crypto from "crypto"; import * as path from "path"; import type { EnsureReadyOptions, @@ -35,7 +36,7 @@ import { log } from "@/node/services/log"; import { runInitHookOnRuntime, runWorkspaceInitHook } from "./initHook"; import { expandTildeForSSH as expandHookPath } from "./tildeExpansion"; import { expandTildeForSSH, cdCommandForSSH } from "./tildeExpansion"; -import { getProjectName, execBuffered } from "@/node/utils/runtime/helpers"; +import { execBuffered } from "@/node/utils/runtime/helpers"; import { getErrorMessage } from "@/common/utils/errors"; import { type SSHRuntimeConfig } from "./sshConnectionPool"; import { getOriginUrlForBundle } from "./gitBundleSync"; @@ -43,11 +44,17 @@ import { gitNoHooksPrefix } from "@/node/utils/gitNoHooksEnv"; import { execFileAsync } from "@/node/utils/disposableExec"; import { syncRuntimeGitSubmodules } from "./submoduleSync"; import type { PtyHandle, PtySessionParams, SSHTransport } from "./transports"; +import { + buildLegacyRemoteProjectLayout, + buildRemoteProjectLayout, + getRemoteWorkspacePath, + getSnapshotMarkerPath, + getWorkspaceMetadataPath, + type RemoteProjectLayout, +} from "./remoteProjectLayout"; +import { projectSyncCoordinator } from "./projectSyncCoordinator"; import { streamToString, shescape } from "./streamUtils"; -/** Name of the shared bare repo directory under each project on the remote. */ -const BASE_REPO_DIR = ".mux-base.git"; - /** Staging namespace for bundle-imported branch refs. Branches land here instead * of refs/heads/* so they don't collide with branches checked out in worktrees. */ const BUNDLE_REF_PREFIX = "refs/mux-bundle/"; @@ -139,29 +146,18 @@ async function waitForProcessExit(proc: ChildProcess): Promise { proc.on("error", (err) => reject(err)); }); } -/** Truncate SSH stderr for error logging (keep first line, max 200 chars) */ -function truncateSSHError(stderr: string): string { - const trimmed = stderr.trim(); - if (!trimmed) return "exit code 255"; - // Take first line only (SSH errors are usually single-line) - const firstLine = trimmed.split("\n")[0]; - if (firstLine.length <= 200) return firstLine; - return firstLine.slice(0, 197) + "..."; -} - // Re-export SSHRuntimeConfig from connection pool (defined there to avoid circular deps) export type { SSHRuntimeConfig } from "./sshConnectionPool"; /** * Compute the path to the shared bare base repo for a project on the remote. - * Convention: //.mux-base.git + * Convention: //.mux-base.git * * Exported for unit testing; runtime code should use the private * `SSHRuntime.getBaseRepoPath()` method instead. */ export function computeBaseRepoPath(srcBaseDir: string, projectPath: string): string { - const projectName = getProjectName(projectPath); - return path.posix.join(srcBaseDir, projectName, BASE_REPO_DIR); + return buildRemoteProjectLayout(srcBaseDir, projectPath).baseRepoPath; } /** @@ -175,8 +171,10 @@ export class SSHRuntime extends RemoteRuntime { private readonly transport: SSHTransport; private readonly ensureReadyProjectPath?: string; private readonly ensureReadyWorkspaceName?: string; + private readonly currentWorkspacePath?: string; /** Cached resolved bgOutputDir (tilde expanded to absolute path) */ private resolvedBgOutputDir: string | null = null; + private readonly projectLayouts = new Map(); constructor( config: SSHRuntimeConfig, @@ -184,6 +182,7 @@ export class SSHRuntime extends RemoteRuntime { options?: { projectPath?: string; workspaceName?: string; + workspacePath?: string; } ) { super(); @@ -193,6 +192,18 @@ export class SSHRuntime extends RemoteRuntime { this.transport = transport; this.ensureReadyProjectPath = options?.projectPath; this.ensureReadyWorkspaceName = options?.workspaceName; + this.currentWorkspacePath = options?.workspacePath; + + if (options?.projectPath && options.workspacePath) { + this.projectLayouts.set( + options.projectPath, + buildRemoteProjectLayout( + this.config.srcBaseDir, + options.projectPath, + path.posix.dirname(options.workspacePath) + ) + ); + } } /** @@ -235,6 +246,94 @@ export class SSHRuntime extends RemoteRuntime { return this.config; } + private getDefaultProjectLayout(projectPath: string): RemoteProjectLayout { + return buildRemoteProjectLayout(this.config.srcBaseDir, projectPath); + } + + private getPreferredProjectLayout(projectPath: string): RemoteProjectLayout { + return this.projectLayouts.get(projectPath) ?? this.getDefaultProjectLayout(projectPath); + } + + private async resolveProjectLayout( + projectPath: string, + workspaceName?: string + ): Promise { + const cached = this.projectLayouts.get(projectPath); + if (cached) { + return cached; + } + + const preferredLayout = this.getDefaultProjectLayout(projectPath); + const legacyLayout = buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath); + const preferredWorkspacePath = + workspaceName != null ? getRemoteWorkspacePath(preferredLayout, workspaceName) : undefined; + const legacyWorkspacePath = + workspaceName != null ? getRemoteWorkspacePath(legacyLayout, workspaceName) : undefined; + + const detectLayoutScript = ` + if ${legacyWorkspacePath ? `test -e ${this.quoteForRemote(legacyWorkspacePath)}` : "false"}; then + echo legacy + elif test -d ${this.quoteForRemote(preferredLayout.projectRoot)}${preferredWorkspacePath ? ` || test -e ${this.quoteForRemote(preferredWorkspacePath)}` : ""}; then + echo preferred + else + echo preferred + fi + `; + + try { + const detection = await execBuffered(this, detectLayoutScript, { + cwd: "/tmp", + timeout: 10, + }); + const layout = detection.stdout.trim() === "legacy" ? legacyLayout : preferredLayout; + this.projectLayouts.set(projectPath, layout); + return layout; + } catch { + this.projectLayouts.set(projectPath, preferredLayout); + return preferredLayout; + } + } + + private getProjectSyncKey(projectId: string): string { + return [ + this.config.host, + this.config.port?.toString() ?? "22", + this.config.identityFile ?? "default", + this.config.srcBaseDir, + projectId, + ].join(":"); + } + + private async computeSnapshotDigest(projectPath: string): Promise { + const refsOutput = await new Promise((resolve, reject) => { + const proc = spawn("git", ["-C", projectPath, "show-ref", "--heads", "--tags"], { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + proc.stdout?.on("data", (chunk: Buffer) => stdoutChunks.push(Buffer.from(chunk))); + proc.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(Buffer.from(chunk))); + proc.once("close", (code) => { + if (code === 0) { + resolve(Buffer.concat(stdoutChunks).toString()); + return; + } + const stderrText = Buffer.concat(stderrChunks).toString().trim(); + reject( + new Error( + stderrText.length > 0 + ? stderrText + : `git show-ref failed with code ${code ?? "unknown"}` + ) + ); + }); + proc.once("error", reject); + }); + + return crypto.createHash("sha256").update(refsOutput).digest("hex"); + } + // ===== RemoteRuntime abstract method implementations ===== protected readonly commandPrefix: string = "SSH"; @@ -251,27 +350,15 @@ export class SSHRuntime extends RemoteRuntime { return cdCommandForSSH(cwd); } - /** - * Handle exit codes for SSH connection pool health tracking. - */ - protected override onExitCode(exitCode: number, _options: ExecOptions, stderr: string): void { - // Connection-level failures should inform transport backoff. The meaning of - // specific exit codes (like 255) is transport-dependent. - if (this.transport.isConnectionFailure(exitCode, stderr)) { - this.transport.reportFailure(truncateSSHError(stderr)); - } else { - this.transport.markHealthy(); - } - } - protected async spawnRemoteProcess( fullCommand: string, - options: ExecOptions + options: ExecOptions & { deadlineMs?: number } ): Promise { return this.transport.spawnRemoteProcess(fullCommand, { forcePTY: options.forcePTY, timeout: options.timeout, abortSignal: options.abortSignal, + deadlineMs: options.deadlineMs, }); } @@ -340,8 +427,20 @@ export class SSHRuntime extends RemoteRuntime { } getWorkspacePath(projectPath: string, workspaceName: string): string { - const projectName = getProjectName(projectPath); - return path.posix.join(this.config.srcBaseDir, projectName, workspaceName); + if ( + this.currentWorkspacePath && + this.ensureReadyProjectPath === projectPath && + this.ensureReadyWorkspaceName === workspaceName + ) { + return this.currentWorkspacePath; + } + + const cachedLayout = this.projectLayouts.get(projectPath); + if (cachedLayout) { + return getRemoteWorkspacePath(cachedLayout, workspaceName); + } + + return getRemoteWorkspacePath(this.getDefaultProjectLayout(projectPath), workspaceName); } /** @@ -349,7 +448,7 @@ export class SSHRuntime extends RemoteRuntime { * All worktree-based workspaces share this object store. */ private getBaseRepoPath(projectPath: string): string { - return computeBaseRepoPath(this.config.srcBaseDir, projectPath); + return this.getPreferredProjectLayout(projectPath).baseRepoPath; } /** @@ -362,7 +461,8 @@ export class SSHRuntime extends RemoteRuntime { initLogger: InitLogger, abortSignal?: AbortSignal ): Promise { - const baseRepoPath = this.getBaseRepoPath(projectPath); + const layout = await this.resolveProjectLayout(projectPath); + const baseRepoPath = layout.baseRepoPath; const baseRepoPathArg = expandTildeForSSH(baseRepoPath); const check = await execBuffered(this, `test -d ${baseRepoPathArg}`, { @@ -510,9 +610,7 @@ export class SSHRuntime extends RemoteRuntime { workspaceName: string, branchName: string ): Promise { - const branchMap = await this.readWorkspaceBranchMap(projectPath); - branchMap[workspaceName] = branchName; - await this.writeWorkspaceBranchMap(projectPath, branchMap); + await this.writeWorkspaceBranchMetadata(projectPath, workspaceName, branchName); } private async updateWorkspaceBranchMapping( @@ -520,19 +618,19 @@ export class SSHRuntime extends RemoteRuntime { oldWorkspaceName: string, newWorkspaceName: string ): Promise { - const branchMap = await this.readWorkspaceBranchMap(projectPath); - const branchName = branchMap[oldWorkspaceName]?.trim() || oldWorkspaceName; - delete branchMap[oldWorkspaceName]; - branchMap[newWorkspaceName] = branchName; - await this.writeWorkspaceBranchMap(projectPath, branchMap); + const branchName = + (await this.readWorkspaceBranchMetadata(projectPath, oldWorkspaceName))?.trim() ?? + oldWorkspaceName; + await this.writeWorkspaceBranchMetadata(projectPath, newWorkspaceName, branchName); + await this.deletePersistedWorkspaceBranchMapping(projectPath, oldWorkspaceName); } private async getPersistedWorkspaceBranchName( projectPath: string, workspaceName: string ): Promise { - const branchName = (await this.readWorkspaceBranchMap(projectPath))[workspaceName]?.trim(); - return branchName || null; + const branchName = (await this.readWorkspaceBranchMetadata(projectPath, workspaceName))?.trim(); + return branchName ?? null; } private async deletePersistedWorkspaceBranchMapping( @@ -540,76 +638,105 @@ export class SSHRuntime extends RemoteRuntime { workspaceName: string ): Promise { try { - const branchMap = await this.readWorkspaceBranchMap(projectPath); - if (!(workspaceName in branchMap)) { - return; - } - delete branchMap[workspaceName]; - await this.writeWorkspaceBranchMap(projectPath, branchMap); + await this.resolveProjectLayout(projectPath, workspaceName); + const metadataPath = getWorkspaceMetadataPath( + this.getPreferredProjectLayout(projectPath), + workspaceName + ); + await execBuffered(this, `rm -f ${this.quoteForRemote(metadataPath)}`, { + cwd: "/tmp", + timeout: 10, + }).catch(() => undefined); } catch { // Best-effort cleanup after delete; future creates overwrite any stale entry. } } - private async readWorkspaceBranchMap(projectPath: string): Promise> { + private async readWorkspaceBranchMetadata( + projectPath: string, + workspaceName: string + ): Promise { try { - const contents = await streamToString( - this.readFile(this.getWorkspaceBranchMapPath(projectPath)) + await this.resolveProjectLayout(projectPath, workspaceName); + const metadataPath = getWorkspaceMetadataPath( + this.getPreferredProjectLayout(projectPath), + workspaceName ); + const contents = await streamToString(this.readFile(metadataPath)); const parsed: unknown = JSON.parse(contents); - if (typeof parsed !== "object" || parsed === null) { - return {}; + if (typeof parsed === "object" && parsed !== null) { + const branchName = + "branchName" in parsed && typeof parsed.branchName === "string" + ? parsed.branchName.trim() + : ""; + if (branchName.length > 0) { + return branchName; + } } - return Object.fromEntries( - Object.entries(parsed).filter(([workspaceName, branchName]) => { - return ( - workspaceName.trim().length > 0 && - typeof branchName === "string" && - branchName.trim().length > 0 - ); - }) - ); } catch { - return {}; + // Fall back to the legacy shared manifest for pre-migration workspaces. } + + return this.readLegacyWorkspaceBranchEntry(projectPath, workspaceName); } - private async writeWorkspaceBranchMap( + private async writeWorkspaceBranchMetadata( projectPath: string, - branchMap: Record + workspaceName: string, + branchName: string ): Promise { - const branchMapPath = this.getWorkspaceBranchMapPath(projectPath); - if (Object.keys(branchMap).length === 0) { - await execBuffered(this, `rm -f ${this.quoteForRemote(branchMapPath)}`, { + const layout = await this.resolveProjectLayout(projectPath, workspaceName); + const metadataPath = getWorkspaceMetadataPath(layout, workspaceName); + const mkdirResult = await execBuffered( + this, + `mkdir -p ${this.quoteForRemote(layout.workspaceMetadataDir)}`, + { cwd: "/tmp", timeout: 10, - }).catch(() => undefined); - return; - } - - const parentDir = path.posix.dirname(branchMapPath); - const mkdirResult = await execBuffered(this, `mkdir -p ${this.quoteForRemote(parentDir)}`, { - cwd: "/tmp", - timeout: 10, - }); + } + ); if (mkdirResult.exitCode !== 0) { throw new Error( - `Failed to prepare remote workspace branch map: ${mkdirResult.stderr || mkdirResult.stdout}` + `Failed to prepare remote workspace metadata: ${mkdirResult.stderr || mkdirResult.stdout}` ); } - const writer = this.writeFile(branchMapPath).getWriter(); + const payload = { + workspaceName, + branchName, + }; + const writer = this.writeFile(metadataPath).getWriter(); try { - await writer.write(new TextEncoder().encode(`${JSON.stringify(branchMap, null, 2)}\n`)); + await writer.write(new TextEncoder().encode(`${JSON.stringify(payload, null, 2)}\n`)); } finally { await writer.close(); } } - private getWorkspaceBranchMapPath(projectPath: string): string { + private async readLegacyWorkspaceBranchEntry( + projectPath: string, + workspaceName: string + ): Promise { + try { + const contents = await streamToString( + this.readFile(this.getLegacyWorkspaceBranchMapPath(projectPath)) + ); + const parsed: unknown = JSON.parse(contents); + if (typeof parsed !== "object" || parsed === null) { + return null; + } + const branchMap = parsed as Record; + const branchName = + typeof branchMap[workspaceName] === "string" ? branchMap[workspaceName].trim() : ""; + return branchName.length > 0 ? branchName : null; + } catch { + return null; + } + } + + private getLegacyWorkspaceBranchMapPath(projectPath: string): string { return path.posix.join( - this.config.srcBaseDir, - getProjectName(projectPath), + buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath).projectRoot, ".mux-workspace-branches.json" ); } @@ -821,10 +948,11 @@ export class SSHRuntime extends RemoteRuntime { return { ready: false, error: "Aborted", errorType: "runtime_start_failed" }; } - const workspacePath = this.getWorkspacePath( + const layout = await this.resolveProjectLayout( this.ensureReadyProjectPath, this.ensureReadyWorkspaceName ); + const workspacePath = getRemoteWorkspacePath(layout, this.ensureReadyWorkspaceName); const gitDir = path.posix.join(workspacePath, ".git"); const gitDirProbe = this.quoteForRemote(gitDir); @@ -981,12 +1109,10 @@ export class SSHRuntime extends RemoteRuntime { */ private async transferBundleToRemote( projectPath: string, + remoteBundlePath: string, initLogger: InitLogger, abortSignal?: AbortSignal ): Promise { - const timestamp = Date.now(); - const remoteBundlePath = `~/.mux-bundle-${timestamp}.bundle`; - await this.transport.acquireConnection({ abortSignal, onWait: (waitMs) => logSSHBackoffWait(initLogger, waitMs), @@ -1080,49 +1206,147 @@ export class SSHRuntime extends RemoteRuntime { initLogger: InitLogger, abortSignal?: AbortSignal ): Promise { - const baseRepoPathArg = await this.ensureBaseRepo(projectPath, initLogger, abortSignal); + if (abortSignal?.aborted) { + throw new Error("Operation aborted"); + } - const remoteBundlePath = await this.transferBundleToRemote( - projectPath, - initLogger, - abortSignal - ); - const remoteBundlePathArg = this.quoteForRemote(remoteBundlePath); + const layout = await this.resolveProjectLayout(projectPath); + const snapshotDigest = await this.computeSnapshotDigest(projectPath); + const snapshotMarkerPath = getSnapshotMarkerPath(layout, snapshotDigest); + const currentSnapshotPath = path.posix.join(layout.snapshotMarkerDir, "current"); + const projectKey = this.getProjectSyncKey(layout.projectId); + const snapshotKey = `${projectKey}:${snapshotDigest}`; + + await projectSyncCoordinator.runSnapshotSync( + { projectKey, snapshotKey, abortSignal }, + async (sharedAbortSignal) => { + const baseRepoPathArg = await this.ensureBaseRepo( + projectPath, + initLogger, + sharedAbortSignal + ); - try { - // Import branches and tags from the bundle into the shared bare repo. - // Branches land in refs/mux-bundle/* (staging namespace) instead of - // refs/heads/* to avoid colliding with branches checked out in existing - // worktrees — git refuses to update any ref checked out in a worktree. - // Tags go directly to refs/tags/* (they're never checked out). - initLogger.logStep("Importing bundle into shared base repository..."); - const fetchResult = await execBuffered( - this, - `git -C ${baseRepoPathArg} fetch ${remoteBundlePathArg} '+refs/heads/*:${BUNDLE_REF_PREFIX}*' '+refs/tags/*:refs/tags/*'`, - { cwd: "/tmp", timeout: 300, abortSignal } - ); - if (fetchResult.exitCode !== 0) { - throw new Error( - `Failed to import bundle into base repo: ${fetchResult.stderr || fetchResult.stdout}` + const snapshotStatusCheck = await execBuffered( + this, + [ + 'current_snapshot=""', + `if test -f ${this.quoteForRemote(currentSnapshotPath)}; then`, + ` current_snapshot=$(tr -d '\\n' < ${this.quoteForRemote(currentSnapshotPath)})`, + "fi", + `if test -f ${this.quoteForRemote(snapshotMarkerPath)} && test "$current_snapshot" = ${shescape.quote(snapshotDigest)}; then`, + ` bundle_ref=$(git -C ${baseRepoPathArg} for-each-ref --count=1 --format='%(refname)' ${shescape.quote(BUNDLE_REF_PREFIX)})`, + ' if test -n "$bundle_ref"; then', + " echo reusable", + " else", + " echo stale-marker", + " fi", + `elif test -f ${this.quoteForRemote(snapshotMarkerPath)} || test -n "$current_snapshot"; then`, + " echo stale-marker", + "else", + " echo missing", + "fi", + ].join("\n"), + { cwd: "/tmp", timeout: 10, abortSignal: sharedAbortSignal } ); - } + const snapshotStatus = snapshotStatusCheck.stdout.trim(); + if (snapshotStatus === "reusable") { + await this.refreshBaseRepoOrigin( + projectPath, + baseRepoPathArg, + initLogger, + sharedAbortSignal + ); + initLogger.logStep("Reusing existing remote project snapshot"); + return; + } + if (snapshotStatus === "stale-marker") { + initLogger.logStep( + "Remote snapshot marker found without matching imported refs; reimporting bundle..." + ); + } - // Keep the bare base repo's origin aligned with the local project so later - // fetchOriginTrunk() calls base new worktrees on the intended remote. - await this.refreshBaseRepoOrigin(projectPath, baseRepoPathArg, initLogger, abortSignal); + const remoteBundlePath = path.posix.join( + "~/.mux-bundles", + layout.projectId, + `${snapshotDigest}.bundle` + ); + const remoteBundlePathArg = this.quoteForRemote(remoteBundlePath); + const remoteBundleParentDir = path.posix.dirname(remoteBundlePath); + const prepareRemoteDirs = await execBuffered( + this, + `mkdir -p ${this.quoteForRemote(remoteBundleParentDir)} ${this.quoteForRemote(layout.snapshotMarkerDir)}`, + { cwd: "/tmp", timeout: 10, abortSignal: sharedAbortSignal } + ); + if (prepareRemoteDirs.exitCode !== 0) { + throw new Error( + `Failed to prepare remote snapshot directories: ${prepareRemoteDirs.stderr || prepareRemoteDirs.stdout}` + ); + } - initLogger.logStep("Repository synced to base successfully"); - } finally { - // Best-effort cleanup of the remote bundle file. - try { - await execBuffered(this, `rm -f ${remoteBundlePathArg}`, { - cwd: "/tmp", - timeout: 10, - }); - } catch { - // Ignore cleanup errors. + await this.transferBundleToRemote( + projectPath, + remoteBundlePath, + initLogger, + sharedAbortSignal + ); + + try { + // Import branches and tags from the bundle into the shared bare repo. + // Branches land in refs/mux-bundle/* (staging namespace) instead of + // refs/heads/* to avoid colliding with branches checked out in existing + // worktrees — git refuses to update any ref checked out in a worktree. + // Tags go directly to refs/tags/* (they're never checked out). + initLogger.logStep("Importing bundle into shared base repository..."); + const fetchResult = await execBuffered( + this, + `git -C ${baseRepoPathArg} fetch ${remoteBundlePathArg} '+refs/heads/*:${BUNDLE_REF_PREFIX}*' '+refs/tags/*:refs/tags/*'`, + { cwd: "/tmp", timeout: 300, abortSignal: sharedAbortSignal } + ); + if (fetchResult.exitCode !== 0) { + throw new Error( + `Failed to import bundle into base repo: ${fetchResult.stderr || fetchResult.stdout}` + ); + } + + await this.refreshBaseRepoOrigin( + projectPath, + baseRepoPathArg, + initLogger, + sharedAbortSignal + ); + + const markerWriter = this.writeFile(snapshotMarkerPath).getWriter(); + try { + await markerWriter.write( + new TextEncoder().encode( + `${JSON.stringify({ snapshotDigest, importedAt: new Date().toISOString() }, null, 2)}\n` + ) + ); + } finally { + await markerWriter.close(); + } + + const currentSnapshotWriter = this.writeFile(currentSnapshotPath).getWriter(); + try { + await currentSnapshotWriter.write(new TextEncoder().encode(`${snapshotDigest}\n`)); + } finally { + await currentSnapshotWriter.close(); + } + + initLogger.logStep("Repository synced to base successfully"); + } finally { + // Best-effort cleanup of the remote bundle file. + try { + await execBuffered(this, `rm -f ${remoteBundlePathArg}`, { + cwd: "/tmp", + timeout: 10, + }); + } catch { + // Ignore cleanup errors. + } + } } - } + ); } /** Get origin URL from local project for setting on the remote base repo. */ @@ -1136,8 +1360,10 @@ export class SSHRuntime extends RemoteRuntime { async createWorkspace(params: WorkspaceCreationParams): Promise { try { const { projectPath, directoryName, initLogger, abortSignal } = params; + const layout = await this.resolveProjectLayout(projectPath, directoryName); + this.projectLayouts.set(projectPath, layout); // Workspace directories follow the persisted workspace name; branch checkout happens later. - const workspacePath = this.getWorkspacePath(projectPath, directoryName); + const workspacePath = getRemoteWorkspacePath(layout, directoryName); // Prepare parent directory for git clone (fast - returns immediately) // Note: git clone will create the workspace directory itself during initWorkspace, @@ -1189,6 +1415,15 @@ export class SSHRuntime extends RemoteRuntime { } async initWorkspace(params: WorkspaceInitParams): Promise { + this.projectLayouts.set( + params.projectPath, + buildRemoteProjectLayout( + this.config.srcBaseDir, + params.projectPath, + path.posix.dirname(params.workspacePath) + ) + ); + // Disable git hooks for untrusted projects (prevents post-checkout execution) const nhp = gitNoHooksPrefix(params.trusted); @@ -1534,9 +1769,9 @@ export class SSHRuntime extends RemoteRuntime { if (abortSignal?.aborted) { return { success: false, error: "Rename operation aborted" }; } - // Compute workspace paths using canonical method - const oldPath = this.getWorkspacePath(projectPath, oldName); - const newPath = this.getWorkspacePath(projectPath, newName); + const layout = await this.resolveProjectLayout(projectPath, oldName); + const oldPath = getRemoteWorkspacePath(layout, oldName); + const newPath = getRemoteWorkspacePath(layout, newName); try { const expandedOldPath = expandTildeForSSH(oldPath); @@ -1609,8 +1844,8 @@ export class SSHRuntime extends RemoteRuntime { // Disable git hooks for untrusted projects const nhp = gitNoHooksPrefix(trusted); - // Compute workspace path using canonical method - const deletedPath = this.getWorkspacePath(projectPath, workspaceName); + const layout = await this.resolveProjectLayout(projectPath, workspaceName); + const deletedPath = getRemoteWorkspacePath(layout, workspaceName); try { // Combine all pre-deletion checks into a single bash script to minimize round trips @@ -1828,9 +2063,9 @@ export class SSHRuntime extends RemoteRuntime { async forkWorkspace(params: WorkspaceForkParams): Promise { const { projectPath, sourceWorkspaceName, newWorkspaceName, initLogger, abortSignal } = params; - // Compute workspace paths using canonical method - const sourceWorkspacePath = this.getWorkspacePath(projectPath, sourceWorkspaceName); - const newWorkspacePath = this.getWorkspacePath(projectPath, newWorkspaceName); + const layout = await this.resolveProjectLayout(projectPath, sourceWorkspaceName); + const sourceWorkspacePath = getRemoteWorkspacePath(layout, sourceWorkspaceName); + const newWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); // For SSH commands, tilde must be expanded using $HOME - plain quoting won't expand it. const sourceWorkspacePathArg = expandTildeForSSH(sourceWorkspacePath); diff --git a/src/node/runtime/openSshMasterPool.test.ts b/src/node/runtime/openSshMasterPool.test.ts new file mode 100644 index 0000000000..a0759b07a5 --- /dev/null +++ b/src/node/runtime/openSshMasterPool.test.ts @@ -0,0 +1,359 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import type { spawn as spawnProcess } from "child_process"; +import { EventEmitter } from "events"; +import { PassThrough } from "stream"; +import { OpenSSHMasterPool, getShardedControlPath } from "./openSshMasterPool"; +import type { SSHConnectionConfig } from "./sshConnectionPool"; + +class FakeChildProcess extends EventEmitter { + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + readonly stdin = new PassThrough(); + pid = 1234; + exitCode: number | null = null; + signalCode: string | null = null; + + kill(_signal?: string): boolean { + this.exitCode ??= 0; + this.emit("exit", this.exitCode, this.signalCode); + this.emit("close", this.exitCode, this.signalCode); + return true; + } +} + +describe("getShardedControlPath", () => { + test("is deterministic per shard and unique across shards", () => { + const config: SSHConnectionConfig = { host: "example.com" }; + expect(getShardedControlPath(config, 0)).toBe(getShardedControlPath(config, 0)); + expect(getShardedControlPath(config, 0)).not.toBe(getShardedControlPath(config, 1)); + }); +}); + +describe("OpenSSHMasterPool", () => { + const masterProcesses = new Map(); + + afterEach(() => { + masterProcesses.clear(); + }); + + test("reuses an existing shard until capacity is reached, then scales out", async () => { + const spawnCalls: Array<{ command: string; args: string[] }> = []; + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 2, + maxShardsPerHost: 4, + sleep: () => Promise.resolve(), + spawnProcess: ((command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + spawnCalls.push({ command, args: normalizedArgs }); + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + const second = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + const third = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + + expect(first.controlPath).toBe(second.controlPath); + expect(third.controlPath).not.toBe(first.controlPath); + expect(spawnCalls.filter((call) => call.args.includes("-M"))).toHaveLength(2); + + first.release(); + second.release(); + third.release(); + pool.clearAll(); + }); + + test("does not lease a shard until its master is ready", async () => { + let ready = false; + let releaseStartupWait: (() => void) | undefined; + const startupWait = new Promise((resolve) => { + releaseStartupWait = resolve; + }); + + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 2, + maxShardsPerHost: 1, + sleep: () => startupWait, + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = ready ? 0 : 1; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const firstPromise = pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + let secondResolved = false; + const secondPromise = pool + .acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }) + .then((lease) => { + secondResolved = true; + return lease; + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(secondResolved).toBe(false); + + ready = true; + releaseStartupWait?.(); + + const first = await firstPromise; + const second = await secondPromise; + expect(first.controlPath).toBe(second.controlPath); + + first.release(); + second.release(); + pool.clearAll(); + }); + + test("waits for shard backoff before reusing a failed master", async () => { + let releaseBackoffWait: (() => void) | undefined; + const backoffWait = new Promise((resolve) => { + releaseBackoffWait = resolve; + }); + + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + sleep: () => backoffWait, + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + first.reportFailure("ssh exited 255"); + first.release(); + + let secondResolved = false; + const secondPromise = pool + .acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }) + .then((lease) => { + secondResolved = true; + return lease; + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(secondResolved).toBe(false); + + const internals = pool as unknown as { + hostGroups: Map }>; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (!shard) { + throw new Error("Expected a tracked shard"); + } + shard.health.backoffUntil = new Date(Date.now() - 1); + releaseBackoffWait?.(); + + const second = await secondPromise; + expect(second.controlPath).toBe(first.controlPath); + + second.release(); + pool.clearAll(); + }); + + test("ensureReadyMaster ignores saturated exec slots when a shard is already ready", async () => { + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + sleep: () => Promise.resolve(), + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const lease = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + + await pool.ensureReadyMaster(config, { maxWaitMs: 0, timeoutMs: 1000 }); + + lease.release(); + pool.clearAll(); + }); + + test("retries transient shard startup failures within the maxWait budget", async () => { + let startupAttempts = 0; + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + sleep: () => { + const internals = pool as unknown as { + hostGroups: Map }>; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (shard?.health.backoffUntil) { + shard.health.backoffUntil = new Date(Date.now() - 1); + } + return Promise.resolve(); + }, + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + startupAttempts += 1; + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg && startupAttempts > 1) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + if (startupAttempts === 1) { + queueMicrotask(() => { + proc.emit("error", new Error("transient startup failure")); + proc.exitCode = 1; + proc.emit("exit", proc.exitCode, null); + proc.emit("close", proc.exitCode, null); + }); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const lease = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + + expect(startupAttempts).toBe(2); + + lease.release(); + pool.clearAll(); + }); + + test("records a failed master startup only once when ssh emits error and exit", async () => { + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + sleep: () => Promise.resolve(), + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + queueMicrotask(() => { + proc.emit("error", new Error("spawn ENOENT")); + proc.exitCode = 1; + proc.emit("exit", proc.exitCode, null); + proc.emit("close", proc.exitCode, null); + }); + return proc as never; + } + + queueMicrotask(() => { + proc.exitCode = 1; + proc.emit("close", proc.exitCode, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + try { + await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + throw new Error("Expected acquireLease to reject"); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("spawn ENOENT"); + } + + const internals = pool as unknown as { + hostGroups: Map }>; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (!shard) { + throw new Error("Expected a tracked shard"); + } + expect(shard.health.consecutiveFailures).toBe(1); + + pool.clearAll(); + }); +}); diff --git a/src/node/runtime/openSshMasterPool.ts b/src/node/runtime/openSshMasterPool.ts new file mode 100644 index 0000000000..2ff89ca465 --- /dev/null +++ b/src/node/runtime/openSshMasterPool.ts @@ -0,0 +1,756 @@ +import * as crypto from "crypto"; +import * as os from "os"; +import * as path from "path"; +import { spawn, type ChildProcess } from "child_process"; +import { HOST_KEY_APPROVAL_TIMEOUT_MS } from "@/common/constants/ssh"; +import { formatSshEndpoint } from "@/common/utils/ssh/formatSshEndpoint"; +import { getErrorMessage } from "@/common/utils/errors"; +import { log } from "@/node/services/log"; +import { + appendOpenSSHHostKeyPolicyArgs, + getSshPromptService, + isInteractiveHostKeyApprovalAvailable, + type ConnectionHealth, + type SSHConnectionConfig, +} from "./sshConnectionPool"; +import { createMediatedAskpassSession } from "./openSshPromptMediation"; + +const DEFAULT_MASTER_START_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_WAIT_MS = 2 * 60 * 1000; +const DEFAULT_MAX_SESSIONS_PER_SHARD = 4; +const DEFAULT_MAX_SHARDS_PER_HOST = 8; +const SHARD_IDLE_TTL_MS = 60_000; +const STARTUP_POLL_INTERVAL_MS = 50; +const BACKOFF_SCHEDULE = [1, 2, 4, 7, 10]; + +type SleepFn = (ms: number, abortSignal?: AbortSignal) => Promise; + +type SpawnFn = typeof spawn; + +interface AcquireLeaseOptions { + timeoutMs?: number; + maxWaitMs?: number; + abortSignal?: AbortSignal; + onWait?: (waitMs: number) => void; +} + +export interface OpenSSHMasterLease { + controlPath: string; + shardId: string; + release(): void; + reportFailure(error: string): void; + markHealthy(): void; +} + +interface MasterShard { + id: number; + shardId: string; + controlPath: string; + process?: ChildProcess; + startup?: Promise; + ready: boolean; + stderr: string; + stopping: boolean; + inflight: number; + lastUsedAt: number; + idleTimer?: ReturnType; + health: ConnectionHealth; +} + +interface HostGroup { + config: SSHConnectionConfig; + shards: MasterShard[]; + nextShardId: number; +} + +interface MasterPoolOptions { + spawnProcess?: SpawnFn; + sleep?: SleepFn; + maxSessionsPerShard?: number; + maxShardsPerHost?: number; + startupPollIntervalMs?: number; + defaultMasterStartTimeoutMs?: number; + defaultMaxWaitMs?: number; + shardIdleTtlMs?: number; +} + +function withJitter(seconds: number): number { + const jitterFactor = 0.8 + Math.random() * 0.4; + return seconds * jitterFactor; +} + +async function sleepWithAbort(ms: number, abortSignal?: AbortSignal): Promise { + if (ms <= 0) { + return; + } + if (abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + resolve(); + }, ms); + + const onAbort = () => { + cleanup(); + reject(new Error("Operation aborted")); + }; + + const cleanup = () => { + clearTimeout(timer); + abortSignal?.removeEventListener("abort", onAbort); + }; + + abortSignal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +function makeConnectionKey(config: SSHConnectionConfig): string { + return [ + os.userInfo().username, + config.host, + config.port?.toString() ?? "22", + config.identityFile ?? "default", + ].join(":"); +} + +function hashKey(key: string): string { + return crypto.createHash("sha256").update(key).digest("hex").slice(0, 12); +} + +export function getShardedControlPath(config: SSHConnectionConfig, shardId = 0): string { + const key = makeConnectionKey(config); + return path.join(os.tmpdir(), `mux-ssh-${hashKey(`${key}:${shardId}`)}`); +} + +function createInitialHealth(): ConnectionHealth { + return { + status: "unknown", + consecutiveFailures: 0, + }; +} + +function isProcessAlive(proc: ChildProcess | undefined): boolean { + return proc != null && proc.exitCode == null && proc.signalCode == null; +} + +export class OpenSSHMasterPool { + private readonly hostGroups = new Map(); + private readonly spawnProcess: SpawnFn; + private readonly sleep: SleepFn; + private readonly maxSessionsPerShard: number; + private readonly maxShardsPerHost: number; + private readonly startupPollIntervalMs: number; + private readonly defaultMasterStartTimeoutMs: number; + private readonly defaultMaxWaitMs: number; + private readonly shardIdleTtlMs: number; + + constructor(options: MasterPoolOptions = {}) { + this.spawnProcess = options.spawnProcess ?? spawn; + this.sleep = options.sleep ?? sleepWithAbort; + this.maxSessionsPerShard = options.maxSessionsPerShard ?? DEFAULT_MAX_SESSIONS_PER_SHARD; + this.maxShardsPerHost = options.maxShardsPerHost ?? DEFAULT_MAX_SHARDS_PER_HOST; + this.startupPollIntervalMs = options.startupPollIntervalMs ?? STARTUP_POLL_INTERVAL_MS; + this.defaultMasterStartTimeoutMs = + options.defaultMasterStartTimeoutMs ?? DEFAULT_MASTER_START_TIMEOUT_MS; + this.defaultMaxWaitMs = options.defaultMaxWaitMs ?? DEFAULT_MAX_WAIT_MS; + this.shardIdleTtlMs = options.shardIdleTtlMs ?? SHARD_IDLE_TTL_MS; + } + + async ensureConnection( + config: SSHConnectionConfig, + options?: AcquireLeaseOptions + ): Promise { + const lease = await this.acquireLease(config, options); + lease.release(); + } + + async ensureReadyMaster( + config: SSHConnectionConfig, + options?: AcquireLeaseOptions + ): Promise { + const maxWaitMs = options?.maxWaitMs ?? this.defaultMaxWaitMs; + const defaultStartTimeoutMs = options?.timeoutMs ?? this.defaultMasterStartTimeoutMs; + const deadlineMs = Date.now() + maxWaitMs; + const key = makeConnectionKey(config); + const hostGroup = this.getOrCreateHostGroup(key, config); + let lastStartError: Error | undefined; + + while (true) { + if (options?.abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + + this.trimExitedShards(hostGroup); + if (this.pickReadyShard(hostGroup)) { + return; + } + + const restartable = hostGroup.shards.find((shard) => { + return ( + !isProcessAlive(shard.process) && + shard.startup == null && + (shard.health.backoffUntil == null || shard.health.backoffUntil.getTime() <= Date.now()) + ); + }); + if (restartable) { + try { + const startupTimeoutMs = + maxWaitMs === 0 + ? defaultStartTimeoutMs + : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); + await this.startShard(hostGroup, restartable, startupTimeoutMs, options?.abortSignal); + return; + } catch (error) { + if (options?.abortSignal?.aborted) { + throw error; + } + lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); + } + } + + if (hostGroup.shards.length < this.maxShardsPerHost) { + const shard = this.createShard(hostGroup); + try { + const startupTimeoutMs = + maxWaitMs === 0 + ? defaultStartTimeoutMs + : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); + await this.startShard(hostGroup, shard, startupTimeoutMs, options?.abortSignal); + return; + } catch (error) { + if (options?.abortSignal?.aborted) { + throw error; + } + lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); + } + } + + const nextBackoffMs = this.getNextBackoffWaitMs(hostGroup); + const remainingMs = deadlineMs - Date.now(); + if (remainingMs <= 0) { + if (lastStartError) { + throw lastStartError; + } + throw new Error( + `SSH master pool for ${config.host} did not become available within ${maxWaitMs}ms` + ); + } + + const waitMs = Math.min( + remainingMs, + nextBackoffMs ?? this.startupPollIntervalMs, + this.startupPollIntervalMs + ); + options?.onWait?.(waitMs); + await this.sleep(waitMs, options?.abortSignal); + } + } + + async acquireLease( + config: SSHConnectionConfig, + options?: AcquireLeaseOptions + ): Promise { + const maxWaitMs = options?.maxWaitMs ?? this.defaultMaxWaitMs; + const defaultStartTimeoutMs = options?.timeoutMs ?? this.defaultMasterStartTimeoutMs; + const deadlineMs = Date.now() + maxWaitMs; + const key = makeConnectionKey(config); + const hostGroup = this.getOrCreateHostGroup(key, config); + let lastStartError: Error | undefined; + + while (true) { + if (options?.abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + + this.trimExitedShards(hostGroup); + const available = this.pickAvailableShard(hostGroup); + if (available) { + return this.reserveShard(available); + } + + const restartable = hostGroup.shards.find((shard) => { + return ( + !isProcessAlive(shard.process) && + shard.startup == null && + (shard.health.backoffUntil == null || shard.health.backoffUntil.getTime() <= Date.now()) + ); + }); + if (restartable) { + try { + const startupTimeoutMs = + maxWaitMs === 0 + ? defaultStartTimeoutMs + : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); + await this.startShard(hostGroup, restartable, startupTimeoutMs, options?.abortSignal); + return this.reserveShard(restartable); + } catch (error) { + if (options?.abortSignal?.aborted) { + throw error; + } + lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); + } + } + + if (hostGroup.shards.length < this.maxShardsPerHost) { + const shard = this.createShard(hostGroup); + try { + const startupTimeoutMs = + maxWaitMs === 0 + ? defaultStartTimeoutMs + : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); + await this.startShard(hostGroup, shard, startupTimeoutMs, options?.abortSignal); + return this.reserveShard(shard); + } catch (error) { + if (options?.abortSignal?.aborted) { + throw error; + } + lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); + } + } + + const nextBackoffMs = this.getNextBackoffWaitMs(hostGroup); + const remainingMs = deadlineMs - Date.now(); + if (remainingMs <= 0) { + if (lastStartError) { + throw lastStartError; + } + throw new Error( + `SSH master pool for ${config.host} did not become available within ${maxWaitMs}ms` + ); + } + + const waitMs = Math.min( + remainingMs, + nextBackoffMs ?? this.startupPollIntervalMs, + this.startupPollIntervalMs + ); + options?.onWait?.(waitMs); + await this.sleep(waitMs, options?.abortSignal); + } + } + + clearAll(): void { + for (const group of this.hostGroups.values()) { + for (const shard of group.shards) { + this.disposeShard(group, shard, { expected: true }); + } + } + this.hostGroups.clear(); + } + + private getOrCreateHostGroup(key: string, config: SSHConnectionConfig): HostGroup { + const existing = this.hostGroups.get(key); + if (existing) { + return existing; + } + + const group: HostGroup = { + config, + shards: [], + nextShardId: 0, + }; + this.hostGroups.set(key, group); + return group; + } + + private createShard(group: HostGroup): MasterShard { + const id = group.nextShardId++; + const shard: MasterShard = { + id, + shardId: `shard-${id}`, + controlPath: getShardedControlPath(group.config, id), + inflight: 0, + lastUsedAt: Date.now(), + ready: false, + stderr: "", + stopping: false, + health: createInitialHealth(), + }; + group.shards.push(shard); + return shard; + } + + private getReadyShards(group: HostGroup): MasterShard[] { + return group.shards.filter((shard) => { + const backoffUntilMs = shard.health.backoffUntil?.getTime(); + return ( + shard.ready && + isProcessAlive(shard.process) && + (backoffUntilMs == null || backoffUntilMs <= Date.now()) + ); + }); + } + + private pickReadyShard(group: HostGroup): MasterShard | undefined { + return this.getReadyShards(group).sort((left, right) => left.inflight - right.inflight)[0]; + } + + private pickAvailableShard(group: HostGroup): MasterShard | undefined { + return this.getReadyShards(group) + .filter((shard) => shard.inflight < this.maxSessionsPerShard) + .sort((left, right) => left.inflight - right.inflight)[0]; + } + + private reserveShard(shard: MasterShard): OpenSSHMasterLease { + clearTimeout(shard.idleTimer); + shard.idleTimer = undefined; + shard.inflight += 1; + shard.lastUsedAt = Date.now(); + + let released = false; + const release = () => { + if (released) { + return; + } + released = true; + shard.inflight = Math.max(0, shard.inflight - 1); + shard.lastUsedAt = Date.now(); + if (shard.inflight === 0) { + this.scheduleIdleDisposal(shard); + } + }; + + return { + controlPath: shard.controlPath, + shardId: shard.shardId, + release, + markHealthy: () => { + shard.health = { + status: "healthy", + consecutiveFailures: 0, + lastSuccess: new Date(), + }; + }, + reportFailure: (error: string) => { + this.recordShardFailure(shard, error); + }, + }; + } + + private scheduleIdleDisposal(shard: MasterShard): void { + clearTimeout(shard.idleTimer); + shard.idleTimer = setTimeout(() => { + const hostGroup = this.findHostGroupForShard(shard); + if (!hostGroup) { + return; + } + if (shard.inflight !== 0) { + return; + } + this.disposeShard(hostGroup, shard, { expected: true }); + }, this.shardIdleTtlMs); + shard.idleTimer.unref?.(); + } + + private findHostGroupForShard(target: MasterShard): HostGroup | undefined { + for (const group of this.hostGroups.values()) { + if (group.shards.includes(target)) { + return group; + } + } + return undefined; + } + + private async startShard( + group: HostGroup, + shard: MasterShard, + timeoutMs: number, + abortSignal?: AbortSignal + ): Promise { + if (shard.startup) { + return shard.startup; + } + + shard.startup = this.startShardInner(group, shard, timeoutMs, abortSignal).finally(() => { + shard.startup = undefined; + }); + return shard.startup; + } + + private async startShardInner( + group: HostGroup, + shard: MasterShard, + timeoutMs: number, + abortSignal?: AbortSignal + ): Promise { + const canPromptInteractively = isInteractiveHostKeyApprovalAvailable(); + const promptService = getSshPromptService(); + let stderr = ""; + let scheduleKill = (_ms: number) => undefined; + const extendDeadline = (ms: number) => scheduleKill(ms); + + const askpass = + canPromptInteractively && promptService + ? await createMediatedAskpassSession({ + sshPromptService: promptService, + promptPolicy: { + allowHostKey: true, + allowCredential: false, + }, + dedupeKey: `${formatSshEndpoint(group.config.host, group.config.port ?? 22)}:${shard.shardId}`, + getStderrContext: () => stderr, + onHostKeyPromptStarted: () => { + extendDeadline(HOST_KEY_APPROVAL_TIMEOUT_MS); + }, + }) + : undefined; + + const connectTimeout = canPromptInteractively + ? Math.ceil(HOST_KEY_APPROVAL_TIMEOUT_MS / 1000) + : Math.min(Math.ceil(timeoutMs / 1000), 15); + const args = this.buildMasterArgs(group.config, shard.controlPath, connectTimeout); + const proc = this.spawnProcess("ssh", args, { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + ...(askpass ? { env: { ...process.env, ...askpass.env } } : {}), + }); + + shard.process = proc; + shard.ready = false; + shard.stderr = ""; + shard.stopping = false; + + let shardFailureRecorded = false; + const recordShardFailureOnce = (error: string) => { + if (shardFailureRecorded) { + return; + } + shardFailureRecorded = true; + this.recordShardFailure(shard, error); + }; + + const markShardUnavailable = (error: string) => { + clearTimeout(shard.idleTimer); + shard.idleTimer = undefined; + shard.process = undefined; + shard.ready = false; + if (shard.stopping) { + shard.stopping = false; + return; + } + recordShardFailureOnce(error); + }; + + proc.stderr?.on("data", (data: Buffer) => { + const chunk = data.toString(); + stderr += chunk; + shard.stderr += chunk; + }); + + const onUnexpectedError = (error: Error) => { + const message = getErrorMessage(error); + stderr = stderr.length > 0 ? `${stderr}\n${message}` : message; + shard.stderr = stderr; + markShardUnavailable(message); + }; + proc.once("error", onUnexpectedError); + + const onUnexpectedExit = (code: number | null, signal: string | null) => { + markShardUnavailable( + stderr.trim() || `SSH master exited unexpectedly (${code ?? signal ?? "unknown"})` + ); + }; + proc.once("exit", onUnexpectedExit); + + let timer: ReturnType | undefined; + scheduleKill = (ms: number) => { + if (timer) { + clearTimeout(timer); + } + timer = setTimeout(() => { + this.disposeShard(group, shard, { expected: false }); + }, ms); + }; + + scheduleKill(timeoutMs); + + try { + await this.waitForMasterReady(group.config, shard, abortSignal); + shard.ready = true; + shard.health = { + status: "healthy", + consecutiveFailures: 0, + lastSuccess: new Date(), + }; + log.debug(`Started OpenSSH master ${shard.shardId} for ${group.config.host}`); + } catch (error) { + shard.ready = false; + recordShardFailureOnce(getErrorMessage(error)); + this.disposeShard(group, shard, { expected: true }); + throw error; + } finally { + if (timer) { + clearTimeout(timer); + } + askpass?.cleanup(); + } + } + + private async waitForMasterReady( + config: SSHConnectionConfig, + shard: MasterShard, + abortSignal?: AbortSignal + ): Promise { + while (true) { + if (abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + if (!isProcessAlive(shard.process)) { + throw new Error(shard.stderr.trim() || "SSH master exited before becoming ready"); + } + + const ready = await this.checkMaster(config, shard.controlPath); + if (ready) { + return; + } + + await this.sleep(this.startupPollIntervalMs, abortSignal); + } + } + + private async checkMaster(config: SSHConnectionConfig, controlPath: string): Promise { + const args: string[] = ["-S", controlPath, "-O", "check"]; + if (config.port) { + args.push("-p", config.port.toString()); + } + if (config.identityFile) { + args.push("-i", config.identityFile); + } + args.push(config.host); + + return new Promise((resolve) => { + const proc = this.spawnProcess("ssh", args, { + stdio: ["ignore", "ignore", "ignore"], + windowsHide: true, + }); + proc.once("close", (code) => resolve(code === 0)); + proc.once("error", () => resolve(false)); + }); + } + + private buildMasterArgs( + config: SSHConnectionConfig, + controlPath: string, + connectTimeout: number + ): string[] { + const args: string[] = ["-M", "-N", "-T"]; + + if (config.port) { + args.push("-p", config.port.toString()); + } + if (config.identityFile) { + args.push("-i", config.identityFile); + } + + args.push("-o", "LogLevel=FATAL"); + args.push("-o", "ControlMaster=yes"); + args.push("-o", `ControlPath=${controlPath}`); + args.push("-o", "ControlPersist=no"); + args.push("-o", `ConnectTimeout=${connectTimeout}`); + args.push("-o", "ServerAliveInterval=5"); + args.push("-o", "ServerAliveCountMax=2"); + appendOpenSSHHostKeyPolicyArgs(args); + args.push(config.host); + + return args; + } + + private recordShardFailure(shard: MasterShard, error: string): void { + const failures = (shard.health.consecutiveFailures ?? 0) + 1; + const backoffIndex = Math.min(failures - 1, BACKOFF_SCHEDULE.length - 1); + const backoffSecs = withJitter(BACKOFF_SCHEDULE[backoffIndex]); + shard.health = { + status: "unhealthy", + lastFailure: new Date(), + lastError: error, + consecutiveFailures: failures, + backoffUntil: new Date(Date.now() + backoffSecs * 1000), + }; + log.warn( + `OpenSSH master ${shard.shardId} failed for ${shard.controlPath}: ${error} (backoff ${backoffSecs.toFixed(1)}s)` + ); + } + + private getNextBackoffWaitMs(group: HostGroup): number | undefined { + const waits = group.shards + .map((shard) => shard.health.backoffUntil?.getTime()) + .filter((value): value is number => value != null) + .map((until) => until - Date.now()) + .filter((value) => value > 0); + if (waits.length === 0) { + return undefined; + } + return Math.min(...waits); + } + + private trimExitedShards(group: HostGroup): void { + group.shards = group.shards.filter((shard) => { + if (isProcessAlive(shard.process) || shard.startup) { + return true; + } + if (shard.inflight > 0) { + return true; + } + const backoffUntil = shard.health.backoffUntil?.getTime(); + return backoffUntil != null && backoffUntil > Date.now(); + }); + } + + private disposeShard(group: HostGroup, shard: MasterShard, options: { expected: boolean }): void { + clearTimeout(shard.idleTimer); + shard.idleTimer = undefined; + shard.ready = false; + shard.stopping = options.expected; + + const masterProcess = shard.process; + if (masterProcess && isProcessAlive(masterProcess)) { + const args: string[] = ["-S", shard.controlPath, "-O", "exit"]; + if (group.config.port) { + args.push("-p", group.config.port.toString()); + } + if (group.config.identityFile) { + args.push("-i", group.config.identityFile); + } + args.push(group.config.host); + + const exitProc = this.spawnProcess("ssh", args, { + stdio: ["ignore", "ignore", "ignore"], + windowsHide: true, + }); + exitProc.once("error", () => { + try { + masterProcess.kill("SIGTERM"); + } catch { + // Ignore process teardown failures. + } + }); + const hardKill = setTimeout(() => { + try { + masterProcess.kill("SIGKILL"); + } catch { + // Ignore process teardown failures. + } + }, 1000); + hardKill.unref?.(); + masterProcess.once("exit", () => clearTimeout(hardKill)); + exitProc.once("close", (code) => { + if (code === 0) { + clearTimeout(hardKill); + return; + } + try { + masterProcess.kill("SIGTERM"); + } catch { + // Ignore process teardown failures. + } + }); + } + + shard.stderr = ""; + } +} + +export const openSshMasterPool = new OpenSSHMasterPool(); diff --git a/src/node/runtime/projectSyncCoordinator.test.ts b/src/node/runtime/projectSyncCoordinator.test.ts new file mode 100644 index 0000000000..12c04ad60a --- /dev/null +++ b/src/node/runtime/projectSyncCoordinator.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, test } from "bun:test"; +import { ProjectSyncCoordinator } from "./projectSyncCoordinator"; + +describe("ProjectSyncCoordinator", () => { + test("does not let one aborted caller cancel other waiters on the same snapshot", async () => { + const coordinator = new ProjectSyncCoordinator(); + const firstAbort = new AbortController(); + let releaseSharedWork: (() => void) | undefined; + let runCount = 0; + + const firstPromise = coordinator.runSnapshotSync( + { + projectKey: "project-a", + snapshotKey: "snapshot-a", + abortSignal: firstAbort.signal, + }, + async (sharedAbortSignal) => { + runCount += 1; + return await new Promise((resolve, reject) => { + releaseSharedWork = resolve; + sharedAbortSignal.addEventListener("abort", () => reject(new Error("shared aborted")), { + once: true, + }); + }); + } + ); + + const secondPromise = coordinator.runSnapshotSync( + { + projectKey: "project-a", + snapshotKey: "snapshot-a", + }, + () => Promise.reject(new Error("expected existing snapshot sync to be reused")) + ); + + firstAbort.abort(); + try { + await firstPromise; + throw new Error("Expected first waiter to abort"); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("Operation aborted"); + } + expect(runCount).toBe(1); + + releaseSharedWork?.(); + await secondPromise; + }); + + test("returns promptly when the last waiter aborts", async () => { + const coordinator = new ProjectSyncCoordinator(); + const abortController = new AbortController(); + + const resultPromise = coordinator.runSnapshotSync( + { + projectKey: "project-b", + snapshotKey: "snapshot-b", + abortSignal: abortController.signal, + }, + async (sharedAbortSignal) => { + return await new Promise((_resolve, reject) => { + sharedAbortSignal.addEventListener("abort", () => reject(new Error("shared aborted")), { + once: true, + }); + }); + } + ); + + abortController.abort(); + try { + await resultPromise; + throw new Error("Expected the caller to abort"); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("Operation aborted"); + } + }); +}); diff --git a/src/node/runtime/projectSyncCoordinator.ts b/src/node/runtime/projectSyncCoordinator.ts new file mode 100644 index 0000000000..c144fdb290 --- /dev/null +++ b/src/node/runtime/projectSyncCoordinator.ts @@ -0,0 +1,128 @@ +interface ProjectSyncOptions { + projectKey: string; + snapshotKey: string; + abortSignal?: AbortSignal; +} + +interface InflightSnapshotSync { + promise: Promise; + abortController: AbortController; + waiters: number; +} + +export class ProjectSyncCoordinator { + private readonly inflightSyncs = new Map(); + private readonly projectTails = new Map>(); + + async runSnapshotSync( + options: ProjectSyncOptions, + fn: (abortSignal: AbortSignal) => Promise + ): Promise { + if (options.abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + + let inflight = this.inflightSyncs.get(options.snapshotKey); + if (inflight?.abortController.signal.aborted && inflight.waiters === 0) { + this.inflightSyncs.delete(options.snapshotKey); + inflight = undefined; + } + + if (!inflight) { + const abortController = new AbortController(); + const promise = this.enqueueProjectMutation(options.projectKey, () => + fn(abortController.signal) + ); + inflight = { + promise, + abortController, + waiters: 0, + }; + this.inflightSyncs.set(options.snapshotKey, inflight); + void promise.then( + () => { + if (this.inflightSyncs.get(options.snapshotKey) === inflight) { + this.inflightSyncs.delete(options.snapshotKey); + } + }, + () => { + if (this.inflightSyncs.get(options.snapshotKey) === inflight) { + this.inflightSyncs.delete(options.snapshotKey); + } + } + ); + } + + inflight.waiters += 1; + let released = false; + const releaseWaiter = () => { + if (released) { + return; + } + released = true; + inflight.waiters = Math.max(0, inflight.waiters - 1); + if (inflight.waiters === 0 && !inflight.abortController.signal.aborted) { + inflight.abortController.abort(); + } + }; + + const callerAbortSignal = options.abortSignal; + let onAbort: (() => void) | undefined; + const callerAbort = callerAbortSignal + ? new Promise((_, reject) => { + onAbort = () => { + releaseWaiter(); + reject(new Error("Operation aborted")); + }; + if (callerAbortSignal.aborted) { + onAbort(); + return; + } + callerAbortSignal.addEventListener("abort", onAbort, { once: true }); + }) + : undefined; + + try { + if (callerAbort) { + await Promise.race([inflight.promise, callerAbort]); + } else { + await inflight.promise; + } + } finally { + if (onAbort) { + options.abortSignal?.removeEventListener("abort", onAbort); + } + releaseWaiter(); + } + } + + async enqueueProjectMutation(projectKey: string, fn: () => Promise): Promise { + const previous = this.projectTails.get(projectKey) ?? Promise.resolve(); + let releaseCurrent: (() => void) | undefined; + const current = new Promise((resolve) => { + releaseCurrent = resolve; + }); + const tail = previous.then( + () => current, + () => current + ); + this.projectTails.set(projectKey, tail); + + try { + await previous.catch(() => undefined); + await fn(); + } finally { + releaseCurrent?.(); + if (this.projectTails.get(projectKey) === tail) { + this.projectTails.delete(projectKey); + } + } + } + + clearAll(): void { + this.inflightSyncs.clear(); + this.projectTails.clear(); + } +} + +export const projectSyncCoordinator = new ProjectSyncCoordinator(); diff --git a/src/node/runtime/remoteProjectLayout.ts b/src/node/runtime/remoteProjectLayout.ts new file mode 100644 index 0000000000..81811603e3 --- /dev/null +++ b/src/node/runtime/remoteProjectLayout.ts @@ -0,0 +1,76 @@ +import * as crypto from "crypto"; +import * as path from "path"; +import { getProjectName } from "@/node/utils/runtime/helpers"; + +export const REMOTE_BASE_REPO_DIR = ".mux-base.git"; +const REMOTE_METADATA_DIR = ".mux-meta"; +const REMOTE_WORKSPACE_METADATA_DIR = "workspaces"; +const REMOTE_SNAPSHOT_MARKER_DIR = "snapshots"; + +export interface RemoteProjectLayout { + projectId: string; + projectRoot: string; + baseRepoPath: string; + workspaceMetadataDir: string; + snapshotMarkerDir: string; +} + +function sanitizeProjectSegment(segment: string): string { + const sanitized = segment + .trim() + .replace(/[^A-Za-z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, ""); + return sanitized.length > 0 ? sanitized : "project"; +} + +function hashText(input: string): string { + return crypto.createHash("sha256").update(input).digest("hex").slice(0, 12); +} + +export function createRemoteProjectId(projectPath: string): string { + const normalizedPath = projectPath.replace(/\\/g, "/"); + const projectSlug = sanitizeProjectSegment(getProjectName(projectPath)); + return `${projectSlug}-${hashText(normalizedPath)}`; +} + +export function buildRemoteProjectLayout( + srcBaseDir: string, + projectPath: string, + projectRootOverride?: string +): RemoteProjectLayout { + const projectId = createRemoteProjectId(projectPath); + const projectRoot = projectRootOverride ?? path.posix.join(srcBaseDir, projectId); + const metadataRoot = path.posix.join(projectRoot, REMOTE_METADATA_DIR); + + return { + projectId, + projectRoot, + baseRepoPath: path.posix.join(projectRoot, REMOTE_BASE_REPO_DIR), + workspaceMetadataDir: path.posix.join(metadataRoot, REMOTE_WORKSPACE_METADATA_DIR), + snapshotMarkerDir: path.posix.join(metadataRoot, REMOTE_SNAPSHOT_MARKER_DIR), + }; +} + +export function buildLegacyRemoteProjectLayout( + srcBaseDir: string, + projectPath: string +): RemoteProjectLayout { + const legacyRoot = path.posix.join(srcBaseDir, getProjectName(projectPath)); + return buildRemoteProjectLayout(srcBaseDir, projectPath, legacyRoot); +} + +export function getRemoteWorkspacePath(layout: RemoteProjectLayout, workspaceName: string): string { + return path.posix.join(layout.projectRoot, workspaceName); +} + +export function getWorkspaceMetadataPath( + layout: RemoteProjectLayout, + workspaceName: string +): string { + return path.posix.join(layout.workspaceMetadataDir, `${hashText(workspaceName)}.json`); +} + +export function getSnapshotMarkerPath(layout: RemoteProjectLayout, snapshotDigest: string): string { + return path.posix.join(layout.snapshotMarkerDir, `${snapshotDigest}.json`); +} diff --git a/src/node/runtime/runtimeFactory.ts b/src/node/runtime/runtimeFactory.ts index 296a6d8ab0..c8dacfc70f 100644 --- a/src/node/runtime/runtimeFactory.ts +++ b/src/node/runtime/runtimeFactory.ts @@ -135,6 +135,7 @@ export function createRuntime(config: RuntimeConfig, options?: CreateRuntimeOpti const runtimeIdentity = { projectPath: options?.projectPath, workspaceName: options?.workspaceName, + workspacePath: options?.workspacePath, }; switch (config.type) { diff --git a/src/node/runtime/sshConnectionPool.ts b/src/node/runtime/sshConnectionPool.ts index cb36d59513..a9dd05b4b6 100644 --- a/src/node/runtime/sshConnectionPool.ts +++ b/src/node/runtime/sshConnectionPool.ts @@ -33,6 +33,10 @@ export function setSshPromptService(svc: SshPromptService | undefined): void { sshPromptService = svc; } +export function getSshPromptService(): SshPromptService | undefined { + return sshPromptService; +} + export function setOpenSSHHostKeyPolicyMode(mode: OpenSSHHostKeyPolicyMode): void { hostKeyPolicyMode = mode; } diff --git a/src/node/runtime/transports/OpenSSHTransport.test.ts b/src/node/runtime/transports/OpenSSHTransport.test.ts index 8d92da011b..34d5311879 100644 --- a/src/node/runtime/transports/OpenSSHTransport.test.ts +++ b/src/node/runtime/transports/OpenSSHTransport.test.ts @@ -1,12 +1,10 @@ import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; import * as childProcess from "child_process"; +import * as ptySpawn from "../ptySpawn"; import { SshPromptService } from "@/node/services/sshPromptService"; -import { - setSshPromptService, - setOpenSSHHostKeyPolicyMode, - sshConnectionPool, -} from "../sshConnectionPool"; +import { setSshPromptService, setOpenSSHHostKeyPolicyMode } from "../sshConnectionPool"; +import { openSshMasterPool } from "../openSshMasterPool"; import { OpenSSHTransport } from "./OpenSSHTransport"; function createMockChildProcess(): ReturnType { @@ -18,15 +16,19 @@ function createMockChildProcess(): ReturnType { describe("OpenSSHTransport.spawnRemoteProcess", () => { let spawnSpy: ReturnType>; - let acquireConnectionSpy: ReturnType>; + let acquireLeaseSpy: ReturnType>; let releaseInteractiveResponder: (() => void) | undefined; beforeEach(() => { spawnSpy = spyOn(childProcess, "spawn").mockImplementation((() => createMockChildProcess()) as unknown as typeof childProcess.spawn); - acquireConnectionSpy = spyOn(sshConnectionPool, "acquireConnection").mockResolvedValue( - undefined - ); + acquireLeaseSpy = spyOn(openSshMasterPool, "acquireLease").mockResolvedValue({ + controlPath: "/tmp/mux-ssh-test-shard", + shardId: "shard-0", + release: mock(() => undefined), + reportFailure: mock(() => undefined), + markHealthy: mock(() => undefined), + }); }); afterEach(() => { @@ -37,7 +39,7 @@ describe("OpenSSHTransport.spawnRemoteProcess", () => { setOpenSSHHostKeyPolicyMode("headless-fallback"); spawnSpy.mockRestore(); - acquireConnectionSpy.mockRestore(); + acquireLeaseSpy.mockRestore(); }); function setSshPromptCapability(configured: boolean): SshPromptService | undefined { @@ -102,3 +104,41 @@ describe("OpenSSHTransport.spawnRemoteProcess", () => { expect(args).not.toContain("UserKnownHostsFile=/dev/null"); }); }); + +describe("OpenSSHTransport.createPtySession", () => { + let ensureReadyMasterSpy: ReturnType>; + let spawnPtyProcessSpy: ReturnType>; + + beforeEach(() => { + ensureReadyMasterSpy = spyOn(openSshMasterPool, "ensureReadyMaster").mockResolvedValue( + undefined + ); + spawnPtyProcessSpy = spyOn(ptySpawn, "spawnPtyProcess").mockReturnValue({ + write: mock(() => undefined), + resize: mock(() => undefined), + kill: mock(() => undefined), + onData: mock(() => ({ dispose: () => undefined })), + onExit: mock(() => ({ dispose: () => undefined })), + } as unknown as ReturnType); + }); + + afterEach(() => { + ensureReadyMasterSpy.mockRestore(); + spawnPtyProcessSpy.mockRestore(); + }); + + test("preflights against a ready master without reserving an exec slot", async () => { + const transport = new OpenSSHTransport({ host: "remote.example.com", port: 2222 }); + + await transport.createPtySession({ + workspacePath: "~/workspace", + cols: 80, + rows: 24, + }); + + expect(ensureReadyMasterSpy).toHaveBeenCalledWith( + { host: "remote.example.com", port: 2222 }, + { maxWaitMs: 0 } + ); + }); +}); diff --git a/src/node/runtime/transports/OpenSSHTransport.ts b/src/node/runtime/transports/OpenSSHTransport.ts index b20f7523a6..ff83ecc5ee 100644 --- a/src/node/runtime/transports/OpenSSHTransport.ts +++ b/src/node/runtime/transports/OpenSSHTransport.ts @@ -3,12 +3,8 @@ import { log } from "@/node/services/log"; import { spawnPtyProcess } from "../ptySpawn"; import { expandTildeForSSH } from "../tildeExpansion"; -import { - appendOpenSSHHostKeyPolicyArgs, - getControlPath, - sshConnectionPool, - type SSHConnectionConfig, -} from "../sshConnectionPool"; +import { appendOpenSSHHostKeyPolicyArgs, type SSHConnectionConfig } from "../sshConnectionPool"; +import { openSshMasterPool } from "../openSshMasterPool"; import type { SpawnResult } from "../RemoteRuntime"; import type { SSHTransport, @@ -19,13 +15,7 @@ import type { } from "./SSHTransport"; export class OpenSSHTransport implements SSHTransport { - private readonly config: SSHConnectionConfig; - private readonly controlPath: string; - - constructor(config: SSHConnectionConfig) { - this.config = config; - this.controlPath = getControlPath(config); - } + constructor(private readonly config: SSHConnectionConfig) {} isConnectionFailure(exitCode: number, _stderr: string): boolean { return exitCode === 255; @@ -36,11 +26,11 @@ export class OpenSSHTransport implements SSHTransport { } markHealthy(): void { - sshConnectionPool.markHealthy(this.config); + // OpenSSH transport reports health through per-process master leases. } - reportFailure(error: string): void { - sshConnectionPool.reportFailure(this.config, error); + reportFailure(_error: string): void { + // OpenSSH transport reports health through per-process master leases. } async acquireConnection(options?: { @@ -49,7 +39,7 @@ export class OpenSSHTransport implements SSHTransport { maxWaitMs?: number; onWait?: (waitMs: number) => void; }): Promise { - await sshConnectionPool.acquireConnection(this.config, { + await openSshMasterPool.ensureConnection(this.config, { abortSignal: options?.abortSignal, timeoutMs: options?.timeoutMs, maxWaitMs: options?.maxWaitMs, @@ -58,42 +48,77 @@ export class OpenSSHTransport implements SSHTransport { } async spawnRemoteProcess(fullCommand: string, options: SpawnOptions): Promise { - await sshConnectionPool.acquireConnection(this.config, { + const remainingWaitMs = + options.deadlineMs != null ? Math.max(0, options.deadlineMs - Date.now()) : undefined; + const lease = await openSshMasterPool.acquireLease(this.config, { abortSignal: options.abortSignal, + timeoutMs: remainingWaitMs, + maxWaitMs: remainingWaitMs, }); // Note: use -tt (not -t) so PTY allocation works even when stdin is a pipe. - const sshArgs: string[] = [options.forcePTY ? "-tt" : "-T", ...this.buildSSHArgs()]; + const sshArgs: string[] = [ + options.forcePTY ? "-tt" : "-T", + ...this.buildBaseSSHArgs(), + "-o", + "ControlMaster=no", + "-o", + `ControlPath=${lease.controlPath}`, + ]; const connectTimeout = options.timeout !== undefined ? Math.min(Math.ceil(options.timeout), 15) : 15; sshArgs.push("-o", `ConnectTimeout=${connectTimeout}`); sshArgs.push("-o", "ServerAliveInterval=5"); sshArgs.push("-o", "ServerAliveCountMax=2"); - // Non-interactive execs must never hang on host-key or password prompts. - // Host-key trust policy is capability-scoped (verification service wired), - // while responder liveness only affects whether prompts can be shown. sshArgs.push("-o", "BatchMode=yes"); appendOpenSSHHostKeyPolicyArgs(sshArgs); - sshArgs.push(this.config.host, fullCommand); - log.debug(`SSH exec on ${this.config.host}`); + log.debug(`SSH exec on ${this.config.host} via ${lease.shardId}`); const process = spawn("ssh", sshArgs, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true, }); - return { process }; + let released = false; + const releaseLease = () => { + if (released) { + return; + } + released = true; + lease.release(); + }; + + return { + process, + onExit: (exitCode, stderr) => { + if (this.isConnectionFailure(exitCode, stderr)) { + lease.reportFailure(stderr.trim() || `SSH exited with code ${exitCode}`); + } else { + lease.markHealthy(); + } + releaseLease(); + }, + onError: (error) => { + lease.reportFailure(error.message); + releaseLease(); + }, + }; } async createPtySession(params: PtySessionParams): Promise { - await sshConnectionPool.acquireConnection(this.config, { maxWaitMs: 0 }); + // PTYs stay on a dedicated direct SSH session so they do not consume pooled master + // capacity reserved for the many short exec/file operations that drive workspace scale. + // Preflight only needs an already-started master (or to bootstrap one), not a free exec slot. + await openSshMasterPool.ensureReadyMaster(this.config, { maxWaitMs: 0 }); - const args: string[] = [...this.buildSSHArgs()]; + const args: string[] = [...this.buildBaseSSHArgs()]; args.push("-o", "ConnectTimeout=15"); args.push("-o", "ServerAliveInterval=5"); args.push("-o", "ServerAliveCountMax=2"); + args.push("-o", "BatchMode=yes"); + appendOpenSSHHostKeyPolicyArgs(args); args.push("-t"); args.push(this.config.host); @@ -113,7 +138,7 @@ export class OpenSSHTransport implements SSHTransport { }); } - private buildSSHArgs(): string[] { + private buildBaseSSHArgs(): string[] { const args: string[] = []; if (this.config.port) { @@ -125,10 +150,6 @@ export class OpenSSHTransport implements SSHTransport { } args.push("-o", "LogLevel=FATAL"); - args.push("-o", "ControlMaster=auto"); - args.push("-o", `ControlPath=${this.controlPath}`); - args.push("-o", "ControlPersist=60"); - return args; } } diff --git a/src/node/runtime/transports/SSH2Transport.ts b/src/node/runtime/transports/SSH2Transport.ts index 3ff1df2db5..dc1960f258 100644 --- a/src/node/runtime/transports/SSH2Transport.ts +++ b/src/node/runtime/transports/SSH2Transport.ts @@ -297,7 +297,17 @@ export class SSH2Transport implements SSHTransport { }); const process = new SSH2ChildProcess(channel) as unknown as ChildProcess; - return { process }; + return { + process, + onExit: (exitCode) => { + if (exitCode === 0) { + ssh2ConnectionPool.markHealthy(this.config); + } + }, + onError: (error) => { + ssh2ConnectionPool.reportFailure(this.config, getErrorMessage(error)); + }, + }; } catch (error) { ssh2ConnectionPool.reportFailure(this.config, getErrorMessage(error)); throw new RuntimeErrorClass( diff --git a/src/node/runtime/transports/SSHTransport.ts b/src/node/runtime/transports/SSHTransport.ts index 43f7aedef4..0e70bd08d0 100644 --- a/src/node/runtime/transports/SSHTransport.ts +++ b/src/node/runtime/transports/SSHTransport.ts @@ -15,6 +15,8 @@ export interface SpawnOptions { forcePTY?: boolean; timeout?: number; abortSignal?: AbortSignal; + /** Absolute client-side deadline (Date.now milliseconds) for queueing + execution. */ + deadlineMs?: number; } export interface PtySessionParams { diff --git a/src/node/services/workspaceProjectRepos.test.ts b/src/node/services/workspaceProjectRepos.test.ts index 1acf0c01dc..730cdfa33d 100644 --- a/src/node/services/workspaceProjectRepos.test.ts +++ b/src/node/services/workspaceProjectRepos.test.ts @@ -38,6 +38,27 @@ describe("getWorkspaceProjectRepos", () => { expect(repos[0]?.storageKey).toBe("..-..-secrets"); }); + it("reuses the persisted workspace path for the current SSH project in multi-project views", () => { + const repos = getWorkspaceProjectRepos({ + workspaceId: "workspace-1", + workspaceName: "main", + workspacePath: "/tmp/legacy/main", + runtimeConfig: { + type: "ssh", + host: "example.com", + srcBaseDir: "/tmp/src", + }, + projectPath: "/tmp/projects/main", + projectName: "main", + projects: [ + { projectPath: "/tmp/projects/main", projectName: "main" }, + { projectPath: "/tmp/projects/other", projectName: "other" }, + ], + }); + + expect(repos[0]?.repoCwd).toBe("/tmp/legacy/main"); + }); + it("disambiguates storage keys when sanitized project names collide", () => { const repos = getWorkspaceProjectRepos({ workspaceId: "workspace-1", diff --git a/src/node/services/workspaceProjectRepos.ts b/src/node/services/workspaceProjectRepos.ts index 52e7129f8f..4523f32ea7 100644 --- a/src/node/services/workspaceProjectRepos.ts +++ b/src/node/services/workspaceProjectRepos.ts @@ -165,6 +165,8 @@ export function getWorkspaceProjectRepos( ? createRuntime(params.runtimeConfig, { projectPath: project.projectPath, workspaceName: params.workspaceName, + workspacePath: + project.projectPath === params.projectPath ? params.workspacePath : undefined, }).getWorkspacePath(project.projectPath, params.workspaceName) : params.workspacePath; diff --git a/tests/e2e/utils/ui.ts b/tests/e2e/utils/ui.ts index aed0a1e389..d04d347e23 100644 --- a/tests/e2e/utils/ui.ts +++ b/tests/e2e/utils/ui.ts @@ -211,7 +211,7 @@ export function createWorkspaceUI(page: Page, context: DemoProjectConfig): Works } for (let attempt = 0; attempt < 3; attempt++) { - await control.click(); + await control.dispatchEvent("click"); try { await expect .poll( @@ -282,8 +282,9 @@ export function createWorkspaceUI(page: Page, context: DemoProjectConfig): Works return labelIndex === -1 ? 0 : labelIndex; }; - // Click paddles until we reach the target level. If the control is still settling after - // a mode switch, clickUntilLabelChanges() retries the interaction before giving up. + // Click paddles until we reach the target level. clickUntilLabelChanges() retries + // the interaction and dispatches DOM clicks directly so transient Linux Electron + // overlays do not interfere with toolbar hit-testing. for (let i = 0; i < allowedLabels.length * 2; i++) { const currentLevel = await getCurrentLevel(); if (currentLevel === clampedTargetLevel) { diff --git a/tests/runtime/runtime.test.ts b/tests/runtime/runtime.test.ts index bc35d99f2d..56fbd4714d 100644 --- a/tests/runtime/runtime.test.ts +++ b/tests/runtime/runtime.test.ts @@ -34,6 +34,10 @@ import { execBuffered, readFileString, writeFileString } from "@/node/utils/runt import type { Runtime } from "@/node/runtime/Runtime"; import { RuntimeError } from "@/node/runtime/Runtime"; import { computeBaseRepoPath, type SSHRuntime } from "@/node/runtime/SSHRuntime"; +import { + buildRemoteProjectLayout, + getRemoteWorkspacePath, +} from "@/node/runtime/remoteProjectLayout"; import { createSSHTransport } from "@/node/runtime/transports"; import { runFullInit } from "@/node/runtime/runtimeFactory"; import { sshConnectionPool } from "@/node/runtime/sshConnectionPool"; @@ -955,14 +959,15 @@ describeIntegration("Runtime integration tests", () => { * SSHRuntime-specific workspace operation tests * WorktreeRuntime workspace tests are covered by the matrix above * - * Note: SSHRuntime.getWorkspacePath uses srcBaseDir + projectName + workspaceName. - * The projectPath argument is only used to extract the project name (basename). - * So the actual workspace path is: /home/testuser/workspace/{projectName}/{workspaceName} + * Note: SSHRuntime derives workspace paths from the hashed remote project layout + * when a persisted workspacePath is not available. + * These tests build the same layout helpers as production code before asserting paths. */ describe("SSHRuntime workspace operations", () => { const testForRuntime = test; const srcBaseDir = "/home/testuser/workspace"; const createSSHRuntime = (): Runtime => createTestRuntime("ssh", srcBaseDir, sshConfig); + const getLayout = (projectPath: string) => buildRemoteProjectLayout(srcBaseDir, projectPath); describe("renameWorkspace", () => { testForRuntime("successfully renames directory", async () => { @@ -972,9 +977,9 @@ describeIntegration("Runtime integration tests", () => { // projectPath is used to extract project name - can be any path ending with projectName const projectPath = `/some/path/${projectName}`; - // The runtime will construct paths as: srcBaseDir/projectName/workspaceName - const oldWorkspacePath = `${srcBaseDir}/${projectName}/worktree-1`; - const newWorkspacePath = `${srcBaseDir}/${projectName}/worktree-renamed`; + const layout = getLayout(projectPath); + const oldWorkspacePath = getRemoteWorkspacePath(layout, "worktree-1"); + const newWorkspacePath = getRemoteWorkspacePath(layout, "worktree-renamed"); // Create the workspace directory structure where the runtime expects it await execBuffered( @@ -1009,7 +1014,7 @@ describeIntegration("Runtime integration tests", () => { } // Cleanup - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1039,8 +1044,9 @@ describeIntegration("Runtime integration tests", () => { const sourceWorkspaceName = "source"; const newWorkspaceName = "forked"; - const sourceWorkspacePath = `${srcBaseDir}/${projectName}/${sourceWorkspaceName}`; - const newWorkspacePath = `${srcBaseDir}/${projectName}/${newWorkspaceName}`; + const layout = getLayout(projectPath); + const sourceWorkspacePath = getRemoteWorkspacePath(layout, sourceWorkspaceName); + const newWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); // Create a source workspace repo with a non-trunk branch checked out. await execBuffered( @@ -1136,7 +1142,7 @@ describeIntegration("Runtime integration tests", () => { expect(initResult.success).toBe(true); // Cleanup - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1148,7 +1154,8 @@ describeIntegration("Runtime integration tests", () => { const runtime = createSSHRuntime(); const projectName = `delete-test-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const workspacePath = `${srcBaseDir}/${projectName}/worktree-delete-test`; + const layout = getLayout(projectPath); + const workspacePath = getRemoteWorkspacePath(layout, "worktree-delete-test"); // Create the workspace directory structure where the runtime expects it await execBuffered( @@ -1182,7 +1189,7 @@ describeIntegration("Runtime integration tests", () => { } // Cleanup - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1213,27 +1220,30 @@ describeIntegration("Runtime integration tests", () => { const srcBaseDir = "/home/testuser/workspace"; const createSSHRuntime = (): SSHRuntime => createTestRuntime("ssh", srcBaseDir, sshConfig) as SSHRuntime; + const getLayout = (projectPath: string) => buildRemoteProjectLayout(srcBaseDir, projectPath); test("computeBaseRepoPath returns correct path", async () => { + const layout = getLayout("/some/path/my-project"); const result = computeBaseRepoPath(srcBaseDir, "/some/path/my-project"); - expect(result).toBe(`${srcBaseDir}/my-project/.mux-base.git`); + expect(result).toBe(layout.baseRepoPath); }, 10000); test("forkWorkspace uses worktree when base repo exists", async () => { const runtime = createSSHRuntime(); const projectName = `wt-fork-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; - const sourceWorkspacePath = `${srcBaseDir}/${projectName}/source`; + const layout = getLayout(projectPath); + const baseRepoPath = layout.baseRepoPath; + const sourceWorkspacePath = getRemoteWorkspacePath(layout, "source"); const newWorkspaceName = "forked-wt"; - const newWorkspacePath = `${srcBaseDir}/${projectName}/${newWorkspaceName}`; + const newWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); try { // 1. Create a bare base repo and populate it with a commit. await execBuffered( runtime, [ - `mkdir -p "${srcBaseDir}/${projectName}"`, + `mkdir -p "${layout.projectRoot}"`, `git init --bare "${baseRepoPath}"`, // Create a temp repo, commit, and push to the bare repo. `TMPCLONE=$(mktemp -d)`, @@ -1308,7 +1318,7 @@ describeIntegration("Runtime integration tests", () => { expect(worktreeList.stdout).toContain(newWorkspaceName); } finally { // Cleanup: remove all worktrees and the project directory. - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1319,9 +1329,10 @@ describeIntegration("Runtime integration tests", () => { const runtime = createSSHRuntime(); const projectName = `wt-legacy-fork-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const sourceWorkspacePath = `${srcBaseDir}/${projectName}/legacy-source`; + const layout = getLayout(projectPath); + const sourceWorkspacePath = getRemoteWorkspacePath(layout, "legacy-source"); const newWorkspaceName = "legacy-forked"; - const newWorkspacePath = `${srcBaseDir}/${projectName}/${newWorkspaceName}`; + const newWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); try { // Create a legacy workspace (standalone git clone, no base repo). @@ -1344,7 +1355,7 @@ describeIntegration("Runtime integration tests", () => { // Verify no base repo exists. const baseCheck = await execBuffered( runtime, - `test -d "${srcBaseDir}/${projectName}/.mux-base.git" && echo "exists" || echo "missing"`, + `test -d "${layout.baseRepoPath}" && echo "exists" || echo "missing"`, { cwd: "/home/testuser", timeout: 30 } ); expect(baseCheck.stdout.trim()).toBe("missing"); @@ -1376,7 +1387,7 @@ describeIntegration("Runtime integration tests", () => { }); expect(fileCheck.stdout.trim()).toBe("legacy content"); } finally { - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1387,17 +1398,18 @@ describeIntegration("Runtime integration tests", () => { const runtime = createSSHRuntime(); const projectName = `wt-mixed-fork-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; - const sourceWorkspacePath = `${srcBaseDir}/${projectName}/legacy-ws`; + const layout = getLayout(projectPath); + const baseRepoPath = layout.baseRepoPath; + const sourceWorkspacePath = getRemoteWorkspacePath(layout, "legacy-ws"); const newWorkspaceName = "forked-mixed"; - const newWorkspacePath = `${srcBaseDir}/${projectName}/${newWorkspaceName}`; + const newWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); try { // 1. Create a bare base repo with a commit on 'main' (simulates a previous initWorkspace). await execBuffered( runtime, [ - `mkdir -p "${srcBaseDir}/${projectName}"`, + `mkdir -p "${layout.projectRoot}"`, `git init --bare "${baseRepoPath}"`, `TMPCLONE=$(mktemp -d)`, `git clone "${baseRepoPath}" "$TMPCLONE/work"`, @@ -1466,7 +1478,7 @@ describeIntegration("Runtime integration tests", () => { }); expect(fileCheck.stdout.trim()).toBe("legacy content"); } finally { - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1477,16 +1489,17 @@ describeIntegration("Runtime integration tests", () => { const runtime = createSSHRuntime(); const projectName = `wt-delete-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; + const layout = getLayout(projectPath); + const baseRepoPath = layout.baseRepoPath; const workspaceName = "to-delete"; - const workspacePath = `${srcBaseDir}/${projectName}/${workspaceName}`; + const workspacePath = getRemoteWorkspacePath(layout, workspaceName); try { // Create bare base repo with a commit. await execBuffered( runtime, [ - `mkdir -p "${srcBaseDir}/${projectName}"`, + `mkdir -p "${layout.projectRoot}"`, `git init --bare "${baseRepoPath}"`, `TMPCLONE=$(mktemp -d)`, `git clone "${baseRepoPath}" "$TMPCLONE/work"`, @@ -1539,7 +1552,7 @@ describeIntegration("Runtime integration tests", () => { }); expect(worktreeList.stdout).not.toContain(workspaceName); } finally { - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1550,7 +1563,8 @@ describeIntegration("Runtime integration tests", () => { const runtime = createSSHRuntime(); const projectName = `wt-del-legacy-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const workspacePath = `${srcBaseDir}/${projectName}/legacy-ws`; + const layout = getLayout(projectPath); + const workspacePath = getRemoteWorkspacePath(layout, "legacy-ws"); try { // Create a legacy workspace (standalone git clone, .git is a directory). @@ -1577,7 +1591,7 @@ describeIntegration("Runtime integration tests", () => { ); expect(afterCheck.stdout.trim()).toBe("missing"); } finally { - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1588,16 +1602,17 @@ describeIntegration("Runtime integration tests", () => { const runtime = createSSHRuntime(); const projectName = `wt-rename-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; - const oldWorkspacePath = `${srcBaseDir}/${projectName}/old-name`; - const newWorkspacePath = `${srcBaseDir}/${projectName}/new-name`; + const layout = getLayout(projectPath); + const baseRepoPath = layout.baseRepoPath; + const oldWorkspacePath = getRemoteWorkspacePath(layout, "old-name"); + const newWorkspacePath = getRemoteWorkspacePath(layout, "new-name"); try { // Set up bare base repo with a commit. await execBuffered( runtime, [ - `mkdir -p "${srcBaseDir}/${projectName}"`, + `mkdir -p "${layout.projectRoot}"`, `git init --bare "${baseRepoPath}"`, `TMPCLONE=$(mktemp -d)`, `git clone "${baseRepoPath}" "$TMPCLONE/work"`, @@ -1649,7 +1664,7 @@ describeIntegration("Runtime integration tests", () => { expect(worktreeList.stdout).toContain("/new-name"); expect(worktreeList.stdout).not.toContain("/old-name"); } finally { - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1830,6 +1845,191 @@ describeIntegration("Runtime integration tests", () => { }); } }, 120000); + + test("initWorkspace reimports when the snapshot marker outlives the base repo", async () => { + const runtime = createSSHRuntime(); + + const projectName = `sync-heal-marker-${Date.now()}-${Math.random().toString(36).substring(7)}`; + const tmpDir = await import("os").then((os) => os.tmpdir()); + const localProjectPath = `${tmpDir}/${projectName}`; + const layout = buildRemoteProjectLayout(srcBaseDir, localProjectPath); + const firstWorkspaceName = "marker-a"; + const secondWorkspaceName = "marker-b"; + const firstWorkspacePath = getRemoteWorkspacePath(layout, firstWorkspaceName); + const secondWorkspacePath = getRemoteWorkspacePath(layout, secondWorkspaceName); + const baseRepoPath = layout.baseRepoPath; + + const { execSync } = await import("child_process"); + try { + execSync( + [ + `mkdir -p "${localProjectPath}"`, + `cd "${localProjectPath}"`, + `git init -b main`, + `git config user.email "test@test.com"`, + `git config user.name "Test"`, + `echo "content" > file.txt`, + `git add file.txt`, + `git commit -m "initial"`, + ].join(" && "), + { stdio: "pipe" } + ); + + const firstInit = await runtime.initWorkspace({ + projectPath: localProjectPath, + branchName: firstWorkspaceName, + trunkBranch: "main", + workspacePath: firstWorkspacePath, + initLogger: noopInitLogger, + }); + if (!firstInit.success) { + throw new Error(`first initWorkspace failed: ${firstInit.error}`); + } + + const snapshotMarkerCheck = await execBuffered( + runtime, + `find "${layout.snapshotMarkerDir}" -type f | head -n 1`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(snapshotMarkerCheck.stdout.trim()).not.toBe(""); + + await execBuffered( + runtime, + `rm -rf "${baseRepoPath}" && git init --bare "${baseRepoPath}"`, + { cwd: "/home/testuser", timeout: 30 } + ); + + const secondInit = await runtime.initWorkspace({ + projectPath: localProjectPath, + branchName: secondWorkspaceName, + trunkBranch: "main", + workspacePath: secondWorkspacePath, + initLogger: noopInitLogger, + }); + if (!secondInit.success) { + throw new Error(`second initWorkspace failed: ${secondInit.error}`); + } + + const baseRefs = await execBuffered( + runtime, + `git -C "${baseRepoPath}" for-each-ref --format='%(refname)' refs/mux-bundle/`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(baseRefs.stdout).toContain("refs/mux-bundle/main"); + + const insideWorkTreeCheck = await execBuffered( + runtime, + `git -C "${secondWorkspacePath}" rev-parse --is-inside-work-tree`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(insideWorkTreeCheck.stdout.trim()).toBe("true"); + } finally { + execSync(`rm -rf "${localProjectPath}"`); + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { + cwd: "/home/testuser", + timeout: 30, + }); + } + }, 120000); + + test("initWorkspace reimports when an older snapshot marker exists but bundle refs were advanced", async () => { + const runtime = createSSHRuntime(); + + const projectName = `sync-heal-history-${Date.now()}-${Math.random().toString(36).substring(7)}`; + const tmpDir = await import("os").then((os) => os.tmpdir()); + const localProjectPath = `${tmpDir}/${projectName}`; + const layout = buildRemoteProjectLayout(srcBaseDir, localProjectPath); + const firstWorkspacePath = getRemoteWorkspacePath(layout, "history-a"); + const secondWorkspacePath = getRemoteWorkspacePath(layout, "history-b"); + const thirdWorkspacePath = getRemoteWorkspacePath(layout, "history-c"); + const baseRepoPath = layout.baseRepoPath; + + const { execSync } = await import("child_process"); + try { + execSync( + [ + `mkdir -p "${localProjectPath}"`, + `cd "${localProjectPath}"`, + `git init -b main`, + `git config user.email "test@test.com"`, + `git config user.name "Test"`, + `echo "version-a" > file.txt`, + `git add file.txt`, + `git commit -m "initial"`, + ].join(" && "), + { stdio: "pipe" } + ); + const firstCommit = execSync(`git -C "${localProjectPath}" rev-parse HEAD`, { + encoding: "utf8", + }).trim(); + + const firstInit = await runtime.initWorkspace({ + projectPath: localProjectPath, + branchName: "history-a", + trunkBranch: "main", + workspacePath: firstWorkspacePath, + initLogger: noopInitLogger, + }); + if (!firstInit.success) { + throw new Error(`first initWorkspace failed: ${firstInit.error}`); + } + + execSync( + [ + `cd "${localProjectPath}"`, + `echo "version-b" > file.txt`, + `git add file.txt`, + `git commit -m "second"`, + ].join(" && "), + { stdio: "pipe" } + ); + + const secondInit = await runtime.initWorkspace({ + projectPath: localProjectPath, + branchName: "history-b", + trunkBranch: "main", + workspacePath: secondWorkspacePath, + initLogger: noopInitLogger, + }); + if (!secondInit.success) { + throw new Error(`second initWorkspace failed: ${secondInit.error}`); + } + + execSync(`git -C "${localProjectPath}" reset --hard ${firstCommit}`, { + stdio: "pipe", + }); + + const thirdInit = await runtime.initWorkspace({ + projectPath: localProjectPath, + branchName: "history-c", + trunkBranch: "main", + workspacePath: thirdWorkspacePath, + initLogger: noopInitLogger, + }); + if (!thirdInit.success) { + throw new Error(`third initWorkspace failed: ${thirdInit.error}`); + } + + const fileCheck = await execBuffered(runtime, `cat "${thirdWorkspacePath}/file.txt"`, { + cwd: "/home/testuser", + timeout: 30, + }); + expect(fileCheck.stdout.trim()).toBe("version-a"); + + const baseHead = await execBuffered( + runtime, + `git -C "${baseRepoPath}" rev-parse refs/mux-bundle/main`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(baseHead.stdout.trim()).toBe(firstCommit); + } finally { + execSync(`rm -rf "${localProjectPath}"`); + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { + cwd: "/home/testuser", + timeout: 30, + }); + } + }, 120000); }); /** @@ -2291,6 +2491,7 @@ describeIntegration("Runtime integration tests", () => { */ describe("CoderSSHRuntime workspace operations", () => { const srcBaseDir = "/home/testuser/src"; + const getLayout = (projectPath: string) => buildRemoteProjectLayout(srcBaseDir, projectPath); // Create a CoderSSHRuntime with mock CoderService const createCoderSSHRuntime = async () => { @@ -2326,7 +2527,8 @@ describeIntegration("Runtime integration tests", () => { const sourceWorkspaceName = "source"; const newWorkspaceName = "forked"; - const sourceWorkspacePath = `${srcBaseDir}/${projectName}/${sourceWorkspaceName}`; + const layout = getLayout(projectPath); + const sourceWorkspacePath = getRemoteWorkspacePath(layout, sourceWorkspaceName); // Create a source workspace repo await execBuffered( @@ -2415,8 +2617,9 @@ describeIntegration("Runtime integration tests", () => { const projectPath = `/some/path/${projectName}`; const sourceWorkspaceName = "source"; const newWorkspaceName = "forked"; - const sourceWorkspacePath = `${srcBaseDir}/${projectName}/${sourceWorkspaceName}`; - const forkedWorkspacePath = `${srcBaseDir}/${projectName}/${newWorkspaceName}`; + const layout = getLayout(projectPath); + const sourceWorkspacePath = getRemoteWorkspacePath(layout, sourceWorkspaceName); + const forkedWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); // Create a source workspace repo await execBuffered( From c600553045b10a76674f4c89413852248f3e1e26 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 16:32:39 -0500 Subject: [PATCH 02/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20legacy?= =?UTF-8?q?=20SSH=20workspace=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve persisted workspace roots for existing SSH workspaces in AI/session callsites and keep the legacy SSH branch manifest updated for downgrade compatibility. --- _Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$102.40`_ --- src/node/runtime/RemoteRuntime.test.ts | 133 ++++++++++ src/node/runtime/RemoteRuntime.ts | 4 +- src/node/runtime/SSHRuntime.test.ts | 146 +++++++++++ src/node/runtime/SSHRuntime.ts | 142 +++++++++-- src/node/runtime/openSshMasterPool.test.ts | 227 +++++++++++++++++- src/node/runtime/openSshMasterPool.ts | 26 +- src/node/runtime/runtimeHelpers.test.ts | 33 +++ src/node/runtime/runtimeHelpers.ts | 10 +- src/node/services/agentSession.ts | 39 ++- src/node/services/aiService.test.ts | 50 +++- src/node/services/aiService.ts | 33 ++- .../services/workspaceProjectRepos.test.ts | 37 +++ src/node/services/workspaceProjectRepos.ts | 47 +++- 13 files changed, 857 insertions(+), 70 deletions(-) create mode 100644 src/node/runtime/RemoteRuntime.test.ts diff --git a/src/node/runtime/RemoteRuntime.test.ts b/src/node/runtime/RemoteRuntime.test.ts new file mode 100644 index 0000000000..f7a8b4a49c --- /dev/null +++ b/src/node/runtime/RemoteRuntime.test.ts @@ -0,0 +1,133 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { describe, expect, test } from "bun:test"; + +import { EXIT_CODE_ABORTED, EXIT_CODE_TIMEOUT } from "@/common/constants/exitCodes"; +import { RemoteRuntime, type SpawnResult } from "./RemoteRuntime"; +import type { + ExecOptions, + WorkspaceCreationParams, + WorkspaceCreationResult, + WorkspaceForkParams, + WorkspaceForkResult, + WorkspaceInitParams, + WorkspaceInitResult, +} from "./Runtime"; + +class FakeChildProcess extends EventEmitter { + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + readonly stdin = new PassThrough(); + pid = 1234; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + + kill(_signal?: string): boolean { + return true; + } +} + +class TestRemoteRuntime extends RemoteRuntime { + protected readonly commandPrefix = "Test"; + + constructor( + private readonly childProcess: FakeChildProcess, + private readonly onExitCalls: Array<[number, string]>, + private readonly onExitCodeCalls: number[] + ) { + super(); + } + + protected spawnRemoteProcess( + _fullCommand: string, + _options: ExecOptions & { deadlineMs?: number } + ): Promise { + return Promise.resolve({ + process: this.childProcess as never, + onExit: (exitCode, stderr) => { + this.onExitCalls.push([exitCode, stderr]); + }, + }); + } + + protected getBasePath(): string { + return "/tmp"; + } + + protected quoteForRemote(targetPath: string): string { + return targetPath; + } + + protected cdCommand(cwd: string): string { + return `cd ${cwd}`; + } + + protected override onExitCode(exitCode: number): void { + this.onExitCodeCalls.push(exitCode); + } + + resolvePath(targetPath: string): Promise { + return Promise.resolve(targetPath); + } + + getWorkspacePath(projectPath: string, workspaceName: string): string { + return `${projectPath}/${workspaceName}`; + } + + createWorkspace(_params: WorkspaceCreationParams): Promise { + throw new Error("unused in test"); + } + + initWorkspace(_params: WorkspaceInitParams): Promise { + throw new Error("unused in test"); + } + + renameWorkspace(): Promise< + { success: true; oldPath: string; newPath: string } | { success: false; error: string } + > { + throw new Error("unused in test"); + } + + deleteWorkspace(): Promise< + { success: true; deletedPath: string } | { success: false; error: string } + > { + throw new Error("unused in test"); + } + + forkWorkspace(_params: WorkspaceForkParams): Promise { + throw new Error("unused in test"); + } +} + +describe("RemoteRuntime synthetic exit handling", () => { + test("does not forward aborted exits to transport onExit hooks", async () => { + const childProcess = new FakeChildProcess(); + const onExitCalls: Array<[number, string]> = []; + const onExitCodeCalls: number[] = []; + const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onExitCodeCalls); + const controller = new AbortController(); + + const stream = await runtime.exec("echo ok", { cwd: "/tmp", abortSignal: controller.signal }); + controller.abort(); + childProcess.emit("close", 0, null); + + expect(await stream.exitCode).toBe(EXIT_CODE_ABORTED); + expect(onExitCalls).toEqual([]); + expect(onExitCodeCalls).toEqual([EXIT_CODE_ABORTED]); + }); + + test("does not forward timed-out exits to transport onExit hooks", async () => { + const childProcess = new FakeChildProcess(); + const onExitCalls: Array<[number, string]> = []; + const onExitCodeCalls: number[] = []; + const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onExitCodeCalls); + + const stream = await runtime.exec("echo ok", { cwd: "/tmp", timeout: 0.01 }); + await new Promise((resolve) => setTimeout(resolve, 20)); + childProcess.emit("close", 0, null); + + expect(await stream.exitCode).toBe(EXIT_CODE_TIMEOUT); + expect(onExitCalls).toEqual([]); + expect(onExitCodeCalls).toEqual([EXIT_CODE_TIMEOUT]); + }); +}); diff --git a/src/node/runtime/RemoteRuntime.ts b/src/node/runtime/RemoteRuntime.ts index 20bb977e36..e7775807ff 100644 --- a/src/node/runtime/RemoteRuntime.ts +++ b/src/node/runtime/RemoteRuntime.ts @@ -179,7 +179,9 @@ export abstract class RemoteRuntime implements Runtime { ? EXIT_CODE_TIMEOUT : (code ?? (signal ? -1 : 0)); - spawnResult.onExit?.(finalExitCode, stderrForErrorReporting); + if (finalExitCode !== EXIT_CODE_ABORTED && finalExitCode !== EXIT_CODE_TIMEOUT) { + spawnResult.onExit?.(finalExitCode, stderrForErrorReporting); + } // Let subclass handle exit code (e.g., SSH connection pool) this.onExitCode(finalExitCode, options, stderrForErrorReporting); diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index ea35d4f6e3..8ab6a6698d 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -1,3 +1,4 @@ +import * as path from "node:path"; import { describe, expect, it, beforeEach, afterEach, spyOn } from "bun:test"; import * as runtimeHelpers from "@/node/utils/runtime/helpers"; import * as disposableExec from "@/node/utils/disposableExec"; @@ -9,6 +10,7 @@ import { getRemoteWorkspacePath, } from "./remoteProjectLayout"; import { createSSHTransport } from "./transports"; +import { projectSyncCoordinator } from "./projectSyncCoordinator"; /** * SSHRuntime unit tests (run with bun test) @@ -717,6 +719,150 @@ describe("SSHRuntime.deleteWorkspace", () => { } }); }); +describe("SSHRuntime branch metadata compatibility", () => { + it("keeps the legacy branch manifest in sync when renaming a legacy workspace", async () => { + type UpdateWorkspaceBranchMapping = ( + projectPath: string, + oldWorkspaceName: string, + newWorkspaceName: string + ) => Promise; + + const config = { host: "example.com", srcBaseDir: "/home/user/src" }; + const projectPath = "/projects/demo"; + const oldWorkspaceName = "review-slot"; + const newWorkspaceName = "renamed-slot"; + const legacyLayout = buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath); + const legacyManifestPath = path.posix.join( + legacyLayout.projectRoot, + ".mux-workspace-branches.json" + ); + const runtime = new SSHRuntime(config, createSSHTransport(config, false), { + projectPath, + workspaceName: oldWorkspaceName, + workspacePath: getRemoteWorkspacePath(legacyLayout, oldWorkspaceName), + }); + const files = new Map([ + [legacyManifestPath, '{"review-slot":"feature-branch"}\n'], + ]); + const readFileSpy = spyOn(runtime, "readFile").mockImplementation((filePath: string) => { + const contents = files.get(filePath); + return new ReadableStream({ + start(controller) { + if (contents === undefined) { + controller.error(new Error(`Missing file: ${filePath}`)); + return; + } + controller.enqueue(new TextEncoder().encode(contents)); + controller.close(); + }, + }); + }); + const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation((filePath: string) => { + const decoder = new TextDecoder(); + let contents = ""; + return new WritableStream({ + write(chunk) { + contents += decoder.decode(chunk, { stream: true }); + }, + close() { + contents += decoder.decode(); + files.set(filePath, contents); + }, + }); + }); + const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( + (_runtime, command) => { + if (command.startsWith("mkdir -p ") || command.startsWith("rm -f ")) { + return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); + } + throw new Error(`Unexpected execBuffered command: ${command}`); + } + ); + + try { + const updateWorkspaceBranchMapping = Reflect.get( + runtime, + "updateWorkspaceBranchMapping" + ) as UpdateWorkspaceBranchMapping; + await updateWorkspaceBranchMapping.call( + runtime, + projectPath, + oldWorkspaceName, + newWorkspaceName + ); + + expect(JSON.parse(files.get(legacyManifestPath) ?? "null")).toEqual({ + [newWorkspaceName]: "feature-branch", + }); + } finally { + readFileSpy.mockRestore(); + writeFileSpy.mockRestore(); + execBufferedSpy.mockRestore(); + projectSyncCoordinator.clearAll(); + } + }); + it("removes stale legacy branch manifest entries even when layout detection falls back to preferred", async () => { + type DeletePersistedWorkspaceBranchMapping = ( + projectPath: string, + workspaceName: string + ) => Promise; + + const config = { host: "example.com", srcBaseDir: "/home/user/src" }; + const projectPath = "/projects/demo"; + const workspaceName = "review-slot"; + const legacyManifestPath = path.posix.join( + buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath).projectRoot, + ".mux-workspace-branches.json" + ); + const runtime = new SSHRuntime(config, createSSHTransport(config, false)); + const files = new Map([ + [legacyManifestPath, '{"review-slot":"feature-branch"}\n'], + ]); + const readFileSpy = spyOn(runtime, "readFile").mockImplementation((filePath: string) => { + const contents = files.get(filePath); + return new ReadableStream({ + start(controller) { + if (contents === undefined) { + controller.error(new Error(`Missing file: ${filePath}`)); + return; + } + controller.enqueue(new TextEncoder().encode(contents)); + controller.close(); + }, + }); + }); + const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( + (_runtime, command) => { + if (command.includes("echo legacy") && command.includes("echo preferred")) { + return Promise.resolve({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }); + } + if (command.startsWith("rm -f ")) { + const pathMatch = /^rm -f\s+(.+)$/.exec(command); + if (pathMatch?.[1]) { + files.delete(pathMatch[1].replace(/^"|"$/g, "")); + } + return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); + } + throw new Error(`Unexpected execBuffered command: ${command}`); + } + ); + + try { + const deletePersistedWorkspaceBranchMapping = Reflect.get( + runtime, + "deletePersistedWorkspaceBranchMapping" + ) as DeletePersistedWorkspaceBranchMapping; + await deletePersistedWorkspaceBranchMapping.call(runtime, projectPath, workspaceName); + + expect(files.has(legacyManifestPath)).toBe(false); + } finally { + readFileSpy.mockRestore(); + execBufferedSpy.mockRestore(); + projectSyncCoordinator.clearAll(); + } + }); +}); + describe("SSHRuntime.ensureReady repository checks", () => { let execBufferedSpy: ReturnType> | null = null; diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index 876f92ae93..b2b382142b 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -638,11 +638,8 @@ export class SSHRuntime extends RemoteRuntime { workspaceName: string ): Promise { try { - await this.resolveProjectLayout(projectPath, workspaceName); - const metadataPath = getWorkspaceMetadataPath( - this.getPreferredProjectLayout(projectPath), - workspaceName - ); + const layout = await this.resolveProjectLayout(projectPath, workspaceName); + const metadataPath = getWorkspaceMetadataPath(layout, workspaceName); await execBuffered(this, `rm -f ${this.quoteForRemote(metadataPath)}`, { cwd: "/tmp", timeout: 10, @@ -650,6 +647,8 @@ export class SSHRuntime extends RemoteRuntime { } catch { // Best-effort cleanup after delete; future creates overwrite any stale entry. } + + await this.deleteLegacyWorkspaceBranchEntry(projectPath, workspaceName).catch(() => undefined); } private async readWorkspaceBranchMetadata( @@ -711,29 +710,142 @@ export class SSHRuntime extends RemoteRuntime { } finally { await writer.close(); } + + await this.mutateLegacyWorkspaceBranchMap(projectPath, layout, (branchMap) => { + branchMap[workspaceName] = branchName; + }); } - private async readLegacyWorkspaceBranchEntry( - projectPath: string, - workspaceName: string - ): Promise { + private usesLegacyProjectLayout(projectPath: string, layout: RemoteProjectLayout): boolean { + return ( + layout.projectRoot === + buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath).projectRoot + ); + } + + private async readLegacyWorkspaceBranchMap(projectPath: string): Promise> { try { const contents = await streamToString( this.readFile(this.getLegacyWorkspaceBranchMapPath(projectPath)) ); const parsed: unknown = JSON.parse(contents); if (typeof parsed !== "object" || parsed === null) { - return null; + return {}; + } + + const branchMap: Record = {}; + for (const [workspaceName, branchName] of Object.entries(parsed)) { + if (typeof branchName !== "string") { + continue; + } + const trimmedWorkspaceName = workspaceName.trim(); + const trimmedBranchName = branchName.trim(); + if (trimmedWorkspaceName.length === 0 || trimmedBranchName.length === 0) { + continue; + } + branchMap[trimmedWorkspaceName] = trimmedBranchName; } - const branchMap = parsed as Record; - const branchName = - typeof branchMap[workspaceName] === "string" ? branchMap[workspaceName].trim() : ""; - return branchName.length > 0 ? branchName : null; + return branchMap; } catch { - return null; + return {}; + } + } + + private async writeLegacyWorkspaceBranchMap( + projectPath: string, + branchMap: Record + ): Promise { + const legacyManifestPath = this.getLegacyWorkspaceBranchMapPath(projectPath); + const normalizedBranchMap = Object.fromEntries( + Object.entries(branchMap).sort(([leftWorkspace], [rightWorkspace]) => + leftWorkspace.localeCompare(rightWorkspace) + ) + ); + const writer = this.writeFile(legacyManifestPath).getWriter(); + try { + await writer.write( + new TextEncoder().encode(`${JSON.stringify(normalizedBranchMap, null, 2)}\n`) + ); + } finally { + await writer.close(); } } + private async deleteLegacyWorkspaceBranchEntry( + projectPath: string, + workspaceName: string + ): Promise { + const legacyManifestPath = this.getLegacyWorkspaceBranchMapPath(projectPath); + const projectKey = this.getProjectSyncKey(this.getDefaultProjectLayout(projectPath).projectId); + await projectSyncCoordinator.enqueueProjectMutation(projectKey, async () => { + const branchMap = await this.readLegacyWorkspaceBranchMap(projectPath); + if (!(workspaceName in branchMap)) { + return; + } + + delete branchMap[workspaceName]; + if (Object.keys(branchMap).length === 0) { + await execBuffered(this, `rm -f ${this.quoteForRemote(legacyManifestPath)}`, { + cwd: "/tmp", + timeout: 10, + }).catch(() => undefined); + return; + } + + await this.writeLegacyWorkspaceBranchMap(projectPath, branchMap); + }); + } + + private async mutateLegacyWorkspaceBranchMap( + projectPath: string, + layout: RemoteProjectLayout, + mutate: (branchMap: Record) => void + ): Promise { + if (!this.usesLegacyProjectLayout(projectPath, layout)) { + return; + } + + const legacyManifestPath = this.getLegacyWorkspaceBranchMapPath(projectPath); + const projectKey = this.getProjectSyncKey(this.getDefaultProjectLayout(projectPath).projectId); + await projectSyncCoordinator.enqueueProjectMutation(projectKey, async () => { + const branchMap = await this.readLegacyWorkspaceBranchMap(projectPath); + mutate(branchMap); + if (Object.keys(branchMap).length === 0) { + await execBuffered(this, `rm -f ${this.quoteForRemote(legacyManifestPath)}`, { + cwd: "/tmp", + timeout: 10, + }).catch(() => undefined); + return; + } + + const mkdirResult = await execBuffered( + this, + `mkdir -p ${this.quoteForRemote(path.posix.dirname(legacyManifestPath))}`, + { + cwd: "/tmp", + timeout: 10, + } + ); + if (mkdirResult.exitCode !== 0) { + throw new Error( + `Failed to prepare legacy workspace branch manifest: ${mkdirResult.stderr || mkdirResult.stdout}` + ); + } + + await this.writeLegacyWorkspaceBranchMap(projectPath, branchMap); + }); + } + + private async readLegacyWorkspaceBranchEntry( + projectPath: string, + workspaceName: string + ): Promise { + const branchName = (await this.readLegacyWorkspaceBranchMap(projectPath))[ + workspaceName + ]?.trim(); + return branchName && branchName.length > 0 ? branchName : null; + } + private getLegacyWorkspaceBranchMapPath(projectPath: string): string { return path.posix.join( buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath).projectRoot, diff --git a/src/node/runtime/openSshMasterPool.test.ts b/src/node/runtime/openSshMasterPool.test.ts index a0759b07a5..e6120ab22c 100644 --- a/src/node/runtime/openSshMasterPool.test.ts +++ b/src/node/runtime/openSshMasterPool.test.ts @@ -212,6 +212,229 @@ describe("OpenSSHMasterPool", () => { pool.clearAll(); }); + test("honors shard backoff waits instead of polling at the startup cadence", async () => { + const sleepCalls: number[] = []; + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + startupPollIntervalMs: 50, + sleep: (waitMs) => { + sleepCalls.push(waitMs); + const internals = pool as unknown as { + hostGroups: Map }>; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (shard?.health.backoffUntil) { + shard.health.backoffUntil = new Date(Date.now() - 1); + } + return Promise.resolve(); + }, + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + sleepCalls.length = 0; + first.reportFailure("ssh exited 255"); + first.release(); + + const second = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + + expect(sleepCalls[0]).toBeGreaterThan(100); + + second.release(); + pool.clearAll(); + }); + + test("polls for free capacity when a healthy shard is saturated even if another shard is backing off", async () => { + const sleepCalls: number[] = []; + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 2, + startupPollIntervalMs: 50, + sleep: (waitMs) => { + sleepCalls.push(waitMs); + return Promise.resolve(); + }, + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + const internals = pool as unknown as { + hostGroups: Map< + string, + { + shards: Array<{ + health: { + backoffUntil?: Date; + status: string; + consecutiveFailures?: number; + lastFailure?: Date; + lastError?: string; + }; + ready: boolean; + inflight: number; + process?: FakeChildProcess; + startup?: Promise; + stopping: boolean; + stderr: string; + id: number; + shardId: string; + controlPath: string; + lastUsedAt: number; + }>; + } + >; + }; + const group = [...internals.hostGroups.values()][0]; + if (!group) { + throw new Error("Expected a tracked host group"); + } + group.shards.push({ + id: 99, + shardId: "shard-99", + controlPath: "/tmp/mux-backoff-shard", + inflight: 0, + lastUsedAt: Date.now(), + ready: false, + stderr: "", + stopping: false, + health: { + status: "unhealthy", + consecutiveFailures: 1, + lastFailure: new Date(), + lastError: "ssh exited 255", + backoffUntil: new Date(Date.now() + 10_000), + }, + }); + + const secondPromise = pool.acquireLease(config, { + maxWaitMs: 1000, + timeoutMs: 1000, + onWait: () => { + first.release(); + }, + }); + const second = await secondPromise; + + expect(sleepCalls[0]).toBe(50); + + second.release(); + pool.clearAll(); + }); + + test("preserves shard failure history across retries after backoff expires", async () => { + let startupAttempts = 0; + const controller = new AbortController(); + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + sleep: () => { + const internals = pool as unknown as { + hostGroups: Map }>; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (shard?.health.backoffUntil) { + shard.health.backoffUntil = new Date(Date.now() - 1); + } + return Promise.resolve(); + }, + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + startupAttempts += 1; + queueMicrotask(() => { + proc.emit("error", new Error(`startup failure ${startupAttempts}`)); + proc.exitCode = 1; + proc.emit("exit", proc.exitCode, null); + proc.emit("close", proc.exitCode, null); + if (startupAttempts === 2) { + controller.abort(); + } + }); + return proc as never; + } + + queueMicrotask(() => { + proc.exitCode = 1; + proc.emit("close", proc.exitCode, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + try { + await pool.acquireLease(config, { + maxWaitMs: 1000, + timeoutMs: 1000, + abortSignal: controller.signal, + }); + throw new Error("Expected acquireLease to reject"); + } catch { + // Expected: we abort after the second failed startup attempt. + } + + const internals = pool as unknown as { + hostGroups: Map }>; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (!shard) { + throw new Error("Expected a tracked shard"); + } + expect(startupAttempts).toBe(2); + expect(shard.health.consecutiveFailures).toBe(2); + + pool.clearAll(); + }); + test("ensureReadyMaster ignores saturated exec slots when a shard is already ready", async () => { const pool = new OpenSSHMasterPool({ maxSessionsPerShard: 1, @@ -338,7 +561,9 @@ describe("OpenSSHMasterPool", () => { const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; try { - await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + // Keep the wait budget below the minimum 0.8s backoff jitter so this assertion only + // verifies one startup attempt, not whether acquireLease later retries within budget. + await pool.acquireLease(config, { maxWaitMs: 100, timeoutMs: 1000 }); throw new Error("Expected acquireLease to reject"); } catch (error) { expect(error).toBeInstanceOf(Error); diff --git a/src/node/runtime/openSshMasterPool.ts b/src/node/runtime/openSshMasterPool.ts index 2ff89ca465..db69d77362 100644 --- a/src/node/runtime/openSshMasterPool.ts +++ b/src/node/runtime/openSshMasterPool.ts @@ -239,11 +239,7 @@ export class OpenSSHMasterPool { ); } - const waitMs = Math.min( - remainingMs, - nextBackoffMs ?? this.startupPollIntervalMs, - this.startupPollIntervalMs - ); + const waitMs = this.getPoolWaitMs(remainingMs, nextBackoffMs); options?.onWait?.(waitMs); await this.sleep(waitMs, options?.abortSignal); } @@ -322,10 +318,10 @@ export class OpenSSHMasterPool { ); } - const waitMs = Math.min( + const waitMs = this.getPoolWaitMs( remainingMs, - nextBackoffMs ?? this.startupPollIntervalMs, - this.startupPollIntervalMs + nextBackoffMs, + this.pickReadyShard(hostGroup) != null ); options?.onWait?.(waitMs); await this.sleep(waitMs, options?.abortSignal); @@ -373,6 +369,17 @@ export class OpenSSHMasterPool { return shard; } + private getPoolWaitMs( + remainingMs: number, + nextBackoffMs: number | undefined, + preferPolling = false + ): number { + return Math.min( + remainingMs, + preferPolling || nextBackoffMs == null ? this.startupPollIntervalMs : nextBackoffMs + ); + } + private getReadyShards(group: HostGroup): MasterShard[] { return group.shards.filter((shard) => { const backoffUntilMs = shard.health.backoffUntil?.getTime(); @@ -694,6 +701,9 @@ export class OpenSSHMasterPool { if (shard.inflight > 0) { return true; } + if (shard.health.status === "unhealthy") { + return true; + } const backoffUntil = shard.health.backoffUntil?.getTime(); return backoffUntil != null && backoffUntil > Date.now(); }); diff --git a/src/node/runtime/runtimeHelpers.test.ts b/src/node/runtime/runtimeHelpers.test.ts index 02fefdc9d7..b8157bd8b3 100644 --- a/src/node/runtime/runtimeHelpers.test.ts +++ b/src/node/runtime/runtimeHelpers.test.ts @@ -19,6 +19,24 @@ describe("createRuntimeForWorkspace", () => { const internal = runtime as unknown as { currentWorkspacePath?: string }; expect(internal.currentWorkspacePath).toBe("/tmp/non-canonical/workspaces/review-1"); }); + + it("seeds ssh runtimes from the persisted workspace root", () => { + const metadata = { + runtimeConfig: { + type: "ssh", + host: "example.com", + srcBaseDir: "/remote/src", + } satisfies RuntimeConfig, + projectPath: "/projects/demo", + name: "review-1", + namedWorkspacePath: "/remote/src/demo/review-1", + }; + + const runtime = createRuntimeForWorkspace(metadata); + expect(runtime.getWorkspacePath(metadata.projectPath, "review-2")).toBe( + "/remote/src/demo/review-2" + ); + }); }); describe("resolveWorkspaceExecutionPath", () => { @@ -37,6 +55,21 @@ describe("resolveWorkspaceExecutionPath", () => { expect(resolveWorkspaceExecutionPath(metadata, runtime)).toBe("/persisted/review-1"); }); + it("falls back to the runtime path when persisted metadata is unavailable", () => { + const metadata = { + runtimeConfig: { + type: "ssh", + host: "example.com", + srcBaseDir: "/remote/src", + } satisfies RuntimeConfig, + projectPath: "/projects/demo", + name: "review-1", + }; + + const runtime = createRuntimeForWorkspace(metadata); + const runtimeWorkspacePath = runtime.getWorkspacePath(metadata.projectPath, metadata.name); + expect(resolveWorkspaceExecutionPath(metadata, runtime)).toBe(runtimeWorkspacePath); + }); it("uses the runtime path for docker workspaces", () => { const metadata = { runtimeConfig: { diff --git a/src/node/runtime/runtimeHelpers.ts b/src/node/runtime/runtimeHelpers.ts index 737c688093..1be433474b 100644 --- a/src/node/runtime/runtimeHelpers.ts +++ b/src/node/runtime/runtimeHelpers.ts @@ -36,10 +36,12 @@ export function resolveWorkspaceExecutionPath( } const persistedWorkspacePath = metadata.namedWorkspacePath?.trim(); - assert( - persistedWorkspacePath, - `Workspace ${metadata.name} is missing its persisted workspace path for runtime ${metadata.runtimeConfig.type}` - ); + if (!persistedWorkspacePath) { + // Some metadata readers and unit tests only carry canonical workspace identity. Fall back to the + // runtime-derived path there, but prefer the persisted path whenever it is available so upgraded + // SSH/devcontainer workspaces keep using their exact checkout root. + return runtimeWorkspacePath; + } if (isLocalProjectRuntime(metadata.runtimeConfig)) { // Project-dir local runtimes always execute directly in the project root. diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 754c5afd7f..0476baca8b 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -63,8 +63,10 @@ import { type ReviewNoteDataForDisplay, type StartupRetrySendOptions, } from "@/common/types/message"; -import { createRuntime } from "@/node/runtime/runtimeFactory"; -import { createRuntimeForWorkspace } from "@/node/runtime/runtimeHelpers"; +import { + createRuntimeForWorkspace, + resolveWorkspaceExecutionPath, +} from "@/node/runtime/runtimeHelpers"; import { hasNonEmptyPlanFile } from "@/node/utils/runtime/helpers"; import { isExecLikeEditingCapableInResolvedChain } from "@/common/utils/agentTools"; import { @@ -1952,20 +1954,12 @@ export class AgentSession { const existing = await this.aiService.getWorkspaceMetadata(this.workspaceId); if (existing.success) { - // Metadata already exists, verify workspace path matches - const metadata = existing.data; - // For in-place workspaces (projectPath === name), use path directly - // Otherwise reconstruct using runtime's worktree pattern - const isInPlace = metadata.projectPath === metadata.name; - const expectedPath = isInPlace - ? metadata.projectPath - : (() => { - const runtime = createRuntime(metadata.runtimeConfig, { - projectPath: metadata.projectPath, - workspaceName: metadata.name, - }); - return runtime.getWorkspacePath(metadata.projectPath, metadata.name); - })(); + // Metadata already exists; use the persisted config entry as the source of truth instead of + // reconstructing a canonical path, because upgraded SSH workspaces may still live under a + // legacy remote layout until an operation explicitly seeds that layout back into the runtime. + const workspace = this.config.findWorkspace(this.workspaceId); + assert(workspace, `Workspace ${this.workspaceId} is missing its persisted config entry`); + const expectedPath = path.resolve(workspace.workspacePath); assert( expectedPath === normalizedWorkspacePath, `Existing metadata workspace path mismatch for ${this.workspaceId}: expected ${expectedPath}, got ${normalizedWorkspacePath}` @@ -4459,11 +4453,10 @@ export class AgentSession { const runtime = createRuntimeForWorkspace(metadata); // In-place workspaces (CLI/benchmarks) have projectPath === name. - // Use the path directly instead of reconstructing via getWorkspacePath. const isInPlace = metadata.projectPath === metadata.name; const workspacePath = isInPlace ? metadata.projectPath - : runtime.getWorkspacePath(metadata.projectPath, metadata.name); + : resolveWorkspaceExecutionPath(metadata, runtime); // When disableWorkspaceAgents is active, use project path for discovery // (only built-in/global agents). Mirrors resolveAgentForStream behavior. @@ -5126,7 +5119,7 @@ export class AgentSession { const metadata = metadataResult.data; const runtime = createRuntimeForWorkspace(metadata); - const workspacePath = runtime.getWorkspacePath(metadata.projectPath, metadata.name); + const workspacePath = resolveWorkspaceExecutionPath(metadata, runtime); const materialized = await materializeFileAtMentions(messageText, { runtime, @@ -5189,17 +5182,13 @@ export class AgentSession { } const metadata = metadataResult.data; - const runtime = createRuntime(metadata.runtimeConfig, { - projectPath: metadata.projectPath, - workspaceName: metadata.name, - }); + const runtime = createRuntimeForWorkspace(metadata); // In-place workspaces (CLI/benchmarks) have projectPath === name. - // Use the path directly instead of reconstructing via getWorkspacePath. const isInPlace = metadata.projectPath === metadata.name; const workspacePath = isInPlace ? metadata.projectPath - : runtime.getWorkspacePath(metadata.projectPath, metadata.name); + : resolveWorkspaceExecutionPath(metadata, runtime); // When workspace agents are disabled, resolve skills from the project path instead of // the worktree so skill invocation uses the same precedence/discovery root as the UI. diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index dd7220e137..024b35d8ef 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -19,6 +19,7 @@ import { InitStateManager } from "./initStateManager"; import { ProviderService } from "./providerService"; import { EXPERIMENT_IDS } from "@/common/constants/experiments"; import { Config } from "@/node/config"; +import * as runtimeFactory from "@/node/runtime/runtimeFactory"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { DisposableTempDir } from "@/node/services/tempDir"; @@ -1815,7 +1816,11 @@ describe("AIService.streamMessage multi-project trust gating", () => { getToolsForModelSpy: ReturnType>; } - function createTrustMetadata(workspaceId: string, projectPaths: string[]): WorkspaceMetadata { + function createTrustMetadata( + workspaceId: string, + projectPaths: string[], + runtimeConfig: WorkspaceMetadata["runtimeConfig"] = { type: "local" } + ): WorkspaceMetadata { const [primaryProjectPath, secondaryProjectPath] = projectPaths; if (!primaryProjectPath) { throw new Error("Expected at least one project path"); @@ -1832,14 +1837,15 @@ describe("AIService.streamMessage multi-project trust gating", () => { { projectPath: secondaryProjectPath, projectName: "project-b" }, ] : undefined, - runtimeConfig: { type: "local" }, + runtimeConfig, }; } function createHarness( muxHomePath: string, metadata: WorkspaceMetadata, - multiProjectExperimentEnabled = true + multiProjectExperimentEnabled = true, + workspacePathOverride?: string ): TrustGatingHarness { const config = new Config(muxHomePath); const historyService = new HistoryService(config); @@ -1949,7 +1955,7 @@ describe("AIService.streamMessage multi-project trust gating", () => { spyOn(initStateManager, "waitForInit").mockResolvedValue(undefined); spyOn(config, "findWorkspace").mockReturnValue({ - workspacePath: metadata.projectPath, + workspacePath: workspacePathOverride ?? metadata.projectPath, projectPath: metadata.projectPath, }); @@ -2032,6 +2038,42 @@ describe("AIService.streamMessage multi-project trust gating", () => { expect(trustedFromFirstGetToolsCall(harness.getToolsForModelSpy)).toBe(false); }); + it("uses the persisted workspace root as cwd for multi-project ssh startup", async () => { + using muxHome = new DisposableTempDir("ai-service-multi-project-persisted-cwd"); + const projectAPath = path.join(muxHome.path, "project-a"); + const projectBPath = path.join(muxHome.path, "project-b"); + await fs.mkdir(projectAPath, { recursive: true }); + await fs.mkdir(projectBPath, { recursive: true }); + + const workspaceId = "workspace-multi-project-persisted-cwd"; + const persistedWorkspacePath = path.join(muxHome.path, "persisted-legacy-workspace-root"); + const metadata = createTrustMetadata(workspaceId, [projectAPath, projectBPath], { + type: "ssh", + host: "example.com", + srcBaseDir: "/remote/src", + }); + const harness = createHarness(muxHome.path, metadata, true, persistedWorkspacePath); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockImplementation( + (_runtimeConfig, options) => new LocalRuntime(options?.projectPath ?? projectAPath) + ); + + try { + await harness.config.editConfig((cfg) => { + cfg.projects.set(projectAPath, { workspaces: [], trusted: true }); + cfg.projects.set(projectBPath, { workspaces: [], trusted: true }); + return cfg; + }); + + await streamOnce(harness, workspaceId); + + const toolConfig = harness.getToolsForModelSpy.mock.calls[0]?.[1] as + | { cwd?: unknown } + | undefined; + expect(toolConfig?.cwd).toBe(persistedWorkspacePath); + } finally { + createRuntimeSpy.mockRestore(); + } + }); it("fails closed before tool setup when the multi-project experiment is disabled", async () => { using muxHome = new DisposableTempDir("ai-service-multi-project-experiment-disabled"); const projectAPath = path.join(muxHome.path, "project-a"); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 2c0e681cb1..827d58b6de 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -23,9 +23,13 @@ import type { SendMessageError } from "@/common/types/errors"; import { getToolsForModel } from "@/common/utils/tools/tools"; import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors"; import { createRuntime } from "@/node/runtime/runtimeFactory"; +import { + createRuntimeForWorkspace, + resolveWorkspaceExecutionPath, +} from "@/node/runtime/runtimeHelpers"; import { MultiProjectRuntime } from "@/node/runtime/multiProjectRuntime"; import { getMuxEnv, getRuntimeType } from "@/node/runtime/initHook"; -import { getSrcBaseDir } from "@/common/types/runtime"; +import { getSrcBaseDir, isSSHRuntime } from "@/common/types/runtime"; import { ContainerManager } from "@/node/multiProject/containerManager"; import { secretsToRecord, type ExternalSecretResolver } from "@/common/types/secrets"; import { mergeMultiProjectSecrets } from "@/node/services/utils/multiProjectSecrets"; @@ -924,10 +928,19 @@ export class AIService extends EventEmitter { }); }; - if (!this.config.findWorkspace(workspaceId)) { + const workspace = this.config.findWorkspace(workspaceId); + if (!workspace) { return Err({ type: "unknown", raw: `Workspace ${workspaceId} not found in config` }); } + const metadataWithPath = { + ...metadata, + // Existing workspaces may still live under the legacy SSH basename layout until the first + // post-upgrade operation reuses their persisted root, so stream startup must seed the runtime + // from config instead of reconstructing a hashed default path before layout detection runs. + namedWorkspacePath: workspace.workspacePath, + }; + const multiProjectExecutionGate = this.ensureMultiProjectRuntimeExecutionEnabled( workspaceId, metadata @@ -949,17 +962,19 @@ export class AIService extends EventEmitter { })), metadata.name ) - : createRuntime(metadata.runtimeConfig, { - projectPath: metadata.projectPath, - workspaceName: metadata.name, - }); + : createRuntimeForWorkspace(metadataWithPath); - // In-place workspaces (CLI/benchmarks) have projectPath === name - // Use path directly instead of reconstructing via getWorkspacePath + // In-place workspaces (CLI/benchmarks) have projectPath === name. const isInPlace = metadata.projectPath === metadata.name; const workspacePath = isInPlace ? metadata.projectPath - : runtime.getWorkspacePath(metadata.projectPath, metadata.name); + : isMultiProject(metadata) && !isSSHRuntime(metadata.runtimeConfig) + ? // Non-SSH multi-project runtimes intentionally start from their shared container root so + // sibling repos stay addressable during agent/tool setup. SSH workspaces are the exception: + // upgraded legacy layouts must reuse the persisted root from config until remote layout + // detection seeds the new hashed paths. + runtime.getWorkspacePath(metadata.projectPath, metadata.name) + : resolveWorkspaceExecutionPath(metadataWithPath, runtime); // Wait for init to complete before any runtime I/O operations // (SSH/devcontainer may not be ready until init finishes pulling the container) diff --git a/src/node/services/workspaceProjectRepos.test.ts b/src/node/services/workspaceProjectRepos.test.ts index 730cdfa33d..5e5e9f5afe 100644 --- a/src/node/services/workspaceProjectRepos.test.ts +++ b/src/node/services/workspaceProjectRepos.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "bun:test"; +import { + buildLegacyRemoteProjectLayout, + getRemoteWorkspacePath, +} from "@/node/runtime/remoteProjectLayout"; import { getWorkspaceProjectRepos } from "@/node/services/workspaceProjectRepos"; describe("getWorkspaceProjectRepos", () => { @@ -59,6 +63,39 @@ describe("getWorkspaceProjectRepos", () => { expect(repos[0]?.repoCwd).toBe("/tmp/legacy/main"); }); + it("derives persisted legacy SSH paths for secondary multi-project repos", () => { + const runtimeConfig = { + type: "ssh", + host: "example.com", + srcBaseDir: "/tmp/src", + } as const; + const workspaceName = "main"; + const primaryProjectPath = "/tmp/projects/main"; + const secondaryProjectPath = "/tmp/projects/other"; + const repos = getWorkspaceProjectRepos({ + workspaceId: "workspace-1", + workspaceName, + workspacePath: getRemoteWorkspacePath( + buildLegacyRemoteProjectLayout(runtimeConfig.srcBaseDir, primaryProjectPath), + workspaceName + ), + runtimeConfig, + projectPath: primaryProjectPath, + projectName: "main", + projects: [ + { projectPath: primaryProjectPath, projectName: "main" }, + { projectPath: secondaryProjectPath, projectName: "other" }, + ], + }); + + expect(repos[1]?.repoCwd).toBe( + getRemoteWorkspacePath( + buildLegacyRemoteProjectLayout(runtimeConfig.srcBaseDir, secondaryProjectPath), + workspaceName + ) + ); + }); + it("disambiguates storage keys when sanitized project names collide", () => { const repos = getWorkspaceProjectRepos({ workspaceId: "workspace-1", diff --git a/src/node/services/workspaceProjectRepos.ts b/src/node/services/workspaceProjectRepos.ts index 4523f32ea7..3eb4c1aad6 100644 --- a/src/node/services/workspaceProjectRepos.ts +++ b/src/node/services/workspaceProjectRepos.ts @@ -2,8 +2,13 @@ import assert from "node:assert/strict"; import * as path from "node:path"; import type { ProjectRef } from "@/common/types/workspace"; -import type { RuntimeConfig } from "@/common/types/runtime"; +import { isSSHRuntime, type RuntimeConfig } from "@/common/types/runtime"; import { PlatformPaths } from "@/common/utils/paths"; +import { + buildLegacyRemoteProjectLayout, + buildRemoteProjectLayout, + getRemoteWorkspacePath, +} from "@/node/runtime/remoteProjectLayout"; import { createRuntime } from "@/node/runtime/runtimeFactory"; export interface WorkspaceProjectRepo { @@ -133,6 +138,43 @@ export function getWorkspaceProjectStorageKeys( return storageKeys; } +function getWorkspacePathHint( + params: WorkspaceProjectRepoParams, + targetProjectPath: string +): string | undefined { + if (targetProjectPath === params.projectPath) { + return params.workspacePath; + } + if (!isSSHRuntime(params.runtimeConfig)) { + return undefined; + } + + const currentProjectRoot = path.posix.dirname(path.posix.normalize(params.workspacePath)); + const primaryLegacyLayout = buildLegacyRemoteProjectLayout( + params.runtimeConfig.srcBaseDir, + params.projectPath + ); + if (currentProjectRoot === primaryLegacyLayout.projectRoot) { + return getRemoteWorkspacePath( + buildLegacyRemoteProjectLayout(params.runtimeConfig.srcBaseDir, targetProjectPath), + params.workspaceName + ); + } + + const primaryPreferredLayout = buildRemoteProjectLayout( + params.runtimeConfig.srcBaseDir, + params.projectPath + ); + if (currentProjectRoot === primaryPreferredLayout.projectRoot) { + return getRemoteWorkspacePath( + buildRemoteProjectLayout(params.runtimeConfig.srcBaseDir, targetProjectPath), + params.workspaceName + ); + } + + return undefined; +} + export function getWorkspaceProjectRepos( params: WorkspaceProjectRepoParams ): WorkspaceProjectRepo[] { @@ -165,8 +207,7 @@ export function getWorkspaceProjectRepos( ? createRuntime(params.runtimeConfig, { projectPath: project.projectPath, workspaceName: params.workspaceName, - workspacePath: - project.projectPath === params.projectPath ? params.workspacePath : undefined, + workspacePath: getWorkspacePathHint(params, project.projectPath), }).getWorkspacePath(project.projectPath, params.workspaceName) : params.workspacePath; From 44b8500e9451691d511008d7a87bb86d72e2f592 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 17:55:44 -0500 Subject: [PATCH 03/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20legacy?= =?UTF-8?q?=20ssh=20workspace=20path=20hints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/SSHRuntime.test.ts | 2 +- src/node/services/workspaceProjectRepos.ts | 4 +- .../workspaceService.multiProject.test.ts | 109 ++++++++++++++++++ src/node/services/workspaceService.test.ts | 72 ++++++++++++ src/node/services/workspaceService.ts | 39 ++++++- 5 files changed, 221 insertions(+), 5 deletions(-) diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index 8ab6a6698d..144fc3ada3 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -269,7 +269,7 @@ describe("SSHRuntime bundle sync reuse", () => { ); expect(localManifestSpy).toHaveBeenCalledWith("/projects/demo"); expect(remoteManifestSpy).toHaveBeenCalledWith( - '"/home/user/src/demo/.mux-base.git"', + JSON.stringify(computeBaseRepoPath("/home/user/src", "/projects/demo")), undefined ); expect(initMessages.some((message) => message.includes("skipping sync"))).toBe(true); diff --git a/src/node/services/workspaceProjectRepos.ts b/src/node/services/workspaceProjectRepos.ts index 3eb4c1aad6..7bddfadf33 100644 --- a/src/node/services/workspaceProjectRepos.ts +++ b/src/node/services/workspaceProjectRepos.ts @@ -138,7 +138,7 @@ export function getWorkspaceProjectStorageKeys( return storageKeys; } -function getWorkspacePathHint( +export function getWorkspacePathHintForProject( params: WorkspaceProjectRepoParams, targetProjectPath: string ): string | undefined { @@ -207,7 +207,7 @@ export function getWorkspaceProjectRepos( ? createRuntime(params.runtimeConfig, { projectPath: project.projectPath, workspaceName: params.workspaceName, - workspacePath: getWorkspacePathHint(params, project.projectPath), + workspacePath: getWorkspacePathHintForProject(params, project.projectPath), }).getWorkspacePath(project.projectPath, params.workspaceName) : params.workspacePath; diff --git a/src/node/services/workspaceService.multiProject.test.ts b/src/node/services/workspaceService.multiProject.test.ts index b83c904e28..476d511c8b 100644 --- a/src/node/services/workspaceService.multiProject.test.ts +++ b/src/node/services/workspaceService.multiProject.test.ts @@ -201,6 +201,115 @@ describe("WorkspaceService executeBash runtime selection", () => { } }); + test("preserves inferred legacy SSH repo roots for multi-project repo-root bash mode", async () => { + const workspaceId = "ws-multi-bash-ssh"; + const workspaceName = "feature-multi-bash-ssh"; + const srcDir = "/tmp/src"; + const projectAPath = "/tmp/project-a"; + const projectBPath = "/tmp/project-b"; + const primaryWorkspacePath = `/tmp/src/project-a/${workspaceName}`; + const metadata: WorkspaceMetadata = { + id: workspaceId, + name: workspaceName, + projectPath: projectAPath, + projectName: "project-a", + projects: [ + { projectPath: projectAPath, projectName: "project-a" }, + { projectPath: projectBPath, projectName: "project-b" }, + ], + runtimeConfig: { type: "ssh", host: "example.com", srcBaseDir: "/tmp/src" }, + }; + const waitForInitMock = mock(() => Promise.resolve()); + const ensureReadyAMock = mock(() => Promise.resolve({ ready: true as const })); + const ensureReadyBMock = mock(() => Promise.resolve({ ready: true as const })); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockImplementation( + (_runtimeConfig, options) => { + if (options?.projectPath === projectAPath) { + expect(options.workspacePath).toBe(primaryWorkspacePath); + return { + ensureReady: ensureReadyAMock, + getWorkspacePath: mock(() => primaryWorkspacePath), + } as unknown as ReturnType; + } + if (options?.projectPath === projectBPath) { + expect(options.workspacePath).toBe(`/tmp/src/project-b/${workspaceName}`); + return { + ensureReady: ensureReadyBMock, + getWorkspacePath: mock(() => `/tmp/src/project-b/${workspaceName}`), + } as unknown as ReturnType; + } + throw new Error(`Unexpected projectPath: ${options?.projectPath ?? "missing"}`); + } + ); + const bashExecuteMock = mock(() => + Promise.resolve({ success: true as const, output: "ok", exitCode: 0, wall_duration_ms: 1 }) + ); + let capturedToolConfig: Parameters[0] | undefined; + const createBashToolSpy = spyOn(bashToolModule, "createBashTool").mockImplementation( + (config) => { + capturedToolConfig = config; + return { + execute: bashExecuteMock, + } as unknown as ReturnType; + } + ); + + const aiService: AIService = { + isStreaming: mock(() => false), + getWorkspaceMetadata: mock(() => Promise.resolve(Ok(metadata))), + on: mock(() => undefined), + off: mock(() => undefined), + } as unknown as AIService; + const workspaceService = new WorkspaceService( + { + srcDir, + getSessionDir: mock(() => "/tmp/test/sessions"), + findWorkspace: mock(() => ({ + projectPath: projectAPath, + workspacePath: primaryWorkspacePath, + })), + loadConfigOrDefault: mock(() => ({ + projects: new Map([ + [projectAPath, { workspaces: [], trusted: true }], + [projectBPath, { workspaces: [], trusted: true }], + ]), + })), + getEffectiveSecrets: mock(() => []), + } as unknown as Config, + historyService, + aiService, + { + on: mock(() => undefined as unknown as InitStateManager), + getInitState: mock(() => undefined), + waitForInit: waitForInitMock, + } as unknown as InitStateManager, + mockExtensionMetadataService as ExtensionMetadataService, + mockBackgroundProcessManager as BackgroundProcessManager, + undefined, + undefined, + undefined, + createMockExperimentsService(true) + ); + + try { + const result = await workspaceService.executeBash(workspaceId, "git status --short", { + cwdMode: "repo-root", + repoRootProjectPath: projectBPath, + }); + + expect(result.success).toBe(true); + expect(waitForInitMock).toHaveBeenCalledWith(workspaceId); + assert(capturedToolConfig); + expect(capturedToolConfig.cwd).toBe(`/tmp/src/project-b/${workspaceName}`); + expect(ensureReadyAMock).toHaveBeenCalledTimes(1); + expect(ensureReadyBMock).toHaveBeenCalledTimes(1); + expect(bashExecuteMock).toHaveBeenCalledTimes(1); + } finally { + createBashToolSpy.mockRestore(); + createRuntimeSpy.mockRestore(); + } + }); + test("lets multi-project script mode target a secondary repo checkout explicitly", async () => { const workspaceId = "ws-multi-bash-repo-root"; const workspaceName = "feature-multi-bash-repo-root"; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 08bd038646..22dce9e810 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -2240,6 +2240,78 @@ describe("WorkspaceService getFileCompletions", () => { expect(execBufferedSpy.mock.calls[0]?.[2].cwd).toBe("/persisted/project-a/ws"); }); + test("preserves inferred legacy SSH paths for multi-project completions", async () => { + interface WorkspaceServiceTestAccess { + getInfo: (workspaceId: string) => Promise; + } + + const svc = workspaceService as unknown as WorkspaceServiceTestAccess; + svc.getInfo = mock(() => + Promise.resolve({ + id: "ws-multi-ssh", + name: "ws", + projectName: "project-a", + projectPath: "/tmp/project-a", + namedWorkspacePath: "/tmp/src/project-a/ws", + runtimeConfig: { type: "ssh", host: "example.com", srcBaseDir: "/tmp/src" }, + projects: [ + { projectPath: "/tmp/project-a", projectName: "project-a" }, + { projectPath: "/tmp/project-b", projectName: "project-b" }, + ], + } satisfies FrontendWorkspaceMetadata) + ); + const config = (workspaceService as unknown as { config: Config }).config; + spyOn(config, "findWorkspace").mockReturnValue({ + projectPath: "/tmp/project-a", + workspacePath: "/tmp/src/project-a/ws", + }); + createRuntimeSpy.mockImplementation((_runtimeConfig, options) => { + const runtimeProjectPath = options?.projectPath; + if (!runtimeProjectPath) { + throw new Error("Expected createRuntime projectPath in legacy SSH completion test"); + } + return { + getWorkspacePath: () => + options.workspacePath ?? `/runtime/${path.basename(runtimeProjectPath)}/ws`, + } as unknown as ReturnType; + }); + + execBufferedSpy.mockImplementation((_runtime, _command, options) => { + if (options.cwd === "/tmp/src/project-a/ws") { + return Promise.resolve({ + stdout: "README.md\n", + stderr: "", + exitCode: 0, + duration: 1, + }); + } + if (options.cwd === "/tmp/src/project-b/ws") { + return Promise.resolve({ + stdout: "src/b.ts\n", + stderr: "", + exitCode: 0, + duration: 1, + }); + } + return Promise.reject(new Error(`Unexpected cwd ${options.cwd}`)); + }); + + const result = await workspaceService.getFileCompletions("ws-multi-ssh", "", 10); + + expect(result.paths).toContain("project-a/README.md"); + expect(result.paths).toContain("project-b/src/b.ts"); + expect(createRuntimeSpy).toHaveBeenNthCalledWith(1, expect.anything(), { + projectPath: "/tmp/project-a", + workspaceName: "ws", + workspacePath: "/tmp/src/project-a/ws", + }); + expect(createRuntimeSpy).toHaveBeenNthCalledWith(2, expect.anything(), { + projectPath: "/tmp/project-b", + workspaceName: "ws", + workspacePath: "/tmp/src/project-b/ws", + }); + }); + test("aggregates multi-project completions using project-prefixed paths", async () => { interface WorkspaceServiceTestAccess { getInfo: (workspaceId: string) => Promise; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index a5e965834b..e4bdffac6e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -39,6 +39,7 @@ import { createRuntimeForWorkspace, resolveWorkspaceExecutionPath, } from "@/node/runtime/runtimeHelpers"; +import { getWorkspacePathHintForProject } from "@/node/services/workspaceProjectRepos"; import { validateWorkspaceName } from "@/common/utils/validation/workspaceValidation"; import { ensurePrivateDir } from "@/node/utils/fs"; import { stripTrailingSlashes } from "@/node/utils/pathUtils"; @@ -6446,7 +6447,8 @@ export class WorkspaceService extends EventEmitter { } private async listWorkspacePathsForFileCompletions( - metadata: FrontendWorkspaceMetadata + metadata: FrontendWorkspaceMetadata, + workspacePath?: string ): Promise { if (!isMultiProject(metadata)) { const runtime = createRuntimeForWorkspace(metadata); @@ -6464,6 +6466,21 @@ export class WorkspaceService extends EventEmitter { const projectRuntime = createRuntime(metadata.runtimeConfig, { projectPath: project.projectPath, workspaceName: metadata.name, + workspacePath: + isSSHRuntime(metadata.runtimeConfig) && workspacePath != null + ? getWorkspacePathHintForProject( + { + workspaceId: metadata.id, + workspaceName: metadata.name, + workspacePath, + runtimeConfig: metadata.runtimeConfig, + projectPath: metadata.projectPath, + projectName: metadata.projectName, + projects: metadata.projects, + }, + project.projectPath + ) + : undefined, }); const projectWorkspacePath = projectRuntime.getWorkspacePath( project.projectPath, @@ -6526,7 +6543,11 @@ export class WorkspaceService extends EventEmitter { const previousIndex = cacheEntry.index; try { - const files = await this.listWorkspacePathsForFileCompletions(metadata); + const workspace = this.config.findWorkspace(workspaceId); + const files = await this.listWorkspacePathsForFileCompletions( + metadata, + workspace?.workspacePath + ); cacheEntry.index = files === null ? previousIndex : buildFileCompletionsIndex(files); cacheEntry.fetchedAt = Date.now(); } catch (error) { @@ -6616,6 +6637,20 @@ export class WorkspaceService extends EventEmitter { runtime: createRuntime(metadata.runtimeConfig, { projectPath: project.projectPath, workspaceName: metadata.name, + workspacePath: isSSHRuntime(metadata.runtimeConfig) + ? getWorkspacePathHintForProject( + { + workspaceId, + workspaceName: metadata.name, + workspacePath: workspace.workspacePath, + runtimeConfig: metadata.runtimeConfig, + projectPath: metadata.projectPath, + projectName: metadata.projectName, + projects: metadata.projects, + }, + project.projectPath + ) + : undefined, }), })) : undefined; From ab184396cc6a95f25d7e06978146e2bd8d4bbd6c Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 18:04:39 -0500 Subject: [PATCH 04/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20release=20ssh=20lea?= =?UTF-8?q?ses=20on=20synthetic=20exits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/RemoteRuntime.test.ts | 12 ++++++++-- src/node/runtime/RemoteRuntime.ts | 3 +++ .../transports/OpenSSHTransport.test.ts | 22 +++++++++++++++++++ .../runtime/transports/OpenSSHTransport.ts | 2 ++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/node/runtime/RemoteRuntime.test.ts b/src/node/runtime/RemoteRuntime.test.ts index f7a8b4a49c..25cc8e6917 100644 --- a/src/node/runtime/RemoteRuntime.test.ts +++ b/src/node/runtime/RemoteRuntime.test.ts @@ -33,6 +33,7 @@ class TestRemoteRuntime extends RemoteRuntime { constructor( private readonly childProcess: FakeChildProcess, private readonly onExitCalls: Array<[number, string]>, + private readonly onCloseCalls: number[], private readonly onExitCodeCalls: number[] ) { super(); @@ -47,6 +48,9 @@ class TestRemoteRuntime extends RemoteRuntime { onExit: (exitCode, stderr) => { this.onExitCalls.push([exitCode, stderr]); }, + onClose: () => { + this.onCloseCalls.push(1); + }, }); } @@ -103,8 +107,9 @@ describe("RemoteRuntime synthetic exit handling", () => { test("does not forward aborted exits to transport onExit hooks", async () => { const childProcess = new FakeChildProcess(); const onExitCalls: Array<[number, string]> = []; + const onCloseCalls: number[] = []; const onExitCodeCalls: number[] = []; - const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onExitCodeCalls); + const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls, onExitCodeCalls); const controller = new AbortController(); const stream = await runtime.exec("echo ok", { cwd: "/tmp", abortSignal: controller.signal }); @@ -113,14 +118,16 @@ describe("RemoteRuntime synthetic exit handling", () => { expect(await stream.exitCode).toBe(EXIT_CODE_ABORTED); expect(onExitCalls).toEqual([]); + expect(onCloseCalls).toEqual([1]); expect(onExitCodeCalls).toEqual([EXIT_CODE_ABORTED]); }); test("does not forward timed-out exits to transport onExit hooks", async () => { const childProcess = new FakeChildProcess(); const onExitCalls: Array<[number, string]> = []; + const onCloseCalls: number[] = []; const onExitCodeCalls: number[] = []; - const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onExitCodeCalls); + const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls, onExitCodeCalls); const stream = await runtime.exec("echo ok", { cwd: "/tmp", timeout: 0.01 }); await new Promise((resolve) => setTimeout(resolve, 20)); @@ -128,6 +135,7 @@ describe("RemoteRuntime synthetic exit handling", () => { expect(await stream.exitCode).toBe(EXIT_CODE_TIMEOUT); expect(onExitCalls).toEqual([]); + expect(onCloseCalls).toEqual([1]); expect(onExitCodeCalls).toEqual([EXIT_CODE_TIMEOUT]); }); }); diff --git a/src/node/runtime/RemoteRuntime.ts b/src/node/runtime/RemoteRuntime.ts index e7775807ff..31e5da32b3 100644 --- a/src/node/runtime/RemoteRuntime.ts +++ b/src/node/runtime/RemoteRuntime.ts @@ -49,6 +49,8 @@ export interface SpawnResult { preExec?: Promise; /** Optional transport-scoped exit handling (e.g., master-pool health accounting). */ onExit?: (exitCode: number, stderr: string) => void; + /** Optional close handling that must run even for synthetic abort/timeout exits. */ + onClose?: () => void; /** Optional transport-scoped spawn error handling. */ onError?: (error: Error) => void; } @@ -182,6 +184,7 @@ export abstract class RemoteRuntime implements Runtime { if (finalExitCode !== EXIT_CODE_ABORTED && finalExitCode !== EXIT_CODE_TIMEOUT) { spawnResult.onExit?.(finalExitCode, stderrForErrorReporting); } + spawnResult.onClose?.(); // Let subclass handle exit code (e.g., SSH connection pool) this.onExitCode(finalExitCode, options, stderrForErrorReporting); diff --git a/src/node/runtime/transports/OpenSSHTransport.test.ts b/src/node/runtime/transports/OpenSSHTransport.test.ts index 34d5311879..e073a43ae3 100644 --- a/src/node/runtime/transports/OpenSSHTransport.test.ts +++ b/src/node/runtime/transports/OpenSSHTransport.test.ts @@ -66,6 +66,28 @@ describe("OpenSSHTransport.spawnRemoteProcess", () => { return args; } + test("releases exec leases on close even when health accounting is skipped", async () => { + const release = mock(() => undefined); + const reportFailure = mock(() => undefined); + const markHealthy = mock(() => undefined); + acquireLeaseSpy.mockResolvedValue({ + controlPath: "/tmp/mux-ssh-test-shard", + shardId: "shard-0", + release, + reportFailure, + markHealthy, + }); + const transport = new OpenSSHTransport({ host: "remote.example.com" }); + const spawnResult = await transport.spawnRemoteProcess("echo ok", {}); + + spawnResult.onClose?.(); + spawnResult.onClose?.(); + + expect(release).toHaveBeenCalledTimes(1); + expect(reportFailure).not.toHaveBeenCalled(); + expect(markHealthy).not.toHaveBeenCalled(); + }); + test("explicit headless (no service) includes host-key fallback options and BatchMode=yes", async () => { setSshPromptCapability(false); setOpenSSHHostKeyPolicyMode("headless-fallback"); diff --git a/src/node/runtime/transports/OpenSSHTransport.ts b/src/node/runtime/transports/OpenSSHTransport.ts index ff83ecc5ee..275df3f6a0 100644 --- a/src/node/runtime/transports/OpenSSHTransport.ts +++ b/src/node/runtime/transports/OpenSSHTransport.ts @@ -98,6 +98,8 @@ export class OpenSSHTransport implements SSHTransport { } else { lease.markHealthy(); } + }, + onClose: () => { releaseLease(); }, onError: (error) => { From 81f1cd3e57ae73070a98c66e6bb33e2293f45ced Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 18:22:47 -0500 Subject: [PATCH 05/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20use=20unique=20temp?= =?UTF-8?q?=20bundle=20paths=20per=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/SSHRuntime.test.ts | 155 ++++++++++++++++++++++++++++ src/node/runtime/SSHRuntime.ts | 4 +- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index 144fc3ada3..e6d9ad5cb0 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -1,3 +1,4 @@ +import * as crypto from "node:crypto"; import * as path from "node:path"; import { describe, expect, it, beforeEach, afterEach, spyOn } from "bun:test"; import * as runtimeHelpers from "@/node/utils/runtime/helpers"; @@ -8,6 +9,7 @@ import { buildLegacyRemoteProjectLayout, buildRemoteProjectLayout, getRemoteWorkspacePath, + getSnapshotMarkerPath, } from "./remoteProjectLayout"; import { createSSHTransport } from "./transports"; import { projectSyncCoordinator } from "./projectSyncCoordinator"; @@ -195,6 +197,65 @@ describe("SSHRuntime bundle sync reuse", () => { ) => Promise; } + interface RuntimeWithSyncProjectToRemote { + syncProjectToRemote: ( + projectPath: string, + workspacePath: string, + initLogger: { + logStep: (message: string) => void; + logStdout: (line: string) => void; + logStderr: (line: string) => void; + logComplete: (exitCode: number) => void; + }, + abortSignal?: AbortSignal + ) => Promise; + } + + interface RuntimeWithEnsureBaseRepo { + ensureBaseRepo: ( + projectPath: string, + initLogger: { + logStep: (message: string) => void; + logStdout: (line: string) => void; + logStderr: (line: string) => void; + logComplete: (exitCode: number) => void; + }, + abortSignal?: AbortSignal + ) => Promise; + } + + interface RuntimeWithComputeSnapshotDigest { + computeSnapshotDigest: (projectPath: string) => Promise; + } + + interface RuntimeWithTransferBundleToRemote { + transferBundleToRemote: ( + projectPath: string, + remoteBundlePath: string, + initLogger: { + logStep: (message: string) => void; + logStdout: (line: string) => void; + logStderr: (line: string) => void; + logComplete: (exitCode: number) => void; + }, + abortSignal?: AbortSignal + ) => Promise; + } + + interface RuntimeWithRefreshBaseRepoOrigin { + refreshBaseRepoOrigin: ( + projectPath: string, + baseRepoPathArg: string, + initLogger: { + logStep: (message: string) => void; + logStdout: (line: string) => void; + logStderr: (line: string) => void; + logComplete: (exitCode: number) => void; + }, + abortSignal?: AbortSignal + ) => Promise; + } + let runtime: SSHRuntime; let execBufferedSpy: ReturnType> | null = null; @@ -279,6 +340,100 @@ describe("SSHRuntime bundle sync reuse", () => { } }); + it("uploads snapshot bundles through a per-attempt temp path", async () => { + const projectPath = "/projects/demo"; + const snapshotDigest = "abc123"; + const layout = buildRemoteProjectLayout("/home/user/src", projectPath); + const baseRepoPathArg = JSON.stringify(layout.baseRepoPath); + const bundleFileName = `${snapshotDigest}.uuid-1234.bundle`; + const expectedRemoteBundlePath = path.posix.join( + "~/.mux-bundles", + layout.projectId, + bundleFileName + ); + const snapshotMarkerPath = getSnapshotMarkerPath(layout, snapshotDigest); + const currentSnapshotPath = path.posix.join(layout.snapshotMarkerDir, "current"); + const writeFileCalls: string[] = []; + const randomUuidSpy = spyOn(crypto, "randomUUID").mockReturnValue("uuid-1234"); + const ensureBaseRepoSpy = spyOn( + runtime as unknown as RuntimeWithEnsureBaseRepo, + "ensureBaseRepo" + ).mockResolvedValue(baseRepoPathArg); + const computeSnapshotDigestSpy = spyOn( + runtime as unknown as RuntimeWithComputeSnapshotDigest, + "computeSnapshotDigest" + ).mockResolvedValue(snapshotDigest); + const transferBundleSpy = spyOn( + runtime as unknown as RuntimeWithTransferBundleToRemote, + "transferBundleToRemote" + ).mockResolvedValue(expectedRemoteBundlePath); + const refreshBaseRepoOriginSpy = spyOn( + runtime as unknown as RuntimeWithRefreshBaseRepoOrigin, + "refreshBaseRepoOrigin" + ).mockResolvedValue(undefined); + const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation((filePath: string) => { + writeFileCalls.push(filePath); + return new WritableStream({ + write() { + return Promise.resolve(); + }, + close() { + return Promise.resolve(); + }, + }); + }); + execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( + (_runtime, command) => { + if (command.includes('current_snapshot=""')) { + return Promise.resolve({ stdout: "missing\n", stderr: "", exitCode: 0, duration: 0 }); + } + if (command.startsWith("mkdir -p ")) { + expect(command).toContain(layout.projectId); + return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); + } + if (command.includes(" fetch ")) { + expect(command).toContain(bundleFileName); + return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); + } + if (command.startsWith("rm -f ")) { + expect(command).toContain(bundleFileName); + return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); + } + throw new Error(`Unexpected execBuffered command: ${command}`); + } + ); + + try { + await (runtime as unknown as RuntimeWithSyncProjectToRemote).syncProjectToRemote( + projectPath, + "/unused/workspace", + initLogger + ); + + expect(transferBundleSpy).toHaveBeenCalledWith( + projectPath, + expectedRemoteBundlePath, + initLogger, + expect.any(AbortSignal) + ); + expect(refreshBaseRepoOriginSpy).toHaveBeenCalledWith( + projectPath, + baseRepoPathArg, + initLogger, + expect.any(AbortSignal) + ); + expect(writeFileCalls).toEqual([snapshotMarkerPath, currentSnapshotPath]); + } finally { + randomUuidSpy.mockRestore(); + ensureBaseRepoSpy.mockRestore(); + computeSnapshotDigestSpy.mockRestore(); + transferBundleSpy.mockRestore(); + refreshBaseRepoOriginSpy.mockRestore(); + writeFileSpy.mockRestore(); + projectSyncCoordinator.clearAll(); + } + }); + it("does not reuse the shared bundle when the remote trunk ref is stale", async () => { execFileAsyncSpy = spyOn(disposableExec, "execFileAsync").mockReturnValue( createMockExecResult(Promise.resolve({ stdout: "abc123\n", stderr: "" })) diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index b2b382142b..2172a19f9a 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -1377,10 +1377,12 @@ export class SSHRuntime extends RemoteRuntime { ); } + // Snapshot markers stay deterministic, but the uploaded bundle itself must use + // a per-attempt temp path so concurrent Mux processes do not stream into the same file. const remoteBundlePath = path.posix.join( "~/.mux-bundles", layout.projectId, - `${snapshotDigest}.bundle` + `${snapshotDigest}.${crypto.randomUUID()}.bundle` ); const remoteBundlePathArg = this.quoteForRemote(remoteBundlePath); const remoteBundleParentDir = path.posix.dirname(remoteBundlePath); From 04fc73145566721e22a4e8fa5cceeffebacbdfe5 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 18:26:21 -0500 Subject: [PATCH 06/16] =?UTF-8?q?=F0=9F=A4=96=20test:=20use=20a=20valid=20?= =?UTF-8?q?uuid=20fixture=20in=20bundle=20temp-path=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/SSHRuntime.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index e6d9ad5cb0..aa4eb72880 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -345,7 +345,8 @@ describe("SSHRuntime bundle sync reuse", () => { const snapshotDigest = "abc123"; const layout = buildRemoteProjectLayout("/home/user/src", projectPath); const baseRepoPathArg = JSON.stringify(layout.baseRepoPath); - const bundleFileName = `${snapshotDigest}.uuid-1234.bundle`; + const bundleUuid = "11111111-1111-1111-1111-111111111111"; + const bundleFileName = `${snapshotDigest}.${bundleUuid}.bundle`; const expectedRemoteBundlePath = path.posix.join( "~/.mux-bundles", layout.projectId, @@ -354,7 +355,7 @@ describe("SSHRuntime bundle sync reuse", () => { const snapshotMarkerPath = getSnapshotMarkerPath(layout, snapshotDigest); const currentSnapshotPath = path.posix.join(layout.snapshotMarkerDir, "current"); const writeFileCalls: string[] = []; - const randomUuidSpy = spyOn(crypto, "randomUUID").mockReturnValue("uuid-1234"); + const randomUuidSpy = spyOn(crypto, "randomUUID").mockReturnValue(bundleUuid); const ensureBaseRepoSpy = spyOn( runtime as unknown as RuntimeWithEnsureBaseRepo, "ensureBaseRepo" From 5198aab2b79802a240d6271e699aa4740f3a6f93 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 18:41:47 -0500 Subject: [PATCH 07/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20dedupe=20askpass=20?= =?UTF-8?q?prompts=20across=20ssh=20shards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/openSshMasterPool.test.ts | 85 +++++++++++++++++++++- src/node/runtime/openSshMasterPool.ts | 2 +- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/src/node/runtime/openSshMasterPool.test.ts b/src/node/runtime/openSshMasterPool.test.ts index e6120ab22c..4029cff64e 100644 --- a/src/node/runtime/openSshMasterPool.test.ts +++ b/src/node/runtime/openSshMasterPool.test.ts @@ -1,9 +1,16 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import type { spawn as spawnProcess } from "child_process"; import { EventEmitter } from "events"; import { PassThrough } from "stream"; +import { formatSshEndpoint } from "@/common/utils/ssh/formatSshEndpoint"; +import { SshPromptService } from "@/node/services/sshPromptService"; +import * as openSshPromptMediation from "./openSshPromptMediation"; import { OpenSSHMasterPool, getShardedControlPath } from "./openSshMasterPool"; -import type { SSHConnectionConfig } from "./sshConnectionPool"; +import { + setOpenSSHHostKeyPolicyMode, + setSshPromptService, + type SSHConnectionConfig, +} from "./sshConnectionPool"; class FakeChildProcess extends EventEmitter { readonly stdout = new PassThrough(); @@ -32,7 +39,13 @@ describe("getShardedControlPath", () => { describe("OpenSSHMasterPool", () => { const masterProcesses = new Map(); + let releaseInteractiveResponder: (() => void) | undefined; + afterEach(() => { + releaseInteractiveResponder?.(); + releaseInteractiveResponder = undefined; + setSshPromptService(undefined); + setOpenSSHHostKeyPolicyMode("headless-fallback"); masterProcesses.clear(); }); @@ -83,6 +96,74 @@ describe("OpenSSHMasterPool", () => { pool.clearAll(); }); + test("reuses one endpoint-scoped askpass dedupe key across shard startups", async () => { + const dedupeKeys: string[] = []; + const promptService = new SshPromptService(); + releaseInteractiveResponder = promptService.registerInteractiveResponder(); + setSshPromptService(promptService); + setOpenSSHHostKeyPolicyMode("strict"); + const askpassSpy = spyOn( + openSshPromptMediation, + "createMediatedAskpassSession" + ).mockImplementation((params) => { + dedupeKeys.push(params.dedupeKey ?? ""); + const mediatedAskpass: Awaited< + ReturnType + > = { + env: {}, + cleanup: mock(() => undefined), + getLastPromptOutcome: () => null, + }; + return Promise.resolve(mediatedAskpass); + }); + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 2, + sleep: () => Promise.resolve(), + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + try { + const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + const second = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); + + expect(first.controlPath).not.toBe(second.controlPath); + expect(dedupeKeys).toEqual([ + formatSshEndpoint(config.host, config.port ?? 22), + formatSshEndpoint(config.host, config.port ?? 22), + ]); + + first.release(); + second.release(); + } finally { + askpassSpy.mockRestore(); + pool.clearAll(); + } + }); + test("does not lease a shard until its master is ready", async () => { let ready = false; let releaseStartupWait: (() => void) | undefined; diff --git a/src/node/runtime/openSshMasterPool.ts b/src/node/runtime/openSshMasterPool.ts index db69d77362..92d8044af5 100644 --- a/src/node/runtime/openSshMasterPool.ts +++ b/src/node/runtime/openSshMasterPool.ts @@ -497,7 +497,7 @@ export class OpenSSHMasterPool { allowHostKey: true, allowCredential: false, }, - dedupeKey: `${formatSshEndpoint(group.config.host, group.config.port ?? 22)}:${shard.shardId}`, + dedupeKey: formatSshEndpoint(group.config.host, group.config.port ?? 22), getStderrContext: () => stderr, onHostKeyPromptStarted: () => { extendDeadline(HOST_KEY_APPROVAL_TIMEOUT_MS); From 09071ca56f1ac2b6a95d5e6f5b542b7ca39e4262 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 19:03:52 -0500 Subject: [PATCH 08/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20truncate=20ssh=20sh?= =?UTF-8?q?ard=20failure=20stderr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transports/OpenSSHTransport.test.ts | 28 +++++++++++++++++++ .../runtime/transports/OpenSSHTransport.ts | 15 +++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/node/runtime/transports/OpenSSHTransport.test.ts b/src/node/runtime/transports/OpenSSHTransport.test.ts index e073a43ae3..6d0060ecc5 100644 --- a/src/node/runtime/transports/OpenSSHTransport.test.ts +++ b/src/node/runtime/transports/OpenSSHTransport.test.ts @@ -88,6 +88,34 @@ describe("OpenSSHTransport.spawnRemoteProcess", () => { expect(markHealthy).not.toHaveBeenCalled(); }); + test("truncates connection-failure stderr before reporting shard health", async () => { + const release = mock(() => undefined); + let reportedError: string | undefined; + const reportFailure = mock((error: string) => { + reportedError = error; + }); + const markHealthy = mock(() => undefined); + acquireLeaseSpy.mockResolvedValue({ + controlPath: "/tmp/mux-ssh-test-shard", + shardId: "shard-0", + release, + reportFailure, + markHealthy, + }); + const transport = new OpenSSHTransport({ host: "remote.example.com" }); + const spawnResult = await transport.spawnRemoteProcess("echo ok", {}); + const longStderr = `${"x".repeat(1100)}\n`; + + spawnResult.onExit?.(255, longStderr); + + if (reportedError == null) { + throw new Error("Expected reportFailure to be called with a string error summary"); + } + expect(reportedError.length).toBeLessThan(longStderr.trim().length); + expect(reportedError.endsWith("…")).toBe(true); + expect(markHealthy).not.toHaveBeenCalled(); + }); + test("explicit headless (no service) includes host-key fallback options and BatchMode=yes", async () => { setSshPromptCapability(false); setOpenSSHHostKeyPolicyMode("headless-fallback"); diff --git a/src/node/runtime/transports/OpenSSHTransport.ts b/src/node/runtime/transports/OpenSSHTransport.ts index 275df3f6a0..3579e3821e 100644 --- a/src/node/runtime/transports/OpenSSHTransport.ts +++ b/src/node/runtime/transports/OpenSSHTransport.ts @@ -14,6 +14,19 @@ import type { PtySessionParams, } from "./SSHTransport"; +const MAX_REPORTED_FAILURE_STDERR_CHARS = 1000; + +function summarizeFailureStderr(stderr: string, exitCode: number): string { + const trimmed = stderr.trim(); + if (trimmed.length === 0) { + return `SSH exited with code ${exitCode}`; + } + if (trimmed.length <= MAX_REPORTED_FAILURE_STDERR_CHARS) { + return trimmed; + } + return `${trimmed.slice(0, MAX_REPORTED_FAILURE_STDERR_CHARS)}…`; +} + export class OpenSSHTransport implements SSHTransport { constructor(private readonly config: SSHConnectionConfig) {} @@ -94,7 +107,7 @@ export class OpenSSHTransport implements SSHTransport { process, onExit: (exitCode, stderr) => { if (this.isConnectionFailure(exitCode, stderr)) { - lease.reportFailure(stderr.trim() || `SSH exited with code ${exitCode}`); + lease.reportFailure(summarizeFailureStderr(stderr, exitCode)); } else { lease.markHealthy(); } From 9cdda18b9e4ded21c494619471405a9e6628c0ed Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 19:28:17 -0500 Subject: [PATCH 09/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20expire=20idle=20mas?= =?UTF-8?q?ters=20started=20by=20ensureReadyMaster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/openSshMasterPool.test.ts | 49 ++++++++++++++++++++++ src/node/runtime/openSshMasterPool.ts | 12 +++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/node/runtime/openSshMasterPool.test.ts b/src/node/runtime/openSshMasterPool.test.ts index 4029cff64e..797ea59ea3 100644 --- a/src/node/runtime/openSshMasterPool.test.ts +++ b/src/node/runtime/openSshMasterPool.test.ts @@ -555,6 +555,55 @@ describe("OpenSSHMasterPool", () => { pool.clearAll(); }); + test("ensureReadyMaster schedules idle cleanup after bootstrapping a shard", async () => { + const pool = new OpenSSHMasterPool({ + maxSessionsPerShard: 1, + maxShardsPerHost: 1, + sleep: () => Promise.resolve(), + spawnProcess: ((_command: string, args?: readonly string[]) => { + const proc = new FakeChildProcess(); + const normalizedArgs = [...(args ?? [])]; + + if (normalizedArgs.includes("-M")) { + const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); + if (controlPathArg) { + masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); + } + return proc as never; + } + + const controlPathIndex = normalizedArgs.indexOf("-S"); + const controlPath = + controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; + queueMicrotask(() => { + if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { + proc.exitCode = 0; + } + proc.emit("close", proc.exitCode ?? 1, null); + }); + return proc as never; + }) as unknown as typeof spawnProcess, + }); + + const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; + await pool.ensureReadyMaster(config, { maxWaitMs: 0, timeoutMs: 1000 }); + + const internals = pool as unknown as { + hostGroups: Map< + string, + { shards: Array<{ inflight: number; idleTimer?: ReturnType }> } + >; + }; + const shard = [...internals.hostGroups.values()][0]?.shards[0]; + if (!shard) { + throw new Error("Expected a tracked shard"); + } + expect(shard.inflight).toBe(0); + expect(shard.idleTimer).toBeDefined(); + + pool.clearAll(); + }); + test("retries transient shard startup failures within the maxWait budget", async () => { let startupAttempts = 0; const pool = new OpenSSHMasterPool({ diff --git a/src/node/runtime/openSshMasterPool.ts b/src/node/runtime/openSshMasterPool.ts index 92d8044af5..f8c484e7dc 100644 --- a/src/node/runtime/openSshMasterPool.ts +++ b/src/node/runtime/openSshMasterPool.ts @@ -184,7 +184,9 @@ export class OpenSSHMasterPool { } this.trimExitedShards(hostGroup); - if (this.pickReadyShard(hostGroup)) { + const readyShard = this.pickReadyShard(hostGroup); + if (readyShard) { + this.scheduleIdleDisposalIfUnused(readyShard); return; } @@ -202,6 +204,7 @@ export class OpenSSHMasterPool { ? defaultStartTimeoutMs : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); await this.startShard(hostGroup, restartable, startupTimeoutMs, options?.abortSignal); + this.scheduleIdleDisposalIfUnused(restartable); return; } catch (error) { if (options?.abortSignal?.aborted) { @@ -219,6 +222,7 @@ export class OpenSSHMasterPool { ? defaultStartTimeoutMs : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); await this.startShard(hostGroup, shard, startupTimeoutMs, options?.abortSignal); + this.scheduleIdleDisposalIfUnused(shard); return; } catch (error) { if (options?.abortSignal?.aborted) { @@ -437,6 +441,12 @@ export class OpenSSHMasterPool { }; } + private scheduleIdleDisposalIfUnused(shard: MasterShard): void { + if (shard.inflight === 0) { + this.scheduleIdleDisposal(shard); + } + } + private scheduleIdleDisposal(shard: MasterShard): void { clearTimeout(shard.idleTimer); shard.idleTimer = setTimeout(() => { From 97e7215ed0fcd6b50601b3aa5a1295a36f6c3b9e Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 19:49:09 -0500 Subject: [PATCH 10/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20reuse=20cached=20ss?= =?UTF-8?q?h=20layouts=20across=20runtime=20instances?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/SSHRuntime.test.ts | 23 ++++++++- src/node/runtime/SSHRuntime.ts | 51 ++++++++++++++----- .../services/utils/forkOrchestrator.test.ts | 11 +++- src/node/services/utils/forkOrchestrator.ts | 1 + src/node/services/workspaceService.ts | 2 + 5 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index aa4eb72880..2ce94990eb 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it, beforeEach, afterEach, spyOn } from "bun:test"; import * as runtimeHelpers from "@/node/utils/runtime/helpers"; import * as disposableExec from "@/node/utils/disposableExec"; import * as submoduleSync from "./submoduleSync"; -import { SSHRuntime, computeBaseRepoPath } from "./SSHRuntime"; +import { SSHRuntime, clearSharedProjectLayoutCache, computeBaseRepoPath } from "./SSHRuntime"; import { buildLegacyRemoteProjectLayout, buildRemoteProjectLayout, @@ -35,6 +35,10 @@ function createMockExecResult( } as unknown as ReturnType; } +afterEach(() => { + clearSharedProjectLayoutCache(); +}); + describe("SSHRuntime constructor", () => { it("should accept tilde in srcBaseDir", () => { // Tildes are now allowed - they will be resolved via resolvePath() @@ -1205,6 +1209,23 @@ describe("SSHRuntime layout detection", () => { expect(detectionCommand).toContain(`test -e "${legacyLayout.projectRoot}/${workspaceName}"`); expect(detectionCommand).not.toContain(`test -d "${legacyLayout.projectRoot}"`); }); + it("reuses a cached legacy layout for fresh runtimes without workspacePath hints", () => { + const config = { host: "example.com", srcBaseDir: "/home/user/src" }; + const projectPath = "/projects/cached-legacy-demo"; + const workspaceName = "legacy-slot"; + const legacyLayout = buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath); + const legacyWorkspacePath = getRemoteWorkspacePath(legacyLayout, workspaceName); + + new SSHRuntime(config, createSSHTransport(config, false), { + projectPath, + workspaceName, + workspacePath: legacyWorkspacePath, + }); + + const freshRuntime = new SSHRuntime(config, createSSHTransport(config, false)); + + expect(freshRuntime.getWorkspacePath(projectPath, workspaceName)).toBe(legacyWorkspacePath); + }); }); describe("computeBaseRepoPath", () => { diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index 2172a19f9a..8a7033899d 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -62,6 +62,18 @@ const BUNDLE_REF_PREFIX = "refs/mux-bundle/"; /** Small backoff for concurrent writers healing the same shared base repo config. */ const BASE_REPO_CONFIG_LOCK_RETRY_DELAYS_MS = [50, 100, 200]; +const sharedProjectLayouts = new Map(); + +function getProjectLayoutCacheKey(config: SSHRuntimeConfig, projectPath: string): string { + return [ + config.host, + config.port?.toString() ?? "22", + config.identityFile ?? "default", + config.srcBaseDir, + projectPath, + ].join(":"); +} + function isGitConfigLockConflict(message: string): boolean { return /could not lock config file/i.test(message); } @@ -166,6 +178,10 @@ export function computeBaseRepoPath(srcBaseDir: string, projectPath: string): st * * Extends RemoteRuntime for shared exec/file operations. */ +export function clearSharedProjectLayoutCache(): void { + sharedProjectLayouts.clear(); +} + export class SSHRuntime extends RemoteRuntime { private readonly config: SSHRuntimeConfig; private readonly transport: SSHTransport; @@ -195,7 +211,7 @@ export class SSHRuntime extends RemoteRuntime { this.currentWorkspacePath = options?.workspacePath; if (options?.projectPath && options.workspacePath) { - this.projectLayouts.set( + this.cacheProjectLayout( options.projectPath, buildRemoteProjectLayout( this.config.srcBaseDir, @@ -250,15 +266,31 @@ export class SSHRuntime extends RemoteRuntime { return buildRemoteProjectLayout(this.config.srcBaseDir, projectPath); } + private getCachedProjectLayout(projectPath: string): RemoteProjectLayout | undefined { + return ( + this.projectLayouts.get(projectPath) ?? + sharedProjectLayouts.get(getProjectLayoutCacheKey(this.config, projectPath)) + ); + } + + private cacheProjectLayout( + projectPath: string, + layout: RemoteProjectLayout + ): RemoteProjectLayout { + this.projectLayouts.set(projectPath, layout); + sharedProjectLayouts.set(getProjectLayoutCacheKey(this.config, projectPath), layout); + return layout; + } + private getPreferredProjectLayout(projectPath: string): RemoteProjectLayout { - return this.projectLayouts.get(projectPath) ?? this.getDefaultProjectLayout(projectPath); + return this.getCachedProjectLayout(projectPath) ?? this.getDefaultProjectLayout(projectPath); } private async resolveProjectLayout( projectPath: string, workspaceName?: string ): Promise { - const cached = this.projectLayouts.get(projectPath); + const cached = this.getCachedProjectLayout(projectPath); if (cached) { return cached; } @@ -286,11 +318,9 @@ export class SSHRuntime extends RemoteRuntime { timeout: 10, }); const layout = detection.stdout.trim() === "legacy" ? legacyLayout : preferredLayout; - this.projectLayouts.set(projectPath, layout); - return layout; + return this.cacheProjectLayout(projectPath, layout); } catch { - this.projectLayouts.set(projectPath, preferredLayout); - return preferredLayout; + return this.cacheProjectLayout(projectPath, preferredLayout); } } @@ -435,12 +465,7 @@ export class SSHRuntime extends RemoteRuntime { return this.currentWorkspacePath; } - const cachedLayout = this.projectLayouts.get(projectPath); - if (cachedLayout) { - return getRemoteWorkspacePath(cachedLayout, workspaceName); - } - - return getRemoteWorkspacePath(this.getDefaultProjectLayout(projectPath), workspaceName); + return getRemoteWorkspacePath(this.getPreferredProjectLayout(projectPath), workspaceName); } /** diff --git a/src/node/services/utils/forkOrchestrator.test.ts b/src/node/services/utils/forkOrchestrator.test.ts index 63e95d56db..e4c45d02d0 100644 --- a/src/node/services/utils/forkOrchestrator.test.ts +++ b/src/node/services/utils/forkOrchestrator.test.ts @@ -153,6 +153,7 @@ describe("orchestrateFork", () => { expect(createRuntimeMock).toHaveBeenCalledWith(DEFAULT_FORKED_RUNTIME_CONFIG, { projectPath: PROJECT_PATH, workspaceName: NEW_WORKSPACE_NAME, + workspacePath: "/workspaces/forked", }); }); @@ -201,6 +202,7 @@ describe("orchestrateFork", () => { expect(createRuntimeMock).toHaveBeenCalledWith(DEFAULT_FORKED_RUNTIME_CONFIG, { projectPath: PROJECT_PATH, workspaceName: NEW_WORKSPACE_NAME, + workspacePath: "/workspaces/created", }); }); @@ -409,6 +411,7 @@ describe("orchestrateFork", () => { expect(createRuntimeMock).toHaveBeenCalledWith(customForkedRuntimeConfig, { projectPath: PROJECT_PATH, workspaceName: NEW_WORKSPACE_NAME, + workspacePath: "/workspaces/created-with-custom-runtime", }); }); @@ -451,10 +454,14 @@ describe("orchestrateFork", () => { expect.objectContaining({ containerName: "mux-demo-source-aaaaaa" }) ); - // createRuntime should also receive the normalized config + // createRuntime should also receive the normalized config and the created workspace path. expect(createRuntimeMock).toHaveBeenCalledWith( expect.objectContaining({ containerName: expectedContainerName }), - { projectPath: PROJECT_PATH, workspaceName: NEW_WORKSPACE_NAME } + { + projectPath: PROJECT_PATH, + workspaceName: NEW_WORKSPACE_NAME, + workspacePath: "/workspaces/new", + } ); }); it("returns Err when create fallback also fails", async () => { diff --git a/src/node/services/utils/forkOrchestrator.ts b/src/node/services/utils/forkOrchestrator.ts index 67c85bd5f3..560373f228 100644 --- a/src/node/services/utils/forkOrchestrator.ts +++ b/src/node/services/utils/forkOrchestrator.ts @@ -577,6 +577,7 @@ export async function orchestrateFork( const targetRuntime = createRuntime(normalizedForkedRuntimeConfig, { projectPath, workspaceName: newWorkspaceName, + workspacePath, }); return Ok({ diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e4bdffac6e..43d0ac097b 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -5168,9 +5168,11 @@ export class WorkspaceService extends EventEmitter { // Copy plan file using explicit source/target runtimes for cross-runtime safety. // Create a fresh source runtime handle because DockerRuntime.forkWorkspace() can // mutate the original runtime's container identity to target the new workspace. + const sourceWorkspace = this.config.findWorkspace(sourceWorkspaceId); const freshSourceRuntime = createRuntime(sourceRuntimeConfig, { projectPath: foundProjectPath, workspaceName: sourceMetadata.name, + workspacePath: sourceWorkspace?.workspacePath, }); await copyPlanFileAcrossRuntimes( freshSourceRuntime, From 95467e375f857cf4bed96eae7fd90bcd0a9f8994 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sun, 5 Apr 2026 20:12:27 -0500 Subject: [PATCH 11/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20prune=20stale=20ref?= =?UTF-8?q?s=20when=20importing=20ssh=20bundles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/SSHRuntime.test.ts | 1 + src/node/runtime/SSHRuntime.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index 2ce94990eb..45bfe11809 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -397,6 +397,7 @@ describe("SSHRuntime bundle sync reuse", () => { return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); } if (command.includes(" fetch ")) { + expect(command).toContain("fetch --prune --prune-tags"); expect(command).toContain(bundleFileName); return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); } diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index 8a7033899d..58c9bc80c7 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -1438,7 +1438,7 @@ export class SSHRuntime extends RemoteRuntime { initLogger.logStep("Importing bundle into shared base repository..."); const fetchResult = await execBuffered( this, - `git -C ${baseRepoPathArg} fetch ${remoteBundlePathArg} '+refs/heads/*:${BUNDLE_REF_PREFIX}*' '+refs/tags/*:refs/tags/*'`, + `git -C ${baseRepoPathArg} fetch --prune --prune-tags ${remoteBundlePathArg} '+refs/heads/*:${BUNDLE_REF_PREFIX}*' '+refs/tags/*:refs/tags/*'`, { cwd: "/tmp", timeout: 300, abortSignal: sharedAbortSignal } ); if (fetchResult.exitCode !== 0) { From 9bac30026b1bb38a61e7b4186272e5999bef516c Mon Sep 17 00:00:00 2001 From: Ammar Date: Mon, 6 Apr 2026 09:18:51 -0500 Subject: [PATCH 12/16] =?UTF-8?q?=F0=9F=A4=96=20refactor:=20simplify=20ssh?= =?UTF-8?q?=20runtime=20scaling=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/runtime/RemoteRuntime.test.ts | 15 +- src/node/runtime/RemoteRuntime.ts | 16 +- src/node/runtime/SSHRuntime.test.ts | 242 +----------------- src/node/runtime/SSHRuntime.ts | 196 +++----------- src/node/runtime/openSshMasterPool.test.ts | 8 +- src/node/runtime/openSshMasterPool.ts | 228 ++++++++--------- src/node/runtime/runtimeHelpers.test.ts | 42 ++- src/node/runtime/runtimeHelpers.ts | 25 ++ .../runtime/transports/OpenSSHTransport.ts | 8 - src/node/runtime/transports/SSH2Transport.ts | 8 - src/node/runtime/transports/SSHTransport.ts | 6 - src/node/services/agentSession.ts | 30 +-- src/node/services/aiService.ts | 29 ++- src/node/services/workspaceService.ts | 4 +- 14 files changed, 250 insertions(+), 607 deletions(-) diff --git a/src/node/runtime/RemoteRuntime.test.ts b/src/node/runtime/RemoteRuntime.test.ts index 25cc8e6917..1a4615c7fd 100644 --- a/src/node/runtime/RemoteRuntime.test.ts +++ b/src/node/runtime/RemoteRuntime.test.ts @@ -33,8 +33,7 @@ class TestRemoteRuntime extends RemoteRuntime { constructor( private readonly childProcess: FakeChildProcess, private readonly onExitCalls: Array<[number, string]>, - private readonly onCloseCalls: number[], - private readonly onExitCodeCalls: number[] + private readonly onCloseCalls: number[] ) { super(); } @@ -66,10 +65,6 @@ class TestRemoteRuntime extends RemoteRuntime { return `cd ${cwd}`; } - protected override onExitCode(exitCode: number): void { - this.onExitCodeCalls.push(exitCode); - } - resolvePath(targetPath: string): Promise { return Promise.resolve(targetPath); } @@ -108,8 +103,7 @@ describe("RemoteRuntime synthetic exit handling", () => { const childProcess = new FakeChildProcess(); const onExitCalls: Array<[number, string]> = []; const onCloseCalls: number[] = []; - const onExitCodeCalls: number[] = []; - const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls, onExitCodeCalls); + const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls); const controller = new AbortController(); const stream = await runtime.exec("echo ok", { cwd: "/tmp", abortSignal: controller.signal }); @@ -119,15 +113,13 @@ describe("RemoteRuntime synthetic exit handling", () => { expect(await stream.exitCode).toBe(EXIT_CODE_ABORTED); expect(onExitCalls).toEqual([]); expect(onCloseCalls).toEqual([1]); - expect(onExitCodeCalls).toEqual([EXIT_CODE_ABORTED]); }); test("does not forward timed-out exits to transport onExit hooks", async () => { const childProcess = new FakeChildProcess(); const onExitCalls: Array<[number, string]> = []; const onCloseCalls: number[] = []; - const onExitCodeCalls: number[] = []; - const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls, onExitCodeCalls); + const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls); const stream = await runtime.exec("echo ok", { cwd: "/tmp", timeout: 0.01 }); await new Promise((resolve) => setTimeout(resolve, 20)); @@ -136,6 +128,5 @@ describe("RemoteRuntime synthetic exit handling", () => { expect(await stream.exitCode).toBe(EXIT_CODE_TIMEOUT); expect(onExitCalls).toEqual([]); expect(onCloseCalls).toEqual([1]); - expect(onExitCodeCalls).toEqual([EXIT_CODE_TIMEOUT]); }); }); diff --git a/src/node/runtime/RemoteRuntime.ts b/src/node/runtime/RemoteRuntime.ts index 31e5da32b3..b531bb68ce 100644 --- a/src/node/runtime/RemoteRuntime.ts +++ b/src/node/runtime/RemoteRuntime.ts @@ -11,7 +11,6 @@ * - spawnRemoteProcess() - how to spawn the external process (ssh/docker) * - getBasePath() - base directory for workspace operations * - quoteForRemote() - path quoting strategy - * - onExitCode() - optional exit code handling (SSH connection pool) */ import type { ChildProcess } from "child_process"; @@ -45,8 +44,6 @@ import { getAtomicWriteTempPath } from "./atomicWriteTempPath"; export interface SpawnResult { /** The spawned child process */ process: ChildProcess; - /** Optional async work to do before exec (e.g., acquire connection) */ - preExec?: Promise; /** Optional transport-scoped exit handling (e.g., master-pool health accounting). */ onExit?: (exitCode: number, stderr: string) => void; /** Optional close handling that must run even for synthetic abort/timeout exits. */ @@ -65,7 +62,7 @@ export abstract class RemoteRuntime implements Runtime { * * @param fullCommand The full shell command to execute (already wrapped in bash -c) * @param options Original exec options - * @returns The spawned process and optional pre-exec work + * @returns The spawned process and optional transport lifecycle hooks */ protected abstract spawnRemoteProcess( fullCommand: string, @@ -90,15 +87,6 @@ export abstract class RemoteRuntime implements Runtime { */ protected abstract cdCommand(cwd: string): string; - /** - * Called when exec completes with an exit code. - * Subclasses can use this for connection pool health tracking. - * @param stderr - Captured stderr for error reporting (e.g., SSH connection failures) - */ - protected onExitCode(_exitCode: number, _options: ExecOptions, _stderr: string): void { - // Default: no-op. SSH overrides to report to connection pool. - } - /** * Command prefix (e.g., "SSH" or "Docker") for logging. */ @@ -185,8 +173,6 @@ export abstract class RemoteRuntime implements Runtime { spawnResult.onExit?.(finalExitCode, stderrForErrorReporting); } spawnResult.onClose?.(); - // Let subclass handle exit code (e.g., SSH connection pool) - this.onExitCode(finalExitCode, options, stderrForErrorReporting); resolve(finalExitCode); }); diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts index 45bfe11809..5c706c941b 100644 --- a/src/node/runtime/SSHRuntime.test.ts +++ b/src/node/runtime/SSHRuntime.test.ts @@ -2,7 +2,6 @@ import * as crypto from "node:crypto"; import * as path from "node:path"; import { describe, expect, it, beforeEach, afterEach, spyOn } from "bun:test"; import * as runtimeHelpers from "@/node/utils/runtime/helpers"; -import * as disposableExec from "@/node/utils/disposableExec"; import * as submoduleSync from "./submoduleSync"; import { SSHRuntime, clearSharedProjectLayoutCache, computeBaseRepoPath } from "./SSHRuntime"; import { @@ -21,20 +20,6 @@ import { projectSyncCoordinator } from "./projectSyncCoordinator"; * worktree-based operations) require Docker and are in tests/runtime/runtime.test.ts. * Run with: TEST_INTEGRATION=1 bun x jest tests/runtime/runtime.test.ts */ -function createMockExecResult( - result: Promise<{ stdout: string; stderr: string }> -): ReturnType { - void result.catch(() => undefined); - return { - result, - get promise() { - return result; - }, - child: {}, - [Symbol.dispose]: () => undefined, - } as unknown as ReturnType; -} - afterEach(() => { clearSharedProjectLayoutCache(); }); @@ -178,29 +163,6 @@ describe("SSHRuntime base repo config normalization", () => { }); describe("SSHRuntime bundle sync reuse", () => { - type ShouldReuseCurrentBundleTrunk = ( - projectPath: string, - trunkBranch: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - - interface RuntimeWithResolveLocalSyncRefManifest { - resolveLocalSyncRefManifest: (projectPath: string) => Promise; - } - - interface RuntimeWithResolveRemoteSyncRefManifest { - resolveRemoteSyncRefManifest: ( - baseRepoPathArg: string, - abortSignal?: AbortSignal - ) => Promise; - } - interface RuntimeWithSyncProjectToRemote { syncProjectToRemote: ( projectPath: string, @@ -263,13 +225,8 @@ describe("SSHRuntime bundle sync reuse", () => { let runtime: SSHRuntime; let execBufferedSpy: ReturnType> | null = null; - let execFileAsyncSpy: ReturnType> | null = - null; - const initMessages: string[] = []; const initLogger = { - logStep: (message: string) => { - initMessages.push(message); - }, + logStep: () => undefined, logStdout: () => undefined, logStderr: () => undefined, logComplete: () => undefined, @@ -278,70 +235,12 @@ describe("SSHRuntime bundle sync reuse", () => { beforeEach(() => { const config = { host: "example.com", srcBaseDir: "/home/user/src" }; runtime = new SSHRuntime(config, createSSHTransport(config, false)); - initMessages.length = 0; }); afterEach(() => { execBufferedSpy?.mockRestore(); execBufferedSpy = null; - execFileAsyncSpy?.mockRestore(); - execFileAsyncSpy = null; - }); - - function getShouldReuseCurrentBundleTrunk(): ShouldReuseCurrentBundleTrunk { - const reuseUnknown: unknown = Reflect.get(runtime, "shouldReuseCurrentBundleTrunk"); - if (typeof reuseUnknown !== "function") { - throw new Error("shouldReuseCurrentBundleTrunk is unavailable"); - } - - return reuseUnknown as ShouldReuseCurrentBundleTrunk; - } - - it("reuses the shared bundle when the remote refs already match local refs", async () => { - execFileAsyncSpy = spyOn(disposableExec, "execFileAsync").mockReturnValue( - createMockExecResult(Promise.resolve({ stdout: "abc123\n", stderr: "" })) - ); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "abc123\n", - stderr: "", - exitCode: 0, - duration: 0, - }); - const localManifestSpy = spyOn( - runtime as unknown as RuntimeWithResolveLocalSyncRefManifest, - "resolveLocalSyncRefManifest" - ).mockResolvedValue("refs/heads/main abc123\nrefs/tags/v1 def456"); - const remoteManifestSpy = spyOn( - runtime as unknown as RuntimeWithResolveRemoteSyncRefManifest, - "resolveRemoteSyncRefManifest" - ).mockResolvedValue("refs/heads/main abc123\nrefs/tags/v1 def456"); - - try { - expect( - await getShouldReuseCurrentBundleTrunk().call(runtime, "/projects/demo", "main", initLogger) - ).toBe(true); - expect(execFileAsyncSpy).toHaveBeenCalledWith("git", [ - "-C", - "/projects/demo", - "rev-parse", - "--verify", - "main", - ]); - expect(execBufferedSpy).toHaveBeenCalledWith( - runtime, - expect.stringContaining("refs/mux-bundle/main"), - expect.objectContaining({ cwd: "/tmp", timeout: 10 }) - ); - expect(localManifestSpy).toHaveBeenCalledWith("/projects/demo"); - expect(remoteManifestSpy).toHaveBeenCalledWith( - JSON.stringify(computeBaseRepoPath("/home/user/src", "/projects/demo")), - undefined - ); - expect(initMessages.some((message) => message.includes("skipping sync"))).toBe(true); - } finally { - localManifestSpy.mockRestore(); - remoteManifestSpy.mockRestore(); - } + projectSyncCoordinator.clearAll(); }); it("uploads snapshot bundles through a per-attempt temp path", async () => { @@ -439,84 +338,6 @@ describe("SSHRuntime bundle sync reuse", () => { projectSyncCoordinator.clearAll(); } }); - - it("does not reuse the shared bundle when the remote trunk ref is stale", async () => { - execFileAsyncSpy = spyOn(disposableExec, "execFileAsync").mockReturnValue( - createMockExecResult(Promise.resolve({ stdout: "abc123\n", stderr: "" })) - ); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "def456\n", - stderr: "", - exitCode: 0, - duration: 0, - }); - - expect( - await getShouldReuseCurrentBundleTrunk().call(runtime, "/projects/demo", "main", initLogger) - ).toBe(false); - expect(initMessages).toHaveLength(0); - }); - - it("does not reuse the shared bundle when non-trunk refs drift", async () => { - execFileAsyncSpy = spyOn(disposableExec, "execFileAsync").mockReturnValue( - createMockExecResult(Promise.resolve({ stdout: "abc123\n", stderr: "" })) - ); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "abc123\n", - stderr: "", - exitCode: 0, - duration: 0, - }); - const localManifestSpy = spyOn( - runtime as unknown as RuntimeWithResolveLocalSyncRefManifest, - "resolveLocalSyncRefManifest" - ).mockResolvedValue("refs/heads/main abc123\nrefs/tags/v2 fedcba"); - const remoteManifestSpy = spyOn( - runtime as unknown as RuntimeWithResolveRemoteSyncRefManifest, - "resolveRemoteSyncRefManifest" - ).mockResolvedValue("refs/heads/main abc123\nrefs/tags/v1 def456"); - - try { - expect( - await getShouldReuseCurrentBundleTrunk().call(runtime, "/projects/demo", "main", initLogger) - ).toBe(false); - expect(initMessages).toHaveLength(0); - } finally { - localManifestSpy.mockRestore(); - remoteManifestSpy.mockRestore(); - } - }); - - it("falls back to sync when the remote bundle probe throws", async () => { - execFileAsyncSpy = spyOn(disposableExec, "execFileAsync").mockReturnValue( - createMockExecResult(Promise.resolve({ stdout: "abc123\n", stderr: "" })) - ); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation(() => { - throw new Error("ssh unavailable"); - }); - - expect( - await getShouldReuseCurrentBundleTrunk().call(runtime, "/projects/demo", "main", initLogger) - ).toBe(false); - expect(initMessages).toHaveLength(0); - }); - - it("does not reuse the shared bundle when the local trunk ref is missing", async () => { - execFileAsyncSpy = spyOn(disposableExec, "execFileAsync").mockReturnValue( - createMockExecResult(Promise.reject(new Error("unknown revision"))) - ); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "abc123\n", - stderr: "", - exitCode: 0, - duration: 0, - }); - - expect( - await getShouldReuseCurrentBundleTrunk().call(runtime, "/projects/demo", "main", initLogger) - ).toBe(false); - expect(execBufferedSpy).not.toHaveBeenCalled(); - }); }); describe("SSHRuntime.prepareWorkspaceCheckout", () => { @@ -541,20 +362,6 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { ) => Promise; } - interface RuntimeWithShouldReuseCurrentBundleTrunk { - shouldReuseCurrentBundleTrunk: ( - projectPath: string, - trunkBranch: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - interface RuntimeWithFetchOriginTrunk { fetchOriginTrunk: ( workspacePath: string, @@ -591,18 +398,6 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { ) => Promise; } - interface RuntimeWithGetOriginUrlForSync { - getOriginUrlForSync: ( - projectPath: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - } - ) => Promise<{ originUrl: string | null }>; - } - interface RuntimeWithCanFastForwardToOrigin { canFastForwardToOrigin: ( workspacePath: string, @@ -632,7 +427,7 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { ) => Promise; } - it("still creates a worktree when bundle sync is skipped for a new workspace", async () => { + it("syncs the project before creating a worktree for a new workspace", async () => { const config = { host: "example.com", srcBaseDir: "/home/user/src" }; const runtime = new SSHRuntime(config, createSSHTransport(config, false)); const initMessages: string[] = []; @@ -650,20 +445,12 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { if (command === "test -d /home/user/src/demo/review-slot") { return Promise.resolve({ stdout: "", stderr: "", exitCode: 1, duration: 0 }); } - if ( - command.includes("remote set-url origin") || - command.includes("remote add origin") || - command.includes('worktree add "/home/user/src/demo/review-slot"') - ) { + if (command.includes('worktree add "/home/user/src/demo/review-slot"')) { return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); } throw new Error(`Unexpected execBuffered command: ${command}`); } ); - const reuseSpy = spyOn( - runtime as unknown as RuntimeWithShouldReuseCurrentBundleTrunk, - "shouldReuseCurrentBundleTrunk" - ).mockResolvedValue(true); const fetchOriginSpy = spyOn( runtime as unknown as RuntimeWithFetchOriginTrunk, "fetchOriginTrunk" @@ -676,10 +463,6 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { runtime as unknown as RuntimeWithEnsureBaseRepo, "ensureBaseRepo" ).mockResolvedValue('"/home/user/src/demo/.mux-base.git"'); - const getOriginUrlSpy = spyOn( - runtime as unknown as RuntimeWithGetOriginUrlForSync, - "getOriginUrlForSync" - ).mockResolvedValue({ originUrl: "git@github.com:coder/mux.git" }); const canFastForwardSpy = spyOn( runtime as unknown as RuntimeWithCanFastForwardToOrigin, "canFastForwardToOrigin" @@ -706,18 +489,16 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { "" ); - expect(reuseSpy).toHaveBeenCalled(); - expect(syncProjectSpy).not.toHaveBeenCalled(); + expect(syncProjectSpy).toHaveBeenCalledWith( + "/projects/demo", + "/home/user/src/demo/review-slot", + initLogger, + undefined + ); expect(ensureBaseRepoSpy).toHaveBeenCalledWith("/projects/demo", initLogger, undefined); expect(fetchOriginSpy).toHaveBeenCalled(); expect(resolveBundleSpy).toHaveBeenCalled(); - expect(getOriginUrlSpy).toHaveBeenCalledWith("/projects/demo", initLogger); expect(canFastForwardSpy).not.toHaveBeenCalled(); - expect(execBufferedSpy).toHaveBeenCalledWith( - runtime, - expect.stringContaining("remote set-url origin 'git@github.com:coder/mux.git'"), - expect.objectContaining({ cwd: "/tmp", timeout: 10 }) - ); expect(execBufferedSpy).toHaveBeenCalledWith( runtime, expect.stringContaining( @@ -728,14 +509,13 @@ describe("SSHRuntime.prepareWorkspaceCheckout", () => { expect(syncSubmodulesSpy).toHaveBeenCalledWith( expect.objectContaining({ workspacePath: "/home/user/src/demo/review-slot" }) ); + expect(initMessages).toContain("Files synced successfully"); expect(initMessages).toContain("Worktree created successfully"); } finally { execBufferedSpy.mockRestore(); - reuseSpy.mockRestore(); fetchOriginSpy.mockRestore(); resolveBundleSpy.mockRestore(); ensureBaseRepoSpy.mockRestore(); - getOriginUrlSpy.mockRestore(); canFastForwardSpy.mockRestore(); syncProjectSpy.mockRestore(); syncSubmodulesSpy.mockRestore(); diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index 58c9bc80c7..a10392b54a 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -41,7 +41,6 @@ import { getErrorMessage } from "@/common/utils/errors"; import { type SSHRuntimeConfig } from "./sshConnectionPool"; import { getOriginUrlForBundle } from "./gitBundleSync"; import { gitNoHooksPrefix } from "@/node/utils/gitNoHooksEnv"; -import { execFileAsync } from "@/node/utils/disposableExec"; import { syncRuntimeGitSubmodules } from "./submoduleSync"; import type { PtyHandle, PtySessionParams, SSHTransport } from "./transports"; import { @@ -915,120 +914,6 @@ export class SSHRuntime extends RemoteRuntime { return null; } - private async resolveLocalRefOid(projectPath: string, ref: string): Promise { - try { - using proc = execFileAsync("git", ["-C", projectPath, "rev-parse", "--verify", ref]); - const { stdout } = await proc.result; - const oid = stdout.trim(); - return oid.length > 0 ? oid : null; - } catch { - return null; - } - } - - private async resolveLocalSyncRefManifest(projectPath: string): Promise { - try { - using proc = execFileAsync("git", [ - "-C", - projectPath, - "for-each-ref", - "--format=%(refname) %(objectname)", - "refs/heads", - "refs/tags", - ]); - const { stdout } = await proc.result; - return stdout - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .sort() - .join("\n"); - } catch { - return null; - } - } - - private async resolveRemoteSyncRefManifest( - baseRepoPathArg: string, - abortSignal?: AbortSignal - ): Promise { - const result = await execBuffered( - this, - `git -C ${baseRepoPathArg} for-each-ref --format='%(refname) %(objectname)' ${BUNDLE_REF_PREFIX} refs/tags`, - { cwd: "/tmp", timeout: 20, abortSignal } - ); - if (result.exitCode !== 0) { - return null; - } - - return result.stdout - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => - line.startsWith(BUNDLE_REF_PREFIX) ? line.replace(BUNDLE_REF_PREFIX, "refs/heads/") : line - ) - .sort() - .join("\n"); - } - - private async shouldReuseCurrentBundleTrunk( - projectPath: string, - trunkBranch: string, - initLogger: InitLogger, - abortSignal?: AbortSignal - ): Promise { - const localTrunkOid = await this.resolveLocalRefOid(projectPath, trunkBranch); - if (localTrunkOid == null) { - return false; - } - - try { - const remoteTrunkRef = `${BUNDLE_REF_PREFIX}${trunkBranch}`; - const baseRepoPathArg = expandTildeForSSH(this.getBaseRepoPath(projectPath)); - const remoteTrunkResult = await execBuffered( - this, - `git -C ${baseRepoPathArg} rev-parse --verify ${shescape.quote(remoteTrunkRef)}`, - { cwd: "/tmp", timeout: 10, abortSignal } - ); - if (remoteTrunkResult.exitCode !== 0) { - return false; - } - - const remoteTrunkOid = remoteTrunkResult.stdout.trim(); - if (remoteTrunkOid.length === 0 || remoteTrunkOid !== localTrunkOid) { - return false; - } - - const localRefManifest = await this.resolveLocalSyncRefManifest(projectPath); - if (localRefManifest == null) { - return false; - } - - const remoteRefManifest = await this.resolveRemoteSyncRefManifest( - baseRepoPathArg, - abortSignal - ); - if (remoteRefManifest == null || remoteRefManifest !== localRefManifest) { - return false; - } - } catch (error) { - log.debug("Failed to probe shared bundle trunk; falling back to sync", { - projectPath, - trunkBranch, - error: getErrorMessage(error), - }); - return false; - } - - // Re-uploading the full bundle adds startup latency even when the shared base already - // has the exact trunk snapshot this workspace will branch from. - initLogger.logStep( - `Remote bundle already matches ${trunkBranch} (${localTrunkOid.slice(0, 12)}); skipping sync` - ); - return true; - } - private async refreshBaseRepoOrigin( projectPath: string, baseRepoPathArg: string, @@ -1620,65 +1505,44 @@ export class SSHRuntime extends RemoteRuntime { needsWorktreeCheckout = true; } - let shouldSyncBaseRepo = needsWorktreeCheckout; - if (shouldSyncBaseRepo) { - shouldSyncBaseRepo = !(await this.shouldReuseCurrentBundleTrunk( - projectPath, - trunkBranch, - initLogger, - abortSignal - )); - } - if (needsWorktreeCheckout) { - if (shouldSyncBaseRepo) { - // SSH workspace initialization owns repo materialization: it syncs the project into - // the shared base repo, checks out the worktree, and then materializes submodules - // before repo-controlled init hooks run. - initLogger.logStep("Syncing project files to remote..."); - const maxSyncAttempts = 3; - for (let attempt = 1; attempt <= maxSyncAttempts; attempt++) { - try { - await this.syncProjectToRemote(projectPath, workspacePath, initLogger, abortSignal); - break; - } catch (error) { - const errorMsg = getErrorMessage(error); - const isRetryable = - errorMsg.includes("pack-objects died") || - errorMsg.includes("Connection reset") || - errorMsg.includes("Connection closed") || - errorMsg.includes("Broken pipe") || - errorMsg.includes("EPIPE"); - - if (!isRetryable || attempt === maxSyncAttempts) { - throw new Error(`Failed to sync project: ${errorMsg}`); - } - - log.info( - `Sync failed (attempt ${attempt}/${maxSyncAttempts}), will retry: ${errorMsg}` - ); - initLogger.logStep( - `Sync failed, retrying (attempt ${attempt + 1}/${maxSyncAttempts})...` - ); - await new Promise((r) => setTimeout(r, attempt * 1000)); + // SSH workspace initialization owns repo materialization: it syncs the project into + // the shared base repo, checks out the worktree, and then materializes submodules + // before repo-controlled init hooks run. + initLogger.logStep("Syncing project files to remote..."); + const maxSyncAttempts = 3; + for (let attempt = 1; attempt <= maxSyncAttempts; attempt++) { + try { + await this.syncProjectToRemote(projectPath, workspacePath, initLogger, abortSignal); + break; + } catch (error) { + const errorMsg = getErrorMessage(error); + const isRetryable = + errorMsg.includes("pack-objects died") || + errorMsg.includes("Connection reset") || + errorMsg.includes("Connection closed") || + errorMsg.includes("Broken pipe") || + errorMsg.includes("EPIPE"); + + if (!isRetryable || attempt === maxSyncAttempts) { + throw new Error(`Failed to sync project: ${errorMsg}`); } + + log.info(`Sync failed (attempt ${attempt}/${maxSyncAttempts}), will retry: ${errorMsg}`); + initLogger.logStep( + `Sync failed, retrying (attempt ${attempt + 1}/${maxSyncAttempts})...` + ); + await new Promise((r) => setTimeout(r, attempt * 1000)); } - initLogger.logStep("Files synced successfully"); } + initLogger.logStep("Files synced successfully"); - // Even when the shared base repo is already current, a brand-new workspace still needs - // git worktree add so the checkout exists before init hooks or submodule sync run. - // Re-enter ensureBaseRepo() here so older shared repos still get their local core.bare - // config normalized before we reuse them for a fresh worktree checkout. + // A brand-new workspace still needs git worktree add so the checkout exists before init hooks + // or submodule sync run. Re-enter ensureBaseRepo() here so older shared repos still get their + // local core.bare config normalized before we reuse them for a fresh worktree checkout. const baseRepoPath = this.getBaseRepoPath(projectPath); const baseRepoPathArg = await this.ensureBaseRepo(projectPath, initLogger, abortSignal); - if (!shouldSyncBaseRepo) { - // Skipping the bundle upload must still refresh origin in case the user repointed - // the local remote without changing trunk commits. - await this.refreshBaseRepoOrigin(projectPath, baseRepoPathArg, initLogger, abortSignal); - } - // Fetch latest from origin in the base repo (best-effort) so new branches // can start from the latest upstream state. const fetchedOrigin = await this.fetchOriginTrunk( diff --git a/src/node/runtime/openSshMasterPool.test.ts b/src/node/runtime/openSshMasterPool.test.ts index 797ea59ea3..8f48945e66 100644 --- a/src/node/runtime/openSshMasterPool.test.ts +++ b/src/node/runtime/openSshMasterPool.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; import type { spawn as spawnProcess } from "child_process"; import { EventEmitter } from "events"; import { PassThrough } from "stream"; @@ -41,6 +41,12 @@ describe("OpenSSHMasterPool", () => { let releaseInteractiveResponder: (() => void) | undefined; + beforeEach(() => { + setSshPromptService(undefined); + setOpenSSHHostKeyPolicyMode("headless-fallback"); + masterProcesses.clear(); + }); + afterEach(() => { releaseInteractiveResponder?.(); releaseInteractiveResponder = undefined; diff --git a/src/node/runtime/openSshMasterPool.ts b/src/node/runtime/openSshMasterPool.ts index f8c484e7dc..cfbb0a9fce 100644 --- a/src/node/runtime/openSshMasterPool.ts +++ b/src/node/runtime/openSshMasterPool.ts @@ -136,6 +136,12 @@ function isProcessAlive(proc: ChildProcess | undefined): boolean { return proc != null && proc.exitCode == null && proc.signalCode == null; } +interface ShardAcquisitionBehavior { + pickReadyShard(group: HostGroup): MasterShard | undefined; + finalizeShard(shard: MasterShard): T; + preferPolling(group: HostGroup): boolean; +} + export class OpenSSHMasterPool { private readonly hostGroups = new Map(); private readonly spawnProcess: SpawnFn; @@ -171,93 +177,35 @@ export class OpenSSHMasterPool { config: SSHConnectionConfig, options?: AcquireLeaseOptions ): Promise { - const maxWaitMs = options?.maxWaitMs ?? this.defaultMaxWaitMs; - const defaultStartTimeoutMs = options?.timeoutMs ?? this.defaultMasterStartTimeoutMs; - const deadlineMs = Date.now() + maxWaitMs; - const key = makeConnectionKey(config); - const hostGroup = this.getOrCreateHostGroup(key, config); - let lastStartError: Error | undefined; - - while (true) { - if (options?.abortSignal?.aborted) { - throw new Error("Operation aborted"); - } - - this.trimExitedShards(hostGroup); - const readyShard = this.pickReadyShard(hostGroup); - if (readyShard) { - this.scheduleIdleDisposalIfUnused(readyShard); - return; - } - - const restartable = hostGroup.shards.find((shard) => { - return ( - !isProcessAlive(shard.process) && - shard.startup == null && - (shard.health.backoffUntil == null || shard.health.backoffUntil.getTime() <= Date.now()) - ); - }); - if (restartable) { - try { - const startupTimeoutMs = - maxWaitMs === 0 - ? defaultStartTimeoutMs - : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); - await this.startShard(hostGroup, restartable, startupTimeoutMs, options?.abortSignal); - this.scheduleIdleDisposalIfUnused(restartable); - return; - } catch (error) { - if (options?.abortSignal?.aborted) { - throw error; - } - lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); - } - } - - if (hostGroup.shards.length < this.maxShardsPerHost) { - const shard = this.createShard(hostGroup); - try { - const startupTimeoutMs = - maxWaitMs === 0 - ? defaultStartTimeoutMs - : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); - await this.startShard(hostGroup, shard, startupTimeoutMs, options?.abortSignal); - this.scheduleIdleDisposalIfUnused(shard); - return; - } catch (error) { - if (options?.abortSignal?.aborted) { - throw error; - } - lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); - } - } - - const nextBackoffMs = this.getNextBackoffWaitMs(hostGroup); - const remainingMs = deadlineMs - Date.now(); - if (remainingMs <= 0) { - if (lastStartError) { - throw lastStartError; - } - throw new Error( - `SSH master pool for ${config.host} did not become available within ${maxWaitMs}ms` - ); - } - - const waitMs = this.getPoolWaitMs(remainingMs, nextBackoffMs); - options?.onWait?.(waitMs); - await this.sleep(waitMs, options?.abortSignal); - } + await this.withAcquiredShard(config, options, { + pickReadyShard: (group) => this.pickReadyShard(group), + finalizeShard: (shard) => { + this.scheduleIdleDisposalIfUnused(shard); + }, + preferPolling: () => false, + }); } async acquireLease( config: SSHConnectionConfig, options?: AcquireLeaseOptions ): Promise { + return this.withAcquiredShard(config, options, { + pickReadyShard: (group) => this.pickAvailableShard(group), + finalizeShard: (shard) => this.reserveShard(shard), + preferPolling: (group) => this.pickReadyShard(group) != null, + }); + } + + private async withAcquiredShard( + config: SSHConnectionConfig, + options: AcquireLeaseOptions | undefined, + behavior: ShardAcquisitionBehavior + ): Promise { const maxWaitMs = options?.maxWaitMs ?? this.defaultMaxWaitMs; const defaultStartTimeoutMs = options?.timeoutMs ?? this.defaultMasterStartTimeoutMs; const deadlineMs = Date.now() + maxWaitMs; - const key = makeConnectionKey(config); - const hostGroup = this.getOrCreateHostGroup(key, config); + const hostGroup = this.getOrCreateHostGroup(makeConnectionKey(config), config); let lastStartError: Error | undefined; while (true) { @@ -266,52 +214,44 @@ export class OpenSSHMasterPool { } this.trimExitedShards(hostGroup); - const available = this.pickAvailableShard(hostGroup); - if (available) { - return this.reserveShard(available); + const readyShard = behavior.pickReadyShard(hostGroup); + if (readyShard) { + return behavior.finalizeShard(readyShard); } - const restartable = hostGroup.shards.find((shard) => { - return ( - !isProcessAlive(shard.process) && - shard.startup == null && - (shard.health.backoffUntil == null || shard.health.backoffUntil.getTime() <= Date.now()) - ); - }); - if (restartable) { - try { - const startupTimeoutMs = - maxWaitMs === 0 - ? defaultStartTimeoutMs - : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); - await this.startShard(hostGroup, restartable, startupTimeoutMs, options?.abortSignal); - return this.reserveShard(restartable); - } catch (error) { - if (options?.abortSignal?.aborted) { - throw error; - } - lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); - } + const restartedShard = await this.tryStartShardForAcquisition( + hostGroup, + this.pickRestartableShard(hostGroup), + defaultStartTimeoutMs, + maxWaitMs, + deadlineMs, + options?.abortSignal, + behavior, + lastStartError + ); + if ("result" in restartedShard) { + return restartedShard.result; } + lastStartError = restartedShard.error; if (hostGroup.shards.length < this.maxShardsPerHost) { - const shard = this.createShard(hostGroup); - try { - const startupTimeoutMs = - maxWaitMs === 0 - ? defaultStartTimeoutMs - : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); - await this.startShard(hostGroup, shard, startupTimeoutMs, options?.abortSignal); - return this.reserveShard(shard); - } catch (error) { - if (options?.abortSignal?.aborted) { - throw error; - } - lastStartError = error instanceof Error ? error : new Error(getErrorMessage(error)); + const createdShard = this.createShard(hostGroup); + const startedShard = await this.tryStartShardForAcquisition( + hostGroup, + createdShard, + defaultStartTimeoutMs, + maxWaitMs, + deadlineMs, + options?.abortSignal, + behavior, + lastStartError + ); + if ("result" in startedShard) { + return startedShard.result; } + lastStartError = startedShard.error; } - const nextBackoffMs = this.getNextBackoffWaitMs(hostGroup); const remainingMs = deadlineMs - Date.now(); if (remainingMs <= 0) { if (lastStartError) { @@ -324,14 +264,66 @@ export class OpenSSHMasterPool { const waitMs = this.getPoolWaitMs( remainingMs, - nextBackoffMs, - this.pickReadyShard(hostGroup) != null + this.getNextBackoffWaitMs(hostGroup), + behavior.preferPolling(hostGroup) ); options?.onWait?.(waitMs); await this.sleep(waitMs, options?.abortSignal); } } + private async tryStartShardForAcquisition( + hostGroup: HostGroup, + shard: MasterShard | undefined, + defaultStartTimeoutMs: number, + maxWaitMs: number, + deadlineMs: number, + abortSignal: AbortSignal | undefined, + behavior: ShardAcquisitionBehavior, + lastStartError: Error | undefined + ): Promise<{ result: T } | { error: Error | undefined }> { + if (!shard) { + return { error: lastStartError }; + } + + try { + await this.startShard( + hostGroup, + shard, + this.getStartupTimeoutMs(defaultStartTimeoutMs, maxWaitMs, deadlineMs), + abortSignal + ); + return { result: behavior.finalizeShard(shard) }; + } catch (error) { + if (abortSignal?.aborted) { + throw error; + } + return { + error: error instanceof Error ? error : new Error(getErrorMessage(error)), + }; + } + } + + private getStartupTimeoutMs( + defaultStartTimeoutMs: number, + maxWaitMs: number, + deadlineMs: number + ): number { + return maxWaitMs === 0 + ? defaultStartTimeoutMs + : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); + } + + private pickRestartableShard(group: HostGroup): MasterShard | undefined { + return group.shards.find((shard) => { + return ( + !isProcessAlive(shard.process) && + shard.startup == null && + (shard.health.backoffUntil == null || shard.health.backoffUntil.getTime() <= Date.now()) + ); + }); + } + clearAll(): void { for (const group of this.hostGroups.values()) { for (const shard of group.shards) { diff --git a/src/node/runtime/runtimeHelpers.test.ts b/src/node/runtime/runtimeHelpers.test.ts index b8157bd8b3..0f0f4a6a74 100644 --- a/src/node/runtime/runtimeHelpers.test.ts +++ b/src/node/runtime/runtimeHelpers.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from "bun:test"; import type { RuntimeConfig } from "@/common/types/runtime"; import { DevcontainerRuntime } from "./DevcontainerRuntime"; -import { createRuntimeForWorkspace, resolveWorkspaceExecutionPath } from "./runtimeHelpers"; +import { + createRuntimeContextForWorkspace, + createRuntimeForWorkspace, + resolveWorkspaceExecutionPath, +} from "./runtimeHelpers"; describe("createRuntimeForWorkspace", () => { it("forwards the persisted workspace path to devcontainer runtimes", () => { @@ -84,4 +88,40 @@ describe("resolveWorkspaceExecutionPath", () => { const runtime = createRuntimeForWorkspace(metadata); expect(resolveWorkspaceExecutionPath(metadata, runtime)).toBe("/src"); }); + + it("returns the project root for in-place workspaces", () => { + const metadata = { + runtimeConfig: { + type: "worktree", + srcBaseDir: "/tmp/src", + } satisfies RuntimeConfig, + projectPath: "/projects/cli", + name: "/projects/cli", + }; + + const runtime = createRuntimeForWorkspace(metadata); + expect(resolveWorkspaceExecutionPath(metadata, runtime)).toBe("/projects/cli"); + }); +}); + +describe("createRuntimeContextForWorkspace", () => { + it("returns a runtime together with the resolved execution path", () => { + const metadata = { + runtimeConfig: { + type: "ssh", + host: "example.com", + srcBaseDir: "/remote/src", + } satisfies RuntimeConfig, + projectPath: "/projects/demo", + name: "review-1", + namedWorkspacePath: "/remote/src/demo/review-1", + }; + + const context = createRuntimeContextForWorkspace(metadata); + + expect(context.workspacePath).toBe("/remote/src/demo/review-1"); + expect(context.runtime.getWorkspacePath(metadata.projectPath, "review-2")).toBe( + "/remote/src/demo/review-2" + ); + }); }); diff --git a/src/node/runtime/runtimeHelpers.ts b/src/node/runtime/runtimeHelpers.ts index 1be433474b..4733166be6 100644 --- a/src/node/runtime/runtimeHelpers.ts +++ b/src/node/runtime/runtimeHelpers.ts @@ -28,6 +28,12 @@ export function resolveWorkspaceExecutionPath( metadata: WorkspaceMetadataForRuntime, runtime: Runtime ): string { + if (metadata.projectPath === metadata.name) { + // In-place workspaces (CLI/benchmarks) execute directly in their project root instead of a + // named sibling checkout, so deriving a worktree path would be reconstructing the wrong shape. + return metadata.projectPath; + } + const runtimeWorkspacePath = runtime.getWorkspacePath(metadata.projectPath, metadata.name); assert(runtimeWorkspacePath, `Workspace ${metadata.name} resolved to an empty runtime path`); @@ -54,6 +60,25 @@ export function resolveWorkspaceExecutionPath( return persistedWorkspacePath; } +export interface WorkspaceRuntimeContext { + runtime: Runtime; + workspacePath: string; +} + +/** + * Recreate an existing workspace runtime together with the execution path that should be used for + * terminals, tool calls, and agent discovery. + */ +export function createRuntimeContextForWorkspace( + metadata: WorkspaceMetadataForRuntime +): WorkspaceRuntimeContext { + const runtime = createRuntimeForWorkspace(metadata); + return { + runtime, + workspacePath: resolveWorkspaceExecutionPath(metadata, runtime), + }; +} + /** * Create a runtime from workspace metadata, ensuring workspace identity is always passed. * diff --git a/src/node/runtime/transports/OpenSSHTransport.ts b/src/node/runtime/transports/OpenSSHTransport.ts index 3579e3821e..df8cfadd5d 100644 --- a/src/node/runtime/transports/OpenSSHTransport.ts +++ b/src/node/runtime/transports/OpenSSHTransport.ts @@ -38,14 +38,6 @@ export class OpenSSHTransport implements SSHTransport { return this.config; } - markHealthy(): void { - // OpenSSH transport reports health through per-process master leases. - } - - reportFailure(_error: string): void { - // OpenSSH transport reports health through per-process master leases. - } - async acquireConnection(options?: { abortSignal?: AbortSignal; timeoutMs?: number; diff --git a/src/node/runtime/transports/SSH2Transport.ts b/src/node/runtime/transports/SSH2Transport.ts index dc1960f258..e28e61dd12 100644 --- a/src/node/runtime/transports/SSH2Transport.ts +++ b/src/node/runtime/transports/SSH2Transport.ts @@ -235,14 +235,6 @@ export class SSH2Transport implements SSHTransport { return this.config; } - markHealthy(): void { - ssh2ConnectionPool.markHealthy(this.config); - } - - reportFailure(error: string): void { - ssh2ConnectionPool.reportFailure(this.config, error); - } - async acquireConnection(options?: { abortSignal?: AbortSignal; timeoutMs?: number; diff --git a/src/node/runtime/transports/SSHTransport.ts b/src/node/runtime/transports/SSHTransport.ts index 0e70bd08d0..9ff4c7a546 100644 --- a/src/node/runtime/transports/SSHTransport.ts +++ b/src/node/runtime/transports/SSHTransport.ts @@ -32,12 +32,6 @@ export interface SSHTransport { /** Determine if an exit code represents a connection-level failure for this transport. */ isConnectionFailure(exitCode: number, stderr: string): boolean; - /** Mark connection as healthy (after successful command). */ - markHealthy(): void; - - /** Report connection failure (triggers backoff). */ - reportFailure(error: string): void; - /** Pre-flight connection check with backoff enforcement. */ acquireConnection(options?: { abortSignal?: AbortSignal; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0476baca8b..0f5bbcacc4 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -64,8 +64,8 @@ import { type StartupRetrySendOptions, } from "@/common/types/message"; import { + createRuntimeContextForWorkspace, createRuntimeForWorkspace, - resolveWorkspaceExecutionPath, } from "@/node/runtime/runtimeHelpers"; import { hasNonEmptyPlanFile } from "@/node/utils/runtime/helpers"; import { isExecLikeEditingCapableInResolvedChain } from "@/common/utils/agentTools"; @@ -3521,14 +3521,7 @@ export class AgentSession { let chain: Awaited> | undefined; for (const agentMetadata of metadataCandidates) { try { - const runtime = createRuntimeForWorkspace(agentMetadata); - - // In-place workspaces (CLI/benchmarks) have projectPath === name. - // Use path directly instead of reconstructing via getWorkspacePath. - const isInPlace = agentMetadata.projectPath === agentMetadata.name; - const workspacePath = isInPlace - ? agentMetadata.projectPath - : runtime.getWorkspacePath(agentMetadata.projectPath, agentMetadata.name); + const { runtime, workspacePath } = createRuntimeContextForWorkspace(agentMetadata); const agentDiscoveryPath = context.options?.disableWorkspaceAgents === true @@ -4450,13 +4443,7 @@ export class AgentSession { } const metadata = metadataResult.data; - const runtime = createRuntimeForWorkspace(metadata); - - // In-place workspaces (CLI/benchmarks) have projectPath === name. - const isInPlace = metadata.projectPath === metadata.name; - const workspacePath = isInPlace - ? metadata.projectPath - : resolveWorkspaceExecutionPath(metadata, runtime); + const { runtime, workspacePath } = createRuntimeContextForWorkspace(metadata); // When disableWorkspaceAgents is active, use project path for discovery // (only built-in/global agents). Mirrors resolveAgentForStream behavior. @@ -5118,8 +5105,7 @@ export class AgentSession { } const metadata = metadataResult.data; - const runtime = createRuntimeForWorkspace(metadata); - const workspacePath = resolveWorkspaceExecutionPath(metadata, runtime); + const { runtime, workspacePath } = createRuntimeContextForWorkspace(metadata); const materialized = await materializeFileAtMentions(messageText, { runtime, @@ -5182,13 +5168,7 @@ export class AgentSession { } const metadata = metadataResult.data; - const runtime = createRuntimeForWorkspace(metadata); - - // In-place workspaces (CLI/benchmarks) have projectPath === name. - const isInPlace = metadata.projectPath === metadata.name; - const workspacePath = isInPlace - ? metadata.projectPath - : resolveWorkspaceExecutionPath(metadata, runtime); + const { runtime, workspacePath } = createRuntimeContextForWorkspace(metadata); // When workspace agents are disabled, resolve skills from the project path instead of // the worktree so skill invocation uses the same precedence/discovery root as the UI. diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 827d58b6de..da03831e96 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -24,7 +24,7 @@ import { getToolsForModel } from "@/common/utils/tools/tools"; import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors"; import { createRuntime } from "@/node/runtime/runtimeFactory"; import { - createRuntimeForWorkspace, + createRuntimeContextForWorkspace, resolveWorkspaceExecutionPath, } from "@/node/runtime/runtimeHelpers"; import { MultiProjectRuntime } from "@/node/runtime/multiProjectRuntime"; @@ -949,8 +949,12 @@ export class AIService extends EventEmitter { return multiProjectExecutionGate; } - const runtime = isMultiProject(metadata) - ? new MultiProjectRuntime( + const singleProjectContext = isMultiProject(metadata) + ? undefined + : createRuntimeContextForWorkspace(metadataWithPath); + const runtime = singleProjectContext + ? singleProjectContext.runtime + : new MultiProjectRuntime( new ContainerManager(getSrcBaseDir(metadata.runtimeConfig) ?? this.config.srcDir), getProjects(metadata).map((project) => ({ projectPath: project.projectPath, @@ -961,20 +965,17 @@ export class AIService extends EventEmitter { }), })), metadata.name - ) - : createRuntimeForWorkspace(metadataWithPath); - - // In-place workspaces (CLI/benchmarks) have projectPath === name. - const isInPlace = metadata.projectPath === metadata.name; - const workspacePath = isInPlace - ? metadata.projectPath - : isMultiProject(metadata) && !isSSHRuntime(metadata.runtimeConfig) - ? // Non-SSH multi-project runtimes intentionally start from their shared container root so + ); + + const workspacePath = + singleProjectContext?.workspacePath ?? + (isSSHRuntime(metadata.runtimeConfig) + ? resolveWorkspaceExecutionPath(metadataWithPath, runtime) + : // Non-SSH multi-project runtimes intentionally start from their shared container root so // sibling repos stay addressable during agent/tool setup. SSH workspaces are the exception: // upgraded legacy layouts must reuse the persisted root from config until remote layout // detection seeds the new hashed paths. - runtime.getWorkspacePath(metadata.projectPath, metadata.name) - : resolveWorkspaceExecutionPath(metadataWithPath, runtime); + runtime.getWorkspacePath(metadata.projectPath, metadata.name)); // Wait for init to complete before any runtime I/O operations // (SSH/devcontainer may not be ready until init finishes pulling the container) diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 43d0ac097b..6a17016b38 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -36,6 +36,7 @@ import { } from "@/node/runtime/runtimeFactory"; import { MultiProjectRuntime } from "@/node/runtime/multiProjectRuntime"; import { + createRuntimeContextForWorkspace, createRuntimeForWorkspace, resolveWorkspaceExecutionPath, } from "@/node/runtime/runtimeHelpers"; @@ -6453,8 +6454,7 @@ export class WorkspaceService extends EventEmitter { workspacePath?: string ): Promise { if (!isMultiProject(metadata)) { - const runtime = createRuntimeForWorkspace(metadata); - const workspacePath = resolveWorkspaceExecutionPath(metadata, runtime); + const { runtime, workspacePath } = createRuntimeContextForWorkspace(metadata); return this.listGitPathsForFileCompletions(runtime, workspacePath); } From e812fc96d5d39b740c2b4ae148c4d062ea71e100 Mon Sep 17 00:00:00 2001 From: Ammar Date: Mon, 6 Apr 2026 10:55:09 -0500 Subject: [PATCH 13/16] =?UTF-8?q?=F0=9F=A4=96=20perf:=20simplify=20ssh=20r?= =?UTF-8?q?untime=20scaling=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the SSH scaling redesign around a simpler sharded OpenSSH transport, hashed project layout, and higher-level integration coverage. Delete the explicit OpenSSH master pool and project sync coordinator, remove remote branch-metadata persistence and legacy layout inference, and keep a serialized per-project sync path with current-snapshot reuse validation. --- _Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$219.50`_ --- src/node/runtime/RemoteRuntime.test.ts | 132 --- src/node/runtime/SSHRuntime.test.ts | 1026 ----------------- src/node/runtime/SSHRuntime.ts | 698 ++++------- src/node/runtime/openSshMasterPool.test.ts | 720 ------------ src/node/runtime/openSshMasterPool.ts | 768 ------------ .../runtime/projectSyncCoordinator.test.ts | 78 -- src/node/runtime/projectSyncCoordinator.ts | 128 -- src/node/runtime/remoteProjectLayout.ts | 32 +- src/node/runtime/runtimeHelpers.test.ts | 8 +- src/node/runtime/sshConnectionPool.ts | 48 +- .../transports/OpenSSHTransport.test.ts | 194 ---- .../runtime/transports/OpenSSHTransport.ts | 62 +- src/node/runtime/transports/SSH2Transport.ts | 6 +- .../agentSession.postCompactionRetry.test.ts | 4 +- src/node/services/aiService.ts | 21 +- .../services/workspaceProjectRepos.test.ts | 103 +- src/node/services/workspaceProjectRepos.ts | 17 +- .../workspaceService.multiProject.test.ts | 4 +- src/node/services/workspaceService.test.ts | 4 +- src/node/services/workspaceService.ts | 34 +- tests/runtime/runtime.test.ts | 105 +- 21 files changed, 545 insertions(+), 3647 deletions(-) delete mode 100644 src/node/runtime/RemoteRuntime.test.ts delete mode 100644 src/node/runtime/SSHRuntime.test.ts delete mode 100644 src/node/runtime/openSshMasterPool.test.ts delete mode 100644 src/node/runtime/openSshMasterPool.ts delete mode 100644 src/node/runtime/projectSyncCoordinator.test.ts delete mode 100644 src/node/runtime/projectSyncCoordinator.ts delete mode 100644 src/node/runtime/transports/OpenSSHTransport.test.ts diff --git a/src/node/runtime/RemoteRuntime.test.ts b/src/node/runtime/RemoteRuntime.test.ts deleted file mode 100644 index 1a4615c7fd..0000000000 --- a/src/node/runtime/RemoteRuntime.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { EventEmitter } from "node:events"; -import { PassThrough } from "node:stream"; -import { describe, expect, test } from "bun:test"; - -import { EXIT_CODE_ABORTED, EXIT_CODE_TIMEOUT } from "@/common/constants/exitCodes"; -import { RemoteRuntime, type SpawnResult } from "./RemoteRuntime"; -import type { - ExecOptions, - WorkspaceCreationParams, - WorkspaceCreationResult, - WorkspaceForkParams, - WorkspaceForkResult, - WorkspaceInitParams, - WorkspaceInitResult, -} from "./Runtime"; - -class FakeChildProcess extends EventEmitter { - readonly stdout = new PassThrough(); - readonly stderr = new PassThrough(); - readonly stdin = new PassThrough(); - pid = 1234; - exitCode: number | null = null; - signalCode: NodeJS.Signals | null = null; - - kill(_signal?: string): boolean { - return true; - } -} - -class TestRemoteRuntime extends RemoteRuntime { - protected readonly commandPrefix = "Test"; - - constructor( - private readonly childProcess: FakeChildProcess, - private readonly onExitCalls: Array<[number, string]>, - private readonly onCloseCalls: number[] - ) { - super(); - } - - protected spawnRemoteProcess( - _fullCommand: string, - _options: ExecOptions & { deadlineMs?: number } - ): Promise { - return Promise.resolve({ - process: this.childProcess as never, - onExit: (exitCode, stderr) => { - this.onExitCalls.push([exitCode, stderr]); - }, - onClose: () => { - this.onCloseCalls.push(1); - }, - }); - } - - protected getBasePath(): string { - return "/tmp"; - } - - protected quoteForRemote(targetPath: string): string { - return targetPath; - } - - protected cdCommand(cwd: string): string { - return `cd ${cwd}`; - } - - resolvePath(targetPath: string): Promise { - return Promise.resolve(targetPath); - } - - getWorkspacePath(projectPath: string, workspaceName: string): string { - return `${projectPath}/${workspaceName}`; - } - - createWorkspace(_params: WorkspaceCreationParams): Promise { - throw new Error("unused in test"); - } - - initWorkspace(_params: WorkspaceInitParams): Promise { - throw new Error("unused in test"); - } - - renameWorkspace(): Promise< - { success: true; oldPath: string; newPath: string } | { success: false; error: string } - > { - throw new Error("unused in test"); - } - - deleteWorkspace(): Promise< - { success: true; deletedPath: string } | { success: false; error: string } - > { - throw new Error("unused in test"); - } - - forkWorkspace(_params: WorkspaceForkParams): Promise { - throw new Error("unused in test"); - } -} - -describe("RemoteRuntime synthetic exit handling", () => { - test("does not forward aborted exits to transport onExit hooks", async () => { - const childProcess = new FakeChildProcess(); - const onExitCalls: Array<[number, string]> = []; - const onCloseCalls: number[] = []; - const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls); - const controller = new AbortController(); - - const stream = await runtime.exec("echo ok", { cwd: "/tmp", abortSignal: controller.signal }); - controller.abort(); - childProcess.emit("close", 0, null); - - expect(await stream.exitCode).toBe(EXIT_CODE_ABORTED); - expect(onExitCalls).toEqual([]); - expect(onCloseCalls).toEqual([1]); - }); - - test("does not forward timed-out exits to transport onExit hooks", async () => { - const childProcess = new FakeChildProcess(); - const onExitCalls: Array<[number, string]> = []; - const onCloseCalls: number[] = []; - const runtime = new TestRemoteRuntime(childProcess, onExitCalls, onCloseCalls); - - const stream = await runtime.exec("echo ok", { cwd: "/tmp", timeout: 0.01 }); - await new Promise((resolve) => setTimeout(resolve, 20)); - childProcess.emit("close", 0, null); - - expect(await stream.exitCode).toBe(EXIT_CODE_TIMEOUT); - expect(onExitCalls).toEqual([]); - expect(onCloseCalls).toEqual([1]); - }); -}); diff --git a/src/node/runtime/SSHRuntime.test.ts b/src/node/runtime/SSHRuntime.test.ts deleted file mode 100644 index 5c706c941b..0000000000 --- a/src/node/runtime/SSHRuntime.test.ts +++ /dev/null @@ -1,1026 +0,0 @@ -import * as crypto from "node:crypto"; -import * as path from "node:path"; -import { describe, expect, it, beforeEach, afterEach, spyOn } from "bun:test"; -import * as runtimeHelpers from "@/node/utils/runtime/helpers"; -import * as submoduleSync from "./submoduleSync"; -import { SSHRuntime, clearSharedProjectLayoutCache, computeBaseRepoPath } from "./SSHRuntime"; -import { - buildLegacyRemoteProjectLayout, - buildRemoteProjectLayout, - getRemoteWorkspacePath, - getSnapshotMarkerPath, -} from "./remoteProjectLayout"; -import { createSSHTransport } from "./transports"; -import { projectSyncCoordinator } from "./projectSyncCoordinator"; - -/** - * SSHRuntime unit tests (run with bun test) - * - * Integration tests for workspace operations (renameWorkspace, deleteWorkspace, forkWorkspace, - * worktree-based operations) require Docker and are in tests/runtime/runtime.test.ts. - * Run with: TEST_INTEGRATION=1 bun x jest tests/runtime/runtime.test.ts - */ -afterEach(() => { - clearSharedProjectLayoutCache(); -}); - -describe("SSHRuntime constructor", () => { - it("should accept tilde in srcBaseDir", () => { - // Tildes are now allowed - they will be resolved via resolvePath() - expect(() => { - const config = { host: "example.com", srcBaseDir: "~/mux" }; - new SSHRuntime(config, createSSHTransport(config, false)); - }).not.toThrow(); - }); - - it("should accept bare tilde in srcBaseDir", () => { - // Tildes are now allowed - they will be resolved via resolvePath() - expect(() => { - const config = { host: "example.com", srcBaseDir: "~" }; - new SSHRuntime(config, createSSHTransport(config, false)); - }).not.toThrow(); - }); - - it("should accept absolute paths in srcBaseDir", () => { - expect(() => { - const config = { host: "example.com", srcBaseDir: "/home/user/mux" }; - new SSHRuntime(config, createSSHTransport(config, false)); - }).not.toThrow(); - }); -}); - -describe("SSHRuntime base repo config normalization", () => { - type NormalizeBaseRepoSharedConfig = ( - baseRepoPathArg: string, - abortSignal?: AbortSignal - ) => Promise; - - let execBufferedSpy: ReturnType> | null = - null; - let runtime: SSHRuntime; - - beforeEach(() => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - runtime = new SSHRuntime(config, createSSHTransport(config, false)); - }); - - afterEach(() => { - execBufferedSpy?.mockRestore(); - execBufferedSpy = null; - }); - - function getNormalizeBaseRepoSharedConfig(): NormalizeBaseRepoSharedConfig { - const normalizeUnknown: unknown = Reflect.get(runtime, "normalizeBaseRepoSharedConfig"); - if (typeof normalizeUnknown !== "function") { - throw new Error("normalizeBaseRepoSharedConfig is unavailable"); - } - - return normalizeUnknown as NormalizeBaseRepoSharedConfig; - } - - function normalizeBaseRepoSharedConfig(): Promise { - return getNormalizeBaseRepoSharedConfig().call( - runtime, - '"/home/user/src/project/.mux-base.git"' - ); - } - - it("removes a local core.bare entry from the shared base repo config", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "", - stderr: "", - exitCode: 0, - duration: 0, - }); - - expect(await normalizeBaseRepoSharedConfig()).toBe(true); - expect(execBufferedSpy).toHaveBeenCalledWith( - runtime, - expect.stringContaining("config --local --unset-all core.bare"), - expect.objectContaining({ cwd: "/tmp", timeout: 10 }) - ); - }); - - it("treats missing local core.bare config as already normalized", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "", - stderr: "", - exitCode: 5, - duration: 0, - }); - - expect(await normalizeBaseRepoSharedConfig()).toBe(false); - }); - - it("treats lock conflicts as a no-op when another writer already removed core.bare", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") - .mockResolvedValueOnce({ - stdout: "", - stderr: "error: could not lock config file config: File exists", - exitCode: 255, - duration: 0, - }) - .mockResolvedValueOnce({ - stdout: "", - stderr: "", - exitCode: 1, - duration: 0, - }); - - expect(await normalizeBaseRepoSharedConfig()).toBe(false); - expect(execBufferedSpy).toHaveBeenNthCalledWith( - 2, - runtime, - expect.stringContaining("config --local --get core.bare"), - expect.objectContaining({ cwd: "/tmp", timeout: 10 }) - ); - }); - - it("retries lock conflicts while the shared core.bare entry still exists", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") - .mockResolvedValueOnce({ - stdout: "", - stderr: "error: could not lock config file config: File exists", - exitCode: 255, - duration: 0, - }) - .mockResolvedValueOnce({ - stdout: "true\n", - stderr: "", - exitCode: 0, - duration: 0, - }) - .mockResolvedValueOnce({ - stdout: "", - stderr: "", - exitCode: 0, - duration: 0, - }); - - expect(await normalizeBaseRepoSharedConfig()).toBe(true); - expect(execBufferedSpy).toHaveBeenCalledTimes(3); - }); -}); - -describe("SSHRuntime bundle sync reuse", () => { - interface RuntimeWithSyncProjectToRemote { - syncProjectToRemote: ( - projectPath: string, - workspacePath: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - interface RuntimeWithEnsureBaseRepo { - ensureBaseRepo: ( - projectPath: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - interface RuntimeWithComputeSnapshotDigest { - computeSnapshotDigest: (projectPath: string) => Promise; - } - - interface RuntimeWithTransferBundleToRemote { - transferBundleToRemote: ( - projectPath: string, - remoteBundlePath: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - interface RuntimeWithRefreshBaseRepoOrigin { - refreshBaseRepoOrigin: ( - projectPath: string, - baseRepoPathArg: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - let runtime: SSHRuntime; - let execBufferedSpy: ReturnType> | null = - null; - const initLogger = { - logStep: () => undefined, - logStdout: () => undefined, - logStderr: () => undefined, - logComplete: () => undefined, - }; - - beforeEach(() => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - runtime = new SSHRuntime(config, createSSHTransport(config, false)); - }); - - afterEach(() => { - execBufferedSpy?.mockRestore(); - execBufferedSpy = null; - projectSyncCoordinator.clearAll(); - }); - - it("uploads snapshot bundles through a per-attempt temp path", async () => { - const projectPath = "/projects/demo"; - const snapshotDigest = "abc123"; - const layout = buildRemoteProjectLayout("/home/user/src", projectPath); - const baseRepoPathArg = JSON.stringify(layout.baseRepoPath); - const bundleUuid = "11111111-1111-1111-1111-111111111111"; - const bundleFileName = `${snapshotDigest}.${bundleUuid}.bundle`; - const expectedRemoteBundlePath = path.posix.join( - "~/.mux-bundles", - layout.projectId, - bundleFileName - ); - const snapshotMarkerPath = getSnapshotMarkerPath(layout, snapshotDigest); - const currentSnapshotPath = path.posix.join(layout.snapshotMarkerDir, "current"); - const writeFileCalls: string[] = []; - const randomUuidSpy = spyOn(crypto, "randomUUID").mockReturnValue(bundleUuid); - const ensureBaseRepoSpy = spyOn( - runtime as unknown as RuntimeWithEnsureBaseRepo, - "ensureBaseRepo" - ).mockResolvedValue(baseRepoPathArg); - const computeSnapshotDigestSpy = spyOn( - runtime as unknown as RuntimeWithComputeSnapshotDigest, - "computeSnapshotDigest" - ).mockResolvedValue(snapshotDigest); - const transferBundleSpy = spyOn( - runtime as unknown as RuntimeWithTransferBundleToRemote, - "transferBundleToRemote" - ).mockResolvedValue(expectedRemoteBundlePath); - const refreshBaseRepoOriginSpy = spyOn( - runtime as unknown as RuntimeWithRefreshBaseRepoOrigin, - "refreshBaseRepoOrigin" - ).mockResolvedValue(undefined); - const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation((filePath: string) => { - writeFileCalls.push(filePath); - return new WritableStream({ - write() { - return Promise.resolve(); - }, - close() { - return Promise.resolve(); - }, - }); - }); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( - (_runtime, command) => { - if (command.includes('current_snapshot=""')) { - return Promise.resolve({ stdout: "missing\n", stderr: "", exitCode: 0, duration: 0 }); - } - if (command.startsWith("mkdir -p ")) { - expect(command).toContain(layout.projectId); - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - if (command.includes(" fetch ")) { - expect(command).toContain("fetch --prune --prune-tags"); - expect(command).toContain(bundleFileName); - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - if (command.startsWith("rm -f ")) { - expect(command).toContain(bundleFileName); - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - throw new Error(`Unexpected execBuffered command: ${command}`); - } - ); - - try { - await (runtime as unknown as RuntimeWithSyncProjectToRemote).syncProjectToRemote( - projectPath, - "/unused/workspace", - initLogger - ); - - expect(transferBundleSpy).toHaveBeenCalledWith( - projectPath, - expectedRemoteBundlePath, - initLogger, - expect.any(AbortSignal) - ); - expect(refreshBaseRepoOriginSpy).toHaveBeenCalledWith( - projectPath, - baseRepoPathArg, - initLogger, - expect.any(AbortSignal) - ); - expect(writeFileCalls).toEqual([snapshotMarkerPath, currentSnapshotPath]); - } finally { - randomUuidSpy.mockRestore(); - ensureBaseRepoSpy.mockRestore(); - computeSnapshotDigestSpy.mockRestore(); - transferBundleSpy.mockRestore(); - refreshBaseRepoOriginSpy.mockRestore(); - writeFileSpy.mockRestore(); - projectSyncCoordinator.clearAll(); - } - }); -}); - -describe("SSHRuntime.prepareWorkspaceCheckout", () => { - interface RuntimeWithPrepareWorkspaceCheckout { - prepareWorkspaceCheckout: ( - params: { - projectPath: string; - branchName: string; - trunkBranch: string; - workspacePath: string; - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }; - abortSignal?: AbortSignal; - env?: Record; - trusted?: boolean; - }, - nhp: string - ) => Promise; - } - - interface RuntimeWithFetchOriginTrunk { - fetchOriginTrunk: ( - workspacePath: string, - trunkBranch: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal, - nhp?: string - ) => Promise; - } - - interface RuntimeWithResolveBundleTrunkRef { - resolveBundleTrunkRef: ( - baseRepoPathArg: string, - trunkBranch: string, - abortSignal?: AbortSignal - ) => Promise; - } - - interface RuntimeWithEnsureBaseRepo { - ensureBaseRepo: ( - projectPath: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - interface RuntimeWithCanFastForwardToOrigin { - canFastForwardToOrigin: ( - workspacePath: string, - localRef: string, - originBranch: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - interface RuntimeWithSyncProjectToRemote { - syncProjectToRemote: ( - projectPath: string, - workspacePath: string, - initLogger: { - logStep: (message: string) => void; - logStdout: (line: string) => void; - logStderr: (line: string) => void; - logComplete: (exitCode: number) => void; - }, - abortSignal?: AbortSignal - ) => Promise; - } - - it("syncs the project before creating a worktree for a new workspace", async () => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const runtime = new SSHRuntime(config, createSSHTransport(config, false)); - const initMessages: string[] = []; - const initLogger = { - logStep: (message: string) => { - initMessages.push(message); - }, - logStdout: () => undefined, - logStderr: () => undefined, - logComplete: () => undefined, - }; - - const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( - (_runtime, command) => { - if (command === "test -d /home/user/src/demo/review-slot") { - return Promise.resolve({ stdout: "", stderr: "", exitCode: 1, duration: 0 }); - } - if (command.includes('worktree add "/home/user/src/demo/review-slot"')) { - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - throw new Error(`Unexpected execBuffered command: ${command}`); - } - ); - const fetchOriginSpy = spyOn( - runtime as unknown as RuntimeWithFetchOriginTrunk, - "fetchOriginTrunk" - ).mockResolvedValue(false); - const resolveBundleSpy = spyOn( - runtime as unknown as RuntimeWithResolveBundleTrunkRef, - "resolveBundleTrunkRef" - ).mockResolvedValue("refs/mux-bundle/main"); - const ensureBaseRepoSpy = spyOn( - runtime as unknown as RuntimeWithEnsureBaseRepo, - "ensureBaseRepo" - ).mockResolvedValue('"/home/user/src/demo/.mux-base.git"'); - const canFastForwardSpy = spyOn( - runtime as unknown as RuntimeWithCanFastForwardToOrigin, - "canFastForwardToOrigin" - ).mockResolvedValue(false); - const syncProjectSpy = spyOn( - runtime as unknown as RuntimeWithSyncProjectToRemote, - "syncProjectToRemote" - ).mockResolvedValue(undefined); - const syncSubmodulesSpy = spyOn(submoduleSync, "syncRuntimeGitSubmodules").mockResolvedValue( - undefined - ); - - try { - await (runtime as unknown as RuntimeWithPrepareWorkspaceCheckout).prepareWorkspaceCheckout( - { - projectPath: "/projects/demo", - branchName: "review-slot", - trunkBranch: "main", - workspacePath: "/home/user/src/demo/review-slot", - initLogger, - env: {}, - trusted: true, - }, - "" - ); - - expect(syncProjectSpy).toHaveBeenCalledWith( - "/projects/demo", - "/home/user/src/demo/review-slot", - initLogger, - undefined - ); - expect(ensureBaseRepoSpy).toHaveBeenCalledWith("/projects/demo", initLogger, undefined); - expect(fetchOriginSpy).toHaveBeenCalled(); - expect(resolveBundleSpy).toHaveBeenCalled(); - expect(canFastForwardSpy).not.toHaveBeenCalled(); - expect(execBufferedSpy).toHaveBeenCalledWith( - runtime, - expect.stringContaining( - "worktree add \"/home/user/src/demo/review-slot\" -B 'review-slot' 'refs/mux-bundle/main'" - ), - expect.objectContaining({ cwd: "/tmp", timeout: 300 }) - ); - expect(syncSubmodulesSpy).toHaveBeenCalledWith( - expect.objectContaining({ workspacePath: "/home/user/src/demo/review-slot" }) - ); - expect(initMessages).toContain("Files synced successfully"); - expect(initMessages).toContain("Worktree created successfully"); - } finally { - execBufferedSpy.mockRestore(); - fetchOriginSpy.mockRestore(); - resolveBundleSpy.mockRestore(); - ensureBaseRepoSpy.mockRestore(); - canFastForwardSpy.mockRestore(); - syncProjectSpy.mockRestore(); - syncSubmodulesSpy.mockRestore(); - } - }); -}); - -describe("SSHRuntime.createWorkspace", () => { - function createExecStream(exitCode = 0) { - return { - stdout: new ReadableStream({ - start(controller) { - controller.close(); - }, - }), - stderr: new ReadableStream({ - start(controller) { - controller.close(); - }, - }), - stdin: new WritableStream(), - exitCode: Promise.resolve(exitCode), - duration: Promise.resolve(0), - }; - } - - it("uses directoryName for the workspace path while preparing the remote parent directory", async () => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const runtime = new SSHRuntime(config, createSSHTransport(config, false)); - const expectedLayout = buildRemoteProjectLayout(config.srcBaseDir, "/projects/demo"); - const expectedWorkspacePath = getRemoteWorkspacePath(expectedLayout, "review-slot"); - const execSpy = spyOn(runtime, "exec").mockImplementation(() => - Promise.resolve(createExecStream()) - ); - const readFileSpy = spyOn(runtime, "readFile").mockImplementation( - () => - new ReadableStream({ - start(controller) { - controller.error(new Error("missing branch metadata")); - }, - }) - ); - const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation( - () => new WritableStream() - ); - - try { - const result = await runtime.createWorkspace({ - projectPath: "/projects/demo", - branchName: "feature-branch", - directoryName: "review-slot", - trunkBranch: "main", - initLogger: { - logStep: () => undefined, - logStdout: () => undefined, - logStderr: () => undefined, - logComplete: () => undefined, - }, - }); - - expect(result).toEqual({ - success: true, - workspacePath: expectedWorkspacePath, - }); - expect(execSpy).toHaveBeenCalledWith( - `mkdir -p ${JSON.stringify(expectedLayout.projectRoot)}`, - { - cwd: "/tmp", - timeout: 10, - abortSignal: undefined, - } - ); - } finally { - execSpy.mockRestore(); - readFileSpy.mockRestore(); - writeFileSpy.mockRestore(); - } - }); -}); - -describe("SSHRuntime.deleteWorkspace", () => { - function createExecStream(exitCode: number) { - return { - stdout: new ReadableStream(), - stderr: new ReadableStream(), - stdin: new WritableStream(), - exitCode: Promise.resolve(exitCode), - duration: Promise.resolve(0), - }; - } - - it("deletes the mapped workspace branch instead of the current remote checkout", async () => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const runtime = new SSHRuntime(config, createSSHTransport(config, false)); - const expectedLayout = buildRemoteProjectLayout(config.srcBaseDir, "/projects/demo"); - const expectedDeletedPath = getRemoteWorkspacePath(expectedLayout, "review-slot"); - const execSpy = spyOn(runtime, "exec").mockImplementation((command) => { - if (command.includes("git diff --quiet") || command.includes("test -d")) { - return Promise.resolve(createExecStream(0)); - } - if (command.includes("worktree remove")) { - return Promise.resolve(createExecStream(0)); - } - throw new Error(`Unexpected exec command: ${command}`); - }); - const readFileSpy = spyOn(runtime, "readFile").mockImplementation( - () => - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode('{"review-slot":"feature-branch"}\n')); - controller.close(); - }, - }) - ); - const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( - (_runtime, command) => { - if (command.startsWith("test -f ")) { - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - if (command.startsWith("rm -f ")) { - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - if (command.includes(" branch -D ")) { - expect(command).toContain("feature-branch"); - expect(command).not.toContain("review-slot"); - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - throw new Error(`Unexpected execBuffered command: ${command}`); - } - ); - - try { - const result = await runtime.deleteWorkspace("/projects/demo", "review-slot", true); - expect(result).toEqual({ - success: true, - deletedPath: expectedDeletedPath, - }); - } finally { - execSpy.mockRestore(); - readFileSpy.mockRestore(); - execBufferedSpy.mockRestore(); - } - }); -}); -describe("SSHRuntime branch metadata compatibility", () => { - it("keeps the legacy branch manifest in sync when renaming a legacy workspace", async () => { - type UpdateWorkspaceBranchMapping = ( - projectPath: string, - oldWorkspaceName: string, - newWorkspaceName: string - ) => Promise; - - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const projectPath = "/projects/demo"; - const oldWorkspaceName = "review-slot"; - const newWorkspaceName = "renamed-slot"; - const legacyLayout = buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath); - const legacyManifestPath = path.posix.join( - legacyLayout.projectRoot, - ".mux-workspace-branches.json" - ); - const runtime = new SSHRuntime(config, createSSHTransport(config, false), { - projectPath, - workspaceName: oldWorkspaceName, - workspacePath: getRemoteWorkspacePath(legacyLayout, oldWorkspaceName), - }); - const files = new Map([ - [legacyManifestPath, '{"review-slot":"feature-branch"}\n'], - ]); - const readFileSpy = spyOn(runtime, "readFile").mockImplementation((filePath: string) => { - const contents = files.get(filePath); - return new ReadableStream({ - start(controller) { - if (contents === undefined) { - controller.error(new Error(`Missing file: ${filePath}`)); - return; - } - controller.enqueue(new TextEncoder().encode(contents)); - controller.close(); - }, - }); - }); - const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation((filePath: string) => { - const decoder = new TextDecoder(); - let contents = ""; - return new WritableStream({ - write(chunk) { - contents += decoder.decode(chunk, { stream: true }); - }, - close() { - contents += decoder.decode(); - files.set(filePath, contents); - }, - }); - }); - const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( - (_runtime, command) => { - if (command.startsWith("mkdir -p ") || command.startsWith("rm -f ")) { - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - throw new Error(`Unexpected execBuffered command: ${command}`); - } - ); - - try { - const updateWorkspaceBranchMapping = Reflect.get( - runtime, - "updateWorkspaceBranchMapping" - ) as UpdateWorkspaceBranchMapping; - await updateWorkspaceBranchMapping.call( - runtime, - projectPath, - oldWorkspaceName, - newWorkspaceName - ); - - expect(JSON.parse(files.get(legacyManifestPath) ?? "null")).toEqual({ - [newWorkspaceName]: "feature-branch", - }); - } finally { - readFileSpy.mockRestore(); - writeFileSpy.mockRestore(); - execBufferedSpy.mockRestore(); - projectSyncCoordinator.clearAll(); - } - }); - it("removes stale legacy branch manifest entries even when layout detection falls back to preferred", async () => { - type DeletePersistedWorkspaceBranchMapping = ( - projectPath: string, - workspaceName: string - ) => Promise; - - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const projectPath = "/projects/demo"; - const workspaceName = "review-slot"; - const legacyManifestPath = path.posix.join( - buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath).projectRoot, - ".mux-workspace-branches.json" - ); - const runtime = new SSHRuntime(config, createSSHTransport(config, false)); - const files = new Map([ - [legacyManifestPath, '{"review-slot":"feature-branch"}\n'], - ]); - const readFileSpy = spyOn(runtime, "readFile").mockImplementation((filePath: string) => { - const contents = files.get(filePath); - return new ReadableStream({ - start(controller) { - if (contents === undefined) { - controller.error(new Error(`Missing file: ${filePath}`)); - return; - } - controller.enqueue(new TextEncoder().encode(contents)); - controller.close(); - }, - }); - }); - const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation( - (_runtime, command) => { - if (command.includes("echo legacy") && command.includes("echo preferred")) { - return Promise.resolve({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }); - } - if (command.startsWith("rm -f ")) { - const pathMatch = /^rm -f\s+(.+)$/.exec(command); - if (pathMatch?.[1]) { - files.delete(pathMatch[1].replace(/^"|"$/g, "")); - } - return Promise.resolve({ stdout: "", stderr: "", exitCode: 0, duration: 0 }); - } - throw new Error(`Unexpected execBuffered command: ${command}`); - } - ); - - try { - const deletePersistedWorkspaceBranchMapping = Reflect.get( - runtime, - "deletePersistedWorkspaceBranchMapping" - ) as DeletePersistedWorkspaceBranchMapping; - await deletePersistedWorkspaceBranchMapping.call(runtime, projectPath, workspaceName); - - expect(files.has(legacyManifestPath)).toBe(false); - } finally { - readFileSpy.mockRestore(); - execBufferedSpy.mockRestore(); - projectSyncCoordinator.clearAll(); - } - }); -}); - -describe("SSHRuntime.ensureReady repository checks", () => { - let execBufferedSpy: ReturnType> | null = - null; - let runtime: SSHRuntime; - - beforeEach(() => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - runtime = new SSHRuntime(config, createSSHTransport(config, false), { - projectPath: "/project", - workspaceName: "ws", - }); - }); - - afterEach(() => { - execBufferedSpy?.mockRestore(); - execBufferedSpy = null; - }); - - it("accepts worktrees where .git is a file", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") - .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: ".git\n", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: "true\n", stderr: "", exitCode: 0, duration: 0 }); - - const result = await runtime.ensureReady(); - - expect(execBufferedSpy).toHaveBeenCalledTimes(4); - const secondCommand = execBufferedSpy?.mock.calls[1]?.[1]; - expect(secondCommand).toContain("test -d"); - expect(secondCommand).toContain("test -f"); - const fourthCommand = execBufferedSpy?.mock.calls[3]?.[1]; - expect(fourthCommand).toContain("rev-parse --is-inside-work-tree"); - expect(result).toEqual({ ready: true }); - }); - - it("returns runtime_not_ready when git reports the workspace is not inside a work tree", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") - .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: ".git\n", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: "false\n", stderr: "", exitCode: 0, duration: 0 }); - - const result = await runtime.ensureReady(); - - expect(result.ready).toBe(false); - if (!result.ready) { - expect(result.errorType).toBe("runtime_not_ready"); - } - }); - - it("returns runtime_not_ready when the repo is missing", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") - .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValue({ - stdout: "", - stderr: "", - exitCode: 1, - duration: 0, - }); - - const result = await runtime.ensureReady(); - - expect(result.ready).toBe(false); - if (!result.ready) { - expect(result.errorType).toBe("runtime_not_ready"); - } - }); - - it("returns runtime_start_failed when git is unavailable", async () => { - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered") - .mockResolvedValueOnce({ stdout: "preferred\n", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 0, duration: 0 }) - .mockResolvedValueOnce({ - stdout: "", - stderr: "command not found", - exitCode: 127, - duration: 0, - }); - - const result = await runtime.ensureReady(); - - expect(result.ready).toBe(false); - if (!result.ready) { - expect(result.errorType).toBe("runtime_start_failed"); - } - }); -}); - -describe("SSHRuntime.resolvePath", () => { - let runtime: SSHRuntime; - let transport: ReturnType; - let acquireConnectionSpy: ReturnType> | null = - null; - let execBufferedSpy: ReturnType> | null = - null; - - beforeEach(() => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - transport = createSSHTransport(config, false); - runtime = new SSHRuntime(config, transport, { - projectPath: "/project", - workspaceName: "ws", - }); - }); - - afterEach(() => { - acquireConnectionSpy?.mockRestore(); - acquireConnectionSpy = null; - execBufferedSpy?.mockRestore(); - execBufferedSpy = null; - }); - - it("passes a 10s timeout and max wait to preflight acquireConnection", async () => { - acquireConnectionSpy = spyOn(transport, "acquireConnection").mockResolvedValue(undefined); - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "/home/user/foo\n", - stderr: "", - exitCode: 0, - duration: 0, - }); - - expect(await runtime.resolvePath("~/foo")).toBe("/home/user/foo"); - expect(acquireConnectionSpy).toHaveBeenCalledWith({ - timeoutMs: 10_000, - maxWaitMs: 10_000, - }); - }); -}); -describe("SSHRuntime project sync coordination", () => { - it("uses srcBaseDir in the per-project sync key", () => { - const projectId = "demo-project-123456789abc"; - const configA = { host: "example.com", srcBaseDir: "/home/user/src-a" }; - const configB = { host: "example.com", srcBaseDir: "/home/user/src-b" }; - const runtimeA = new SSHRuntime(configA, createSSHTransport(configA, false)); - const runtimeB = new SSHRuntime(configB, createSSHTransport(configB, false)); - - const getProjectSyncKey = (runtime: SSHRuntime): ((projectIdArg: string) => string) => { - const maybeMethod: unknown = Reflect.get(runtime, "getProjectSyncKey"); - if (typeof maybeMethod !== "function") { - throw new Error("getProjectSyncKey is unavailable"); - } - return maybeMethod as (projectIdArg: string) => string; - }; - - expect(getProjectSyncKey(runtimeA).call(runtimeA, projectId)).not.toBe( - getProjectSyncKey(runtimeB).call(runtimeB, projectId) - ); - }); -}); - -describe("SSHRuntime layout detection", () => { - let execBufferedSpy: ReturnType> | null = - null; - - afterEach(() => { - execBufferedSpy?.mockRestore(); - execBufferedSpy = null; - }); - - it("does not treat legacy root existence alone as evidence of a legacy layout", async () => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const projectPath = "/projects/demo"; - const workspaceName = "fresh-workspace"; - const runtime = new SSHRuntime(config, createSSHTransport(config, false)); - const preferredLayout = buildRemoteProjectLayout(config.srcBaseDir, projectPath); - const legacyLayout = buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath); - - execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockResolvedValue({ - stdout: "preferred\n", - stderr: "", - exitCode: 0, - duration: 0, - }); - - const resolveProjectLayout = Reflect.get(runtime, "resolveProjectLayout") as ( - projectPathArg: string, - workspaceNameArg?: string - ) => Promise<{ projectRoot: string }>; - const layout = await resolveProjectLayout.call(runtime, projectPath, workspaceName); - - expect(layout.projectRoot).toBe(preferredLayout.projectRoot); - const detectionCommand = execBufferedSpy.mock.calls[0]?.[1]; - expect(detectionCommand).toContain(`test -e "${legacyLayout.projectRoot}/${workspaceName}"`); - expect(detectionCommand).not.toContain(`test -d "${legacyLayout.projectRoot}"`); - }); - it("reuses a cached legacy layout for fresh runtimes without workspacePath hints", () => { - const config = { host: "example.com", srcBaseDir: "/home/user/src" }; - const projectPath = "/projects/cached-legacy-demo"; - const workspaceName = "legacy-slot"; - const legacyLayout = buildLegacyRemoteProjectLayout(config.srcBaseDir, projectPath); - const legacyWorkspacePath = getRemoteWorkspacePath(legacyLayout, workspaceName); - - new SSHRuntime(config, createSSHTransport(config, false), { - projectPath, - workspaceName, - workspacePath: legacyWorkspacePath, - }); - - const freshRuntime = new SSHRuntime(config, createSSHTransport(config, false)); - - expect(freshRuntime.getWorkspacePath(projectPath, workspaceName)).toBe(legacyWorkspacePath); - }); -}); - -describe("computeBaseRepoPath", () => { - it("computes the correct bare repo path", () => { - const layout = buildRemoteProjectLayout("~/mux", "/Users/me/code/my-project"); - const result = computeBaseRepoPath("~/mux", "/Users/me/code/my-project"); - expect(result).toBe(layout.baseRepoPath); - expect(result).toMatch(/^~\/mux\/my-project-[a-f0-9]{12}\/\.mux-base\.git$/); - }); - - it("handles absolute srcBaseDir", () => { - const layout = buildRemoteProjectLayout("/home/user/src", "/code/repo"); - const result = computeBaseRepoPath("/home/user/src", "/code/repo"); - expect(result).toBe(layout.baseRepoPath); - expect(result).toMatch(/^\/home\/user\/src\/repo-[a-f0-9]{12}\/\.mux-base\.git$/); - }); -}); diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index a10392b54a..97780d0503 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -41,17 +41,14 @@ import { getErrorMessage } from "@/common/utils/errors"; import { type SSHRuntimeConfig } from "./sshConnectionPool"; import { getOriginUrlForBundle } from "./gitBundleSync"; import { gitNoHooksPrefix } from "@/node/utils/gitNoHooksEnv"; +import { execFileAsync } from "@/node/utils/disposableExec"; import { syncRuntimeGitSubmodules } from "./submoduleSync"; import type { PtyHandle, PtySessionParams, SSHTransport } from "./transports"; import { - buildLegacyRemoteProjectLayout, buildRemoteProjectLayout, getRemoteWorkspacePath, - getSnapshotMarkerPath, - getWorkspaceMetadataPath, type RemoteProjectLayout, } from "./remoteProjectLayout"; -import { projectSyncCoordinator } from "./projectSyncCoordinator"; import { streamToString, shescape } from "./streamUtils"; /** Staging namespace for bundle-imported branch refs. Branches land here instead @@ -61,16 +58,57 @@ const BUNDLE_REF_PREFIX = "refs/mux-bundle/"; /** Small backoff for concurrent writers healing the same shared base repo config. */ const BASE_REPO_CONFIG_LOCK_RETRY_DELAYS_MS = [50, 100, 200]; -const sharedProjectLayouts = new Map(); +const sharedProjectSyncTails = new Map>(); -function getProjectLayoutCacheKey(config: SSHRuntimeConfig, projectPath: string): string { - return [ - config.host, - config.port?.toString() ?? "22", - config.identityFile ?? "default", - config.srcBaseDir, - projectPath, - ].join(":"); +async function enqueueProjectSync( + projectKey: string, + abortSignal: AbortSignal | undefined, + fn: () => Promise +): Promise { + const previous = sharedProjectSyncTails.get(projectKey) ?? Promise.resolve(); + let releaseCurrent: (() => void) | undefined; + const current = new Promise((resolve) => { + releaseCurrent = resolve; + }); + const tail = previous.then( + () => current, + () => current + ); + sharedProjectSyncTails.set(projectKey, tail); + void tail.finally(() => { + if (sharedProjectSyncTails.get(projectKey) === tail) { + sharedProjectSyncTails.delete(projectKey); + } + }); + + let onAbort: (() => void) | undefined; + const waitForPrevious = previous.catch(() => undefined); + const waitForTurn = abortSignal + ? Promise.race([ + waitForPrevious, + new Promise((_, reject) => { + onAbort = () => reject(new Error("Operation aborted")); + if (abortSignal.aborted) { + onAbort(); + return; + } + abortSignal.addEventListener("abort", onAbort, { once: true }); + }), + ]) + : waitForPrevious; + + try { + await waitForTurn; + if (abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + await fn(); + } finally { + if (onAbort) { + abortSignal?.removeEventListener("abort", onAbort); + } + releaseCurrent?.(); + } } function isGitConfigLockConflict(message: string): boolean { @@ -177,10 +215,6 @@ export function computeBaseRepoPath(srcBaseDir: string, projectPath: string): st * * Extends RemoteRuntime for shared exec/file operations. */ -export function clearSharedProjectLayoutCache(): void { - sharedProjectLayouts.clear(); -} - export class SSHRuntime extends RemoteRuntime { private readonly config: SSHRuntimeConfig; private readonly transport: SSHTransport; @@ -189,7 +223,6 @@ export class SSHRuntime extends RemoteRuntime { private readonly currentWorkspacePath?: string; /** Cached resolved bgOutputDir (tilde expanded to absolute path) */ private resolvedBgOutputDir: string | null = null; - private readonly projectLayouts = new Map(); constructor( config: SSHRuntimeConfig, @@ -208,17 +241,6 @@ export class SSHRuntime extends RemoteRuntime { this.ensureReadyProjectPath = options?.projectPath; this.ensureReadyWorkspaceName = options?.workspaceName; this.currentWorkspacePath = options?.workspacePath; - - if (options?.projectPath && options.workspacePath) { - this.cacheProjectLayout( - options.projectPath, - buildRemoteProjectLayout( - this.config.srcBaseDir, - options.projectPath, - path.posix.dirname(options.workspacePath) - ) - ); - } } /** @@ -261,68 +283,10 @@ export class SSHRuntime extends RemoteRuntime { return this.config; } - private getDefaultProjectLayout(projectPath: string): RemoteProjectLayout { + private getProjectLayout(projectPath: string): RemoteProjectLayout { return buildRemoteProjectLayout(this.config.srcBaseDir, projectPath); } - private getCachedProjectLayout(projectPath: string): RemoteProjectLayout | undefined { - return ( - this.projectLayouts.get(projectPath) ?? - sharedProjectLayouts.get(getProjectLayoutCacheKey(this.config, projectPath)) - ); - } - - private cacheProjectLayout( - projectPath: string, - layout: RemoteProjectLayout - ): RemoteProjectLayout { - this.projectLayouts.set(projectPath, layout); - sharedProjectLayouts.set(getProjectLayoutCacheKey(this.config, projectPath), layout); - return layout; - } - - private getPreferredProjectLayout(projectPath: string): RemoteProjectLayout { - return this.getCachedProjectLayout(projectPath) ?? this.getDefaultProjectLayout(projectPath); - } - - private async resolveProjectLayout( - projectPath: string, - workspaceName?: string - ): Promise { - const cached = this.getCachedProjectLayout(projectPath); - if (cached) { - return cached; - } - - const preferredLayout = this.getDefaultProjectLayout(projectPath); - const legacyLayout = buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath); - const preferredWorkspacePath = - workspaceName != null ? getRemoteWorkspacePath(preferredLayout, workspaceName) : undefined; - const legacyWorkspacePath = - workspaceName != null ? getRemoteWorkspacePath(legacyLayout, workspaceName) : undefined; - - const detectLayoutScript = ` - if ${legacyWorkspacePath ? `test -e ${this.quoteForRemote(legacyWorkspacePath)}` : "false"}; then - echo legacy - elif test -d ${this.quoteForRemote(preferredLayout.projectRoot)}${preferredWorkspacePath ? ` || test -e ${this.quoteForRemote(preferredWorkspacePath)}` : ""}; then - echo preferred - else - echo preferred - fi - `; - - try { - const detection = await execBuffered(this, detectLayoutScript, { - cwd: "/tmp", - timeout: 10, - }); - const layout = detection.stdout.trim() === "legacy" ? legacyLayout : preferredLayout; - return this.cacheProjectLayout(projectPath, layout); - } catch { - return this.cacheProjectLayout(projectPath, preferredLayout); - } - } - private getProjectSyncKey(projectId: string): string { return [ this.config.host, @@ -464,7 +428,7 @@ export class SSHRuntime extends RemoteRuntime { return this.currentWorkspacePath; } - return getRemoteWorkspacePath(this.getPreferredProjectLayout(projectPath), workspaceName); + return getRemoteWorkspacePath(this.getProjectLayout(projectPath), workspaceName); } /** @@ -472,7 +436,7 @@ export class SSHRuntime extends RemoteRuntime { * All worktree-based workspaces share this object store. */ private getBaseRepoPath(projectPath: string): string { - return this.getPreferredProjectLayout(projectPath).baseRepoPath; + return this.getProjectLayout(projectPath).baseRepoPath; } /** @@ -485,7 +449,7 @@ export class SSHRuntime extends RemoteRuntime { initLogger: InitLogger, abortSignal?: AbortSignal ): Promise { - const layout = await this.resolveProjectLayout(projectPath); + const layout = this.getProjectLayout(projectPath); const baseRepoPath = layout.baseRepoPath; const baseRepoPathArg = expandTildeForSSH(baseRepoPath); @@ -629,254 +593,6 @@ export class SSHRuntime extends RemoteRuntime { } } - private async persistWorkspaceBranchMapping( - projectPath: string, - workspaceName: string, - branchName: string - ): Promise { - await this.writeWorkspaceBranchMetadata(projectPath, workspaceName, branchName); - } - - private async updateWorkspaceBranchMapping( - projectPath: string, - oldWorkspaceName: string, - newWorkspaceName: string - ): Promise { - const branchName = - (await this.readWorkspaceBranchMetadata(projectPath, oldWorkspaceName))?.trim() ?? - oldWorkspaceName; - await this.writeWorkspaceBranchMetadata(projectPath, newWorkspaceName, branchName); - await this.deletePersistedWorkspaceBranchMapping(projectPath, oldWorkspaceName); - } - - private async getPersistedWorkspaceBranchName( - projectPath: string, - workspaceName: string - ): Promise { - const branchName = (await this.readWorkspaceBranchMetadata(projectPath, workspaceName))?.trim(); - return branchName ?? null; - } - - private async deletePersistedWorkspaceBranchMapping( - projectPath: string, - workspaceName: string - ): Promise { - try { - const layout = await this.resolveProjectLayout(projectPath, workspaceName); - const metadataPath = getWorkspaceMetadataPath(layout, workspaceName); - await execBuffered(this, `rm -f ${this.quoteForRemote(metadataPath)}`, { - cwd: "/tmp", - timeout: 10, - }).catch(() => undefined); - } catch { - // Best-effort cleanup after delete; future creates overwrite any stale entry. - } - - await this.deleteLegacyWorkspaceBranchEntry(projectPath, workspaceName).catch(() => undefined); - } - - private async readWorkspaceBranchMetadata( - projectPath: string, - workspaceName: string - ): Promise { - try { - await this.resolveProjectLayout(projectPath, workspaceName); - const metadataPath = getWorkspaceMetadataPath( - this.getPreferredProjectLayout(projectPath), - workspaceName - ); - const contents = await streamToString(this.readFile(metadataPath)); - const parsed: unknown = JSON.parse(contents); - if (typeof parsed === "object" && parsed !== null) { - const branchName = - "branchName" in parsed && typeof parsed.branchName === "string" - ? parsed.branchName.trim() - : ""; - if (branchName.length > 0) { - return branchName; - } - } - } catch { - // Fall back to the legacy shared manifest for pre-migration workspaces. - } - - return this.readLegacyWorkspaceBranchEntry(projectPath, workspaceName); - } - - private async writeWorkspaceBranchMetadata( - projectPath: string, - workspaceName: string, - branchName: string - ): Promise { - const layout = await this.resolveProjectLayout(projectPath, workspaceName); - const metadataPath = getWorkspaceMetadataPath(layout, workspaceName); - const mkdirResult = await execBuffered( - this, - `mkdir -p ${this.quoteForRemote(layout.workspaceMetadataDir)}`, - { - cwd: "/tmp", - timeout: 10, - } - ); - if (mkdirResult.exitCode !== 0) { - throw new Error( - `Failed to prepare remote workspace metadata: ${mkdirResult.stderr || mkdirResult.stdout}` - ); - } - - const payload = { - workspaceName, - branchName, - }; - const writer = this.writeFile(metadataPath).getWriter(); - try { - await writer.write(new TextEncoder().encode(`${JSON.stringify(payload, null, 2)}\n`)); - } finally { - await writer.close(); - } - - await this.mutateLegacyWorkspaceBranchMap(projectPath, layout, (branchMap) => { - branchMap[workspaceName] = branchName; - }); - } - - private usesLegacyProjectLayout(projectPath: string, layout: RemoteProjectLayout): boolean { - return ( - layout.projectRoot === - buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath).projectRoot - ); - } - - private async readLegacyWorkspaceBranchMap(projectPath: string): Promise> { - try { - const contents = await streamToString( - this.readFile(this.getLegacyWorkspaceBranchMapPath(projectPath)) - ); - const parsed: unknown = JSON.parse(contents); - if (typeof parsed !== "object" || parsed === null) { - return {}; - } - - const branchMap: Record = {}; - for (const [workspaceName, branchName] of Object.entries(parsed)) { - if (typeof branchName !== "string") { - continue; - } - const trimmedWorkspaceName = workspaceName.trim(); - const trimmedBranchName = branchName.trim(); - if (trimmedWorkspaceName.length === 0 || trimmedBranchName.length === 0) { - continue; - } - branchMap[trimmedWorkspaceName] = trimmedBranchName; - } - return branchMap; - } catch { - return {}; - } - } - - private async writeLegacyWorkspaceBranchMap( - projectPath: string, - branchMap: Record - ): Promise { - const legacyManifestPath = this.getLegacyWorkspaceBranchMapPath(projectPath); - const normalizedBranchMap = Object.fromEntries( - Object.entries(branchMap).sort(([leftWorkspace], [rightWorkspace]) => - leftWorkspace.localeCompare(rightWorkspace) - ) - ); - const writer = this.writeFile(legacyManifestPath).getWriter(); - try { - await writer.write( - new TextEncoder().encode(`${JSON.stringify(normalizedBranchMap, null, 2)}\n`) - ); - } finally { - await writer.close(); - } - } - - private async deleteLegacyWorkspaceBranchEntry( - projectPath: string, - workspaceName: string - ): Promise { - const legacyManifestPath = this.getLegacyWorkspaceBranchMapPath(projectPath); - const projectKey = this.getProjectSyncKey(this.getDefaultProjectLayout(projectPath).projectId); - await projectSyncCoordinator.enqueueProjectMutation(projectKey, async () => { - const branchMap = await this.readLegacyWorkspaceBranchMap(projectPath); - if (!(workspaceName in branchMap)) { - return; - } - - delete branchMap[workspaceName]; - if (Object.keys(branchMap).length === 0) { - await execBuffered(this, `rm -f ${this.quoteForRemote(legacyManifestPath)}`, { - cwd: "/tmp", - timeout: 10, - }).catch(() => undefined); - return; - } - - await this.writeLegacyWorkspaceBranchMap(projectPath, branchMap); - }); - } - - private async mutateLegacyWorkspaceBranchMap( - projectPath: string, - layout: RemoteProjectLayout, - mutate: (branchMap: Record) => void - ): Promise { - if (!this.usesLegacyProjectLayout(projectPath, layout)) { - return; - } - - const legacyManifestPath = this.getLegacyWorkspaceBranchMapPath(projectPath); - const projectKey = this.getProjectSyncKey(this.getDefaultProjectLayout(projectPath).projectId); - await projectSyncCoordinator.enqueueProjectMutation(projectKey, async () => { - const branchMap = await this.readLegacyWorkspaceBranchMap(projectPath); - mutate(branchMap); - if (Object.keys(branchMap).length === 0) { - await execBuffered(this, `rm -f ${this.quoteForRemote(legacyManifestPath)}`, { - cwd: "/tmp", - timeout: 10, - }).catch(() => undefined); - return; - } - - const mkdirResult = await execBuffered( - this, - `mkdir -p ${this.quoteForRemote(path.posix.dirname(legacyManifestPath))}`, - { - cwd: "/tmp", - timeout: 10, - } - ); - if (mkdirResult.exitCode !== 0) { - throw new Error( - `Failed to prepare legacy workspace branch manifest: ${mkdirResult.stderr || mkdirResult.stdout}` - ); - } - - await this.writeLegacyWorkspaceBranchMap(projectPath, branchMap); - }); - } - - private async readLegacyWorkspaceBranchEntry( - projectPath: string, - workspaceName: string - ): Promise { - const branchName = (await this.readLegacyWorkspaceBranchMap(projectPath))[ - workspaceName - ]?.trim(); - return branchName && branchName.length > 0 ? branchName : null; - } - - private getLegacyWorkspaceBranchMapPath(projectPath: string): string { - return path.posix.join( - buildLegacyRemoteProjectLayout(this.config.srcBaseDir, projectPath).projectRoot, - ".mux-workspace-branches.json" - ); - } - /** * Resolve the bundle staging ref for the trunk branch. * Returns refs/mux-bundle/ if it exists, otherwise falls back @@ -914,6 +630,55 @@ export class SSHRuntime extends RemoteRuntime { return null; } + private async resolveLocalSyncRefManifest(projectPath: string): Promise { + try { + using proc = execFileAsync("git", ["-C", projectPath, "show-ref", "--heads", "--tags"]); + const { stdout } = await proc.result; + return stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .sort() + .join("\n"); + } catch { + return null; + } + } + + private async resolveRemoteSyncRefManifest( + baseRepoPathArg: string, + abortSignal?: AbortSignal + ): Promise { + const result = await execBuffered( + this, + `git -C ${baseRepoPathArg} for-each-ref --format='%(objectname) %(refname)' ${BUNDLE_REF_PREFIX} refs/tags`, + { cwd: "/tmp", timeout: 20, abortSignal } + ); + if (result.exitCode !== 0) { + return null; + } + + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + const separator = line.indexOf(" "); + if (separator === -1) { + return line; + } + + const oid = line.slice(0, separator); + const refName = line.slice(separator + 1); + const normalizedRefName = refName.startsWith(BUNDLE_REF_PREFIX) + ? refName.replace(BUNDLE_REF_PREFIX, "refs/heads/") + : refName; + return `${oid} ${normalizedRefName}`; + }) + .sort() + .join("\n"); + } + private async refreshBaseRepoOrigin( projectPath: string, baseRepoPathArg: string, @@ -970,11 +735,10 @@ export class SSHRuntime extends RemoteRuntime { return { ready: false, error: "Aborted", errorType: "runtime_start_failed" }; } - const layout = await this.resolveProjectLayout( + const workspacePath = this.getWorkspacePath( this.ensureReadyProjectPath, this.ensureReadyWorkspaceName ); - const workspacePath = getRemoteWorkspacePath(layout, this.ensureReadyWorkspaceName); const gitDir = path.posix.join(workspacePath, ".git"); const gitDirProbe = this.quoteForRemote(gitDir); @@ -1232,145 +996,122 @@ export class SSHRuntime extends RemoteRuntime { throw new Error("Operation aborted"); } - const layout = await this.resolveProjectLayout(projectPath); - const snapshotDigest = await this.computeSnapshotDigest(projectPath); - const snapshotMarkerPath = getSnapshotMarkerPath(layout, snapshotDigest); - const currentSnapshotPath = path.posix.join(layout.snapshotMarkerDir, "current"); + const layout = this.getProjectLayout(projectPath); + const currentSnapshotPath = layout.currentSnapshotPath; const projectKey = this.getProjectSyncKey(layout.projectId); - const snapshotKey = `${projectKey}:${snapshotDigest}`; - await projectSyncCoordinator.runSnapshotSync( - { projectKey, snapshotKey, abortSignal }, - async (sharedAbortSignal) => { - const baseRepoPathArg = await this.ensureBaseRepo( - projectPath, - initLogger, - sharedAbortSignal - ); + await enqueueProjectSync(projectKey, abortSignal, async () => { + if (abortSignal?.aborted) { + throw new Error("Operation aborted"); + } - const snapshotStatusCheck = await execBuffered( - this, - [ - 'current_snapshot=""', - `if test -f ${this.quoteForRemote(currentSnapshotPath)}; then`, - ` current_snapshot=$(tr -d '\\n' < ${this.quoteForRemote(currentSnapshotPath)})`, - "fi", - `if test -f ${this.quoteForRemote(snapshotMarkerPath)} && test "$current_snapshot" = ${shescape.quote(snapshotDigest)}; then`, - ` bundle_ref=$(git -C ${baseRepoPathArg} for-each-ref --count=1 --format='%(refname)' ${shescape.quote(BUNDLE_REF_PREFIX)})`, - ' if test -n "$bundle_ref"; then', - " echo reusable", - " else", - " echo stale-marker", - " fi", - `elif test -f ${this.quoteForRemote(snapshotMarkerPath)} || test -n "$current_snapshot"; then`, - " echo stale-marker", - "else", - " echo missing", - "fi", - ].join("\n"), - { cwd: "/tmp", timeout: 10, abortSignal: sharedAbortSignal } - ); - const snapshotStatus = snapshotStatusCheck.stdout.trim(); - if (snapshotStatus === "reusable") { - await this.refreshBaseRepoOrigin( - projectPath, - baseRepoPathArg, - initLogger, - sharedAbortSignal - ); + const snapshotDigest = await this.computeSnapshotDigest(projectPath); + const baseRepoPathArg = await this.ensureBaseRepo(projectPath, initLogger, abortSignal); + + const snapshotStatusCheck = await execBuffered( + this, + [ + 'current_snapshot=""', + `if test -f ${this.quoteForRemote(currentSnapshotPath)}; then`, + ` current_snapshot=$(tr -d '\\n' < ${this.quoteForRemote(currentSnapshotPath)})`, + "fi", + `if test "$current_snapshot" = ${shescape.quote(snapshotDigest)}; then`, + ` bundle_ref=$(git -C ${baseRepoPathArg} for-each-ref --count=1 --format='%(refname)' ${shescape.quote(BUNDLE_REF_PREFIX)})`, + ' if test -n "$bundle_ref"; then', + " echo reusable", + " else", + " echo stale-current", + " fi", + "else", + " echo missing", + "fi", + ].join("\n"), + { cwd: "/tmp", timeout: 10, abortSignal } + ); + const snapshotStatus = snapshotStatusCheck.stdout.trim(); + if (snapshotStatus === "reusable") { + const localRefManifest = await this.resolveLocalSyncRefManifest(projectPath); + const remoteRefManifest = + localRefManifest == null + ? null + : await this.resolveRemoteSyncRefManifest(baseRepoPathArg, abortSignal); + if (localRefManifest != null && remoteRefManifest === localRefManifest) { + await this.refreshBaseRepoOrigin(projectPath, baseRepoPathArg, initLogger, abortSignal); initLogger.logStep("Reusing existing remote project snapshot"); return; } - if (snapshotStatus === "stale-marker") { - initLogger.logStep( - "Remote snapshot marker found without matching imported refs; reimporting bundle..." - ); - } + initLogger.logStep( + "Remote snapshot marker drifted from imported refs; reimporting bundle..." + ); + } + if (snapshotStatus === "stale-current") { + initLogger.logStep( + "Remote snapshot marker found without matching imported refs; reimporting bundle..." + ); + } - // Snapshot markers stay deterministic, but the uploaded bundle itself must use - // a per-attempt temp path so concurrent Mux processes do not stream into the same file. - const remoteBundlePath = path.posix.join( - "~/.mux-bundles", - layout.projectId, - `${snapshotDigest}.${crypto.randomUUID()}.bundle` + // Snapshot markers stay deterministic, but the uploaded bundle itself must use + // a per-attempt temp path so concurrent Mux processes do not stream into the same file. + const remoteBundlePath = path.posix.join( + "~/.mux-bundles", + layout.projectId, + `${snapshotDigest}.${crypto.randomUUID()}.bundle` + ); + const remoteBundlePathArg = this.quoteForRemote(remoteBundlePath); + const remoteBundleParentDir = path.posix.dirname(remoteBundlePath); + const prepareRemoteDirs = await execBuffered( + this, + `mkdir -p ${this.quoteForRemote(remoteBundleParentDir)} ${this.quoteForRemote(path.posix.dirname(currentSnapshotPath))}`, + { cwd: "/tmp", timeout: 10, abortSignal } + ); + if (prepareRemoteDirs.exitCode !== 0) { + throw new Error( + `Failed to prepare remote snapshot directories: ${prepareRemoteDirs.stderr || prepareRemoteDirs.stdout}` ); - const remoteBundlePathArg = this.quoteForRemote(remoteBundlePath); - const remoteBundleParentDir = path.posix.dirname(remoteBundlePath); - const prepareRemoteDirs = await execBuffered( + } + + await this.transferBundleToRemote(projectPath, remoteBundlePath, initLogger, abortSignal); + + try { + // Import branches and tags from the bundle into the shared bare repo. + // Branches land in refs/mux-bundle/* (staging namespace) instead of + // refs/heads/* to avoid colliding with branches checked out in existing + // worktrees — git refuses to update any ref checked out in a worktree. + // Tags go directly to refs/tags/* (they're never checked out). + initLogger.logStep("Importing bundle into shared base repository..."); + const fetchResult = await execBuffered( this, - `mkdir -p ${this.quoteForRemote(remoteBundleParentDir)} ${this.quoteForRemote(layout.snapshotMarkerDir)}`, - { cwd: "/tmp", timeout: 10, abortSignal: sharedAbortSignal } + `git -C ${baseRepoPathArg} fetch --prune --prune-tags ${remoteBundlePathArg} '+refs/heads/*:${BUNDLE_REF_PREFIX}*' '+refs/tags/*:refs/tags/*'`, + { cwd: "/tmp", timeout: 300, abortSignal } ); - if (prepareRemoteDirs.exitCode !== 0) { + if (fetchResult.exitCode !== 0) { throw new Error( - `Failed to prepare remote snapshot directories: ${prepareRemoteDirs.stderr || prepareRemoteDirs.stdout}` + `Failed to import bundle into base repo: ${fetchResult.stderr || fetchResult.stdout}` ); } - await this.transferBundleToRemote( - projectPath, - remoteBundlePath, - initLogger, - sharedAbortSignal - ); + await this.refreshBaseRepoOrigin(projectPath, baseRepoPathArg, initLogger, abortSignal); + const currentSnapshotWriter = this.writeFile(currentSnapshotPath).getWriter(); try { - // Import branches and tags from the bundle into the shared bare repo. - // Branches land in refs/mux-bundle/* (staging namespace) instead of - // refs/heads/* to avoid colliding with branches checked out in existing - // worktrees — git refuses to update any ref checked out in a worktree. - // Tags go directly to refs/tags/* (they're never checked out). - initLogger.logStep("Importing bundle into shared base repository..."); - const fetchResult = await execBuffered( - this, - `git -C ${baseRepoPathArg} fetch --prune --prune-tags ${remoteBundlePathArg} '+refs/heads/*:${BUNDLE_REF_PREFIX}*' '+refs/tags/*:refs/tags/*'`, - { cwd: "/tmp", timeout: 300, abortSignal: sharedAbortSignal } - ); - if (fetchResult.exitCode !== 0) { - throw new Error( - `Failed to import bundle into base repo: ${fetchResult.stderr || fetchResult.stdout}` - ); - } - - await this.refreshBaseRepoOrigin( - projectPath, - baseRepoPathArg, - initLogger, - sharedAbortSignal - ); - - const markerWriter = this.writeFile(snapshotMarkerPath).getWriter(); - try { - await markerWriter.write( - new TextEncoder().encode( - `${JSON.stringify({ snapshotDigest, importedAt: new Date().toISOString() }, null, 2)}\n` - ) - ); - } finally { - await markerWriter.close(); - } - - const currentSnapshotWriter = this.writeFile(currentSnapshotPath).getWriter(); - try { - await currentSnapshotWriter.write(new TextEncoder().encode(`${snapshotDigest}\n`)); - } finally { - await currentSnapshotWriter.close(); - } - - initLogger.logStep("Repository synced to base successfully"); + await currentSnapshotWriter.write(new TextEncoder().encode(`${snapshotDigest}\n`)); } finally { - // Best-effort cleanup of the remote bundle file. - try { - await execBuffered(this, `rm -f ${remoteBundlePathArg}`, { - cwd: "/tmp", - timeout: 10, - }); - } catch { - // Ignore cleanup errors. - } + await currentSnapshotWriter.close(); + } + + initLogger.logStep("Repository synced to base successfully"); + } finally { + // Best-effort cleanup of the remote bundle file. + try { + await execBuffered(this, `rm -f ${remoteBundlePathArg}`, { + cwd: "/tmp", + timeout: 10, + }); + } catch { + // Ignore cleanup errors. } } - ); + }); } /** Get origin URL from local project for setting on the remote base repo. */ @@ -1384,8 +1125,7 @@ export class SSHRuntime extends RemoteRuntime { async createWorkspace(params: WorkspaceCreationParams): Promise { try { const { projectPath, directoryName, initLogger, abortSignal } = params; - const layout = await this.resolveProjectLayout(projectPath, directoryName); - this.projectLayouts.set(projectPath, layout); + const layout = this.getProjectLayout(projectPath); // Workspace directories follow the persisted workspace name; branch checkout happens later. const workspacePath = getRemoteWorkspacePath(layout, directoryName); @@ -1423,7 +1163,6 @@ export class SSHRuntime extends RemoteRuntime { }; } - await this.persistWorkspaceBranchMapping(projectPath, directoryName, params.branchName); initLogger.logStep("Remote workspace prepared"); return { @@ -1439,15 +1178,6 @@ export class SSHRuntime extends RemoteRuntime { } async initWorkspace(params: WorkspaceInitParams): Promise { - this.projectLayouts.set( - params.projectPath, - buildRemoteProjectLayout( - this.config.srcBaseDir, - params.projectPath, - path.posix.dirname(params.workspacePath) - ) - ); - // Disable git hooks for untrusted projects (prevents post-checkout execution) const nhp = gitNoHooksPrefix(params.trusted); @@ -1772,9 +1502,8 @@ export class SSHRuntime extends RemoteRuntime { if (abortSignal?.aborted) { return { success: false, error: "Rename operation aborted" }; } - const layout = await this.resolveProjectLayout(projectPath, oldName); - const oldPath = getRemoteWorkspacePath(layout, oldName); - const newPath = getRemoteWorkspacePath(layout, newName); + const oldPath = this.getWorkspacePath(projectPath, oldName); + const newPath = path.posix.join(path.posix.dirname(oldPath), newName); try { const expandedOldPath = expandTildeForSSH(oldPath); @@ -1822,7 +1551,6 @@ export class SSHRuntime extends RemoteRuntime { }; } - await this.updateWorkspaceBranchMapping(projectPath, oldName, newName); return { success: true, oldPath, newPath }; } catch (error) { return { @@ -1847,8 +1575,7 @@ export class SSHRuntime extends RemoteRuntime { // Disable git hooks for untrusted projects const nhp = gitNoHooksPrefix(trusted); - const layout = await this.resolveProjectLayout(projectPath, workspaceName); - const deletedPath = getRemoteWorkspacePath(layout, workspaceName); + const deletedPath = this.getWorkspacePath(projectPath, workspaceName); try { // Combine all pre-deletion checks into a single bash script to minimize round trips @@ -1948,7 +1675,6 @@ export class SSHRuntime extends RemoteRuntime { // Handle check results if (checkExitCode === 3) { // Directory doesn't exist - deletion is idempotent (success). - await this.deletePersistedWorkspaceBranchMapping(projectPath, workspaceName); return { success: true, deletedPath }; } @@ -1982,8 +1708,7 @@ export class SSHRuntime extends RemoteRuntime { }; } - const branchToDelete = - (await this.getPersistedWorkspaceBranchName(projectPath, workspaceName)) ?? workspaceName; + const branchToDelete = await this.resolveCheckedOutBranch(deletedPath, abortSignal, 10); // Detect if workspace is a worktree (.git is a file) vs a legacy full clone (.git is a directory). const isWorktree = await this.isWorktreeWorkspace(deletedPath, abortSignal); @@ -2056,7 +1781,6 @@ export class SSHRuntime extends RemoteRuntime { } } - await this.deletePersistedWorkspaceBranchMapping(projectPath, workspaceName); return { success: true, deletedPath }; } catch (error) { return { success: false, error: `Failed to delete directory: ${getErrorMessage(error)}` }; @@ -2066,9 +1790,11 @@ export class SSHRuntime extends RemoteRuntime { async forkWorkspace(params: WorkspaceForkParams): Promise { const { projectPath, sourceWorkspaceName, newWorkspaceName, initLogger, abortSignal } = params; - const layout = await this.resolveProjectLayout(projectPath, sourceWorkspaceName); - const sourceWorkspacePath = getRemoteWorkspacePath(layout, sourceWorkspaceName); - const newWorkspacePath = getRemoteWorkspacePath(layout, newWorkspaceName); + const sourceWorkspacePath = this.getWorkspacePath(projectPath, sourceWorkspaceName); + const newWorkspacePath = path.posix.join( + path.posix.dirname(sourceWorkspacePath), + newWorkspaceName + ); // For SSH commands, tilde must be expanded using $HOME - plain quoting won't expand it. const sourceWorkspacePathArg = expandTildeForSSH(sourceWorkspacePath); diff --git a/src/node/runtime/openSshMasterPool.test.ts b/src/node/runtime/openSshMasterPool.test.ts deleted file mode 100644 index 8f48945e66..0000000000 --- a/src/node/runtime/openSshMasterPool.test.ts +++ /dev/null @@ -1,720 +0,0 @@ -import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; -import type { spawn as spawnProcess } from "child_process"; -import { EventEmitter } from "events"; -import { PassThrough } from "stream"; -import { formatSshEndpoint } from "@/common/utils/ssh/formatSshEndpoint"; -import { SshPromptService } from "@/node/services/sshPromptService"; -import * as openSshPromptMediation from "./openSshPromptMediation"; -import { OpenSSHMasterPool, getShardedControlPath } from "./openSshMasterPool"; -import { - setOpenSSHHostKeyPolicyMode, - setSshPromptService, - type SSHConnectionConfig, -} from "./sshConnectionPool"; - -class FakeChildProcess extends EventEmitter { - readonly stdout = new PassThrough(); - readonly stderr = new PassThrough(); - readonly stdin = new PassThrough(); - pid = 1234; - exitCode: number | null = null; - signalCode: string | null = null; - - kill(_signal?: string): boolean { - this.exitCode ??= 0; - this.emit("exit", this.exitCode, this.signalCode); - this.emit("close", this.exitCode, this.signalCode); - return true; - } -} - -describe("getShardedControlPath", () => { - test("is deterministic per shard and unique across shards", () => { - const config: SSHConnectionConfig = { host: "example.com" }; - expect(getShardedControlPath(config, 0)).toBe(getShardedControlPath(config, 0)); - expect(getShardedControlPath(config, 0)).not.toBe(getShardedControlPath(config, 1)); - }); -}); - -describe("OpenSSHMasterPool", () => { - const masterProcesses = new Map(); - - let releaseInteractiveResponder: (() => void) | undefined; - - beforeEach(() => { - setSshPromptService(undefined); - setOpenSSHHostKeyPolicyMode("headless-fallback"); - masterProcesses.clear(); - }); - - afterEach(() => { - releaseInteractiveResponder?.(); - releaseInteractiveResponder = undefined; - setSshPromptService(undefined); - setOpenSSHHostKeyPolicyMode("headless-fallback"); - masterProcesses.clear(); - }); - - test("reuses an existing shard until capacity is reached, then scales out", async () => { - const spawnCalls: Array<{ command: string; args: string[] }> = []; - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 2, - maxShardsPerHost: 4, - sleep: () => Promise.resolve(), - spawnProcess: ((command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - spawnCalls.push({ command, args: normalizedArgs }); - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - const second = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - const third = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - - expect(first.controlPath).toBe(second.controlPath); - expect(third.controlPath).not.toBe(first.controlPath); - expect(spawnCalls.filter((call) => call.args.includes("-M"))).toHaveLength(2); - - first.release(); - second.release(); - third.release(); - pool.clearAll(); - }); - - test("reuses one endpoint-scoped askpass dedupe key across shard startups", async () => { - const dedupeKeys: string[] = []; - const promptService = new SshPromptService(); - releaseInteractiveResponder = promptService.registerInteractiveResponder(); - setSshPromptService(promptService); - setOpenSSHHostKeyPolicyMode("strict"); - const askpassSpy = spyOn( - openSshPromptMediation, - "createMediatedAskpassSession" - ).mockImplementation((params) => { - dedupeKeys.push(params.dedupeKey ?? ""); - const mediatedAskpass: Awaited< - ReturnType - > = { - env: {}, - cleanup: mock(() => undefined), - getLastPromptOutcome: () => null, - }; - return Promise.resolve(mediatedAskpass); - }); - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 2, - sleep: () => Promise.resolve(), - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - try { - const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - const second = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - - expect(first.controlPath).not.toBe(second.controlPath); - expect(dedupeKeys).toEqual([ - formatSshEndpoint(config.host, config.port ?? 22), - formatSshEndpoint(config.host, config.port ?? 22), - ]); - - first.release(); - second.release(); - } finally { - askpassSpy.mockRestore(); - pool.clearAll(); - } - }); - - test("does not lease a shard until its master is ready", async () => { - let ready = false; - let releaseStartupWait: (() => void) | undefined; - const startupWait = new Promise((resolve) => { - releaseStartupWait = resolve; - }); - - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 2, - maxShardsPerHost: 1, - sleep: () => startupWait, - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = ready ? 0 : 1; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const firstPromise = pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - let secondResolved = false; - const secondPromise = pool - .acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }) - .then((lease) => { - secondResolved = true; - return lease; - }); - - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(secondResolved).toBe(false); - - ready = true; - releaseStartupWait?.(); - - const first = await firstPromise; - const second = await secondPromise; - expect(first.controlPath).toBe(second.controlPath); - - first.release(); - second.release(); - pool.clearAll(); - }); - - test("waits for shard backoff before reusing a failed master", async () => { - let releaseBackoffWait: (() => void) | undefined; - const backoffWait = new Promise((resolve) => { - releaseBackoffWait = resolve; - }); - - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - sleep: () => backoffWait, - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - first.reportFailure("ssh exited 255"); - first.release(); - - let secondResolved = false; - const secondPromise = pool - .acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }) - .then((lease) => { - secondResolved = true; - return lease; - }); - - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(secondResolved).toBe(false); - - const internals = pool as unknown as { - hostGroups: Map }>; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (!shard) { - throw new Error("Expected a tracked shard"); - } - shard.health.backoffUntil = new Date(Date.now() - 1); - releaseBackoffWait?.(); - - const second = await secondPromise; - expect(second.controlPath).toBe(first.controlPath); - - second.release(); - pool.clearAll(); - }); - - test("honors shard backoff waits instead of polling at the startup cadence", async () => { - const sleepCalls: number[] = []; - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - startupPollIntervalMs: 50, - sleep: (waitMs) => { - sleepCalls.push(waitMs); - const internals = pool as unknown as { - hostGroups: Map }>; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (shard?.health.backoffUntil) { - shard.health.backoffUntil = new Date(Date.now() - 1); - } - return Promise.resolve(); - }, - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - sleepCalls.length = 0; - first.reportFailure("ssh exited 255"); - first.release(); - - const second = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - - expect(sleepCalls[0]).toBeGreaterThan(100); - - second.release(); - pool.clearAll(); - }); - - test("polls for free capacity when a healthy shard is saturated even if another shard is backing off", async () => { - const sleepCalls: number[] = []; - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 2, - startupPollIntervalMs: 50, - sleep: (waitMs) => { - sleepCalls.push(waitMs); - return Promise.resolve(); - }, - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const first = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - const internals = pool as unknown as { - hostGroups: Map< - string, - { - shards: Array<{ - health: { - backoffUntil?: Date; - status: string; - consecutiveFailures?: number; - lastFailure?: Date; - lastError?: string; - }; - ready: boolean; - inflight: number; - process?: FakeChildProcess; - startup?: Promise; - stopping: boolean; - stderr: string; - id: number; - shardId: string; - controlPath: string; - lastUsedAt: number; - }>; - } - >; - }; - const group = [...internals.hostGroups.values()][0]; - if (!group) { - throw new Error("Expected a tracked host group"); - } - group.shards.push({ - id: 99, - shardId: "shard-99", - controlPath: "/tmp/mux-backoff-shard", - inflight: 0, - lastUsedAt: Date.now(), - ready: false, - stderr: "", - stopping: false, - health: { - status: "unhealthy", - consecutiveFailures: 1, - lastFailure: new Date(), - lastError: "ssh exited 255", - backoffUntil: new Date(Date.now() + 10_000), - }, - }); - - const secondPromise = pool.acquireLease(config, { - maxWaitMs: 1000, - timeoutMs: 1000, - onWait: () => { - first.release(); - }, - }); - const second = await secondPromise; - - expect(sleepCalls[0]).toBe(50); - - second.release(); - pool.clearAll(); - }); - - test("preserves shard failure history across retries after backoff expires", async () => { - let startupAttempts = 0; - const controller = new AbortController(); - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - sleep: () => { - const internals = pool as unknown as { - hostGroups: Map }>; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (shard?.health.backoffUntil) { - shard.health.backoffUntil = new Date(Date.now() - 1); - } - return Promise.resolve(); - }, - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - startupAttempts += 1; - queueMicrotask(() => { - proc.emit("error", new Error(`startup failure ${startupAttempts}`)); - proc.exitCode = 1; - proc.emit("exit", proc.exitCode, null); - proc.emit("close", proc.exitCode, null); - if (startupAttempts === 2) { - controller.abort(); - } - }); - return proc as never; - } - - queueMicrotask(() => { - proc.exitCode = 1; - proc.emit("close", proc.exitCode, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - try { - await pool.acquireLease(config, { - maxWaitMs: 1000, - timeoutMs: 1000, - abortSignal: controller.signal, - }); - throw new Error("Expected acquireLease to reject"); - } catch { - // Expected: we abort after the second failed startup attempt. - } - - const internals = pool as unknown as { - hostGroups: Map }>; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (!shard) { - throw new Error("Expected a tracked shard"); - } - expect(startupAttempts).toBe(2); - expect(shard.health.consecutiveFailures).toBe(2); - - pool.clearAll(); - }); - - test("ensureReadyMaster ignores saturated exec slots when a shard is already ready", async () => { - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - sleep: () => Promise.resolve(), - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const lease = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - - await pool.ensureReadyMaster(config, { maxWaitMs: 0, timeoutMs: 1000 }); - - lease.release(); - pool.clearAll(); - }); - - test("ensureReadyMaster schedules idle cleanup after bootstrapping a shard", async () => { - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - sleep: () => Promise.resolve(), - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - await pool.ensureReadyMaster(config, { maxWaitMs: 0, timeoutMs: 1000 }); - - const internals = pool as unknown as { - hostGroups: Map< - string, - { shards: Array<{ inflight: number; idleTimer?: ReturnType }> } - >; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (!shard) { - throw new Error("Expected a tracked shard"); - } - expect(shard.inflight).toBe(0); - expect(shard.idleTimer).toBeDefined(); - - pool.clearAll(); - }); - - test("retries transient shard startup failures within the maxWait budget", async () => { - let startupAttempts = 0; - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - sleep: () => { - const internals = pool as unknown as { - hostGroups: Map }>; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (shard?.health.backoffUntil) { - shard.health.backoffUntil = new Date(Date.now() - 1); - } - return Promise.resolve(); - }, - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - startupAttempts += 1; - const controlPathArg = normalizedArgs.find((arg) => arg.startsWith("ControlPath=")); - if (controlPathArg && startupAttempts > 1) { - masterProcesses.set(controlPathArg.slice("ControlPath=".length), proc); - } - if (startupAttempts === 1) { - queueMicrotask(() => { - proc.emit("error", new Error("transient startup failure")); - proc.exitCode = 1; - proc.emit("exit", proc.exitCode, null); - proc.emit("close", proc.exitCode, null); - }); - } - return proc as never; - } - - const controlPathIndex = normalizedArgs.indexOf("-S"); - const controlPath = - controlPathIndex >= 0 ? normalizedArgs[controlPathIndex + 1] : undefined; - queueMicrotask(() => { - if (normalizedArgs.includes("check") && controlPath && masterProcesses.has(controlPath)) { - proc.exitCode = 0; - } - proc.emit("close", proc.exitCode ?? 1, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - const lease = await pool.acquireLease(config, { maxWaitMs: 1000, timeoutMs: 1000 }); - - expect(startupAttempts).toBe(2); - - lease.release(); - pool.clearAll(); - }); - - test("records a failed master startup only once when ssh emits error and exit", async () => { - const pool = new OpenSSHMasterPool({ - maxSessionsPerShard: 1, - maxShardsPerHost: 1, - sleep: () => Promise.resolve(), - spawnProcess: ((_command: string, args?: readonly string[]) => { - const proc = new FakeChildProcess(); - const normalizedArgs = [...(args ?? [])]; - - if (normalizedArgs.includes("-M")) { - queueMicrotask(() => { - proc.emit("error", new Error("spawn ENOENT")); - proc.exitCode = 1; - proc.emit("exit", proc.exitCode, null); - proc.emit("close", proc.exitCode, null); - }); - return proc as never; - } - - queueMicrotask(() => { - proc.exitCode = 1; - proc.emit("close", proc.exitCode, null); - }); - return proc as never; - }) as unknown as typeof spawnProcess, - }); - - const config: SSHConnectionConfig = { host: "remote.example.com", port: 22 }; - try { - // Keep the wait budget below the minimum 0.8s backoff jitter so this assertion only - // verifies one startup attempt, not whether acquireLease later retries within budget. - await pool.acquireLease(config, { maxWaitMs: 100, timeoutMs: 1000 }); - throw new Error("Expected acquireLease to reject"); - } catch (error) { - expect(error).toBeInstanceOf(Error); - expect((error as Error).message).toContain("spawn ENOENT"); - } - - const internals = pool as unknown as { - hostGroups: Map }>; - }; - const shard = [...internals.hostGroups.values()][0]?.shards[0]; - if (!shard) { - throw new Error("Expected a tracked shard"); - } - expect(shard.health.consecutiveFailures).toBe(1); - - pool.clearAll(); - }); -}); diff --git a/src/node/runtime/openSshMasterPool.ts b/src/node/runtime/openSshMasterPool.ts deleted file mode 100644 index cfbb0a9fce..0000000000 --- a/src/node/runtime/openSshMasterPool.ts +++ /dev/null @@ -1,768 +0,0 @@ -import * as crypto from "crypto"; -import * as os from "os"; -import * as path from "path"; -import { spawn, type ChildProcess } from "child_process"; -import { HOST_KEY_APPROVAL_TIMEOUT_MS } from "@/common/constants/ssh"; -import { formatSshEndpoint } from "@/common/utils/ssh/formatSshEndpoint"; -import { getErrorMessage } from "@/common/utils/errors"; -import { log } from "@/node/services/log"; -import { - appendOpenSSHHostKeyPolicyArgs, - getSshPromptService, - isInteractiveHostKeyApprovalAvailable, - type ConnectionHealth, - type SSHConnectionConfig, -} from "./sshConnectionPool"; -import { createMediatedAskpassSession } from "./openSshPromptMediation"; - -const DEFAULT_MASTER_START_TIMEOUT_MS = 10_000; -const DEFAULT_MAX_WAIT_MS = 2 * 60 * 1000; -const DEFAULT_MAX_SESSIONS_PER_SHARD = 4; -const DEFAULT_MAX_SHARDS_PER_HOST = 8; -const SHARD_IDLE_TTL_MS = 60_000; -const STARTUP_POLL_INTERVAL_MS = 50; -const BACKOFF_SCHEDULE = [1, 2, 4, 7, 10]; - -type SleepFn = (ms: number, abortSignal?: AbortSignal) => Promise; - -type SpawnFn = typeof spawn; - -interface AcquireLeaseOptions { - timeoutMs?: number; - maxWaitMs?: number; - abortSignal?: AbortSignal; - onWait?: (waitMs: number) => void; -} - -export interface OpenSSHMasterLease { - controlPath: string; - shardId: string; - release(): void; - reportFailure(error: string): void; - markHealthy(): void; -} - -interface MasterShard { - id: number; - shardId: string; - controlPath: string; - process?: ChildProcess; - startup?: Promise; - ready: boolean; - stderr: string; - stopping: boolean; - inflight: number; - lastUsedAt: number; - idleTimer?: ReturnType; - health: ConnectionHealth; -} - -interface HostGroup { - config: SSHConnectionConfig; - shards: MasterShard[]; - nextShardId: number; -} - -interface MasterPoolOptions { - spawnProcess?: SpawnFn; - sleep?: SleepFn; - maxSessionsPerShard?: number; - maxShardsPerHost?: number; - startupPollIntervalMs?: number; - defaultMasterStartTimeoutMs?: number; - defaultMaxWaitMs?: number; - shardIdleTtlMs?: number; -} - -function withJitter(seconds: number): number { - const jitterFactor = 0.8 + Math.random() * 0.4; - return seconds * jitterFactor; -} - -async function sleepWithAbort(ms: number, abortSignal?: AbortSignal): Promise { - if (ms <= 0) { - return; - } - if (abortSignal?.aborted) { - throw new Error("Operation aborted"); - } - - await new Promise((resolve, reject) => { - const timer = setTimeout(() => { - cleanup(); - resolve(); - }, ms); - - const onAbort = () => { - cleanup(); - reject(new Error("Operation aborted")); - }; - - const cleanup = () => { - clearTimeout(timer); - abortSignal?.removeEventListener("abort", onAbort); - }; - - abortSignal?.addEventListener("abort", onAbort, { once: true }); - }); -} - -function makeConnectionKey(config: SSHConnectionConfig): string { - return [ - os.userInfo().username, - config.host, - config.port?.toString() ?? "22", - config.identityFile ?? "default", - ].join(":"); -} - -function hashKey(key: string): string { - return crypto.createHash("sha256").update(key).digest("hex").slice(0, 12); -} - -export function getShardedControlPath(config: SSHConnectionConfig, shardId = 0): string { - const key = makeConnectionKey(config); - return path.join(os.tmpdir(), `mux-ssh-${hashKey(`${key}:${shardId}`)}`); -} - -function createInitialHealth(): ConnectionHealth { - return { - status: "unknown", - consecutiveFailures: 0, - }; -} - -function isProcessAlive(proc: ChildProcess | undefined): boolean { - return proc != null && proc.exitCode == null && proc.signalCode == null; -} - -interface ShardAcquisitionBehavior { - pickReadyShard(group: HostGroup): MasterShard | undefined; - finalizeShard(shard: MasterShard): T; - preferPolling(group: HostGroup): boolean; -} - -export class OpenSSHMasterPool { - private readonly hostGroups = new Map(); - private readonly spawnProcess: SpawnFn; - private readonly sleep: SleepFn; - private readonly maxSessionsPerShard: number; - private readonly maxShardsPerHost: number; - private readonly startupPollIntervalMs: number; - private readonly defaultMasterStartTimeoutMs: number; - private readonly defaultMaxWaitMs: number; - private readonly shardIdleTtlMs: number; - - constructor(options: MasterPoolOptions = {}) { - this.spawnProcess = options.spawnProcess ?? spawn; - this.sleep = options.sleep ?? sleepWithAbort; - this.maxSessionsPerShard = options.maxSessionsPerShard ?? DEFAULT_MAX_SESSIONS_PER_SHARD; - this.maxShardsPerHost = options.maxShardsPerHost ?? DEFAULT_MAX_SHARDS_PER_HOST; - this.startupPollIntervalMs = options.startupPollIntervalMs ?? STARTUP_POLL_INTERVAL_MS; - this.defaultMasterStartTimeoutMs = - options.defaultMasterStartTimeoutMs ?? DEFAULT_MASTER_START_TIMEOUT_MS; - this.defaultMaxWaitMs = options.defaultMaxWaitMs ?? DEFAULT_MAX_WAIT_MS; - this.shardIdleTtlMs = options.shardIdleTtlMs ?? SHARD_IDLE_TTL_MS; - } - - async ensureConnection( - config: SSHConnectionConfig, - options?: AcquireLeaseOptions - ): Promise { - const lease = await this.acquireLease(config, options); - lease.release(); - } - - async ensureReadyMaster( - config: SSHConnectionConfig, - options?: AcquireLeaseOptions - ): Promise { - await this.withAcquiredShard(config, options, { - pickReadyShard: (group) => this.pickReadyShard(group), - finalizeShard: (shard) => { - this.scheduleIdleDisposalIfUnused(shard); - }, - preferPolling: () => false, - }); - } - - async acquireLease( - config: SSHConnectionConfig, - options?: AcquireLeaseOptions - ): Promise { - return this.withAcquiredShard(config, options, { - pickReadyShard: (group) => this.pickAvailableShard(group), - finalizeShard: (shard) => this.reserveShard(shard), - preferPolling: (group) => this.pickReadyShard(group) != null, - }); - } - - private async withAcquiredShard( - config: SSHConnectionConfig, - options: AcquireLeaseOptions | undefined, - behavior: ShardAcquisitionBehavior - ): Promise { - const maxWaitMs = options?.maxWaitMs ?? this.defaultMaxWaitMs; - const defaultStartTimeoutMs = options?.timeoutMs ?? this.defaultMasterStartTimeoutMs; - const deadlineMs = Date.now() + maxWaitMs; - const hostGroup = this.getOrCreateHostGroup(makeConnectionKey(config), config); - let lastStartError: Error | undefined; - - while (true) { - if (options?.abortSignal?.aborted) { - throw new Error("Operation aborted"); - } - - this.trimExitedShards(hostGroup); - const readyShard = behavior.pickReadyShard(hostGroup); - if (readyShard) { - return behavior.finalizeShard(readyShard); - } - - const restartedShard = await this.tryStartShardForAcquisition( - hostGroup, - this.pickRestartableShard(hostGroup), - defaultStartTimeoutMs, - maxWaitMs, - deadlineMs, - options?.abortSignal, - behavior, - lastStartError - ); - if ("result" in restartedShard) { - return restartedShard.result; - } - lastStartError = restartedShard.error; - - if (hostGroup.shards.length < this.maxShardsPerHost) { - const createdShard = this.createShard(hostGroup); - const startedShard = await this.tryStartShardForAcquisition( - hostGroup, - createdShard, - defaultStartTimeoutMs, - maxWaitMs, - deadlineMs, - options?.abortSignal, - behavior, - lastStartError - ); - if ("result" in startedShard) { - return startedShard.result; - } - lastStartError = startedShard.error; - } - - const remainingMs = deadlineMs - Date.now(); - if (remainingMs <= 0) { - if (lastStartError) { - throw lastStartError; - } - throw new Error( - `SSH master pool for ${config.host} did not become available within ${maxWaitMs}ms` - ); - } - - const waitMs = this.getPoolWaitMs( - remainingMs, - this.getNextBackoffWaitMs(hostGroup), - behavior.preferPolling(hostGroup) - ); - options?.onWait?.(waitMs); - await this.sleep(waitMs, options?.abortSignal); - } - } - - private async tryStartShardForAcquisition( - hostGroup: HostGroup, - shard: MasterShard | undefined, - defaultStartTimeoutMs: number, - maxWaitMs: number, - deadlineMs: number, - abortSignal: AbortSignal | undefined, - behavior: ShardAcquisitionBehavior, - lastStartError: Error | undefined - ): Promise<{ result: T } | { error: Error | undefined }> { - if (!shard) { - return { error: lastStartError }; - } - - try { - await this.startShard( - hostGroup, - shard, - this.getStartupTimeoutMs(defaultStartTimeoutMs, maxWaitMs, deadlineMs), - abortSignal - ); - return { result: behavior.finalizeShard(shard) }; - } catch (error) { - if (abortSignal?.aborted) { - throw error; - } - return { - error: error instanceof Error ? error : new Error(getErrorMessage(error)), - }; - } - } - - private getStartupTimeoutMs( - defaultStartTimeoutMs: number, - maxWaitMs: number, - deadlineMs: number - ): number { - return maxWaitMs === 0 - ? defaultStartTimeoutMs - : Math.min(defaultStartTimeoutMs, Math.max(1, deadlineMs - Date.now())); - } - - private pickRestartableShard(group: HostGroup): MasterShard | undefined { - return group.shards.find((shard) => { - return ( - !isProcessAlive(shard.process) && - shard.startup == null && - (shard.health.backoffUntil == null || shard.health.backoffUntil.getTime() <= Date.now()) - ); - }); - } - - clearAll(): void { - for (const group of this.hostGroups.values()) { - for (const shard of group.shards) { - this.disposeShard(group, shard, { expected: true }); - } - } - this.hostGroups.clear(); - } - - private getOrCreateHostGroup(key: string, config: SSHConnectionConfig): HostGroup { - const existing = this.hostGroups.get(key); - if (existing) { - return existing; - } - - const group: HostGroup = { - config, - shards: [], - nextShardId: 0, - }; - this.hostGroups.set(key, group); - return group; - } - - private createShard(group: HostGroup): MasterShard { - const id = group.nextShardId++; - const shard: MasterShard = { - id, - shardId: `shard-${id}`, - controlPath: getShardedControlPath(group.config, id), - inflight: 0, - lastUsedAt: Date.now(), - ready: false, - stderr: "", - stopping: false, - health: createInitialHealth(), - }; - group.shards.push(shard); - return shard; - } - - private getPoolWaitMs( - remainingMs: number, - nextBackoffMs: number | undefined, - preferPolling = false - ): number { - return Math.min( - remainingMs, - preferPolling || nextBackoffMs == null ? this.startupPollIntervalMs : nextBackoffMs - ); - } - - private getReadyShards(group: HostGroup): MasterShard[] { - return group.shards.filter((shard) => { - const backoffUntilMs = shard.health.backoffUntil?.getTime(); - return ( - shard.ready && - isProcessAlive(shard.process) && - (backoffUntilMs == null || backoffUntilMs <= Date.now()) - ); - }); - } - - private pickReadyShard(group: HostGroup): MasterShard | undefined { - return this.getReadyShards(group).sort((left, right) => left.inflight - right.inflight)[0]; - } - - private pickAvailableShard(group: HostGroup): MasterShard | undefined { - return this.getReadyShards(group) - .filter((shard) => shard.inflight < this.maxSessionsPerShard) - .sort((left, right) => left.inflight - right.inflight)[0]; - } - - private reserveShard(shard: MasterShard): OpenSSHMasterLease { - clearTimeout(shard.idleTimer); - shard.idleTimer = undefined; - shard.inflight += 1; - shard.lastUsedAt = Date.now(); - - let released = false; - const release = () => { - if (released) { - return; - } - released = true; - shard.inflight = Math.max(0, shard.inflight - 1); - shard.lastUsedAt = Date.now(); - if (shard.inflight === 0) { - this.scheduleIdleDisposal(shard); - } - }; - - return { - controlPath: shard.controlPath, - shardId: shard.shardId, - release, - markHealthy: () => { - shard.health = { - status: "healthy", - consecutiveFailures: 0, - lastSuccess: new Date(), - }; - }, - reportFailure: (error: string) => { - this.recordShardFailure(shard, error); - }, - }; - } - - private scheduleIdleDisposalIfUnused(shard: MasterShard): void { - if (shard.inflight === 0) { - this.scheduleIdleDisposal(shard); - } - } - - private scheduleIdleDisposal(shard: MasterShard): void { - clearTimeout(shard.idleTimer); - shard.idleTimer = setTimeout(() => { - const hostGroup = this.findHostGroupForShard(shard); - if (!hostGroup) { - return; - } - if (shard.inflight !== 0) { - return; - } - this.disposeShard(hostGroup, shard, { expected: true }); - }, this.shardIdleTtlMs); - shard.idleTimer.unref?.(); - } - - private findHostGroupForShard(target: MasterShard): HostGroup | undefined { - for (const group of this.hostGroups.values()) { - if (group.shards.includes(target)) { - return group; - } - } - return undefined; - } - - private async startShard( - group: HostGroup, - shard: MasterShard, - timeoutMs: number, - abortSignal?: AbortSignal - ): Promise { - if (shard.startup) { - return shard.startup; - } - - shard.startup = this.startShardInner(group, shard, timeoutMs, abortSignal).finally(() => { - shard.startup = undefined; - }); - return shard.startup; - } - - private async startShardInner( - group: HostGroup, - shard: MasterShard, - timeoutMs: number, - abortSignal?: AbortSignal - ): Promise { - const canPromptInteractively = isInteractiveHostKeyApprovalAvailable(); - const promptService = getSshPromptService(); - let stderr = ""; - let scheduleKill = (_ms: number) => undefined; - const extendDeadline = (ms: number) => scheduleKill(ms); - - const askpass = - canPromptInteractively && promptService - ? await createMediatedAskpassSession({ - sshPromptService: promptService, - promptPolicy: { - allowHostKey: true, - allowCredential: false, - }, - dedupeKey: formatSshEndpoint(group.config.host, group.config.port ?? 22), - getStderrContext: () => stderr, - onHostKeyPromptStarted: () => { - extendDeadline(HOST_KEY_APPROVAL_TIMEOUT_MS); - }, - }) - : undefined; - - const connectTimeout = canPromptInteractively - ? Math.ceil(HOST_KEY_APPROVAL_TIMEOUT_MS / 1000) - : Math.min(Math.ceil(timeoutMs / 1000), 15); - const args = this.buildMasterArgs(group.config, shard.controlPath, connectTimeout); - const proc = this.spawnProcess("ssh", args, { - stdio: ["ignore", "pipe", "pipe"], - windowsHide: true, - ...(askpass ? { env: { ...process.env, ...askpass.env } } : {}), - }); - - shard.process = proc; - shard.ready = false; - shard.stderr = ""; - shard.stopping = false; - - let shardFailureRecorded = false; - const recordShardFailureOnce = (error: string) => { - if (shardFailureRecorded) { - return; - } - shardFailureRecorded = true; - this.recordShardFailure(shard, error); - }; - - const markShardUnavailable = (error: string) => { - clearTimeout(shard.idleTimer); - shard.idleTimer = undefined; - shard.process = undefined; - shard.ready = false; - if (shard.stopping) { - shard.stopping = false; - return; - } - recordShardFailureOnce(error); - }; - - proc.stderr?.on("data", (data: Buffer) => { - const chunk = data.toString(); - stderr += chunk; - shard.stderr += chunk; - }); - - const onUnexpectedError = (error: Error) => { - const message = getErrorMessage(error); - stderr = stderr.length > 0 ? `${stderr}\n${message}` : message; - shard.stderr = stderr; - markShardUnavailable(message); - }; - proc.once("error", onUnexpectedError); - - const onUnexpectedExit = (code: number | null, signal: string | null) => { - markShardUnavailable( - stderr.trim() || `SSH master exited unexpectedly (${code ?? signal ?? "unknown"})` - ); - }; - proc.once("exit", onUnexpectedExit); - - let timer: ReturnType | undefined; - scheduleKill = (ms: number) => { - if (timer) { - clearTimeout(timer); - } - timer = setTimeout(() => { - this.disposeShard(group, shard, { expected: false }); - }, ms); - }; - - scheduleKill(timeoutMs); - - try { - await this.waitForMasterReady(group.config, shard, abortSignal); - shard.ready = true; - shard.health = { - status: "healthy", - consecutiveFailures: 0, - lastSuccess: new Date(), - }; - log.debug(`Started OpenSSH master ${shard.shardId} for ${group.config.host}`); - } catch (error) { - shard.ready = false; - recordShardFailureOnce(getErrorMessage(error)); - this.disposeShard(group, shard, { expected: true }); - throw error; - } finally { - if (timer) { - clearTimeout(timer); - } - askpass?.cleanup(); - } - } - - private async waitForMasterReady( - config: SSHConnectionConfig, - shard: MasterShard, - abortSignal?: AbortSignal - ): Promise { - while (true) { - if (abortSignal?.aborted) { - throw new Error("Operation aborted"); - } - if (!isProcessAlive(shard.process)) { - throw new Error(shard.stderr.trim() || "SSH master exited before becoming ready"); - } - - const ready = await this.checkMaster(config, shard.controlPath); - if (ready) { - return; - } - - await this.sleep(this.startupPollIntervalMs, abortSignal); - } - } - - private async checkMaster(config: SSHConnectionConfig, controlPath: string): Promise { - const args: string[] = ["-S", controlPath, "-O", "check"]; - if (config.port) { - args.push("-p", config.port.toString()); - } - if (config.identityFile) { - args.push("-i", config.identityFile); - } - args.push(config.host); - - return new Promise((resolve) => { - const proc = this.spawnProcess("ssh", args, { - stdio: ["ignore", "ignore", "ignore"], - windowsHide: true, - }); - proc.once("close", (code) => resolve(code === 0)); - proc.once("error", () => resolve(false)); - }); - } - - private buildMasterArgs( - config: SSHConnectionConfig, - controlPath: string, - connectTimeout: number - ): string[] { - const args: string[] = ["-M", "-N", "-T"]; - - if (config.port) { - args.push("-p", config.port.toString()); - } - if (config.identityFile) { - args.push("-i", config.identityFile); - } - - args.push("-o", "LogLevel=FATAL"); - args.push("-o", "ControlMaster=yes"); - args.push("-o", `ControlPath=${controlPath}`); - args.push("-o", "ControlPersist=no"); - args.push("-o", `ConnectTimeout=${connectTimeout}`); - args.push("-o", "ServerAliveInterval=5"); - args.push("-o", "ServerAliveCountMax=2"); - appendOpenSSHHostKeyPolicyArgs(args); - args.push(config.host); - - return args; - } - - private recordShardFailure(shard: MasterShard, error: string): void { - const failures = (shard.health.consecutiveFailures ?? 0) + 1; - const backoffIndex = Math.min(failures - 1, BACKOFF_SCHEDULE.length - 1); - const backoffSecs = withJitter(BACKOFF_SCHEDULE[backoffIndex]); - shard.health = { - status: "unhealthy", - lastFailure: new Date(), - lastError: error, - consecutiveFailures: failures, - backoffUntil: new Date(Date.now() + backoffSecs * 1000), - }; - log.warn( - `OpenSSH master ${shard.shardId} failed for ${shard.controlPath}: ${error} (backoff ${backoffSecs.toFixed(1)}s)` - ); - } - - private getNextBackoffWaitMs(group: HostGroup): number | undefined { - const waits = group.shards - .map((shard) => shard.health.backoffUntil?.getTime()) - .filter((value): value is number => value != null) - .map((until) => until - Date.now()) - .filter((value) => value > 0); - if (waits.length === 0) { - return undefined; - } - return Math.min(...waits); - } - - private trimExitedShards(group: HostGroup): void { - group.shards = group.shards.filter((shard) => { - if (isProcessAlive(shard.process) || shard.startup) { - return true; - } - if (shard.inflight > 0) { - return true; - } - if (shard.health.status === "unhealthy") { - return true; - } - const backoffUntil = shard.health.backoffUntil?.getTime(); - return backoffUntil != null && backoffUntil > Date.now(); - }); - } - - private disposeShard(group: HostGroup, shard: MasterShard, options: { expected: boolean }): void { - clearTimeout(shard.idleTimer); - shard.idleTimer = undefined; - shard.ready = false; - shard.stopping = options.expected; - - const masterProcess = shard.process; - if (masterProcess && isProcessAlive(masterProcess)) { - const args: string[] = ["-S", shard.controlPath, "-O", "exit"]; - if (group.config.port) { - args.push("-p", group.config.port.toString()); - } - if (group.config.identityFile) { - args.push("-i", group.config.identityFile); - } - args.push(group.config.host); - - const exitProc = this.spawnProcess("ssh", args, { - stdio: ["ignore", "ignore", "ignore"], - windowsHide: true, - }); - exitProc.once("error", () => { - try { - masterProcess.kill("SIGTERM"); - } catch { - // Ignore process teardown failures. - } - }); - const hardKill = setTimeout(() => { - try { - masterProcess.kill("SIGKILL"); - } catch { - // Ignore process teardown failures. - } - }, 1000); - hardKill.unref?.(); - masterProcess.once("exit", () => clearTimeout(hardKill)); - exitProc.once("close", (code) => { - if (code === 0) { - clearTimeout(hardKill); - return; - } - try { - masterProcess.kill("SIGTERM"); - } catch { - // Ignore process teardown failures. - } - }); - } - - shard.stderr = ""; - } -} - -export const openSshMasterPool = new OpenSSHMasterPool(); diff --git a/src/node/runtime/projectSyncCoordinator.test.ts b/src/node/runtime/projectSyncCoordinator.test.ts deleted file mode 100644 index 12c04ad60a..0000000000 --- a/src/node/runtime/projectSyncCoordinator.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { ProjectSyncCoordinator } from "./projectSyncCoordinator"; - -describe("ProjectSyncCoordinator", () => { - test("does not let one aborted caller cancel other waiters on the same snapshot", async () => { - const coordinator = new ProjectSyncCoordinator(); - const firstAbort = new AbortController(); - let releaseSharedWork: (() => void) | undefined; - let runCount = 0; - - const firstPromise = coordinator.runSnapshotSync( - { - projectKey: "project-a", - snapshotKey: "snapshot-a", - abortSignal: firstAbort.signal, - }, - async (sharedAbortSignal) => { - runCount += 1; - return await new Promise((resolve, reject) => { - releaseSharedWork = resolve; - sharedAbortSignal.addEventListener("abort", () => reject(new Error("shared aborted")), { - once: true, - }); - }); - } - ); - - const secondPromise = coordinator.runSnapshotSync( - { - projectKey: "project-a", - snapshotKey: "snapshot-a", - }, - () => Promise.reject(new Error("expected existing snapshot sync to be reused")) - ); - - firstAbort.abort(); - try { - await firstPromise; - throw new Error("Expected first waiter to abort"); - } catch (error) { - expect(error).toBeInstanceOf(Error); - expect((error as Error).message).toContain("Operation aborted"); - } - expect(runCount).toBe(1); - - releaseSharedWork?.(); - await secondPromise; - }); - - test("returns promptly when the last waiter aborts", async () => { - const coordinator = new ProjectSyncCoordinator(); - const abortController = new AbortController(); - - const resultPromise = coordinator.runSnapshotSync( - { - projectKey: "project-b", - snapshotKey: "snapshot-b", - abortSignal: abortController.signal, - }, - async (sharedAbortSignal) => { - return await new Promise((_resolve, reject) => { - sharedAbortSignal.addEventListener("abort", () => reject(new Error("shared aborted")), { - once: true, - }); - }); - } - ); - - abortController.abort(); - try { - await resultPromise; - throw new Error("Expected the caller to abort"); - } catch (error) { - expect(error).toBeInstanceOf(Error); - expect((error as Error).message).toContain("Operation aborted"); - } - }); -}); diff --git a/src/node/runtime/projectSyncCoordinator.ts b/src/node/runtime/projectSyncCoordinator.ts deleted file mode 100644 index c144fdb290..0000000000 --- a/src/node/runtime/projectSyncCoordinator.ts +++ /dev/null @@ -1,128 +0,0 @@ -interface ProjectSyncOptions { - projectKey: string; - snapshotKey: string; - abortSignal?: AbortSignal; -} - -interface InflightSnapshotSync { - promise: Promise; - abortController: AbortController; - waiters: number; -} - -export class ProjectSyncCoordinator { - private readonly inflightSyncs = new Map(); - private readonly projectTails = new Map>(); - - async runSnapshotSync( - options: ProjectSyncOptions, - fn: (abortSignal: AbortSignal) => Promise - ): Promise { - if (options.abortSignal?.aborted) { - throw new Error("Operation aborted"); - } - - let inflight = this.inflightSyncs.get(options.snapshotKey); - if (inflight?.abortController.signal.aborted && inflight.waiters === 0) { - this.inflightSyncs.delete(options.snapshotKey); - inflight = undefined; - } - - if (!inflight) { - const abortController = new AbortController(); - const promise = this.enqueueProjectMutation(options.projectKey, () => - fn(abortController.signal) - ); - inflight = { - promise, - abortController, - waiters: 0, - }; - this.inflightSyncs.set(options.snapshotKey, inflight); - void promise.then( - () => { - if (this.inflightSyncs.get(options.snapshotKey) === inflight) { - this.inflightSyncs.delete(options.snapshotKey); - } - }, - () => { - if (this.inflightSyncs.get(options.snapshotKey) === inflight) { - this.inflightSyncs.delete(options.snapshotKey); - } - } - ); - } - - inflight.waiters += 1; - let released = false; - const releaseWaiter = () => { - if (released) { - return; - } - released = true; - inflight.waiters = Math.max(0, inflight.waiters - 1); - if (inflight.waiters === 0 && !inflight.abortController.signal.aborted) { - inflight.abortController.abort(); - } - }; - - const callerAbortSignal = options.abortSignal; - let onAbort: (() => void) | undefined; - const callerAbort = callerAbortSignal - ? new Promise((_, reject) => { - onAbort = () => { - releaseWaiter(); - reject(new Error("Operation aborted")); - }; - if (callerAbortSignal.aborted) { - onAbort(); - return; - } - callerAbortSignal.addEventListener("abort", onAbort, { once: true }); - }) - : undefined; - - try { - if (callerAbort) { - await Promise.race([inflight.promise, callerAbort]); - } else { - await inflight.promise; - } - } finally { - if (onAbort) { - options.abortSignal?.removeEventListener("abort", onAbort); - } - releaseWaiter(); - } - } - - async enqueueProjectMutation(projectKey: string, fn: () => Promise): Promise { - const previous = this.projectTails.get(projectKey) ?? Promise.resolve(); - let releaseCurrent: (() => void) | undefined; - const current = new Promise((resolve) => { - releaseCurrent = resolve; - }); - const tail = previous.then( - () => current, - () => current - ); - this.projectTails.set(projectKey, tail); - - try { - await previous.catch(() => undefined); - await fn(); - } finally { - releaseCurrent?.(); - if (this.projectTails.get(projectKey) === tail) { - this.projectTails.delete(projectKey); - } - } - } - - clearAll(): void { - this.inflightSyncs.clear(); - this.projectTails.clear(); - } -} - -export const projectSyncCoordinator = new ProjectSyncCoordinator(); diff --git a/src/node/runtime/remoteProjectLayout.ts b/src/node/runtime/remoteProjectLayout.ts index 81811603e3..555072d75e 100644 --- a/src/node/runtime/remoteProjectLayout.ts +++ b/src/node/runtime/remoteProjectLayout.ts @@ -4,15 +4,13 @@ import { getProjectName } from "@/node/utils/runtime/helpers"; export const REMOTE_BASE_REPO_DIR = ".mux-base.git"; const REMOTE_METADATA_DIR = ".mux-meta"; -const REMOTE_WORKSPACE_METADATA_DIR = "workspaces"; -const REMOTE_SNAPSHOT_MARKER_DIR = "snapshots"; +const REMOTE_CURRENT_SNAPSHOT_FILE = "current-snapshot"; export interface RemoteProjectLayout { projectId: string; projectRoot: string; baseRepoPath: string; - workspaceMetadataDir: string; - snapshotMarkerDir: string; + currentSnapshotPath: string; } function sanitizeProjectSegment(segment: string): string { @@ -41,14 +39,16 @@ export function buildRemoteProjectLayout( ): RemoteProjectLayout { const projectId = createRemoteProjectId(projectPath); const projectRoot = projectRootOverride ?? path.posix.join(srcBaseDir, projectId); - const metadataRoot = path.posix.join(projectRoot, REMOTE_METADATA_DIR); return { projectId, projectRoot, baseRepoPath: path.posix.join(projectRoot, REMOTE_BASE_REPO_DIR), - workspaceMetadataDir: path.posix.join(metadataRoot, REMOTE_WORKSPACE_METADATA_DIR), - snapshotMarkerDir: path.posix.join(metadataRoot, REMOTE_SNAPSHOT_MARKER_DIR), + currentSnapshotPath: path.posix.join( + projectRoot, + REMOTE_METADATA_DIR, + REMOTE_CURRENT_SNAPSHOT_FILE + ), }; } @@ -56,21 +56,13 @@ export function buildLegacyRemoteProjectLayout( srcBaseDir: string, projectPath: string ): RemoteProjectLayout { - const legacyRoot = path.posix.join(srcBaseDir, getProjectName(projectPath)); - return buildRemoteProjectLayout(srcBaseDir, projectPath, legacyRoot); + return buildRemoteProjectLayout( + srcBaseDir, + projectPath, + path.posix.join(srcBaseDir, getProjectName(projectPath)) + ); } export function getRemoteWorkspacePath(layout: RemoteProjectLayout, workspaceName: string): string { return path.posix.join(layout.projectRoot, workspaceName); } - -export function getWorkspaceMetadataPath( - layout: RemoteProjectLayout, - workspaceName: string -): string { - return path.posix.join(layout.workspaceMetadataDir, `${hashText(workspaceName)}.json`); -} - -export function getSnapshotMarkerPath(layout: RemoteProjectLayout, snapshotDigest: string): string { - return path.posix.join(layout.snapshotMarkerDir, `${snapshotDigest}.json`); -} diff --git a/src/node/runtime/runtimeHelpers.test.ts b/src/node/runtime/runtimeHelpers.test.ts index 0f0f4a6a74..74963d2629 100644 --- a/src/node/runtime/runtimeHelpers.test.ts +++ b/src/node/runtime/runtimeHelpers.test.ts @@ -37,8 +37,8 @@ describe("createRuntimeForWorkspace", () => { }; const runtime = createRuntimeForWorkspace(metadata); - expect(runtime.getWorkspacePath(metadata.projectPath, "review-2")).toBe( - "/remote/src/demo/review-2" + expect(runtime.getWorkspacePath(metadata.projectPath, "review-2")).toMatch( + /^\/remote\/src\/demo-[a-f0-9]{12}\/review-2$/ ); }); }); @@ -120,8 +120,8 @@ describe("createRuntimeContextForWorkspace", () => { const context = createRuntimeContextForWorkspace(metadata); expect(context.workspacePath).toBe("/remote/src/demo/review-1"); - expect(context.runtime.getWorkspacePath(metadata.projectPath, "review-2")).toBe( - "/remote/src/demo/review-2" + expect(context.runtime.getWorkspacePath(metadata.projectPath, "review-2")).toMatch( + /^\/remote\/src\/demo-[a-f0-9]{12}\/review-2$/ ); }); }); diff --git a/src/node/runtime/sshConnectionPool.ts b/src/node/runtime/sshConnectionPool.ts index a9dd05b4b6..f63ceecae2 100644 --- a/src/node/runtime/sshConnectionPool.ts +++ b/src/node/runtime/sshConnectionPool.ts @@ -138,6 +138,12 @@ export interface AcquireConnectionOptions { */ onWait?: (waitMs: number) => void; + /** + * Optional explicit ControlPath to probe/bootstrap before returning. When omitted, + * the default host-scoped ControlPath is used. + */ + controlPath?: string; + /** * Test seam. * @@ -183,6 +189,7 @@ async function sleepWithAbort(ms: number, abortSignal?: AbortSignal): Promise(); + private readyControlPaths = new Map>(); private inflight = new Map>(); /** @@ -214,6 +221,7 @@ export class SSHConnectionPool { const shouldWait = maxWaitMs > 0; const key = makeConnectionKey(config); + const requestedControlPath = options.controlPath ?? getControlPath(config); const startTime = Date.now(); while (true) { @@ -253,13 +261,21 @@ export class SSHConnectionPool { // Return immediately if known healthy and not stale. if (health?.status === "healthy") { const age = Date.now() - (health.lastSuccess?.getTime() ?? 0); - if (age < HEALTHY_TTL_MS) { + const specificMasterReady = + options.controlPath == null ? true : this.isControlPathReady(key, requestedControlPath); + if (age < HEALTHY_TTL_MS && specificMasterReady) { log.debug(`SSH connection to ${config.host} is known healthy, skipping probe`); return; } - log.debug( - `SSH connection to ${config.host} health is stale (${Math.round(age / 1000)}s), re-probing` - ); + if (!specificMasterReady) { + log.debug( + `SSH connection to ${config.host} is healthy, but ControlPath ${requestedControlPath} is not ready; bootstrapping it now` + ); + } else { + log.debug( + `SSH connection to ${config.host} health is stale (${Math.round(age / 1000)}s), re-probing` + ); + } } // Check for inflight probe - singleflighting. @@ -280,7 +296,7 @@ export class SSHConnectionPool { // Start new probe. log.debug(`SSH connection to ${config.host} needs probe, starting health check`); - const probe = this.probeConnection(config, timeoutMs, key); + const probe = this.probeConnection(config, timeoutMs, key, requestedControlPath); this.inflight.set(key, probe); try { @@ -304,6 +320,20 @@ export class SSHConnectionPool { } } + private isControlPathReady(key: string, controlPath: string): boolean { + return this.readyControlPaths.get(key)?.has(controlPath) === true; + } + + private markControlPathReady(key: string, controlPath: string): void { + const readyPaths = this.readyControlPaths.get(key) ?? new Set(); + readyPaths.add(controlPath); + this.readyControlPaths.set(key, readyPaths); + } + + private clearReadyControlPaths(key: string): void { + this.readyControlPaths.delete(key); + } + /** * Get current health status for a connection */ @@ -329,6 +359,7 @@ export class SSHConnectionPool { health.backoffUntil = undefined; health.consecutiveFailures = 0; health.status = "unknown"; + this.clearReadyControlPaths(key); log.info(`Reset backoff for SSH connection to ${config.host}`); } } @@ -372,6 +403,7 @@ export class SSHConnectionPool { const backoffIndex = Math.min(failures - 1, BACKOFF_SCHEDULE.length - 1); const backoffSecs = withJitter(BACKOFF_SCHEDULE[backoffIndex]); + this.clearReadyControlPaths(key); this.health.set(key, { status: "unhealthy", lastFailure: new Date(), @@ -391,6 +423,7 @@ export class SSHConnectionPool { */ clearAllHealth(): void { this.health.clear(); + this.readyControlPaths.clear(); this.inflight.clear(); } @@ -400,9 +433,9 @@ export class SSHConnectionPool { private async probeConnection( config: SSHConnectionConfig, timeoutMs: number, - key: string + key: string, + controlPath = getControlPath(config) ): Promise { - const controlPath = getControlPath(config); const promptService = sshPromptService; const canPromptInteractively = isInteractiveHostKeyApprovalAvailable(); @@ -508,6 +541,7 @@ export class SSHConnectionPool { if (code === 0) { this.markHealthyByKey(key); + this.markControlPathReady(key, controlPath); log.debug(`SSH probe to ${config.host} succeeded`); resolve(); } else { diff --git a/src/node/runtime/transports/OpenSSHTransport.test.ts b/src/node/runtime/transports/OpenSSHTransport.test.ts deleted file mode 100644 index 6d0060ecc5..0000000000 --- a/src/node/runtime/transports/OpenSSHTransport.test.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; -import * as childProcess from "child_process"; - -import * as ptySpawn from "../ptySpawn"; -import { SshPromptService } from "@/node/services/sshPromptService"; -import { setSshPromptService, setOpenSSHHostKeyPolicyMode } from "../sshConnectionPool"; -import { openSshMasterPool } from "../openSshMasterPool"; -import { OpenSSHTransport } from "./OpenSSHTransport"; - -function createMockChildProcess(): ReturnType { - return { - on: mock(() => undefined), - pid: 12345, - } as unknown as ReturnType; -} - -describe("OpenSSHTransport.spawnRemoteProcess", () => { - let spawnSpy: ReturnType>; - let acquireLeaseSpy: ReturnType>; - let releaseInteractiveResponder: (() => void) | undefined; - - beforeEach(() => { - spawnSpy = spyOn(childProcess, "spawn").mockImplementation((() => - createMockChildProcess()) as unknown as typeof childProcess.spawn); - acquireLeaseSpy = spyOn(openSshMasterPool, "acquireLease").mockResolvedValue({ - controlPath: "/tmp/mux-ssh-test-shard", - shardId: "shard-0", - release: mock(() => undefined), - reportFailure: mock(() => undefined), - markHealthy: mock(() => undefined), - }); - }); - - afterEach(() => { - releaseInteractiveResponder?.(); - releaseInteractiveResponder = undefined; - // Reset to a configured service without responders so state does not leak across tests. - setSshPromptService(new SshPromptService()); - setOpenSSHHostKeyPolicyMode("headless-fallback"); - - spawnSpy.mockRestore(); - acquireLeaseSpy.mockRestore(); - }); - - function setSshPromptCapability(configured: boolean): SshPromptService | undefined { - releaseInteractiveResponder?.(); - releaseInteractiveResponder = undefined; - - if (!configured) { - setSshPromptService(undefined); - return undefined; - } - - const service = new SshPromptService(); - setSshPromptService(service); - return service; - } - - async function runSpawnRemoteProcess(): Promise { - const transport = new OpenSSHTransport({ host: "remote.example.com" }); - await transport.spawnRemoteProcess("echo ok", {}); - - expect(spawnSpy).toHaveBeenCalledTimes(1); - const [command, args] = spawnSpy.mock.calls[0] as [string, string[], childProcess.SpawnOptions]; - expect(command).toBe("ssh"); - return args; - } - - test("releases exec leases on close even when health accounting is skipped", async () => { - const release = mock(() => undefined); - const reportFailure = mock(() => undefined); - const markHealthy = mock(() => undefined); - acquireLeaseSpy.mockResolvedValue({ - controlPath: "/tmp/mux-ssh-test-shard", - shardId: "shard-0", - release, - reportFailure, - markHealthy, - }); - const transport = new OpenSSHTransport({ host: "remote.example.com" }); - const spawnResult = await transport.spawnRemoteProcess("echo ok", {}); - - spawnResult.onClose?.(); - spawnResult.onClose?.(); - - expect(release).toHaveBeenCalledTimes(1); - expect(reportFailure).not.toHaveBeenCalled(); - expect(markHealthy).not.toHaveBeenCalled(); - }); - - test("truncates connection-failure stderr before reporting shard health", async () => { - const release = mock(() => undefined); - let reportedError: string | undefined; - const reportFailure = mock((error: string) => { - reportedError = error; - }); - const markHealthy = mock(() => undefined); - acquireLeaseSpy.mockResolvedValue({ - controlPath: "/tmp/mux-ssh-test-shard", - shardId: "shard-0", - release, - reportFailure, - markHealthy, - }); - const transport = new OpenSSHTransport({ host: "remote.example.com" }); - const spawnResult = await transport.spawnRemoteProcess("echo ok", {}); - const longStderr = `${"x".repeat(1100)}\n`; - - spawnResult.onExit?.(255, longStderr); - - if (reportedError == null) { - throw new Error("Expected reportFailure to be called with a string error summary"); - } - expect(reportedError.length).toBeLessThan(longStderr.trim().length); - expect(reportedError.endsWith("…")).toBe(true); - expect(markHealthy).not.toHaveBeenCalled(); - }); - - test("explicit headless (no service) includes host-key fallback options and BatchMode=yes", async () => { - setSshPromptCapability(false); - setOpenSSHHostKeyPolicyMode("headless-fallback"); - - const args = await runSpawnRemoteProcess(); - - expect(args).toContain("BatchMode=yes"); - expect(args).toContain("StrictHostKeyChecking=no"); - expect(args).toContain("UserKnownHostsFile=/dev/null"); - expect(args.indexOf("StrictHostKeyChecking=no")).toBeGreaterThan(args.indexOf("BatchMode=yes")); - expect(args.indexOf("UserKnownHostsFile=/dev/null")).toBeGreaterThan( - args.indexOf("StrictHostKeyChecking=no") - ); - }); - - test("service configured keeps BatchMode=yes but excludes host-key fallback options", async () => { - const service = setSshPromptCapability(true); - releaseInteractiveResponder = service?.registerInteractiveResponder(); - setOpenSSHHostKeyPolicyMode("strict"); - - const args = await runSpawnRemoteProcess(); - - expect(args).toContain("BatchMode=yes"); - expect(args).not.toContain("StrictHostKeyChecking=no"); - expect(args).not.toContain("UserKnownHostsFile=/dev/null"); - }); - - test("service configured without active responder still excludes host-key fallback options", async () => { - setSshPromptCapability(true); - setOpenSSHHostKeyPolicyMode("strict"); - - const args = await runSpawnRemoteProcess(); - - expect(args).toContain("BatchMode=yes"); - expect(args).not.toContain("StrictHostKeyChecking=no"); - expect(args).not.toContain("UserKnownHostsFile=/dev/null"); - }); -}); - -describe("OpenSSHTransport.createPtySession", () => { - let ensureReadyMasterSpy: ReturnType>; - let spawnPtyProcessSpy: ReturnType>; - - beforeEach(() => { - ensureReadyMasterSpy = spyOn(openSshMasterPool, "ensureReadyMaster").mockResolvedValue( - undefined - ); - spawnPtyProcessSpy = spyOn(ptySpawn, "spawnPtyProcess").mockReturnValue({ - write: mock(() => undefined), - resize: mock(() => undefined), - kill: mock(() => undefined), - onData: mock(() => ({ dispose: () => undefined })), - onExit: mock(() => ({ dispose: () => undefined })), - } as unknown as ReturnType); - }); - - afterEach(() => { - ensureReadyMasterSpy.mockRestore(); - spawnPtyProcessSpy.mockRestore(); - }); - - test("preflights against a ready master without reserving an exec slot", async () => { - const transport = new OpenSSHTransport({ host: "remote.example.com", port: 2222 }); - - await transport.createPtySession({ - workspacePath: "~/workspace", - cols: 80, - rows: 24, - }); - - expect(ensureReadyMasterSpy).toHaveBeenCalledWith( - { host: "remote.example.com", port: 2222 }, - { maxWaitMs: 0 } - ); - }); -}); diff --git a/src/node/runtime/transports/OpenSSHTransport.ts b/src/node/runtime/transports/OpenSSHTransport.ts index df8cfadd5d..43990d52b6 100644 --- a/src/node/runtime/transports/OpenSSHTransport.ts +++ b/src/node/runtime/transports/OpenSSHTransport.ts @@ -1,10 +1,12 @@ import { spawn } from "child_process"; -import { log } from "@/node/services/log"; import { spawnPtyProcess } from "../ptySpawn"; import { expandTildeForSSH } from "../tildeExpansion"; -import { appendOpenSSHHostKeyPolicyArgs, type SSHConnectionConfig } from "../sshConnectionPool"; -import { openSshMasterPool } from "../openSshMasterPool"; +import { + appendOpenSSHHostKeyPolicyArgs, + sshConnectionPool, + type SSHConnectionConfig, +} from "../sshConnectionPool"; import type { SpawnResult } from "../RemoteRuntime"; import type { SSHTransport, @@ -15,6 +17,8 @@ import type { } from "./SSHTransport"; const MAX_REPORTED_FAILURE_STDERR_CHARS = 1000; +const OPENSSH_EXEC_SHARD_COUNT = 4; +const nextShardByConnection = new Map(); function summarizeFailureStderr(stderr: string, exitCode: number): string { const trimmed = stderr.trim(); @@ -27,6 +31,13 @@ function summarizeFailureStderr(stderr: string, exitCode: number): string { return `${trimmed.slice(0, MAX_REPORTED_FAILURE_STDERR_CHARS)}…`; } +function getShardedControlPath(config: SSHConnectionConfig): string { + const baseControlPath = sshConnectionPool.getControlPath(config); + const nextShard = nextShardByConnection.get(baseControlPath) ?? 0; + nextShardByConnection.set(baseControlPath, (nextShard + 1) % OPENSSH_EXEC_SHARD_COUNT); + return `${baseControlPath}-${nextShard}`; +} + export class OpenSSHTransport implements SSHTransport { constructor(private readonly config: SSHConnectionConfig) {} @@ -44,7 +55,7 @@ export class OpenSSHTransport implements SSHTransport { maxWaitMs?: number; onWait?: (waitMs: number) => void; }): Promise { - await openSshMasterPool.ensureConnection(this.config, { + await sshConnectionPool.acquireConnection(this.config, { abortSignal: options?.abortSignal, timeoutMs: options?.timeoutMs, maxWaitMs: options?.maxWaitMs, @@ -55,20 +66,25 @@ export class OpenSSHTransport implements SSHTransport { async spawnRemoteProcess(fullCommand: string, options: SpawnOptions): Promise { const remainingWaitMs = options.deadlineMs != null ? Math.max(0, options.deadlineMs - Date.now()) : undefined; - const lease = await openSshMasterPool.acquireLease(this.config, { + const controlPath = getShardedControlPath(this.config); + await sshConnectionPool.acquireConnection(this.config, { abortSignal: options.abortSignal, timeoutMs: remainingWaitMs, maxWaitMs: remainingWaitMs, + controlPath, }); - // Note: use -tt (not -t) so PTY allocation works even when stdin is a pipe. + // Shard short-lived SSH execs across a few deterministic ControlPaths so the host no longer + // funnels all multiplexed sessions through one implicit master socket. const sshArgs: string[] = [ options.forcePTY ? "-tt" : "-T", ...this.buildBaseSSHArgs(), "-o", - "ControlMaster=no", + "ControlMaster=auto", + "-o", + `ControlPath=${controlPath}`, "-o", - `ControlPath=${lease.controlPath}`, + "ControlPersist=60", ]; const connectTimeout = @@ -80,52 +96,34 @@ export class OpenSSHTransport implements SSHTransport { appendOpenSSHHostKeyPolicyArgs(sshArgs); sshArgs.push(this.config.host, fullCommand); - log.debug(`SSH exec on ${this.config.host} via ${lease.shardId}`); const process = spawn("ssh", sshArgs, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true, }); - let released = false; - const releaseLease = () => { - if (released) { - return; - } - released = true; - lease.release(); - }; - return { process, onExit: (exitCode, stderr) => { if (this.isConnectionFailure(exitCode, stderr)) { - lease.reportFailure(summarizeFailureStderr(stderr, exitCode)); - } else { - lease.markHealthy(); + sshConnectionPool.reportFailure(this.config, summarizeFailureStderr(stderr, exitCode)); + return; } - }, - onClose: () => { - releaseLease(); + sshConnectionPool.markHealthy(this.config); }, onError: (error) => { - lease.reportFailure(error.message); - releaseLease(); + sshConnectionPool.reportFailure(this.config, error.message); }, }; } async createPtySession(params: PtySessionParams): Promise { - // PTYs stay on a dedicated direct SSH session so they do not consume pooled master - // capacity reserved for the many short exec/file operations that drive workspace scale. - // Preflight only needs an already-started master (or to bootstrap one), not a free exec slot. - await openSshMasterPool.ensureReadyMaster(this.config, { maxWaitMs: 0 }); + await this.acquireConnection({ maxWaitMs: 0 }); const args: string[] = [...this.buildBaseSSHArgs()]; + args.push("-o", "ControlMaster=no"); args.push("-o", "ConnectTimeout=15"); args.push("-o", "ServerAliveInterval=5"); args.push("-o", "ServerAliveCountMax=2"); - args.push("-o", "BatchMode=yes"); - appendOpenSSHHostKeyPolicyArgs(args); args.push("-t"); args.push(this.config.host); diff --git a/src/node/runtime/transports/SSH2Transport.ts b/src/node/runtime/transports/SSH2Transport.ts index e28e61dd12..7da6350acb 100644 --- a/src/node/runtime/transports/SSH2Transport.ts +++ b/src/node/runtime/transports/SSH2Transport.ts @@ -291,10 +291,8 @@ export class SSH2Transport implements SSHTransport { const process = new SSH2ChildProcess(channel) as unknown as ChildProcess; return { process, - onExit: (exitCode) => { - if (exitCode === 0) { - ssh2ConnectionPool.markHealthy(this.config); - } + onExit: () => { + ssh2ConnectionPool.markHealthy(this.config); }, onError: (error) => { ssh2ConnectionPool.reportFailure(this.config, getErrorMessage(error)); diff --git a/src/node/services/agentSession.postCompactionRetry.test.ts b/src/node/services/agentSession.postCompactionRetry.test.ts index 68d10d6c63..341ce38424 100644 --- a/src/node/services/agentSession.postCompactionRetry.test.ts +++ b/src/node/services/agentSession.postCompactionRetry.test.ts @@ -272,7 +272,9 @@ describe("AgentSession execSubagentHardRestart", () => { name: "child", projectName: "proj", projectPath: "/tmp/proj", - namedWorkspacePath: "/tmp/proj/child", + // Project-dir local runtimes execute in the project root, so persisted workspace paths + // must match projectPath instead of pointing at a synthetic sibling checkout. + namedWorkspacePath: "/tmp/proj", runtimeConfig: { type: "local" }, parentWorkspaceId, agentId: "exec", diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index da03831e96..1ecbfc95ef 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -27,6 +27,7 @@ import { createRuntimeContextForWorkspace, resolveWorkspaceExecutionPath, } from "@/node/runtime/runtimeHelpers"; +import { getWorkspacePathHintForProject } from "@/node/services/workspaceProjectRepos"; import { MultiProjectRuntime } from "@/node/runtime/multiProjectRuntime"; import { getMuxEnv, getRuntimeType } from "@/node/runtime/initHook"; import { getSrcBaseDir, isSSHRuntime } from "@/common/types/runtime"; @@ -935,9 +936,9 @@ export class AIService extends EventEmitter { const metadataWithPath = { ...metadata, - // Existing workspaces may still live under the legacy SSH basename layout until the first - // post-upgrade operation reuses their persisted root, so stream startup must seed the runtime - // from config instead of reconstructing a hashed default path before layout detection runs. + // Existing SSH workspaces may still live at a persisted root that differs from the canonical + // hashed project layout, so stream startup seeds the runtime from config for the current + // workspace instead of always reconstructing the path from project metadata. namedWorkspacePath: workspace.workspacePath, }; @@ -962,6 +963,20 @@ export class AIService extends EventEmitter { runtime: createRuntime(metadata.runtimeConfig, { projectPath: project.projectPath, workspaceName: metadata.name, + workspacePath: isSSHRuntime(metadata.runtimeConfig) + ? getWorkspacePathHintForProject( + { + workspaceId, + workspaceName: metadata.name, + workspacePath: workspace.workspacePath, + runtimeConfig: metadata.runtimeConfig, + projectPath: metadata.projectPath, + projectName: metadata.projectName, + projects: metadata.projects, + }, + project.projectPath + ) + : undefined, }), })), metadata.name diff --git a/src/node/services/workspaceProjectRepos.test.ts b/src/node/services/workspaceProjectRepos.test.ts index 5e5e9f5afe..31d8755472 100644 --- a/src/node/services/workspaceProjectRepos.test.ts +++ b/src/node/services/workspaceProjectRepos.test.ts @@ -2,9 +2,13 @@ import { describe, expect, it } from "bun:test"; import { buildLegacyRemoteProjectLayout, + buildRemoteProjectLayout, getRemoteWorkspacePath, } from "@/node/runtime/remoteProjectLayout"; -import { getWorkspaceProjectRepos } from "@/node/services/workspaceProjectRepos"; +import { + getWorkspacePathHintForProject, + getWorkspaceProjectRepos, +} from "@/node/services/workspaceProjectRepos"; describe("getWorkspaceProjectRepos", () => { it("treats an empty project list as a single-project fallback", () => { @@ -42,28 +46,35 @@ describe("getWorkspaceProjectRepos", () => { expect(repos[0]?.storageKey).toBe("..-..-secrets"); }); - it("reuses the persisted workspace path for the current SSH project in multi-project views", () => { + it("reuses the persisted workspace path for the current SSH project when it matches a known project layout", () => { + const runtimeConfig = { + type: "ssh", + host: "example.com", + srcBaseDir: "/tmp/src", + } as const; + const primaryProjectPath = "/tmp/projects/main"; + const workspaceName = "main"; + const workspacePath = getRemoteWorkspacePath( + buildLegacyRemoteProjectLayout(runtimeConfig.srcBaseDir, primaryProjectPath), + workspaceName + ); const repos = getWorkspaceProjectRepos({ workspaceId: "workspace-1", - workspaceName: "main", - workspacePath: "/tmp/legacy/main", - runtimeConfig: { - type: "ssh", - host: "example.com", - srcBaseDir: "/tmp/src", - }, - projectPath: "/tmp/projects/main", + workspaceName, + workspacePath, + runtimeConfig, + projectPath: primaryProjectPath, projectName: "main", projects: [ - { projectPath: "/tmp/projects/main", projectName: "main" }, + { projectPath: primaryProjectPath, projectName: "main" }, { projectPath: "/tmp/projects/other", projectName: "other" }, ], }); - expect(repos[0]?.repoCwd).toBe("/tmp/legacy/main"); + expect(repos[0]?.repoCwd).toBe(workspacePath); }); - it("derives persisted legacy SSH paths for secondary multi-project repos", () => { + it("derives hashed SSH paths for secondary multi-project repos", () => { const runtimeConfig = { type: "ssh", host: "example.com", @@ -75,10 +86,7 @@ describe("getWorkspaceProjectRepos", () => { const repos = getWorkspaceProjectRepos({ workspaceId: "workspace-1", workspaceName, - workspacePath: getRemoteWorkspacePath( - buildLegacyRemoteProjectLayout(runtimeConfig.srcBaseDir, primaryProjectPath), - workspaceName - ), + workspacePath: "/tmp/legacy/main", runtimeConfig, projectPath: primaryProjectPath, projectName: "main", @@ -89,6 +97,43 @@ describe("getWorkspaceProjectRepos", () => { }); expect(repos[1]?.repoCwd).toBe( + getRemoteWorkspacePath( + buildRemoteProjectLayout(runtimeConfig.srcBaseDir, secondaryProjectPath), + workspaceName + ) + ); + }); + + it("derives legacy SSH path hints for sibling multi-project repos when the persisted root is legacy-shaped", () => { + const runtimeConfig = { + type: "ssh", + host: "example.com", + srcBaseDir: "/tmp/src", + } as const; + const primaryProjectPath = "/tmp/projects/main"; + const secondaryProjectPath = "/tmp/projects/other"; + const workspaceName = "main"; + + const hint = getWorkspacePathHintForProject( + { + workspaceId: "workspace-1", + workspaceName, + workspacePath: getRemoteWorkspacePath( + buildLegacyRemoteProjectLayout(runtimeConfig.srcBaseDir, primaryProjectPath), + workspaceName + ), + runtimeConfig, + projectPath: primaryProjectPath, + projectName: "main", + projects: [ + { projectPath: primaryProjectPath, projectName: "main" }, + { projectPath: secondaryProjectPath, projectName: "other" }, + ], + }, + secondaryProjectPath + ); + + expect(hint).toBe( getRemoteWorkspacePath( buildLegacyRemoteProjectLayout(runtimeConfig.srcBaseDir, secondaryProjectPath), workspaceName @@ -96,6 +141,30 @@ describe("getWorkspaceProjectRepos", () => { ); }); + it("returns no SSH path hint when the persisted root is not a project checkout", () => { + const runtimeConfig = { + type: "ssh", + host: "example.com", + srcBaseDir: "/tmp/src", + } as const; + const primaryProjectPath = "/tmp/projects/main"; + + const hint = getWorkspacePathHintForProject( + { + workspaceId: "workspace-1", + workspaceName: "main", + workspacePath: "/tmp/src/containers/main", + runtimeConfig, + projectPath: primaryProjectPath, + projectName: "main", + projects: [{ projectPath: primaryProjectPath, projectName: "main" }], + }, + primaryProjectPath + ); + + expect(hint).toBeUndefined(); + }); + it("disambiguates storage keys when sanitized project names collide", () => { const repos = getWorkspaceProjectRepos({ workspaceId: "workspace-1", diff --git a/src/node/services/workspaceProjectRepos.ts b/src/node/services/workspaceProjectRepos.ts index 7bddfadf33..8a5677b42d 100644 --- a/src/node/services/workspaceProjectRepos.ts +++ b/src/node/services/workspaceProjectRepos.ts @@ -142,9 +142,6 @@ export function getWorkspacePathHintForProject( params: WorkspaceProjectRepoParams, targetProjectPath: string ): string | undefined { - if (targetProjectPath === params.projectPath) { - return params.workspacePath; - } if (!isSSHRuntime(params.runtimeConfig)) { return undefined; } @@ -203,13 +200,17 @@ export function getWorkspaceProjectRepos( const isMultiProject = projectStorageKeys.length > 1; const repos = projectStorageKeys.map((project) => { - const repoCwd = isMultiProject - ? createRuntime(params.runtimeConfig, { + const sshWorkspacePathHint = isMultiProject + ? getWorkspacePathHintForProject(params, project.projectPath) + : undefined; + + const repoCwd = !isMultiProject + ? params.workspacePath + : (sshWorkspacePathHint ?? + createRuntime(params.runtimeConfig, { projectPath: project.projectPath, workspaceName: params.workspaceName, - workspacePath: getWorkspacePathHintForProject(params, project.projectPath), - }).getWorkspacePath(project.projectPath, params.workspaceName) - : params.workspacePath; + }).getWorkspacePath(project.projectPath, params.workspaceName)); assert( repoCwd.trim().length > 0, diff --git a/src/node/services/workspaceService.multiProject.test.ts b/src/node/services/workspaceService.multiProject.test.ts index 476d511c8b..665e5805cc 100644 --- a/src/node/services/workspaceService.multiProject.test.ts +++ b/src/node/services/workspaceService.multiProject.test.ts @@ -181,10 +181,12 @@ describe("WorkspaceService executeBash runtime selection", () => { expect(createRuntimeSpy).toHaveBeenNthCalledWith(1, metadata.runtimeConfig, { projectPath: projectAPath, workspaceName, + workspacePath: undefined, }); expect(createRuntimeSpy).toHaveBeenNthCalledWith(2, metadata.runtimeConfig, { projectPath: projectBPath, workspaceName, + workspacePath: undefined, }); assert(capturedToolConfig); expect(capturedToolConfig.runtime).toBeInstanceOf(MultiProjectRuntime); @@ -201,7 +203,7 @@ describe("WorkspaceService executeBash runtime selection", () => { } }); - test("preserves inferred legacy SSH repo roots for multi-project repo-root bash mode", async () => { + test("preserves the current SSH repo root and derives sibling legacy repo roots for multi-project repo-root bash mode when the persisted root matches that layout", async () => { const workspaceId = "ws-multi-bash-ssh"; const workspaceName = "feature-multi-bash-ssh"; const srcDir = "/tmp/src"; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 22dce9e810..4871dd08ca 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -2240,7 +2240,7 @@ describe("WorkspaceService getFileCompletions", () => { expect(execBufferedSpy.mock.calls[0]?.[2].cwd).toBe("/persisted/project-a/ws"); }); - test("preserves inferred legacy SSH paths for multi-project completions", async () => { + test("preserves the current SSH workspace path and derives sibling legacy paths for multi-project completions when the persisted root matches that layout", async () => { interface WorkspaceServiceTestAccess { getInfo: (workspaceId: string) => Promise; } @@ -2268,7 +2268,7 @@ describe("WorkspaceService getFileCompletions", () => { createRuntimeSpy.mockImplementation((_runtimeConfig, options) => { const runtimeProjectPath = options?.projectPath; if (!runtimeProjectPath) { - throw new Error("Expected createRuntime projectPath in legacy SSH completion test"); + throw new Error("Expected createRuntime projectPath in SSH completion test"); } return { getWorkspacePath: () => diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6a17016b38..c06c5a395e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2832,6 +2832,8 @@ export class WorkspaceService extends EventEmitter { const metadata = metadataResult.data; const configSnapshot = this.config.loadConfigOrDefault(); + const persistedWorkspacePath = this.config.findWorkspace(workspaceId)?.workspacePath; + if (isMultiProject(metadata)) { const projects = getProjects(metadata); const deleteErrors: string[] = []; @@ -2846,6 +2848,20 @@ export class WorkspaceService extends EventEmitter { const runtime = createRuntime(metadata.runtimeConfig, { projectPath: project.projectPath, workspaceName: metadata.name, + workspacePath: persistedWorkspacePath + ? getWorkspacePathHintForProject( + { + workspaceId, + workspaceName: metadata.name, + workspacePath: persistedWorkspacePath, + runtimeConfig: metadata.runtimeConfig, + projectPath: metadata.projectPath, + projectName: metadata.projectName, + projects: metadata.projects, + }, + project.projectPath + ) + : undefined, }); const trusted = configSnapshot.projects.get(stripTrailingSlashes(project.projectPath))?.trusted ?? @@ -2966,6 +2982,7 @@ export class WorkspaceService extends EventEmitter { const runtime = createRuntime(metadata.runtimeConfig, { projectPath, workspaceName: metadata.name, + workspacePath: persistedWorkspacePath, }); // Delete workspace from runtime first - if this fails with force=false, we abort @@ -3461,6 +3478,7 @@ export class WorkspaceService extends EventEmitter { const rollbackRuntime = createRuntime(oldMetadata.runtimeConfig, { projectPath: renamedProject.projectPath, workspaceName: newName, + workspacePath: renamedProject.newWorkspacePath, }); const rollbackTrusted = configSnapshot.projects.get(stripTrailingSlashes(renamedProject.projectPath)) @@ -3494,6 +3512,18 @@ export class WorkspaceService extends EventEmitter { const runtime = createRuntime(oldMetadata.runtimeConfig, { projectPath: project.projectPath, workspaceName: oldName, + workspacePath: getWorkspacePathHintForProject( + { + workspaceId, + workspaceName: oldName, + workspacePath: workspace.workspacePath, + runtimeConfig: oldMetadata.runtimeConfig, + projectPath: oldMetadata.projectPath, + projectName: oldMetadata.projectName, + projects: oldMetadata.projects, + }, + project.projectPath + ), }); const trusted = @@ -3616,6 +3646,7 @@ export class WorkspaceService extends EventEmitter { const runtime = createRuntime(oldMetadata.runtimeConfig, { projectPath: configProjectPath, workspaceName: oldName, + workspacePath: workspace.workspacePath, }); const trusted = @@ -5011,9 +5042,11 @@ export class WorkspaceService extends EventEmitter { return Err(resolvedNameValidation.error ?? "Invalid workspace name"); } + const sourceWorkspace = this.config.findWorkspace(sourceWorkspaceId); const sourceRuntime = createRuntime(sourceRuntimeConfig, { projectPath: foundProjectPath, workspaceName: sourceMetadata.name, + workspacePath: sourceWorkspace?.workspacePath, }); const newWorkspaceId = this.config.generateStableId(); @@ -5169,7 +5202,6 @@ export class WorkspaceService extends EventEmitter { // Copy plan file using explicit source/target runtimes for cross-runtime safety. // Create a fresh source runtime handle because DockerRuntime.forkWorkspace() can // mutate the original runtime's container identity to target the new workspace. - const sourceWorkspace = this.config.findWorkspace(sourceWorkspaceId); const freshSourceRuntime = createRuntime(sourceRuntimeConfig, { projectPath: foundProjectPath, workspaceName: sourceMetadata.name, diff --git a/tests/runtime/runtime.test.ts b/tests/runtime/runtime.test.ts index 56fbd4714d..65fd0c6279 100644 --- a/tests/runtime/runtime.test.ts +++ b/tests/runtime/runtime.test.ts @@ -1598,7 +1598,7 @@ describeIntegration("Runtime integration tests", () => { } }, 60000); - test("renameWorkspace uses git worktree move for worktree-based workspaces", async () => { + test("renameWorkspace uses git worktree move and deleteWorkspace still cleans up the renamed branch", async () => { const runtime = createSSHRuntime(); const projectName = `wt-rename-${Date.now()}-${Math.random().toString(36).substring(7)}`; const projectPath = `/some/path/${projectName}`; @@ -1663,6 +1663,23 @@ describeIntegration("Runtime integration tests", () => { }); expect(worktreeList.stdout).toContain("/new-name"); expect(worktreeList.stdout).not.toContain("/old-name"); + + const deleteResult = await runtime.deleteWorkspace(projectPath, "new-name", true); + expect(deleteResult.success).toBe(true); + + const deletedPathCheck = await execBuffered( + runtime, + `test -d "${newWorkspacePath}" && echo "exists" || echo "missing"`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(deletedPathCheck.stdout.trim()).toBe("missing"); + + const branchCheck = await execBuffered( + runtime, + `git -C "${baseRepoPath}" branch --list old-name`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(branchCheck.stdout.trim()).toBe(""); } finally { await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", @@ -1670,6 +1687,63 @@ describeIntegration("Runtime integration tests", () => { }); } }, 60000); + + test("exec handles a concurrent burst on one SSH host", async () => { + const runtime = createSSHRuntime(); + const projectName = `ssh-burst-${Date.now()}-${Math.random().toString(36).substring(7)}`; + const tmpDir = await import("os").then((os) => os.tmpdir()); + const localProjectPath = `${tmpDir}/${projectName}`; + const layout = getLayout(localProjectPath); + const workspaceName = "burst-ws"; + const workspacePath = getRemoteWorkspacePath(layout, workspaceName); + const { execSync } = await import("child_process"); + + try { + execSync( + [ + `mkdir -p "${localProjectPath}"`, + `cd "${localProjectPath}"`, + `git init -b main`, + `git config user.email "test@test.com"`, + `git config user.name "Test"`, + `echo "content" > file.txt`, + `git add file.txt`, + `git commit -m "initial"`, + ].join(" && "), + { stdio: "pipe" } + ); + + const initResult = await runtime.initWorkspace({ + projectPath: localProjectPath, + branchName: workspaceName, + trunkBranch: "main", + workspacePath, + initLogger: noopInitLogger, + }); + if (!initResult.success) { + throw new Error(`initWorkspace failed: ${initResult.error}`); + } + + const results = await Promise.all( + Array.from({ length: 12 }, async (_value, index) => { + const result = await execBuffered( + runtime, + `printf '%s' ${JSON.stringify(String(index))} > burst-${index}.txt && cat burst-${index}.txt`, + { cwd: workspacePath, timeout: 30 } + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe(String(index)); + }) + ); + expect(results).toHaveLength(12); + } finally { + execSync(`rm -rf "${localProjectPath}"`); + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { + cwd: "/home/testuser", + timeout: 30, + }); + } + }, 120000); }); /** @@ -1688,14 +1762,13 @@ describeIntegration("Runtime integration tests", () => { test("initWorkspace does not populate refs/remotes/origin in the base repo from the bundle", async () => { const runtime = createSSHRuntime(); - // projectName must match the basename of the local project path so that - // getBaseRepoPath(localProjectPath) resolves to the expected baseRepoPath. const projectName = `sync-no-remotes-${Date.now()}-${Math.random().toString(36).substring(7)}`; const tmpDir = await import("os").then((os) => os.tmpdir()); const localProjectPath = `${tmpDir}/${projectName}`; + const layout = buildRemoteProjectLayout(srcBaseDir, localProjectPath); const branchName = "test-ws"; - const workspacePath = `${srcBaseDir}/${projectName}/${branchName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; + const workspacePath = getRemoteWorkspacePath(layout, branchName); + const baseRepoPath = layout.baseRepoPath; const { execSync } = await import("child_process"); try { @@ -1757,7 +1830,7 @@ describeIntegration("Runtime integration tests", () => { expect(baseRefs.stdout).not.toContain("refs/remotes/origin/main"); expect(baseRefs.stdout).not.toContain("refs/remotes/origin/stale-branch"); } finally { - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1773,9 +1846,10 @@ describeIntegration("Runtime integration tests", () => { const projectName = `sync-heal-bare-${Date.now()}-${Math.random().toString(36).substring(7)}`; const tmpDir = await import("os").then((os) => os.tmpdir()); const localProjectPath = `${tmpDir}/${projectName}`; + const layout = buildRemoteProjectLayout(srcBaseDir, localProjectPath); const branchName = "worktree-heal"; - const workspacePath = `${srcBaseDir}/${projectName}/${branchName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; + const workspacePath = getRemoteWorkspacePath(layout, branchName); + const baseRepoPath = layout.baseRepoPath; const { execSync } = await import("child_process"); try { @@ -1795,7 +1869,7 @@ describeIntegration("Runtime integration tests", () => { await execBuffered( runtime, - `mkdir -p "${srcBaseDir}/${projectName}" && git init --bare "${baseRepoPath}"`, + `mkdir -p "${layout.projectRoot}" && git init --bare "${baseRepoPath}"`, { cwd: "/home/testuser", timeout: 30 } ); @@ -1839,7 +1913,7 @@ describeIntegration("Runtime integration tests", () => { expect(workspaceCoreBareCheck.exitCode).toBe(1); } finally { execSync(`rm -rf "${localProjectPath}"`); - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); @@ -1888,7 +1962,7 @@ describeIntegration("Runtime integration tests", () => { const snapshotMarkerCheck = await execBuffered( runtime, - `find "${layout.snapshotMarkerDir}" -type f | head -n 1`, + `test -f "${layout.currentSnapshotPath}" && cat "${layout.currentSnapshotPath}"`, { cwd: "/home/testuser", timeout: 30 } ); expect(snapshotMarkerCheck.stdout.trim()).not.toBe(""); @@ -2052,9 +2126,10 @@ describeIntegration("Runtime integration tests", () => { const localProjectPath = `${tmpDir}/${projectName}`; const wsAName = "ws-a"; const wsBName = "ws-b"; - const wsAPath = `${srcBaseDir}/${projectName}/${wsAName}`; - const wsBPath = `${srcBaseDir}/${projectName}/${wsBName}`; - const baseRepoPath = `${srcBaseDir}/${projectName}/.mux-base.git`; + const layout = buildRemoteProjectLayout(srcBaseDir, localProjectPath); + const wsAPath = getRemoteWorkspacePath(layout, wsAName); + const wsBPath = getRemoteWorkspacePath(layout, wsBName); + const baseRepoPath = layout.baseRepoPath; const { execSync } = await import("child_process"); @@ -2158,7 +2233,7 @@ describeIntegration("Runtime integration tests", () => { expect(worktreeList.stdout).toContain(wsBName); } finally { execSync(`rm -rf "${localProjectPath}"`); - await execBuffered(runtime, `rm -rf "${srcBaseDir}/${projectName}"`, { + await execBuffered(runtime, `rm -rf "${layout.projectRoot}"`, { cwd: "/home/testuser", timeout: 30, }); From ed8fe808e5e1e05476955061ffe0f4eac2eaaf3f Mon Sep 17 00:00:00 2001 From: Ammar Date: Mon, 6 Apr 2026 13:47:31 -0500 Subject: [PATCH 14/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20recheck=20SSH=20sha?= =?UTF-8?q?rd=20readiness=20after=20inflight=20probes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loop after waiting on another host probe so callers re-validate the requested sharded ControlPath before returning, and cover the regression with a targeted singleflight test. --- _Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$269.00`_ --- src/node/runtime/sshConnectionPool.test.ts | 62 ++++++++++++++++++++++ src/node/runtime/sshConnectionPool.ts | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/node/runtime/sshConnectionPool.test.ts b/src/node/runtime/sshConnectionPool.test.ts index 3ae59fdcbd..ecbaa5238c 100644 --- a/src/node/runtime/sshConnectionPool.test.ts +++ b/src/node/runtime/sshConnectionPool.test.ts @@ -457,6 +457,68 @@ describe("SSHConnectionPool", () => { expect(pool.getConnectionHealth(config)?.consecutiveFailures).toBe(1); }); + test("re-checks requested ControlPath after waiting for another probe", async () => { + const pool = new SSHConnectionPool(); + const config: SSHRuntimeConfig = { + host: "test.example.com", + srcBaseDir: "/work", + }; + const privatePool = pool as unknown as { + probeConnection: ( + config: SSHRuntimeConfig, + timeoutMs: number, + key: string, + controlPath: string + ) => Promise; + markHealthyByKey: (key: string) => void; + markControlPathReady: (key: string, controlPath: string) => void; + }; + + const probeCalls: string[] = []; + let releaseFirstProbe!: () => void; + let signalFirstProbeStarted!: () => void; + const firstProbeStarted = new Promise((resolve) => { + signalFirstProbeStarted = resolve; + }); + + const probeSpy = spyOn(privatePool, "probeConnection").mockImplementation( + async (_config, _timeoutMs, key, controlPath) => { + probeCalls.push(controlPath); + if (probeCalls.length === 1) { + signalFirstProbeStarted(); + await new Promise((resolve) => { + releaseFirstProbe = () => { + privatePool.markHealthyByKey(key); + privatePool.markControlPathReady(key, controlPath); + resolve(); + }; + }); + return; + } + + privatePool.markHealthyByKey(key); + privatePool.markControlPathReady(key, controlPath); + } + ); + + const firstAcquire = pool.acquireConnection(config, { + controlPath: "/tmp/mux-control-a", + maxWaitMs: 0, + }); + await firstProbeStarted; + + const secondAcquire = pool.acquireConnection(config, { + controlPath: "/tmp/mux-control-b", + maxWaitMs: 0, + }); + releaseFirstProbe(); + + await Promise.all([firstAcquire, secondAcquire]); + + expect(probeCalls).toEqual(["/tmp/mux-control-a", "/tmp/mux-control-b"]); + probeSpy.mockRestore(); + }); + test("callers waking from backoff share single probe (herd only released on success)", async () => { const pool = new SSHConnectionPool(); const config: SSHRuntimeConfig = { diff --git a/src/node/runtime/sshConnectionPool.ts b/src/node/runtime/sshConnectionPool.ts index f63ceecae2..c5d161c73e 100644 --- a/src/node/runtime/sshConnectionPool.ts +++ b/src/node/runtime/sshConnectionPool.ts @@ -284,7 +284,7 @@ export class SSHConnectionPool { log.debug(`SSH connection to ${config.host} has inflight probe, waiting...`); try { await existing; - return; + continue; } catch (error) { // Probe failed; if we're in wait mode we'll loop and sleep through the backoff. if (!shouldWait) { From 58a9293d0d4f887b916eb1ff94852aa7c5d8ca84 Mon Sep 17 00:00:00 2001 From: Ammar Date: Mon, 6 Apr 2026 14:00:45 -0500 Subject: [PATCH 15/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20require=20persisted?= =?UTF-8?q?=20SSH=20workspace=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fail fast when SSH runtime helpers are asked to reconstruct an execution root without a persisted workspace path, while preserving the runtime-path fallback for non-SSH unit-test scenarios. --- _Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$269.00`_ --- src/node/runtime/runtimeHelpers.test.ts | 18 +++++++++++++++++- src/node/runtime/runtimeHelpers.ts | 19 +++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/node/runtime/runtimeHelpers.test.ts b/src/node/runtime/runtimeHelpers.test.ts index 74963d2629..1d4024b1b1 100644 --- a/src/node/runtime/runtimeHelpers.test.ts +++ b/src/node/runtime/runtimeHelpers.test.ts @@ -59,7 +59,7 @@ describe("resolveWorkspaceExecutionPath", () => { expect(resolveWorkspaceExecutionPath(metadata, runtime)).toBe("/persisted/review-1"); }); - it("falls back to the runtime path when persisted metadata is unavailable", () => { + it("requires the persisted path for SSH workspaces", () => { const metadata = { runtimeConfig: { type: "ssh", @@ -70,6 +70,22 @@ describe("resolveWorkspaceExecutionPath", () => { name: "review-1", }; + const runtime = createRuntimeForWorkspace(metadata); + expect(() => resolveWorkspaceExecutionPath(metadata, runtime)).toThrow( + /missing a persisted workspace path/ + ); + }); + + it("falls back to the runtime path for non-SSH workspaces when persisted metadata is unavailable", () => { + const metadata = { + runtimeConfig: { + type: "worktree", + srcBaseDir: "/tmp/src", + } satisfies RuntimeConfig, + projectPath: "/projects/demo", + name: "review-1", + }; + const runtime = createRuntimeForWorkspace(metadata); const runtimeWorkspacePath = runtime.getWorkspacePath(metadata.projectPath, metadata.name); expect(resolveWorkspaceExecutionPath(metadata, runtime)).toBe(runtimeWorkspacePath); diff --git a/src/node/runtime/runtimeHelpers.ts b/src/node/runtime/runtimeHelpers.ts index 4733166be6..bd1cc54ea8 100644 --- a/src/node/runtime/runtimeHelpers.ts +++ b/src/node/runtime/runtimeHelpers.ts @@ -1,5 +1,10 @@ import assert from "@/common/utils/assert"; -import { isDockerRuntime, isLocalProjectRuntime, type RuntimeConfig } from "@/common/types/runtime"; +import { + isDockerRuntime, + isLocalProjectRuntime, + isSSHRuntime, + type RuntimeConfig, +} from "@/common/types/runtime"; import type { Runtime } from "./Runtime"; import { createRuntime } from "./runtimeFactory"; @@ -43,9 +48,15 @@ export function resolveWorkspaceExecutionPath( const persistedWorkspacePath = metadata.namedWorkspacePath?.trim(); if (!persistedWorkspacePath) { - // Some metadata readers and unit tests only carry canonical workspace identity. Fall back to the - // runtime-derived path there, but prefer the persisted path whenever it is available so upgraded - // SSH/devcontainer workspaces keep using their exact checkout root. + // SSH workspaces must keep using the persisted checkout root from config so upgraded legacy + // workspaces do not silently fall back to the reconstructed hashed path and miss their real cwd. + assert( + !isSSHRuntime(metadata.runtimeConfig), + `SSH workspace ${metadata.name} is missing a persisted workspace path` + ); + + // Other runtimes can still fall back to their canonical derived path when only identity metadata + // is available (for example in narrow unit tests). return runtimeWorkspacePath; } From 1590dc6e6982f20eb955d9d724524c93f70c7f3b Mon Sep 17 00:00:00 2001 From: Ammar Date: Mon, 6 Apr 2026 14:18:42 -0500 Subject: [PATCH 16/16] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20legacy?= =?UTF-8?q?=20SSH=20worktrees=20and=20probe=20budgets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use each existing SSH worktree's reported common git dir for rename/delete operations so upgraded legacy layouts keep working, and bound follow-up shard probes to the remaining acquire budget after waiting on an inflight probe. --- _Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$269.00`_ --- src/node/runtime/SSHRuntime.ts | 43 ++++++++- src/node/runtime/sshConnectionPool.test.ts | 71 ++++++++++++++ src/node/runtime/sshConnectionPool.ts | 91 ++++++++++++++++-- tests/runtime/runtime.test.ts | 103 ++++++++++++++++++++- 4 files changed, 294 insertions(+), 14 deletions(-) diff --git a/src/node/runtime/SSHRuntime.ts b/src/node/runtime/SSHRuntime.ts index 97780d0503..6849f2da13 100644 --- a/src/node/runtime/SSHRuntime.ts +++ b/src/node/runtime/SSHRuntime.ts @@ -554,6 +554,35 @@ export class SSHRuntime extends RemoteRuntime { return false; } + private async resolveWorktreeBaseRepoPath( + projectPath: string, + workspacePath: string, + abortSignal?: AbortSignal + ): Promise { + const fallbackBaseRepoPath = this.getBaseRepoPath(projectPath); + + try { + const result = await execBuffered( + this, + `git -C ${this.quoteForRemote(workspacePath)} rev-parse --path-format=absolute --git-common-dir`, + { + cwd: "/tmp", + timeout: 10, + abortSignal, + } + ); + const resolvedBaseRepoPath = result.stdout.trim(); + if (result.exitCode === 0 && resolvedBaseRepoPath.length > 0) { + return resolvedBaseRepoPath; + } + } catch { + // Fall back to the canonical hashed layout when the existing workspace cannot report its + // common git dir (for example, if the directory is already partially missing/corrupted). + } + + return fallbackBaseRepoPath; + } + /** * Detect whether a remote workspace is a git worktree (`.git` is a file) * vs a legacy full clone (`.git` is a directory). @@ -1514,8 +1543,11 @@ export class SSHRuntime extends RemoteRuntime { let moveCommand: string; if (isWorktree) { - // Worktree: use `git worktree move` to keep base repo metadata consistent. - const baseRepoPathArg = expandTildeForSSH(this.getBaseRepoPath(projectPath)); + // Worktree: use `git worktree move` to keep the workspace registered in whichever + // shared base repo originally created it, including upgraded legacy SSH layouts. + const baseRepoPathArg = expandTildeForSSH( + await this.resolveWorktreeBaseRepoPath(projectPath, oldPath, abortSignal) + ); moveCommand = `git -C ${baseRepoPathArg} worktree move ${expandedOldPath} ${expandedNewPath}`; } else { // Legacy full clone: plain mv. @@ -1714,8 +1746,11 @@ export class SSHRuntime extends RemoteRuntime { const isWorktree = await this.isWorktreeWorkspace(deletedPath, abortSignal); if (isWorktree) { - // Worktree: use `git worktree remove` to clean up the base repo's worktree metadata. - const baseRepoPathArg = expandTildeForSSH(this.getBaseRepoPath(projectPath)); + // Worktree: use `git worktree remove` against the actual common git dir for this + // workspace so upgraded legacy SSH worktrees keep their original base repo metadata. + const baseRepoPathArg = expandTildeForSSH( + await this.resolveWorktreeBaseRepoPath(projectPath, deletedPath, abortSignal) + ); const removeCmd = force ? `${nhp}git -C ${baseRepoPathArg} worktree remove --force ${this.quoteForRemote(deletedPath)}` : `${nhp}git -C ${baseRepoPathArg} worktree remove ${this.quoteForRemote(deletedPath)}`; diff --git a/src/node/runtime/sshConnectionPool.test.ts b/src/node/runtime/sshConnectionPool.test.ts index ecbaa5238c..cb4ee5ebca 100644 --- a/src/node/runtime/sshConnectionPool.test.ts +++ b/src/node/runtime/sshConnectionPool.test.ts @@ -519,6 +519,77 @@ describe("SSHConnectionPool", () => { probeSpy.mockRestore(); }); + test("caps the follow-up probe timeout to the remaining wait budget", async () => { + const pool = new SSHConnectionPool(); + const config: SSHRuntimeConfig = { + host: "test.example.com", + srcBaseDir: "/work", + }; + const privatePool = pool as unknown as { + probeConnection: ( + config: SSHRuntimeConfig, + timeoutMs: number, + key: string, + controlPath: string + ) => Promise; + markHealthyByKey: (key: string) => void; + markControlPathReady: (key: string, controlPath: string) => void; + }; + + const probeCalls: Array<{ controlPath: string; timeoutMs: number }> = []; + let releaseFirstProbe!: () => void; + let signalFirstProbeStarted!: () => void; + const firstProbeStarted = new Promise((resolve) => { + signalFirstProbeStarted = resolve; + }); + let now = 1_000; + const nowSpy = spyOn(Date, "now").mockImplementation(() => now); + + const probeSpy = spyOn(privatePool, "probeConnection").mockImplementation( + async (_config, timeoutMs, key, controlPath) => { + probeCalls.push({ controlPath, timeoutMs }); + if (probeCalls.length === 1) { + signalFirstProbeStarted(); + await new Promise((resolve) => { + releaseFirstProbe = () => { + now += 25; + privatePool.markHealthyByKey(key); + privatePool.markControlPathReady(key, controlPath); + resolve(); + }; + }); + return; + } + + privatePool.markHealthyByKey(key); + privatePool.markControlPathReady(key, controlPath); + } + ); + + const firstAcquire = pool.acquireConnection(config, { + controlPath: "/tmp/mux-control-a", + timeoutMs: 30, + maxWaitMs: 30, + }); + await firstProbeStarted; + + const secondAcquire = pool.acquireConnection(config, { + controlPath: "/tmp/mux-control-b", + timeoutMs: 30, + maxWaitMs: 30, + }); + releaseFirstProbe(); + + await Promise.all([firstAcquire, secondAcquire]); + + expect(probeCalls).toEqual([ + { controlPath: "/tmp/mux-control-a", timeoutMs: 30 }, + { controlPath: "/tmp/mux-control-b", timeoutMs: 5 }, + ]); + probeSpy.mockRestore(); + nowSpy.mockRestore(); + }); + test("callers waking from backoff share single probe (herd only released on success)", async () => { const pool = new SSHConnectionPool(); const config: SSHRuntimeConfig = { diff --git a/src/node/runtime/sshConnectionPool.ts b/src/node/runtime/sshConnectionPool.ts index c5d161c73e..fb9157b1d1 100644 --- a/src/node/runtime/sshConnectionPool.ts +++ b/src/node/runtime/sshConnectionPool.ts @@ -178,6 +178,53 @@ async function sleepWithAbort(ms: number, abortSignal?: AbortSignal): Promise( + promise: Promise, + timeoutMs: number, + abortSignal?: AbortSignal, + timeoutError?: Error +): Promise { + if (abortSignal?.aborted) { + throw new Error("Operation aborted"); + } + + return await new Promise((resolve, reject) => { + let settled = false; + + const cleanup = () => { + clearTimeout(timer); + abortSignal?.removeEventListener("abort", onAbort); + }; + + const finish = (handler: () => void) => { + if (settled) { + return; + } + settled = true; + cleanup(); + handler(); + }; + + const onAbort = () => { + finish(() => reject(new Error("Operation aborted"))); + }; + + const timer = setTimeout(() => { + finish(() => reject(timeoutError ?? new Error("Operation timed out"))); + }, timeoutMs); + + abortSignal?.addEventListener("abort", onAbort); + promise.then( + (value) => { + finish(() => resolve(value)); + }, + (error) => { + finish(() => reject(error instanceof Error ? error : new Error(String(error)))); + } + ); + }); +} + /** * SSH Connection Pool * @@ -223,6 +270,13 @@ export class SSHConnectionPool { const key = makeConnectionKey(config); const requestedControlPath = options.controlPath ?? getControlPath(config); const startTime = Date.now(); + const getRemainingWaitBudgetMs = (): number => + Math.max(0, maxWaitMs - (Date.now() - startTime)); + const createWaitBudgetExceededError = (lastError?: string): Error => + new Error( + `SSH connection to ${config.host} did not become healthy within ${maxWaitMs}ms. ` + + `Last error: ${lastError ?? "unknown"}` + ); while (true) { if (options.abortSignal?.aborted) { @@ -243,13 +297,9 @@ export class SSHConnectionPool { ); } - const elapsedMs = Date.now() - startTime; - const budgetMs = Math.max(0, maxWaitMs - elapsedMs); + const budgetMs = getRemainingWaitBudgetMs(); if (budgetMs <= 0) { - throw new Error( - `SSH connection to ${config.host} did not become healthy within ${maxWaitMs}ms. ` + - `Last error: ${health.lastError ?? "unknown"}` - ); + throw createWaitBudgetExceededError(health.lastError); } const waitMs = Math.min(remainingMs, budgetMs); @@ -283,11 +333,28 @@ export class SSHConnectionPool { if (existing) { log.debug(`SSH connection to ${config.host} has inflight probe, waiting...`); try { - await existing; + if (shouldWait) { + const budgetMs = getRemainingWaitBudgetMs(); + if (budgetMs <= 0) { + throw createWaitBudgetExceededError(health?.lastError); + } + await waitForPromiseWithTimeout( + existing, + budgetMs, + options.abortSignal, + createWaitBudgetExceededError(health?.lastError) + ); + } else { + await existing; + } continue; } catch (error) { // Probe failed; if we're in wait mode we'll loop and sleep through the backoff. - if (!shouldWait) { + if ( + !shouldWait || + (error instanceof Error && + error.message.includes(`did not become healthy within ${maxWaitMs}ms`)) + ) { throw error; } continue; @@ -295,8 +362,14 @@ export class SSHConnectionPool { } // Start new probe. + const probeTimeoutMs = shouldWait + ? Math.min(timeoutMs, getRemainingWaitBudgetMs()) + : timeoutMs; + if (probeTimeoutMs <= 0) { + throw createWaitBudgetExceededError(health?.lastError); + } log.debug(`SSH connection to ${config.host} needs probe, starting health check`); - const probe = this.probeConnection(config, timeoutMs, key, requestedControlPath); + const probe = this.probeConnection(config, probeTimeoutMs, key, requestedControlPath); this.inflight.set(key, probe); try { diff --git a/tests/runtime/runtime.test.ts b/tests/runtime/runtime.test.ts index 65fd0c6279..1bad6f3b37 100644 --- a/tests/runtime/runtime.test.ts +++ b/tests/runtime/runtime.test.ts @@ -33,8 +33,9 @@ import { import { execBuffered, readFileString, writeFileString } from "@/node/utils/runtime/helpers"; import type { Runtime } from "@/node/runtime/Runtime"; import { RuntimeError } from "@/node/runtime/Runtime"; -import { computeBaseRepoPath, type SSHRuntime } from "@/node/runtime/SSHRuntime"; +import { computeBaseRepoPath, SSHRuntime } from "@/node/runtime/SSHRuntime"; import { + buildLegacyRemoteProjectLayout, buildRemoteProjectLayout, getRemoteWorkspacePath, } from "@/node/runtime/remoteProjectLayout"; @@ -1688,6 +1689,106 @@ describeIntegration("Runtime integration tests", () => { } }, 60000); + test("renameWorkspace and deleteWorkspace keep using the legacy base repo for upgraded SSH worktrees", async () => { + if (!sshConfig) { + throw new Error("SSH config unavailable"); + } + + const config = { + host: "testuser@localhost", + srcBaseDir, + identityFile: sshConfig.privateKeyPath, + port: sshConfig.port, + }; + const runtime = new SSHRuntime(config, createSSHTransport(config, false), { + projectPath: "/unused", + workspaceName: "unused", + }); + const projectName = `wt-legacy-rename-${Date.now()}-${Math.random().toString(36).substring(7)}`; + const projectPath = `/some/path/${projectName}`; + const legacyLayout = buildLegacyRemoteProjectLayout(srcBaseDir, projectPath); + const baseRepoPath = legacyLayout.baseRepoPath; + const oldWorkspacePath = getRemoteWorkspacePath(legacyLayout, "old-name"); + const newWorkspacePath = getRemoteWorkspacePath(legacyLayout, "new-name"); + const legacyRuntime = new SSHRuntime(config, createSSHTransport(config, false), { + projectPath, + workspaceName: "old-name", + workspacePath: oldWorkspacePath, + }); + + try { + await execBuffered( + runtime, + [ + `mkdir -p "${legacyLayout.projectRoot}"`, + `git init --bare "${baseRepoPath}"`, + `TMPCLONE=$(mktemp -d)`, + `git clone "${baseRepoPath}" "$TMPCLONE/work"`, + `cd "$TMPCLONE/work"`, + `git config user.email "test@test.com"`, + `git config user.name "Test"`, + `echo "x" > x.txt && git add x.txt && git commit -m "init"`, + `git push origin HEAD:main`, + `rm -rf "$TMPCLONE"`, + ].join(" && "), + { cwd: "/home/testuser", timeout: 30 } + ); + + await execBuffered( + runtime, + `git -C "${baseRepoPath}" worktree add "${oldWorkspacePath}" -b old-name main`, + { cwd: "/home/testuser", timeout: 30 } + ); + + const renameResult = await legacyRuntime.renameWorkspace( + projectPath, + "old-name", + "new-name" + ); + expect(renameResult.success).toBe(true); + if (!renameResult.success) return; + + const legacyWorktreeList = await execBuffered( + runtime, + `git -C "${baseRepoPath}" worktree list`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(legacyWorktreeList.stdout).toContain("/new-name"); + expect(legacyWorktreeList.stdout).not.toContain("/old-name"); + + const renamedLegacyRuntime = new SSHRuntime(config, createSSHTransport(config, false), { + projectPath, + workspaceName: "new-name", + workspacePath: newWorkspacePath, + }); + const deleteResult = await renamedLegacyRuntime.deleteWorkspace( + projectPath, + "new-name", + true + ); + expect(deleteResult.success).toBe(true); + + const deletedPathCheck = await execBuffered( + runtime, + `test -d "${newWorkspacePath}" && echo "exists" || echo "missing"`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(deletedPathCheck.stdout.trim()).toBe("missing"); + + const branchCheck = await execBuffered( + runtime, + `git -C "${baseRepoPath}" branch --list old-name`, + { cwd: "/home/testuser", timeout: 30 } + ); + expect(branchCheck.stdout.trim()).toBe(""); + } finally { + await execBuffered(runtime, `rm -rf "${legacyLayout.projectRoot}"`, { + cwd: "/home/testuser", + timeout: 30, + }); + } + }, 60000); + test("exec handles a concurrent burst on one SSH host", async () => { const runtime = createSSHRuntime(); const projectName = `ssh-burst-${Date.now()}-${Math.random().toString(36).substring(7)}`;