From 16968af5c7e5a8180a9bae56a1e6fc9cdca1d737 Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 14 Jul 2026 12:30:18 -0700 Subject: [PATCH 1/5] feat(core): add shared Antfly lifecycle with log capture and Podman support Consolidates three divergent copies of the start/recover logic (CLI utils, MCP entry point, adapter registry) into packages/core/src/antfly/lifecycle.ts. Spawned Antfly output is captured to ~/.antfly/antfly.log (rotated at 5MB) instead of stdio:'ignore', and container fallbacks detect Docker or Podman. Orchestration is dependency-injected for testability; 18 unit tests. --- .../src/antfly/__tests__/lifecycle.test.ts | 256 ++++++++++++ packages/core/src/antfly/index.ts | 1 + packages/core/src/antfly/lifecycle.ts | 382 ++++++++++++++++++ packages/core/src/index.ts | 1 + 4 files changed, 640 insertions(+) create mode 100644 packages/core/src/antfly/__tests__/lifecycle.test.ts create mode 100644 packages/core/src/antfly/index.ts create mode 100644 packages/core/src/antfly/lifecycle.ts diff --git a/packages/core/src/antfly/__tests__/lifecycle.test.ts b/packages/core/src/antfly/__tests__/lifecycle.test.ts new file mode 100644 index 0000000..3457368 --- /dev/null +++ b/packages/core/src/antfly/__tests__/lifecycle.test.ts @@ -0,0 +1,256 @@ +/** + * Tests for the shared Antfly lifecycle module. + * + * This module consolidates three previously divergent copies of the + * start/recover logic (CLI utils, MCP server bin, adapter registry). + * Regression context: the MCP server's copy spawned Antfly with + * stdio:'ignore' (startup failures were undiagnosable) and fell back to + * Docker only (dead code on Podman machines), then crashed the whole MCP + * server when both paths failed. + */ + +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + antflyUnavailableMessage, + buildSwarmArgs, + containerRunCommand, + containerStartCommand, + detectContainerRuntime, + ensureAntfly, + getAntflyLogPath, + modelPresentInOutput, + startAntflyProcess, +} from '../lifecycle'; + +describe('buildSwarmArgs', () => { + it('pins data-dir and all port flags so every caller starts an identical server', () => { + const args = buildSwarmArgs('/home/user/.antfly'); + expect(args[0]).toBe('swarm'); + expect(args).toContain('--data-dir'); + expect(args[args.indexOf('--data-dir') + 1]).toBe('/home/user/.antfly'); + expect(args).toContain('--metadata-api'); + expect(args).toContain('--store-api'); + expect(args).toContain('--metadata-raft'); + expect(args).toContain('--store-raft'); + expect(args).toContain('--health-port'); + }); +}); + +describe('getAntflyLogPath', () => { + it('places the log inside the antfly data dir', () => { + expect(getAntflyLogPath('/data/antfly')).toBe(join('/data/antfly', 'antfly.log')); + }); +}); + +describe('startAntflyProcess', () => { + let dir: string; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it('captures child stdout/stderr into the log file instead of discarding it', async () => { + dir = mkdtempSync(join(tmpdir(), 'antfly-lifecycle-')); + const logPath = join(dir, 'antfly.log'); + + startAntflyProcess({ + command: process.execPath, + args: ['-e', 'console.error("boom from child"); console.log("hello from child")'], + logPath, + }); + + // Detached child writes asynchronously — poll briefly. + const deadline = Date.now() + 5000; + let content = ''; + while (Date.now() < deadline) { + try { + content = readFileSync(logPath, 'utf-8'); + if (content.includes('boom from child') && content.includes('hello from child')) break; + } catch { + // file not created yet + } + await new Promise((r) => setTimeout(r, 100)); + } + + expect(content).toContain('boom from child'); + expect(content).toContain('hello from child'); + // Header line marks each start attempt so repeated failures are separable. + expect(content).toContain('[dev-agent] starting'); + }); +}); + +describe('detectContainerRuntime', () => { + it('prefers docker when available', () => { + const probe = (cmd: string) => cmd.startsWith('docker'); + expect(detectContainerRuntime(probe)).toBe('docker'); + }); + + it('falls back to podman when docker is unavailable', () => { + const probe = (cmd: string) => cmd.startsWith('podman'); + expect(detectContainerRuntime(probe)).toBe('podman'); + }); + + it('returns null when neither runtime responds', () => { + expect(detectContainerRuntime(() => false)).toBe(null); + }); +}); + +describe('container command builders', () => { + it('uses the detected runtime binary for start', () => { + expect(containerStartCommand('podman', 'dev-agent-antfly')).toBe( + 'podman start dev-agent-antfly' + ); + expect(containerStartCommand('docker', 'dev-agent-antfly')).toBe( + 'docker start dev-agent-antfly' + ); + }); + + it('uses the detected runtime binary for run', () => { + const cmd = containerRunCommand('podman', { + name: 'dev-agent-antfly', + image: 'ghcr.io/antflydb/antfly:latest', + port: 18080, + }); + expect(cmd.startsWith('podman run ')).toBe(true); + expect(cmd).toContain('--name dev-agent-antfly'); + expect(cmd).toContain('-p 18080:8080'); + expect(cmd).toContain('ghcr.io/antflydb/antfly:latest'); + }); +}); + +describe('ensureAntfly orchestration', () => { + const URL = 'http://localhost:18080/api/v1'; + + it('returns immediately when the server is already reachable', async () => { + const startNative = vi.fn(); + const url = await ensureAntfly({ + quiet: true, + deps: { + isReady: async () => true, + startNative, + }, + }); + expect(url).toBe(URL); + expect(startNative).not.toHaveBeenCalled(); + }); + + it('starts the native binary when installed and waits for readiness', async () => { + let started = false; + const url = await ensureAntfly({ + quiet: true, + deps: { + isReady: async () => started, + hasNative: () => true, + startNative: async () => { + started = true; + }, + waitForReady: async () => {}, + }, + }); + expect(url).toBe(URL); + expect(started).toBe(true); + }); + + it('falls back to podman when native is missing and docker is unavailable', async () => { + const commands: string[] = []; + let started = false; + await ensureAntfly({ + quiet: true, + deps: { + isReady: async () => started, + hasNative: () => false, + detectRuntime: () => 'podman', + containerExists: () => true, + runCommand: (cmd: string) => { + commands.push(cmd); + started = true; + }, + waitForReady: async () => {}, + }, + }); + expect(commands).toEqual(['podman start dev-agent-antfly']); + }); + + it('creates a new container when none exists', async () => { + const commands: string[] = []; + await ensureAntfly({ + quiet: true, + deps: { + isReady: async () => false, + hasNative: () => false, + detectRuntime: () => 'podman', + containerExists: () => false, + runCommand: (cmd: string) => { + commands.push(cmd); + }, + waitForReady: async () => {}, + }, + }); + expect(commands).toHaveLength(1); + expect(commands[0].startsWith('podman run ')).toBe(true); + }); + + it('throws actionable guidance when nothing can start Antfly', async () => { + await expect( + ensureAntfly({ + quiet: true, + deps: { + isReady: async () => false, + hasNative: () => false, + detectRuntime: () => null, + }, + }) + ).rejects.toThrow(/dev setup/); + }); +}); + +describe('antflyUnavailableMessage', () => { + it('points at the doctor command and the captured log', () => { + const msg = antflyUnavailableMessage('/home/user/.antfly/antfly.log'); + expect(msg).toContain('dev doctor'); + expect(msg).toContain('/home/user/.antfly/antfly.log'); + }); +}); + +// Ported from packages/cli/src/utils/__tests__/antfly.test.ts, now importing +// the real implementation instead of a mirrored copy. +describe('modelPresentInOutput', () => { + const FULL_NAME = 'BAAI/bge-small-en-v1.5'; + + const PRESENT_OUTPUT = `Local models in /Users/dev/.antfly/models: + +NAME TYPE SIZE VARIANTS SOURCE +BAAI/bge-small-en-v1.5 embedder 127.8 MB BAAI/bge-small-en-v1.5 +`; + + const EMPTY_OUTPUT = `Local models in /Users/dev/.antfly/models: + +NAME TYPE SIZE VARIANTS SOURCE +No models found locally. +`; + + const OTHER_MODEL_OUTPUT = `Local models in /Users/dev/.antfly/models: + +NAME TYPE SIZE VARIANTS SOURCE +vendor/other-bge-small-en-v1.5 embedder 200.0 MB vendor/other-bge-small-en-v1.5 +`; + + it('returns true when full model name is present', () => { + expect(modelPresentInOutput(FULL_NAME, PRESENT_OUTPUT)).toBe(true); + }); + + it('returns false when models directory is empty', () => { + expect(modelPresentInOutput(FULL_NAME, EMPTY_OUTPUT)).toBe(false); + }); + + it('returns false when a different model shares the short name as a suffix', () => { + expect(modelPresentInOutput(FULL_NAME, OTHER_MODEL_OUTPUT)).toBe(false); + }); + + it('returns true when only the short name is present as a standalone token', () => { + expect(modelPresentInOutput(FULL_NAME, 'bge-small-en-v1.5 embedder 127 MB')).toBe(true); + }); +}); diff --git a/packages/core/src/antfly/index.ts b/packages/core/src/antfly/index.ts new file mode 100644 index 0000000..b55497a --- /dev/null +++ b/packages/core/src/antfly/index.ts @@ -0,0 +1 @@ +export * from './lifecycle'; diff --git a/packages/core/src/antfly/lifecycle.ts b/packages/core/src/antfly/lifecycle.ts new file mode 100644 index 0000000..67a42fe --- /dev/null +++ b/packages/core/src/antfly/lifecycle.ts @@ -0,0 +1,382 @@ +/** + * Antfly server lifecycle management — the single shared implementation. + * + * Consumed by the CLI (`dev setup`, `dev index`, `dev doctor`), the MCP server + * entry point (startup auto-start), and the adapter registry (mid-session + * recovery). Native binary first, container runtime (docker or podman) + * fallback. The user never needs to run `antfly` directly. + * + * All spawned Antfly output is captured to a log file (see getAntflyLogPath) + * so failed starts are diagnosable after the fact. + */ + +import { execSync, spawn } from 'node:child_process'; +import { closeSync, mkdirSync, openSync, renameSync, statSync, writeSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +export const DEFAULT_ANTFLY_URL = 'http://localhost:18080/api/v1'; +export const ANTFLY_CONTAINER_NAME = 'dev-agent-antfly'; +export const ANTFLY_CONTAINER_IMAGE = 'ghcr.io/antflydb/antfly:latest'; +const ANTFLY_PORT = 18080; +const STARTUP_TIMEOUT_MS = 30_000; +const POLL_INTERVAL_MS = 500; +const LOG_ROTATE_BYTES = 5 * 1024 * 1024; + +export type ContainerRuntime = 'docker' | 'podman'; + +export function getAntflyUrl(): string { + return process.env.ANTFLY_URL ?? DEFAULT_ANTFLY_URL; +} + +/** + * `antfly swarm` uses `--data-dir` (default: ~/.antfly) as its root for all + * storage, including Termite models at {data-dir}/models. We always pass it + * explicitly so model tooling and the server agree on paths. + */ +export function getAntflyDataDir(): string { + return process.env.ANTFLY_DATA_DIR ?? join(homedir(), '.antfly'); +} + +export function getTermiteModelsDir(): string { + return join(getAntflyDataDir(), 'models'); +} + +export function getAntflyLogPath(dataDir: string = getAntflyDataDir()): string { + return join(dataDir, 'antfly.log'); +} + +/** + * Canonical `antfly swarm` arguments. Custom ports avoid 8080 conflicts + * (Docker, other services): metadata-api on 18080 (our default URL), + * store-api on 18381, raft on 19017/19021, health on 14200. + */ +export function buildSwarmArgs(dataDir: string = getAntflyDataDir()): string[] { + return [ + 'swarm', + '--data-dir', + dataDir, + '--metadata-api', + `http://0.0.0.0:${ANTFLY_PORT}`, + '--store-api', + 'http://0.0.0.0:18381', + '--metadata-raft', + 'http://0.0.0.0:19017', + '--store-raft', + 'http://0.0.0.0:19021', + '--health-port', + '14200', + ]; +} + +/** Probe returning true when a command exits 0. */ +export type CommandProbe = (command: string) => boolean; + +const defaultProbe: CommandProbe = (command) => { + try { + execSync(command, { stdio: 'pipe', timeout: 5000 }); + return true; + } catch { + return false; + } +}; + +export function hasNativeBinary(probe: CommandProbe = defaultProbe): boolean { + return probe('antfly --version'); +} + +export function detectContainerRuntime( + probe: CommandProbe = defaultProbe +): ContainerRuntime | null { + if (probe('docker info')) return 'docker'; + if (probe('podman info')) return 'podman'; + return null; +} + +export function containerStartCommand(runtime: ContainerRuntime, name: string): string { + return `${runtime} start ${name}`; +} + +export function containerRunCommand( + runtime: ContainerRuntime, + options: { name: string; image: string; port: number } +): string { + return ( + `${runtime} run -d --name ${options.name} -p ${options.port}:8080 ` + + `-m 8g --platform linux/amd64 ${options.image} swarm` + ); +} + +export function containerExists( + runtime: ContainerRuntime, + name: string = ANTFLY_CONTAINER_NAME +): boolean { + try { + const result = execSync(`${runtime} ps -a --filter name=${name} --format "{{.Names}}"`, { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + return result.trim() === name; + } catch { + return false; + } +} + +export async function isServerReady(url: string = getAntflyUrl()): Promise { + const baseUrl = url.replace('/api/v1', ''); + try { + const resp = await fetch(`${baseUrl}/api/v1/tables`, { signal: AbortSignal.timeout(3000) }); + return resp.ok; + } catch { + return false; + } +} + +export interface StartAntflyProcessOptions { + /** Override binary for tests. Defaults to `antfly`. */ + command?: string; + /** Override args for tests. Defaults to buildSwarmArgs(). */ + args?: string[]; + /** Where child stdout/stderr land. Defaults to getAntflyLogPath(). */ + logPath?: string; +} + +/** + * Spawn a detached Antfly process with its output captured to the log file. + * Never spawn with stdio:'ignore' — a failed start with discarded output is + * undiagnosable (the exact failure mode that motivated this module). + */ +export function startAntflyProcess(options: StartAntflyProcessOptions = {}): void { + const logPath = options.logPath ?? getAntflyLogPath(); + const fd = openLogFile(logPath); + writeSync( + fd, + `[dev-agent] starting ${options.command ?? 'antfly'} at ${new Date().toISOString()}\n` + ); + + const child = spawn(options.command ?? 'antfly', options.args ?? buildSwarmArgs(), { + detached: true, + stdio: ['ignore', fd, fd], + }); + child.unref(); + // The child holds its own copy of the descriptor. + closeSync(fd); +} + +function openLogFile(logPath: string): number { + mkdirSync(join(logPath, '..'), { recursive: true }); + try { + if (statSync(logPath).size > LOG_ROTATE_BYTES) { + renameSync(logPath, `${logPath}.old`); + } + } catch { + // no existing log + } + return openSync(logPath, 'a'); +} + +/** Error message for tool responses when Antfly cannot be reached/recovered. */ +export function antflyUnavailableMessage(logPath: string = getAntflyLogPath()): string { + return ( + 'Antfly search backend is not running and could not be started. ' + + 'Run `dev doctor` to diagnose, or `dev setup` to reinstall. ' + + `Startup output is captured in ${logPath}.` + ); +} + +/** Injectable boundaries for ensureAntfly — defaults hit the real system. */ +export interface LifecycleDeps { + isReady: (url: string) => Promise; + hasNative: () => boolean; + startNative: () => Promise | void; + detectRuntime: () => ContainerRuntime | null; + containerExists: (runtime: ContainerRuntime) => boolean; + runCommand: (command: string) => void; + waitForReady: (url: string) => Promise; + logger?: { info: (msg: string) => void }; +} + +export interface EnsureAntflyOptions { + quiet?: boolean; + deps?: Partial; +} + +/** + * Ensure Antfly is running, auto-starting if needed. + * Priority: already running → native binary → container runtime → error. + */ +export async function ensureAntfly(options: EnsureAntflyOptions = {}): Promise { + const url = getAntflyUrl(); + const deps: LifecycleDeps = { + isReady: isServerReady, + hasNative: () => hasNativeBinary(), + startNative: () => startAntflyProcess(), + detectRuntime: () => detectContainerRuntime(), + containerExists: (runtime) => containerExists(runtime), + runCommand: (command) => { + execSync(command, { stdio: 'pipe' }); + }, + waitForReady: (u) => waitForServer(u, deps.isReady), + ...options.deps, + }; + const info = (msg: string) => { + if (options.quiet) return; + deps.logger?.info(msg); + }; + + if (await deps.isReady(url)) { + return url; + } + + if (deps.hasNative()) { + info('Starting Antfly server...'); + await deps.startNative(); + await deps.waitForReady(url); + info(`Antfly running on ${url}`); + return url; + } + + const runtime = deps.detectRuntime(); + if (runtime) { + if (deps.containerExists(runtime)) { + info(`Starting Antfly container (${runtime})...`); + deps.runCommand(containerStartCommand(runtime, ANTFLY_CONTAINER_NAME)); + } else { + info(`Starting Antfly via ${runtime}...`); + deps.runCommand( + containerRunCommand(runtime, { + name: ANTFLY_CONTAINER_NAME, + image: ANTFLY_CONTAINER_IMAGE, + port: ANTFLY_PORT, + }) + ); + } + await deps.waitForReady(url); + info(`Antfly running on ${url}`); + return url; + } + + throw new Error( + 'Antfly is not installed. Run `dev setup` to install:\n' + + ' brew install --cask antflydb/antfly/antfly' + ); +} + +/** + * Poll until the server responds, then return. On timeout, check for a port + * conflict (the most common cause) before giving generic guidance. + */ +export async function waitForServer( + url: string = getAntflyUrl(), + isReady: (url: string) => Promise = isServerReady +): Promise { + const start = Date.now(); + while (Date.now() - start < STARTUP_TIMEOUT_MS) { + if (await isReady(url)) return; + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } + + try { + const lsof = execSync(`lsof -i :${ANTFLY_PORT} -t`, { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }).trim(); + if (lsof) { + throw new Error( + `Port ${ANTFLY_PORT} is already in use (pid: ${lsof}).\n` + + ` Check: lsof -i :${ANTFLY_PORT}\n` + + ` Or set: ANTFLY_URL=http://localhost:/api/v1` + ); + } + } catch (e) { + if (e instanceof Error && e.message.includes('Port')) throw e; + } + + throw new Error( + `Antfly server did not start within ${STARTUP_TIMEOUT_MS / 1000}s.\n` + + ` Check the startup log: ${getAntflyLogPath()}\n` + + ` Then: dev doctor` + ); +} + +/** Antfly native binary version, or null when not installed. */ +export function getNativeVersion(): string | null { + try { + return execSync('antfly --version', { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }).trim(); + } catch { + return null; + } +} + +/** + * Pull a Termite embedding model (native binary), targeting the models dir + * used by the running swarm server. + */ +export function pullModel(model: string): void { + execSync(`antfly termite pull --models-dir ${getTermiteModelsDir()} ${model}`, { + stdio: 'inherit', + }); +} + +/** Pull a Termite model inside the container. */ +export function pullModelContainer(runtime: ContainerRuntime, model: string): void { + execSync(`${runtime} exec ${ANTFLY_CONTAINER_NAME} /antfly termite pull ${model}`, { + stdio: 'inherit', + }); +} + +/** Check model presence in the server's models dir (native binary). */ +export function hasModel(model: string): boolean { + try { + const output = execSync(`antfly termite list --models-dir ${getTermiteModelsDir()}`, { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + return modelPresentInOutput(model, output); + } catch { + return false; + } +} + +/** Check model presence inside the container. */ +export function hasModelContainer(runtime: ContainerRuntime, model: string): boolean { + try { + const output = execSync(`${runtime} exec ${ANTFLY_CONTAINER_NAME} /antfly termite list`, { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + return modelPresentInOutput(model, output); + } catch { + return false; + } +} + +/** Container runtime's total allocated memory in bytes (for setup warnings). */ +export function getContainerMemoryBytes(runtime: ContainerRuntime): number | null { + try { + const output = execSync(`${runtime} info 2>&1`, { encoding: 'utf-8', timeout: 5000 }); + const match = output.match(/memTotal:\s*(\d+)/); + return match ? Number(match[1]) : null; + } catch { + return null; + } +} + +/** + * Return true when the model name is present in `antfly termite list` output. + * + * Strategy (most-specific first): + * 1. Full name exact match — "BAAI/bge-small-en-v1.5" appears verbatim. + * 2. Short name word-boundary — "bge-small-en-v1.5" appears as a whole token + * (not as a suffix of a different model name). + */ +export function modelPresentInOutput(model: string, output: string): boolean { + if (output.includes(model)) return true; + + const shortName = model.split('/').pop() ?? model; + const escaped = shortName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`(? Date: Tue, 14 Jul 2026 12:30:29 -0700 Subject: [PATCH 2/5] feat(mcp): gate backend init so the server survives a down Antfly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stdio handshake previously waited on Antfly auto-start, indexer init, and watcher catchup — a cold or unstartable Antfly killed the process with a bare -32000. BackendGate starts the server immediately, makes the first tool call wait for backend init, turns init failures into legible tool errors pointing at dev doctor, and retries init on the next call so a running session self-heals. The registry's recovery and the entry point's auto-start now use the shared core lifecycle (the old copies were missing --data-dir, Podman support, and port-conflict detection). --- packages/mcp-server/bin/dev-agent-mcp.ts | 187 ++++++------------ .../src/adapters/adapter-registry.ts | 70 +------ .../src/server/__tests__/backend-gate.test.ts | 125 ++++++++++++ .../mcp-server/src/server/backend-gate.ts | 106 ++++++++++ 4 files changed, 306 insertions(+), 182 deletions(-) create mode 100644 packages/mcp-server/src/server/__tests__/backend-gate.test.ts create mode 100644 packages/mcp-server/src/server/backend-gate.ts diff --git a/packages/mcp-server/bin/dev-agent-mcp.ts b/packages/mcp-server/bin/dev-agent-mcp.ts index ba899f1..35b02bd 100644 --- a/packages/mcp-server/bin/dev-agent-mcp.ts +++ b/packages/mcp-server/bin/dev-agent-mcp.ts @@ -6,6 +6,7 @@ import { createPatternMatcher, + ensureAntfly, ensureStorageDirectory, getStorageFilePaths, getStoragePath, @@ -20,6 +21,7 @@ import { SearchAdapter, StatusAdapter, } from '../src/adapters/built-in'; +import { BackendGate } from '../src/server/backend-gate'; import { MCPServer } from '../src/server/mcp-server'; import type { FileWatcherHandle } from '../src/watcher'; import { createIncrementalIndexer, getEventsSince, startFileWatcher } from '../src/watcher'; @@ -145,114 +147,75 @@ async function startupCatchup( await watcher.writeSnapshot(repositoryPath, snapshotPath); } -/** - * Check if Antfly server is reachable. - */ -async function isAntflyReady(): Promise { - const url = process.env.ANTFLY_URL ?? 'http://localhost:18080/api/v1'; - const baseUrl = url.replace('/api/v1', ''); - try { - const resp = await fetch(`${baseUrl}/api/v1/tables`, { signal: AbortSignal.timeout(3000) }); - return resp.ok; - } catch { - return false; - } -} - -/** - * Try to start Antfly if not running (native first, then Docker). - */ -async function tryStartAntfly(): Promise { - const { execSync, spawn } = await import('node:child_process'); - - // Try native - try { - execSync('antfly --version', { stdio: 'pipe', timeout: 5000 }); - const child = spawn( - 'antfly', - [ - 'swarm', - '--metadata-api', - 'http://0.0.0.0:18080', - '--store-api', - 'http://0.0.0.0:18381', - '--metadata-raft', - 'http://0.0.0.0:19017', - '--store-raft', - 'http://0.0.0.0:19021', - '--health-port', - '14200', - ], - { detached: true, stdio: 'ignore' } - ); - child.unref(); - console.error('[MCP] Starting Antfly server (native)...'); - - // Wait for ready - const start = Date.now(); - while (Date.now() - start < 30_000) { - if (await isAntflyReady()) return; - await new Promise((r) => setTimeout(r, 500)); - } - throw new Error('Antfly did not start in 30s'); - } catch { - // Try Docker - try { - execSync('docker info', { stdio: 'pipe', timeout: 5000 }); - try { - execSync('docker start dev-agent-antfly', { stdio: 'pipe' }); - } catch { - execSync( - 'docker run -d --name dev-agent-antfly -p 18080:8080 -m 8g --platform linux/amd64 ghcr.io/antflydb/antfly:latest swarm', - { stdio: 'pipe' } - ); - } - console.error('[MCP] Starting Antfly server (Docker)...'); - - const start = Date.now(); - while (Date.now() - start < 30_000) { - if (await isAntflyReady()) return; - await new Promise((r) => setTimeout(r, 500)); - } - throw new Error('Antfly did not start in 30s'); - } catch { - // Neither available — will fail at indexer.initialize() - } - } -} - async function main() { let watcherHandle: FileWatcherHandle | undefined; try { - // Ensure Antfly is running before initializing - if (!(await isAntflyReady())) { - await tryStartAntfly(); - } - - // Get centralized storage paths + // Get centralized storage paths (filesystem only — no Antfly needed) const storagePath = await getStoragePath(repositoryPath); await ensureStorageDirectory(storagePath); const filePaths = getStorageFilePaths(storagePath); - // Initialize repository indexer with centralized storage + // Construct (but do not initialize) the indexer — construction is + // IO-free, so the MCP handshake never waits on the Antfly backend. const indexer = new RepositoryIndexer({ repositoryPath, vectorStorePath: filePaths.vectors, }); - await indexer.initialize(); + // Backend initialization runs behind the gate: the server starts and + // completes its handshake immediately; the first tool call waits for + // this, and a failure here becomes a legible tool error — never a + // process exit (the old behavior surfaced as a bare -32000). + const gate = new BackendGate(async () => { + console.error('[MCP] Initializing backend...'); + try { + await ensureAntfly({ + deps: { logger: { info: (msg: string) => console.error(`[MCP] ${msg}`) } }, + }); - // Update metadata - await saveMetadata(storagePath, repositoryPath); + await indexer.initialize(); + await saveMetadata(storagePath, repositoryPath); - // Startup catchup: index or update since last snapshot - await startupCatchup( - indexer, - repositoryPath, - filePaths.watcherSnapshot, - filePaths.dependencyGraph - ); + // Startup catchup: index or update since last snapshot + await startupCatchup( + indexer, + repositoryPath, + filePaths.watcherSnapshot, + filePaths.dependencyGraph + ); + + // Start file watcher only once the backend is usable + const incrementalIndexer = createIncrementalIndexer({ + repositoryIndexer: indexer, + repositoryPath, + graphPath: filePaths.dependencyGraph, + logger: { + info: console.error.bind(console), + warn: console.error.bind(console), + error: console.error.bind(console), + }, + }); + + watcherHandle = await startFileWatcher({ + repositoryPath, + snapshotPath: filePaths.watcherSnapshot, + onChanges: async (changed, deleted) => { + await incrementalIndexer.onChanges(changed, deleted); + // Write snapshot after each successful incremental update + await watcherHandle?.writeSnapshot(); + }, + onError: (err) => { + console.error('[MCP] File watcher error:', err); + }, + }); + + console.error('[MCP] Backend ready; file watcher started'); + } catch (error) { + console.error('[MCP] Backend initialization failed:', error); + throw error; + } + }); // Create services const searchService = new SearchService({ repositoryPath }); @@ -297,7 +260,8 @@ async function main() { defaultTokenBudget: 2000, }); - // Create MCP server with 5 adapters (health merged into status) + // Create MCP server with 5 adapters (health merged into status), each + // gated on backend readiness const server = new MCPServer({ serverInfo: { name: 'dev-agent', @@ -308,38 +272,17 @@ async function main() { logLevel, }, transport: 'stdio', - adapters: [searchAdapter, statusAdapter, inspectAdapter, refsAdapter, mapAdapter], + adapters: [searchAdapter, statusAdapter, inspectAdapter, refsAdapter, mapAdapter].map( + (adapter) => gate.wrap(adapter) + ), }); - // Start server + // Start server first — the handshake must not wait on the backend await server.start(); + console.error('[MCP] Server started; backend initializing in background'); - // Start file watcher for automatic incremental re-indexing - const incrementalIndexer = createIncrementalIndexer({ - repositoryIndexer: indexer, - repositoryPath, - graphPath: filePaths.dependencyGraph, - logger: { - info: console.error.bind(console), - warn: console.error.bind(console), - error: console.error.bind(console), - }, - }); - - watcherHandle = await startFileWatcher({ - repositoryPath, - snapshotPath: filePaths.watcherSnapshot, - onChanges: async (changed, deleted) => { - await incrementalIndexer.onChanges(changed, deleted); - // Write snapshot after each successful incremental update - await watcherHandle?.writeSnapshot(); - }, - onError: (err) => { - console.error('[MCP] File watcher error:', err); - }, - }); - - console.error('[MCP] File watcher started'); + // Kick off backend init (Antfly, index, watcher) without blocking + gate.start(); // Handle graceful shutdown const shutdown = async () => { diff --git a/packages/mcp-server/src/adapters/adapter-registry.ts b/packages/mcp-server/src/adapters/adapter-registry.ts index 32f5296..be10539 100644 --- a/packages/mcp-server/src/adapters/adapter-registry.ts +++ b/packages/mcp-server/src/adapters/adapter-registry.ts @@ -3,6 +3,7 @@ * Manages adapter lifecycle and tool execution routing */ +import { antflyUnavailableMessage, ensureAntfly } from '@prosdevlab/dev-agent-core'; import { ErrorCode } from '../server/protocol/types'; import { RateLimiter } from '../server/utils/rate-limiter'; import type { ToolAdapter } from './tool-adapter'; @@ -179,12 +180,10 @@ export class AdapterRegistry { success: false, error: { code: String(ErrorCode.ToolExecutionError), - message: this.isAntflyError(msg) - ? 'Antfly server is not reachable. Run `dev setup` to restart it.' - : msg, + message: this.isAntflyError(msg) ? antflyUnavailableMessage() : msg, recoverable: true, suggestion: this.isAntflyError(msg) - ? 'Run `dev setup` to restart the Antfly server' + ? 'Run `dev doctor` to diagnose the search backend' : 'Check the tool arguments and try again', }, }; @@ -278,65 +277,16 @@ export class AdapterRegistry { } /** - * Attempt to recover Antfly by restarting it (native first, Docker fallback) + * Attempt to recover Antfly by restarting it via the shared lifecycle + * (native first, container runtime fallback; output captured to the + * Antfly log file). */ private async tryRecoverAntfly(): Promise { - const { execSync, spawn } = await import('node:child_process'); - const antflyUrl = process.env.ANTFLY_URL ?? 'http://localhost:18080/api/v1'; - const baseUrl = antflyUrl.replace('/api/v1', ''); - - const isReady = async () => { - try { - const resp = await fetch(`${baseUrl}/api/v1/tables`, { - signal: AbortSignal.timeout(3000), - }); - return resp.ok; - } catch { - return false; - } - }; - - // Try native try { - execSync('antfly --version', { stdio: 'pipe', timeout: 5000 }); - const child = spawn( - 'antfly', - [ - 'swarm', - '--metadata-api', - 'http://0.0.0.0:18080', - '--store-api', - 'http://0.0.0.0:18381', - '--metadata-raft', - 'http://0.0.0.0:19017', - '--store-raft', - 'http://0.0.0.0:19021', - '--health-port', - '14200', - ], - { detached: true, stdio: 'ignore' } - ); - child.unref(); - - const start = Date.now(); - while (Date.now() - start < 15_000) { - if (await isReady()) return; - await new Promise((r) => setTimeout(r, 500)); - } - } catch { - // Try Docker - try { - execSync('docker start dev-agent-antfly', { stdio: 'pipe' }); - const start = Date.now(); - while (Date.now() - start < 15_000) { - if (await isReady()) return; - await new Promise((r) => setTimeout(r, 500)); - } - } catch { - // Neither worked - } + await ensureAntfly({ quiet: true }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to recover Antfly server: ${detail}`); } - - throw new Error('Failed to recover Antfly server'); } } diff --git a/packages/mcp-server/src/server/__tests__/backend-gate.test.ts b/packages/mcp-server/src/server/__tests__/backend-gate.test.ts new file mode 100644 index 0000000..3193886 --- /dev/null +++ b/packages/mcp-server/src/server/__tests__/backend-gate.test.ts @@ -0,0 +1,125 @@ +/** + * Tests for BackendGate — graceful MCP startup when the Antfly backend is + * down or slow. + * + * Regression context: the MCP server used to run Antfly auto-start, indexer + * init, and watcher catchup BEFORE completing the MCP handshake. A cold or + * unstartable Antfly killed the whole server with a bare -32000 "connection + * closed" and no diagnosable trail. The gate lets the server start + * immediately and turns backend failures into legible per-tool errors. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { MockAdapter } from '../../adapters/__tests__/mock-adapter'; +import type { ToolExecutionContext } from '../../adapters/types'; +import { BackendGate } from '../backend-gate'; + +function makeContext(): ToolExecutionContext { + return { + logger: { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + }, + config: { + repositoryPath: '/test/repo', + }, + }; +} + +describe('BackendGate', () => { + it('passes tool calls through once the backend initialized successfully', async () => { + const gate = new BackendGate(async () => {}); + const wrapped = gate.wrap(new MockAdapter()); + + const result = await wrapped.execute({ message: 'hi' }, makeContext()); + + expect(result.success).toBe(true); + }); + + it('makes the first tool call wait for a pending initialization', async () => { + let release: () => void = () => {}; + const initDone = new Promise((r) => { + release = r; + }); + const gate = new BackendGate(() => initDone); + const wrapped = gate.wrap(new MockAdapter()); + gate.start(); + + const pending = wrapped.execute({ message: 'hi' }, makeContext()); + let settled = false; + void pending.then(() => { + settled = true; + }); + await new Promise((r) => setTimeout(r, 20)); + expect(settled).toBe(false); + + release(); + const result = await pending; + expect(result.success).toBe(true); + }); + + it('returns a legible error result instead of crashing when init fails', async () => { + const gate = new BackendGate(async () => { + throw new Error('fetch failed'); + }); + const wrapped = gate.wrap(new MockAdapter()); + gate.start(); + + const result = await wrapped.execute({ message: 'hi' }, makeContext()); + + expect(result.success).toBe(false); + expect(result.error?.message).toContain('Antfly'); + expect(result.error?.message).toContain('dev doctor'); + expect(result.error?.recoverable).toBe(true); + }); + + it('retries initialization on the next call after a failure (self-healing)', async () => { + let attempts = 0; + const gate = new BackendGate(async () => { + attempts++; + if (attempts === 1) throw new Error('fetch failed'); + }); + const wrapped = gate.wrap(new MockAdapter()); + + const first = await wrapped.execute({ message: 'hi' }, makeContext()); + expect(first.success).toBe(false); + + const second = await wrapped.execute({ message: 'hi' }, makeContext()); + expect(second.success).toBe(true); + expect(attempts).toBe(2); + }); + + it('initializes only once across concurrent calls and repeated successes', async () => { + const init = vi.fn(async () => {}); + const gate = new BackendGate(init); + const wrapped = gate.wrap(new MockAdapter()); + + await Promise.all([ + wrapped.execute({ message: 'a' }, makeContext()), + wrapped.execute({ message: 'b' }, makeContext()), + ]); + await wrapped.execute({ message: 'c' }, makeContext()); + + expect(init).toHaveBeenCalledTimes(1); + }); + + it('start() kicks off init in the background without throwing on failure', async () => { + const gate = new BackendGate(async () => { + throw new Error('nope'); + }); + expect(() => gate.start()).not.toThrow(); + // Allow the rejected promise to settle — must not become an unhandled rejection. + await new Promise((r) => setTimeout(r, 10)); + }); + + it('delegates tool definition and metadata to the inner adapter', () => { + const gate = new BackendGate(async () => {}); + const inner = new MockAdapter('my_tool'); + const wrapped = gate.wrap(inner); + + expect(wrapped.getToolDefinition().name).toBe('my_tool'); + expect(wrapped.metadata.name).toBe(inner.metadata.name); + }); +}); diff --git a/packages/mcp-server/src/server/backend-gate.ts b/packages/mcp-server/src/server/backend-gate.ts new file mode 100644 index 0000000..99cf87d --- /dev/null +++ b/packages/mcp-server/src/server/backend-gate.ts @@ -0,0 +1,106 @@ +/** + * BackendGate — decouples MCP handshake from backend readiness. + * + * The MCP server must complete its stdio handshake quickly even when the + * Antfly backend is down or the index needs a long catchup. The gate wraps + * every tool adapter: the first tool call waits for backend initialization; + * a failed initialization becomes a legible per-tool error (never a process + * exit) and is retried on the next call so the server self-heals once the + * backend is reachable again. + */ + +import { antflyUnavailableMessage } from '@prosdevlab/dev-agent-core'; +import { ToolAdapter } from '../adapters/tool-adapter'; +import type { + AdapterContext, + AdapterMetadata, + ToolDefinition, + ToolExecutionContext, + ToolResult, +} from '../adapters/types'; + +export class BackendGate { + private current: Promise | null = null; + private ready = false; + + constructor(private readonly init: () => Promise) {} + + /** + * Kick off initialization in the background. Failures are logged by the + * caller's init function and surfaced per tool call — never thrown here. + */ + start(): void { + void this.ensureReady().catch(() => { + // Swallowed: the failure is stored via `current` reset and re-thrown + // to the tool call that next awaits ensureReady(). + }); + } + + /** Resolve when the backend is ready; reject with the init error if not. */ + async ensureReady(): Promise { + if (this.ready) return; + if (!this.current) { + this.current = this.init().then( + () => { + this.ready = true; + }, + (error) => { + // Reset so the next call retries initialization (self-healing). + this.current = null; + throw error; + } + ); + } + return this.current; + } + + /** Wrap an adapter so its execution waits on (and reports) backend state. */ + wrap(adapter: ToolAdapter): ToolAdapter { + return new GatedToolAdapter(this, adapter); + } +} + +class GatedToolAdapter extends ToolAdapter { + readonly metadata: AdapterMetadata; + + constructor( + private readonly gate: BackendGate, + private readonly inner: ToolAdapter + ) { + super(); + this.metadata = inner.metadata; + if (inner.validate) this.validate = inner.validate.bind(inner); + if (inner.estimateTokens) this.estimateTokens = inner.estimateTokens.bind(inner); + } + + getToolDefinition(): ToolDefinition { + return this.inner.getToolDefinition(); + } + + initialize(context: AdapterContext): Promise { + return this.inner.initialize(context); + } + + async shutdown(): Promise { + await this.inner.shutdown?.(); + } + + async execute(args: Record, context: ToolExecutionContext): Promise { + try { + await this.gate.ensureReady(); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + context.logger.error('Backend initialization failed', { error: detail }); + return { + success: false, + error: { + code: 'BACKEND_UNAVAILABLE', + message: `${antflyUnavailableMessage()} (${detail})`, + recoverable: true, + suggestion: 'Run `dev doctor` to diagnose, then retry this tool call.', + }, + }; + } + return this.inner.execute(args, context); + } +} From 059e779b6d3f33a0b263734040bdf5be0542cc80 Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 14 Jul 2026 12:30:32 -0700 Subject: [PATCH 3/5] feat(cli): add dev doctor and auto-start backend in search/refs/map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dev doctor diagnoses the stack from the CLI — Antfly install, server reachability with port-conflict detection, embedding model, repository index — and points at the captured startup log. Health checks previously lived only in the dev_status MCP tool, which is unreachable exactly when the backend is broken. search/refs/map now auto-start Antfly like index does, and setup/reset are container-runtime aware (Docker or Podman). CLI antfly utils now re-export the shared core lifecycle. --- packages/cli/src/cli.ts | 2 + packages/cli/src/commands/doctor.ts | 89 +++++ packages/cli/src/commands/map.ts | 4 + packages/cli/src/commands/refs.ts | 3 + packages/cli/src/commands/reset.ts | 11 +- packages/cli/src/commands/search.ts | 3 + packages/cli/src/commands/setup.ts | 65 ++-- .../cli/src/utils/__tests__/antfly.test.ts | 92 ------ .../cli/src/utils/__tests__/doctor.test.ts | 110 +++++++ packages/cli/src/utils/antfly.ts | 310 ++---------------- packages/cli/src/utils/doctor.ts | 112 +++++++ 11 files changed, 390 insertions(+), 411 deletions(-) create mode 100644 packages/cli/src/commands/doctor.ts delete mode 100644 packages/cli/src/utils/__tests__/antfly.test.ts create mode 100644 packages/cli/src/utils/__tests__/doctor.test.ts create mode 100644 packages/cli/src/utils/doctor.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index a34ee84..891e057 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -4,6 +4,7 @@ import chalk from 'chalk'; import { Command } from 'commander'; import { cleanCommand } from './commands/clean.js'; import { compactCommand } from './commands/compact.js'; +import { doctorCommand } from './commands/doctor.js'; import { indexCommand } from './commands/index.js'; import { mapCommand } from './commands/map.js'; import { mcpCommand } from './commands/mcp.js'; @@ -35,6 +36,7 @@ program.addCommand(storageCommand); program.addCommand(mcpCommand); program.addCommand(setupCommand); program.addCommand(resetCommand); +program.addCommand(doctorCommand); // Show help if no command provided if (process.argv.length === 2) { diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts new file mode 100644 index 0000000..8c74577 --- /dev/null +++ b/packages/cli/src/commands/doctor.ts @@ -0,0 +1,89 @@ +/** + * dev doctor — Diagnose the dev-agent stack from the CLI. + * + * Works with nothing else running: checks the Antfly install, server + * reachability (including port conflicts), embedding model, and repository + * index, and points at the captured startup log on failure. + */ + +import { execSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import * as path from 'node:path'; +import { getStorageFilePaths, getStoragePath } from '@prosdevlab/dev-agent-core'; +import chalk from 'chalk'; +import { Command } from 'commander'; +import { + detectContainerRuntime, + getAntflyLogPath, + getAntflyUrl, + getNativeVersion, + hasModel, + hasModelContainer, + isServerReady, +} from '../utils/antfly.js'; +import { loadConfig } from '../utils/config.js'; +import { runDoctorChecks } from '../utils/doctor.js'; + +const DEFAULT_MODEL = 'BAAI/bge-small-en-v1.5'; + +function antflyPort(): string { + try { + return new URL(getAntflyUrl()).port || '18080'; + } catch { + return '18080'; + } +} + +export const doctorCommand = new Command('doctor') + .description('Diagnose the dev-agent stack: Antfly install, server, model, index') + .action(async () => { + const config = await loadConfig(); + const repositoryPath = path.resolve( + config?.repository?.path || config?.repositoryPath || process.cwd() + ); + + const checks = await runDoctorChecks({ + nativeVersion: getNativeVersion, + containerRuntime: () => detectContainerRuntime(), + serverReady: () => isServerReady(), + portOwner: () => { + try { + const out = execSync(`lsof -i :${antflyPort()} -t`, { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }).trim(); + return out || null; + } catch { + return null; + } + }, + modelPresent: () => { + if (getNativeVersion()) return hasModel(DEFAULT_MODEL); + const runtime = detectContainerRuntime(); + if (runtime) return hasModelContainer(runtime, DEFAULT_MODEL); + return false; + }, + indexExists: async () => { + const storagePath = await getStoragePath(repositoryPath); + return existsSync(getStorageFilePaths(storagePath).dependencyGraph); + }, + logPath: () => getAntflyLogPath(), + }); + + console.log(); + for (const check of checks) { + const mark = check.ok ? chalk.green('✓') : chalk.red('✗'); + console.log(` ${mark} ${check.name} — ${check.detail}`); + if (!check.ok && check.hint) { + console.log(` ${chalk.dim(check.hint)}`); + } + } + console.log(); + + if (checks.every((c) => c.ok)) { + console.log(chalk.green(' All checks passed.\n')); + } else { + console.log(chalk.red(' Some checks failed — see hints above.\n')); + process.exit(1); + } + }); diff --git a/packages/cli/src/commands/map.ts b/packages/cli/src/commands/map.ts index 4930c81..bfb2e8a 100644 --- a/packages/cli/src/commands/map.ts +++ b/packages/cli/src/commands/map.ts @@ -18,6 +18,7 @@ import { createLogger } from '@prosdevlab/kero'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; +import { ensureAntfly } from '../utils/antfly.js'; import { loadConfig } from '../utils/config.js'; import { logger } from '../utils/logger.js'; @@ -81,6 +82,9 @@ Use Case: vectorStorePath: filePaths.vectors, }); + // Auto-start the search backend if it isn't running + await ensureAntfly({ quiet: true }); + // Skip embedder initialization for read-only map generation (10-20x faster) mapLogger.info('Initializing indexer (skipping embedder for fast read-only access)'); await indexer.initialize({ skipEmbedder: true }); diff --git a/packages/cli/src/commands/refs.ts b/packages/cli/src/commands/refs.ts index cb24cdd..b038d4f 100644 --- a/packages/cli/src/commands/refs.ts +++ b/packages/cli/src/commands/refs.ts @@ -14,6 +14,7 @@ import { import chalk from 'chalk'; import { Command, Option } from 'commander'; import ora from 'ora'; +import { ensureAntfly } from '../utils/antfly.js'; import { loadConfig } from '../utils/config.js'; import { logger } from '../utils/logger.js'; @@ -55,6 +56,8 @@ export const refsCommand = new Command('refs') languages: config?.repository?.languages || config?.languages, }); + // Auto-start the search backend if it isn't running + await ensureAntfly({ quiet: true }); await indexer.initialize(); const direction = options.direction as RefDirection; diff --git a/packages/cli/src/commands/reset.ts b/packages/cli/src/commands/reset.ts index 7de7849..d415eab 100644 --- a/packages/cli/src/commands/reset.ts +++ b/packages/cli/src/commands/reset.ts @@ -9,9 +9,7 @@ import { execSync } from 'node:child_process'; import * as readline from 'node:readline'; import { Command } from 'commander'; import ora from 'ora'; -import { hasDocker, isContainerExists } from '../utils/antfly.js'; - -const CONTAINER_NAME = 'dev-agent-antfly'; +import { ANTFLY_CONTAINER_NAME, containerExists, detectContainerRuntime } from '../utils/antfly.js'; async function confirm(question: string): Promise { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); @@ -41,15 +39,16 @@ export const resetCommand = new Command('reset') try { // ── Stop and remove Antfly ── - if (hasDocker() && isContainerExists()) { + const runtime = detectContainerRuntime(); + if (runtime && containerExists(runtime)) { spinner.start('Stopping Antfly container...'); try { - execSync(`docker stop ${CONTAINER_NAME}`, { stdio: 'pipe' }); + execSync(`${runtime} stop ${ANTFLY_CONTAINER_NAME}`, { stdio: 'pipe' }); } catch { // Already stopped } try { - execSync(`docker rm ${CONTAINER_NAME}`, { stdio: 'pipe' }); + execSync(`${runtime} rm ${ANTFLY_CONTAINER_NAME}`, { stdio: 'pipe' }); } catch { // Already removed } diff --git a/packages/cli/src/commands/search.ts b/packages/cli/src/commands/search.ts index bf4d5e9..bf56122 100644 --- a/packages/cli/src/commands/search.ts +++ b/packages/cli/src/commands/search.ts @@ -8,6 +8,7 @@ import { import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; +import { ensureAntfly } from '../utils/antfly.js'; import { loadConfig } from '../utils/config.js'; import { prepareFileForSearch } from '../utils/file.js'; import { logger } from '../utils/logger.js'; @@ -45,6 +46,8 @@ export const searchCommand = new Command('search') languages: config?.repository?.languages || config?.languages, }); + // Auto-start the search backend if it isn't running + await ensureAntfly({ quiet: true }); await indexer.initialize(); // If --similar-to is provided, use file content as the search query diff --git a/packages/cli/src/commands/setup.ts b/packages/cli/src/commands/setup.ts index 420c0d5..27341d3 100644 --- a/packages/cli/src/commands/setup.ts +++ b/packages/cli/src/commands/setup.ts @@ -1,8 +1,9 @@ /** * dev setup — One-time setup for dev-agent's search backend * - * Native-first, Docker fallback. Handles installation, model download, - * and server startup so users never need to run `antfly` directly. + * Native-first, container runtime (Docker or Podman) fallback. Handles + * installation, model download, and server startup so users never need to + * run `antfly` directly. */ import { execSync, spawn } from 'node:child_process'; @@ -10,17 +11,19 @@ import * as readline from 'node:readline'; import { Command } from 'commander'; import ora from 'ora'; import { + ANTFLY_CONTAINER_IMAGE, + type ContainerRuntime, + detectContainerRuntime, ensureAntfly, getAntflyUrl, - getDockerMemoryBytes, + getContainerMemoryBytes, getNativeVersion, - hasDocker, hasModel, - hasModelDocker, + hasModelContainer, hasNativeBinary, isServerReady, pullModel, - pullModelDocker, + pullModelContainer, } from '../utils/antfly.js'; const DEFAULT_MODEL = 'BAAI/bge-small-en-v1.5'; @@ -35,14 +38,14 @@ async function confirm(question: string): Promise { }); } -function dockerPull(image: string): Promise { +function containerPull(runtime: ContainerRuntime, image: string): Promise { return new Promise((resolve, reject) => { - const child = spawn('docker', ['pull', '--platform', 'linux/amd64', image], { + const child = spawn(runtime, ['pull', '--platform', 'linux/amd64', image], { stdio: 'pipe', }); child.on('close', (code) => { if (code === 0) resolve(); - else reject(new Error(`docker pull exited with code ${code}`)); + else reject(new Error(`${runtime} pull exited with code ${code}`)); }); child.on('error', reject); }); @@ -70,10 +73,14 @@ function ensureModel(spinner: ReturnType, model: string): void { } } -function ensureModelDocker(spinner: ReturnType, model: string): void { - if (!hasModelDocker(model)) { +function ensureModelContainer( + spinner: ReturnType, + runtime: ContainerRuntime, + model: string +): void { + if (!hasModelContainer(runtime, model)) { console.log(` Pulling embedding model: ${model}`); - pullModelDocker(model); + pullModelContainer(runtime, model); spinner.succeed(`Embedding model ready: ${model}`); } else { spinner.succeed(`Embedding model ready: ${model}`); @@ -83,7 +90,7 @@ function ensureModelDocker(spinner: ReturnType, model: string): void export const setupCommand = new Command('setup') .description('One-time setup: install search backend and embedding model') .option('--model ', 'Termite embedding model', DEFAULT_MODEL) - .option('--docker', 'Use Docker instead of native binary', false) + .option('--docker', 'Use a container runtime (Docker or Podman) instead of native binary', false) .action(async (options) => { const model = options.model as string; const useDocker = options.docker as boolean; @@ -94,11 +101,12 @@ export const setupCommand = new Command('setup') if (await isServerReady()) { spinner.succeed('Antfly already running'); - // Ensure model — detect if running via Docker or native + // Ensure model — detect if running via container or native + const runtime = detectContainerRuntime(); if (hasNativeBinary() && !useDocker) { ensureModel(spinner, model); - } else if (hasDocker()) { - ensureModelDocker(spinner, model); + } else if (runtime) { + ensureModelContainer(spinner, runtime, model); } console.log('\n Setup complete!'); @@ -106,24 +114,27 @@ export const setupCommand = new Command('setup') return; } - // ── Docker (explicit flag) ── + // ── Container runtime (explicit flag) ── if (useDocker) { - if (!hasDocker()) { - spinner.fail('Docker is not available. Install Docker or run without --docker.'); + const runtime = detectContainerRuntime(); + if (!runtime) { + spinner.fail( + 'No container runtime available. Install Docker or Podman, or run without --docker.' + ); process.exit(1); } - const dockerMem = getDockerMemoryBytes(); - if (dockerMem && dockerMem < 4 * 1024 * 1024 * 1024) { - const memGB = (dockerMem / (1024 * 1024 * 1024)).toFixed(1); + const containerMem = getContainerMemoryBytes(runtime); + if (containerMem && containerMem < 4 * 1024 * 1024 * 1024) { + const memGB = (containerMem / (1024 * 1024 * 1024)).toFixed(1); spinner.warn( - `Docker has only ${memGB}GB memory. Increase to 8GB+ in Docker Desktop → Settings → Resources.` + `${runtime} has only ${memGB}GB memory. Increase the VM allocation to 8GB+.` ); } spinner.start('Pulling Antfly image...'); try { - await dockerPull(getDockerImage()); + await containerPull(runtime, ANTFLY_CONTAINER_IMAGE); spinner.succeed('Antfly image ready'); } catch { spinner.succeed('Antfly image available'); @@ -133,7 +144,7 @@ export const setupCommand = new Command('setup') await ensureAntfly({ quiet: true }); spinner.succeed(`Antfly running on ${getAntflyUrl()}`); - ensureModelDocker(spinner, model); + ensureModelContainer(spinner, runtime, model); } else if (hasNativeBinary()) { // ── Native (default) ── const version = getNativeVersion(); @@ -182,7 +193,3 @@ export const setupCommand = new Command('setup') process.exit(1); } }); - -function getDockerImage(): string { - return 'ghcr.io/antflydb/antfly:latest'; -} diff --git a/packages/cli/src/utils/__tests__/antfly.test.ts b/packages/cli/src/utils/__tests__/antfly.test.ts deleted file mode 100644 index c2d3888..0000000 --- a/packages/cli/src/utils/__tests__/antfly.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Tests for antfly utility helpers. - * - * Regression for: hasModel() false positive when antfly termite list defaulted - * to ~/.termite/models (different from the server's ~/.antfly/models), causing - * "Embedding model ready" in `dev setup` but "model not found" in `dev index`. - */ - -import { describe, expect, it } from 'vitest'; - -// modelPresentInOutput is not exported — test via the exported path by extracting -// the pure logic into a local copy that mirrors the implementation exactly. -// This keeps the test focused on the matching logic without requiring CLI env. - -function modelPresentInOutput(model: string, output: string): boolean { - if (output.includes(model)) return true; - - const shortName = model.split('/').pop() ?? model; - const escaped = shortName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - return new RegExp(`(? { - const FULL_NAME = 'BAAI/bge-small-en-v1.5'; - const SHORT_NAME = 'bge-small-en-v1.5'; - - // Simulates `antfly termite list --models-dir ~/.antfly/models` output when - // the model IS present (full name in NAME column, also in SOURCE column). - const PRESENT_OUTPUT = `Local models in /Users/dev/.antfly/models: - -NAME TYPE SIZE VARIANTS SOURCE -BAAI/bge-small-en-v1.5 embedder 127.8 MB BAAI/bge-small-en-v1.5 -`; - - // Output when NO models are installed (the bug scenario: server's models-dir - // is empty, but ~/.termite/models has the model — the old code read the wrong - // directory and would never see "No models found"). - const EMPTY_OUTPUT = `Local models in /Users/dev/.antfly/models: - -NAME TYPE SIZE VARIANTS SOURCE -No models found locally. - -Use 'antfly termite pull ' to download models. -Use 'antfly termite list --remote' to see available models. -`; - - // Output with a DIFFERENT model that happens to contain the short name as a - // suffix — the old substring check would incorrectly return true here. - const OTHER_MODEL_OUTPUT = `Local models in /Users/dev/.antfly/models: - -NAME TYPE SIZE VARIANTS SOURCE -vendor/other-bge-small-en-v1.5 embedder 200.0 MB vendor/other-bge-small-en-v1.5 -`; - - it('returns true when full model name is present in output', () => { - expect(modelPresentInOutput(FULL_NAME, PRESENT_OUTPUT)).toBe(true); - }); - - it('returns true when only short name is present as a standalone token', () => { - const outputWithShortName = `Local models:\n\n${SHORT_NAME} embedder 127 MB\n`; - expect(modelPresentInOutput(FULL_NAME, outputWithShortName)).toBe(true); - }); - - it('returns false when models directory is empty (server has no models)', () => { - // This is the core regression: old code checked ~/.termite/models which had - // the model, new code checks ~/.antfly/models which was empty. When empty, - // hasModel must return false so pullModel is invoked. - expect(modelPresentInOutput(FULL_NAME, EMPTY_OUTPUT)).toBe(false); - }); - - it('returns false when a different model shares the short name as a suffix', () => { - // Old bug: output.includes("bge-small-en-v1.5") matched - // "vendor/other-bge-small-en-v1.5" — false positive. - expect(modelPresentInOutput(FULL_NAME, OTHER_MODEL_OUTPUT)).toBe(false); - }); - - it('returns false for completely unrelated output', () => { - expect(modelPresentInOutput(FULL_NAME, 'No models found locally.')).toBe(false); - }); - - it('handles model names without an org prefix', () => { - // model = "mxbai-embed-large-v1" (no slash) - const bareModel = 'mxbai-embed-large-v1'; - const output = `NAME TYPE\nmxbai-embed-large-v1 embedder\n`; - expect(modelPresentInOutput(bareModel, output)).toBe(true); - }); - - it('handles bare model not present', () => { - const bareModel = 'mxbai-embed-large-v1'; - expect(modelPresentInOutput(bareModel, EMPTY_OUTPUT)).toBe(false); - }); -}); diff --git a/packages/cli/src/utils/__tests__/doctor.test.ts b/packages/cli/src/utils/__tests__/doctor.test.ts new file mode 100644 index 0000000..634e060 --- /dev/null +++ b/packages/cli/src/utils/__tests__/doctor.test.ts @@ -0,0 +1,110 @@ +/** + * Tests for the `dev doctor` check runner. + * + * Context: health checks previously lived only in the dev_status MCP tool — + * useless when the MCP server itself is down because Antfly won't start. + * `dev doctor` is the CLI escape hatch; runDoctorChecks is its pure core + * with all system probes injected. + */ + +import { describe, expect, it } from 'vitest'; +import { type DoctorProbes, runDoctorChecks } from '../doctor.js'; + +function healthyProbes(overrides: Partial = {}): DoctorProbes { + return { + nativeVersion: () => 'antfly 1.4.0', + containerRuntime: () => null, + serverReady: async () => true, + portOwner: () => null, + modelPresent: () => true, + indexExists: async () => true, + logPath: () => '/home/user/.antfly/antfly.log', + ...overrides, + }; +} + +describe('runDoctorChecks', () => { + it('reports all checks ok on a healthy system', async () => { + const checks = await runDoctorChecks(healthyProbes()); + + expect(checks.length).toBeGreaterThanOrEqual(4); + expect(checks.every((c) => c.ok)).toBe(true); + }); + + it('fails the install check with setup guidance when nothing is installed', async () => { + const checks = await runDoctorChecks( + healthyProbes({ + nativeVersion: () => null, + containerRuntime: () => null, + serverReady: async () => false, + modelPresent: () => false, + }) + ); + + const install = checks.find((c) => c.name.includes('installed')); + expect(install?.ok).toBe(false); + expect(install?.hint).toContain('dev setup'); + }); + + it('treats a container runtime as a valid install when the binary is missing', async () => { + const checks = await runDoctorChecks( + healthyProbes({ + nativeVersion: () => null, + containerRuntime: () => 'podman', + }) + ); + + const install = checks.find((c) => c.name.includes('installed')); + expect(install?.ok).toBe(true); + expect(install?.detail).toContain('podman'); + }); + + it('surfaces a port conflict when the server is down but the port is taken', async () => { + const checks = await runDoctorChecks( + healthyProbes({ + serverReady: async () => false, + portOwner: () => '12345', + }) + ); + + const server = checks.find((c) => c.name.includes('server')); + expect(server?.ok).toBe(false); + expect(server?.detail).toContain('12345'); + }); + + it('points at the startup log when the server is down with no port conflict', async () => { + const checks = await runDoctorChecks( + healthyProbes({ + serverReady: async () => false, + }) + ); + + const server = checks.find((c) => c.name.includes('server')); + expect(server?.ok).toBe(false); + expect(server?.hint).toContain('/home/user/.antfly/antfly.log'); + }); + + it('fails the index check with `dev index` guidance when no index exists', async () => { + const checks = await runDoctorChecks( + healthyProbes({ + indexExists: async () => false, + }) + ); + + const index = checks.find((c) => c.name.includes('index')); + expect(index?.ok).toBe(false); + expect(index?.hint).toContain('dev index'); + }); + + it('skips the model check when neither binary nor runtime can list models', async () => { + const checks = await runDoctorChecks( + healthyProbes({ + nativeVersion: () => null, + containerRuntime: () => null, + }) + ); + + const model = checks.find((c) => c.name.includes('model')); + expect(model).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/utils/antfly.ts b/packages/cli/src/utils/antfly.ts index bb31d52..320573a 100644 --- a/packages/cli/src/utils/antfly.ts +++ b/packages/cli/src/utils/antfly.ts @@ -1,286 +1,28 @@ /** - * Antfly server lifecycle management + * Antfly server lifecycle — re-exported from core. * - * Docker-first, native fallback. The user never needs to run `antfly` directly. - */ - -import { execSync, spawn } from 'node:child_process'; -import { homedir } from 'node:os'; -import { join } from 'node:path'; -import { logger } from './logger.js'; - -const DEFAULT_ANTFLY_URL = process.env.ANTFLY_URL ?? 'http://localhost:18080/api/v1'; -const CONTAINER_NAME = 'dev-agent-antfly'; -const DOCKER_IMAGE = 'ghcr.io/antflydb/antfly:latest'; -const DOCKER_PORT = 18080; -const STARTUP_TIMEOUT_MS = 30_000; -const POLL_INTERVAL_MS = 500; - -/** - * The Termite models directory used by the running Antfly swarm server. - * - * `antfly swarm` uses `--data-dir` (default: ~/.antfly) as its root for all - * storage, including Termite models at {data-dir}/models. - * `antfly termite list/pull` defaults to --models-dir ~/.termite/models, which - * is a DIFFERENT path. We must always pass --models-dir explicitly so that - * `pullModel` and `hasModel` operate on the same directory the server uses. - */ -const ANTFLY_DATA_DIR = process.env.ANTFLY_DATA_DIR ?? join(homedir(), '.antfly'); -const TERMITE_MODELS_DIR = join(ANTFLY_DATA_DIR, 'models'); - -/** - * Ensure antfly is running. Auto-starts if needed. - * - * Priority: Docker container → native binary → error with guidance. - */ -export async function ensureAntfly(options?: { quiet?: boolean }): Promise { - const url = getAntflyUrl(); - - // 1. Already running? - if (await isServerReady(url)) { - return url; - } - - // 2. Try native first (no VM overhead, better performance) - if (hasNativeBinary()) { - if (!options?.quiet) logger.info('Starting Antfly server...'); - // Use custom ports to avoid 8080 conflicts (Docker, other services). - // metadata-api on 18080 (our default), store-api on 18381, raft on 19017/19021. - // --data-dir is passed explicitly so the server's embedded Termite node stores - // models in the same directory that pullModel/hasModel use (TERMITE_MODELS_DIR). - const child = spawn( - 'antfly', - [ - 'swarm', - '--data-dir', - ANTFLY_DATA_DIR, - '--metadata-api', - 'http://0.0.0.0:18080', - '--store-api', - 'http://0.0.0.0:18381', - '--metadata-raft', - 'http://0.0.0.0:19017', - '--store-raft', - 'http://0.0.0.0:19021', - '--health-port', - '14200', - ], - { - detached: true, - stdio: 'ignore', - } - ); - child.unref(); - - await waitForServer(url); - if (!options?.quiet) logger.info(`Antfly running on ${url}`); - return url; - } - - // 3. Docker fallback - if (hasDocker()) { - if (isContainerExists()) { - if (!options?.quiet) logger.info('Starting Antfly container...'); - execSync(`docker start ${CONTAINER_NAME}`, { stdio: 'pipe' }); - } else { - if (!options?.quiet) logger.info('Starting Antfly via Docker...'); - execSync( - `docker run -d --name ${CONTAINER_NAME} -p ${DOCKER_PORT}:8080 -m 8g --platform linux/amd64 ${DOCKER_IMAGE} swarm`, - { stdio: 'pipe' } - ); - } - - await waitForServer(url); - if (!options?.quiet) logger.info(`Antfly running on ${url}`); - return url; - } - - // 4. Nothing available - throw new Error( - 'Antfly is not installed. Run `dev setup` to install:\n' + - ' brew install --cask antflydb/antfly/antfly' - ); -} - -export function getAntflyUrl(): string { - return process.env.ANTFLY_URL ?? DEFAULT_ANTFLY_URL; -} - -export function hasDocker(): boolean { - try { - execSync('docker info', { stdio: 'pipe', timeout: 5000 }); - return true; - } catch { - return false; - } -} - -/** - * Get Docker's total allocated memory in bytes. - */ -export function getDockerMemoryBytes(): number | null { - try { - const output = execSync('docker info 2>&1', { encoding: 'utf-8', timeout: 5000 }); - const match = output.match(/memTotal:\s*(\d+)/); - return match ? Number(match[1]) : null; - } catch { - return null; - } -} - -export function hasNativeBinary(): boolean { - try { - execSync('antfly --version', { stdio: 'pipe', timeout: 5000 }); - return true; - } catch { - return false; - } -} - -export function isContainerExists(): boolean { - try { - const result = execSync(`docker ps -a --filter name=${CONTAINER_NAME} --format "{{.Names}}"`, { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }); - return result.trim() === CONTAINER_NAME; - } catch { - return false; - } -} - -export async function isServerReady(url?: string): Promise { - const baseUrl = (url ?? getAntflyUrl()).replace('/api/v1', ''); - try { - const resp = await fetch(`${baseUrl}/api/v1/tables`, { signal: AbortSignal.timeout(3000) }); - return resp.ok; - } catch { - return false; - } -} - -async function waitForServer(url: string): Promise { - const start = Date.now(); - while (Date.now() - start < STARTUP_TIMEOUT_MS) { - if (await isServerReady(url)) return; - await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); - } - // Check if port is in use by another process - try { - const { execSync: exec } = await import('node:child_process'); - const lsof = exec(`lsof -i :${DOCKER_PORT} -t`, { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }).trim(); - if (lsof) { - throw new Error( - `Port ${DOCKER_PORT} is already in use (pid: ${lsof}).\n` + - ` Check: lsof -i :${DOCKER_PORT}\n` + - ` Or set: ANTFLY_URL=http://localhost:/api/v1` - ); - } - } catch (e) { - if (e instanceof Error && e.message.includes('Port')) throw e; - } - - throw new Error( - `Antfly server did not start within ${STARTUP_TIMEOUT_MS / 1000}s.\n` + - ` Try: dev reset && dev setup` - ); -} - -/** - * Get the antfly version (native binary). - */ -export function getNativeVersion(): string | null { - try { - return execSync('antfly --version', { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }).trim(); - } catch { - return null; - } -} - -/** - * Pull a Termite embedding model (native binary). - * - * Always targets TERMITE_MODELS_DIR so the model ends up in the same directory - * the running Antfly swarm server uses for its embedded Termite node. - */ -export function pullModel(model: string): void { - execSync(`antfly termite pull --models-dir ${TERMITE_MODELS_DIR} ${model}`, { - stdio: 'inherit', - }); -} - -/** - * Pull a Termite embedding model inside the Docker container. - * Uses stdio: 'inherit' so Antfly's native progress output shows through. - */ -export function pullModelDocker(model: string): void { - execSync(`docker exec ${CONTAINER_NAME} /antfly termite pull ${model}`, { stdio: 'inherit' }); -} - -/** - * Check if a Termite model is available in the directory used by the running - * Antfly swarm server (TERMITE_MODELS_DIR = ~/.antfly/models by default). - * - * Checks for the full model name first (e.g. "BAAI/bge-small-en-v1.5"), then - * the short name as a whole word (e.g. "bge-small-en-v1.5"). Previously used - * a simple substring match on the short name, which caused false positives when - * `antfly termite list` defaulted to ~/.termite/models — a different directory - * from the one the server reads, so the model appeared present but was not - * available to the server during embedding. - */ -export function hasModel(model: string): boolean { - try { - const output = execSync(`antfly termite list --models-dir ${TERMITE_MODELS_DIR}`, { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }); - return modelPresentInOutput(model, output); - } catch { - return false; - } -} - -/** - * Check if a Termite model is available inside the Docker container. - * - * Checks for the full model name first (e.g. "BAAI/bge-small-en-v1.5"), then - * the short name as a whole word (e.g. "bge-small-en-v1.5"). Simple substring - * matching on the short name was causing false positives when other models or - * partial download records shared the suffix. - */ -export function hasModelDocker(model: string): boolean { - try { - const output = execSync(`docker exec ${CONTAINER_NAME} /antfly termite list`, { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }); - return modelPresentInOutput(model, output); - } catch { - return false; - } -} - -/** - * Return true when the model name is present in `antfly termite list` output. - * - * Strategy (most-specific first): - * 1. Full name exact match — "BAAI/bge-small-en-v1.5" appears verbatim. - * 2. Short name word-boundary — "bge-small-en-v1.5" appears as a whole token - * (not as a suffix of a different model name). - */ -function modelPresentInOutput(model: string, output: string): boolean { - // Full name check (covers "BAAI/bge-small-en-v1.5" style output) - if (output.includes(model)) return true; - - // Short name check with word-boundary anchors so "bge-small-en-v1.5" does not - // match inside "other-bge-small-en-v1.5" or a partial download entry. - const shortName = model.split('/').pop() ?? model; - const escaped = shortName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - return new RegExp(`(? string | null; + /** Detected container runtime, or null. */ + containerRuntime: () => ContainerRuntime | null; + /** Is the Antfly HTTP API responding? */ + serverReady: () => Promise; + /** PID(s) holding the Antfly port, or null when free. */ + portOwner: () => string | null; + /** Is the embedding model present in the server's models dir? */ + modelPresent: () => boolean; + /** Does an index exist for the current repository? */ + indexExists: () => Promise; + /** Path to the captured Antfly startup log. */ + logPath: () => string; +} + +export async function runDoctorChecks(probes: DoctorProbes): Promise { + const checks: DoctorCheck[] = []; + + const version = probes.nativeVersion(); + const runtime = probes.containerRuntime(); + + if (version) { + checks.push({ + name: 'Antfly installed', + ok: true, + detail: `native binary (${version})`, + }); + } else if (runtime) { + checks.push({ + name: 'Antfly installed', + ok: true, + detail: `container runtime available (${runtime})`, + }); + } else { + checks.push({ + name: 'Antfly installed', + ok: false, + detail: 'no native binary and no container runtime (docker/podman) found', + hint: 'Run `dev setup` to install Antfly', + }); + } + + const ready = await probes.serverReady(); + if (ready) { + checks.push({ name: 'Antfly server', ok: true, detail: 'reachable' }); + } else { + const owner = probes.portOwner(); + if (owner) { + checks.push({ + name: 'Antfly server', + ok: false, + detail: `not responding, but the port is held by another process (pid: ${owner})`, + hint: 'Free the port or set ANTFLY_URL to a different port, then retry', + }); + } else { + checks.push({ + name: 'Antfly server', + ok: false, + detail: 'not running', + hint: `Run \`dev setup\` to start it; startup output is captured in ${probes.logPath()}`, + }); + } + } + + // Model presence can only be probed when something can run `antfly termite list`. + if (version || runtime) { + const model = probes.modelPresent(); + checks.push( + model + ? { name: 'Embedding model', ok: true, detail: 'present' } + : { + name: 'Embedding model', + ok: false, + detail: 'not found in the server models directory', + hint: 'Run `dev setup` to pull the embedding model', + } + ); + } + + const hasIndex = await probes.indexExists(); + checks.push( + hasIndex + ? { name: 'Repository index', ok: true, detail: 'present' } + : { + name: 'Repository index', + ok: false, + detail: 'no index found for this repository', + hint: 'Run `dev index` to build it', + } + ); + + return checks; +} From ec3f82235c64bb3dc382341c039e2a023915812c Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 14 Jul 2026 12:30:41 -0700 Subject: [PATCH 4/5] chore: add changeset and release notes for backend resilience --- .changeset/antfly-lifecycle-resilience.md | 12 ++++++++++++ website/content/latest-version.ts | 10 +++++----- website/content/updates/index.mdx | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 .changeset/antfly-lifecycle-resilience.md diff --git a/.changeset/antfly-lifecycle-resilience.md b/.changeset/antfly-lifecycle-resilience.md new file mode 100644 index 0000000..4aeb5e2 --- /dev/null +++ b/.changeset/antfly-lifecycle-resilience.md @@ -0,0 +1,12 @@ +--- +'@prosdevlab/dev-agent': minor +--- + +Antfly lifecycle resilience: graceful MCP startup, `dev doctor`, Podman support, captured startup logs + +- **MCP server no longer dies when Antfly is down.** The stdio handshake completes immediately; backend init (Antfly auto-start, index load, watcher catchup) runs behind a gate. Tool calls wait for it, and a failed init returns a legible error pointing at `dev doctor` — with automatic retry on the next call once the backend is reachable. +- **New `dev doctor` command** diagnoses the stack from the CLI (install, server reachability, port conflicts, embedding model, repository index) — works when the MCP server itself can't start. +- **Podman support.** All container fallbacks detect Docker or Podman instead of assuming Docker. +- **Antfly startup output is captured** to `~/.antfly/antfly.log` (rotated at 5MB) instead of being discarded, so failed starts are diagnosable. +- **`dev search`, `dev refs`, and `dev map` auto-start the backend** like `dev index` already did. +- One shared lifecycle implementation in core replaces three divergent copies (CLI utils, MCP entry point, adapter registry) — the MCP copies were missing `--data-dir` and port-conflict detection. diff --git a/website/content/latest-version.ts b/website/content/latest-version.ts index 5f8f2cc..9a17e97 100644 --- a/website/content/latest-version.ts +++ b/website/content/latest-version.ts @@ -4,10 +4,10 @@ */ export const latestVersion = { - version: '0.12.2', - title: 'Reverse Callee Index for dev_refs', - date: 'April 2, 2026', + version: '0.13.0', + title: 'Backend Resilience + dev doctor', + date: 'July 14, 2026', summary: - 'dev_refs callers finally work — reverse callee index with 4,000+ entries, compound keys, class aggregation, and O(1) lookups.', - link: '/updates#v0122--reverse-callee-index-for-dev_refs', + 'The MCP server survives a down search backend — graceful startup, self-healing tool calls, new dev doctor health checks, Podman support, and captured Antfly logs.', + link: '/updates#v0130--backend-resilience--dev-doctor', } as const; diff --git a/website/content/updates/index.mdx b/website/content/updates/index.mdx index 9ecfdcf..0f1e187 100644 --- a/website/content/updates/index.mdx +++ b/website/content/updates/index.mdx @@ -9,6 +9,21 @@ What's new in dev-agent. We ship improvements regularly to help AI assistants un --- +## v0.13.0 — Backend Resilience + dev doctor + +*July 14, 2026* + +**The MCP server survives a down search backend, and `dev doctor` tells you why.** + +- **Graceful MCP startup:** the server completes its handshake instantly even when Antfly is down or the index needs a long catchup. Tool calls wait for the backend, and failures return a clear error instead of the connection dying with `-32000` +- **Self-healing:** a failed backend init is retried on the next tool call — start Antfly (or fix the issue) and the running MCP session recovers without a restart +- **New `dev doctor`:** CLI health checks for the Antfly install, server reachability, port conflicts, embedding model, and repository index — works when nothing else does +- **Podman support:** container fallbacks now detect Docker or Podman +- **Captured startup logs:** Antfly's output lands in `~/.antfly/antfly.log` instead of being discarded, so failed starts are diagnosable +- **Consistent auto-start:** `dev search`, `dev refs`, and `dev map` now start the backend automatically, matching `dev index` + +--- + ## v0.12.2 — Reverse Callee Index for dev_refs *April 2, 2026* From 898c1ca6a05919af1fc88327b072c370efde1990 Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 14 Jul 2026 12:38:51 -0700 Subject: [PATCH 5/5] fix: address code review findings with defense-in-depth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Model names are validated against an allowlist pattern AND all termite pull/list calls use execFileSync argv arrays (no shell), so neither dev setup --model input nor ANTFLY_DATA_DIR can inject commands or break on paths with spaces - startAntflyProcess closes the log fd in finally — BackendGate retries init on every failed tool call, so a leaked fd per attempt would accumulate toward EMFILE - MCP shutdown awaits gate.waitForIdle() before closing the indexer, eliminating the use-after-close race with an in-flight backend init; waitForIdle never triggers a new init attempt - The gated init closure guards indexer.initialize() and the file watcher so a retry after partial failure cannot double-init or leak a second watcher - GatedToolAdapter delegates healthCheck; dev doctor memoizes container runtime detection (avoids up to ~20s of redundant docker/podman probes) --- packages/cli/src/commands/doctor.ts | 12 ++- .../src/antfly/__tests__/lifecycle.test.ts | 32 ++++++++ packages/core/src/antfly/lifecycle.ts | 74 +++++++++++++------ packages/mcp-server/bin/dev-agent-mcp.ts | 59 +++++++++------ .../src/server/__tests__/backend-gate.test.ts | 38 ++++++++++ .../mcp-server/src/server/backend-gate.ts | 12 +++ 6 files changed, 178 insertions(+), 49 deletions(-) diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts index 8c74577..783e36a 100644 --- a/packages/cli/src/commands/doctor.ts +++ b/packages/cli/src/commands/doctor.ts @@ -42,9 +42,17 @@ export const doctorCommand = new Command('doctor') config?.repository?.path || config?.repositoryPath || process.cwd() ); + // Memoized: docker/podman probes carry a 5s timeout each, and two of the + // checks below need the runtime. + let cachedRuntime: ReturnType | undefined; + const runtimeOnce = () => { + if (cachedRuntime === undefined) cachedRuntime = detectContainerRuntime(); + return cachedRuntime; + }; + const checks = await runDoctorChecks({ nativeVersion: getNativeVersion, - containerRuntime: () => detectContainerRuntime(), + containerRuntime: runtimeOnce, serverReady: () => isServerReady(), portOwner: () => { try { @@ -59,7 +67,7 @@ export const doctorCommand = new Command('doctor') }, modelPresent: () => { if (getNativeVersion()) return hasModel(DEFAULT_MODEL); - const runtime = detectContainerRuntime(); + const runtime = runtimeOnce(); if (runtime) return hasModelContainer(runtime, DEFAULT_MODEL); return false; }, diff --git a/packages/core/src/antfly/__tests__/lifecycle.test.ts b/packages/core/src/antfly/__tests__/lifecycle.test.ts index 3457368..c7212a9 100644 --- a/packages/core/src/antfly/__tests__/lifecycle.test.ts +++ b/packages/core/src/antfly/__tests__/lifecycle.test.ts @@ -15,6 +15,7 @@ import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { antflyUnavailableMessage, + assertValidModelName, buildSwarmArgs, containerRunCommand, containerStartCommand, @@ -82,6 +83,37 @@ describe('startAntflyProcess', () => { }); }); +describe('startAntflyProcess error path', () => { + it('throws (and does not hang) when the spawn command is invalid', () => { + const dir = mkdtempSync(join(tmpdir(), 'antfly-lifecycle-err-')); + try { + // Empty command makes spawn throw synchronously — the log fd must be + // released via finally, not leaked (BackendGate retries init on every + // failed tool call, so a leak here accumulates toward EMFILE). + expect(() => + startAntflyProcess({ command: '', args: [], logPath: join(dir, 'antfly.log') }) + ).toThrow(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('assertValidModelName', () => { + it('accepts real Termite model names', () => { + expect(() => assertValidModelName('BAAI/bge-small-en-v1.5')).not.toThrow(); + expect(() => assertValidModelName('mxbai-embed-large-v1')).not.toThrow(); + }); + + it('rejects shell metacharacters and option-like names', () => { + expect(() => assertValidModelName('x; rm -rf ~')).toThrow(/model name/i); + expect(() => assertValidModelName('$(whoami)')).toThrow(/model name/i); + expect(() => assertValidModelName('a && b')).toThrow(/model name/i); + expect(() => assertValidModelName('--models-dir=/tmp/evil')).toThrow(/model name/i); + expect(() => assertValidModelName('')).toThrow(/model name/i); + }); +}); + describe('detectContainerRuntime', () => { it('prefers docker when available', () => { const probe = (cmd: string) => cmd.startsWith('docker'); diff --git a/packages/core/src/antfly/lifecycle.ts b/packages/core/src/antfly/lifecycle.ts index 67a42fe..380a676 100644 --- a/packages/core/src/antfly/lifecycle.ts +++ b/packages/core/src/antfly/lifecycle.ts @@ -10,7 +10,7 @@ * so failed starts are diagnosable after the fact. */ -import { execSync, spawn } from 'node:child_process'; +import { execFileSync, execSync, spawn } from 'node:child_process'; import { closeSync, mkdirSync, openSync, renameSync, statSync, writeSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; @@ -149,18 +149,23 @@ export interface StartAntflyProcessOptions { export function startAntflyProcess(options: StartAntflyProcessOptions = {}): void { const logPath = options.logPath ?? getAntflyLogPath(); const fd = openLogFile(logPath); - writeSync( - fd, - `[dev-agent] starting ${options.command ?? 'antfly'} at ${new Date().toISOString()}\n` - ); - - const child = spawn(options.command ?? 'antfly', options.args ?? buildSwarmArgs(), { - detached: true, - stdio: ['ignore', fd, fd], - }); - child.unref(); - // The child holds its own copy of the descriptor. - closeSync(fd); + // The finally is load-bearing: callers (BackendGate) retry on every failed + // tool call, so a leaked fd per attempt would accumulate toward EMFILE. + try { + writeSync( + fd, + `[dev-agent] starting ${options.command ?? 'antfly'} at ${new Date().toISOString()}\n` + ); + + const child = spawn(options.command ?? 'antfly', options.args ?? buildSwarmArgs(), { + detached: true, + stdio: ['ignore', fd, fd], + }); + child.unref(); + } finally { + // The child holds its own copy of the descriptor. + closeSync(fd); + } } function openLogFile(logPath: string): number { @@ -311,30 +316,51 @@ export function getNativeVersion(): string | null { } } +/** + * Validate a Termite model name before it reaches a child process. + * + * Names look like "BAAI/bge-small-en-v1.5" or "mxbai-embed-large-v1". + * Rejecting anything else blocks shell metacharacters and option-like + * values ("--models-dir=...") even though the exec calls below already use + * argv arrays (defense-in-depth: the value also flows into container exec). + */ +export function assertValidModelName(model: string): void { + if (!/^[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(model)) { + throw new Error( + `Invalid model name: ${JSON.stringify(model)}. ` + + 'Expected letters, digits, ".", "_", "-", "/" (e.g. "BAAI/bge-small-en-v1.5").' + ); + } +} + /** * Pull a Termite embedding model (native binary), targeting the models dir * used by the running swarm server. */ export function pullModel(model: string): void { - execSync(`antfly termite pull --models-dir ${getTermiteModelsDir()} ${model}`, { + assertValidModelName(model); + execFileSync('antfly', ['termite', 'pull', '--models-dir', getTermiteModelsDir(), model], { stdio: 'inherit', }); } /** Pull a Termite model inside the container. */ export function pullModelContainer(runtime: ContainerRuntime, model: string): void { - execSync(`${runtime} exec ${ANTFLY_CONTAINER_NAME} /antfly termite pull ${model}`, { + assertValidModelName(model); + execFileSync(runtime, ['exec', ANTFLY_CONTAINER_NAME, '/antfly', 'termite', 'pull', model], { stdio: 'inherit', }); } /** Check model presence in the server's models dir (native binary). */ export function hasModel(model: string): boolean { + assertValidModelName(model); try { - const output = execSync(`antfly termite list --models-dir ${getTermiteModelsDir()}`, { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }); + const output = execFileSync( + 'antfly', + ['termite', 'list', '--models-dir', getTermiteModelsDir()], + { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] } + ); return modelPresentInOutput(model, output); } catch { return false; @@ -343,11 +369,13 @@ export function hasModel(model: string): boolean { /** Check model presence inside the container. */ export function hasModelContainer(runtime: ContainerRuntime, model: string): boolean { + assertValidModelName(model); try { - const output = execSync(`${runtime} exec ${ANTFLY_CONTAINER_NAME} /antfly termite list`, { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }); + const output = execFileSync( + runtime, + ['exec', ANTFLY_CONTAINER_NAME, '/antfly', 'termite', 'list'], + { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] } + ); return modelPresentInOutput(model, output); } catch { return false; diff --git a/packages/mcp-server/bin/dev-agent-mcp.ts b/packages/mcp-server/bin/dev-agent-mcp.ts index 35b02bd..f3e11be 100644 --- a/packages/mcp-server/bin/dev-agent-mcp.ts +++ b/packages/mcp-server/bin/dev-agent-mcp.ts @@ -149,6 +149,7 @@ async function startupCatchup( async function main() { let watcherHandle: FileWatcherHandle | undefined; + let indexerInitialized = false; try { // Get centralized storage paths (filesystem only — no Antfly needed) @@ -167,6 +168,8 @@ async function main() { // completes its handshake immediately; the first tool call waits for // this, and a failure here becomes a legible tool error — never a // process exit (the old behavior surfaced as a bare -32000). + // The gate retries this closure after a failure, so every step is + // guarded to be safe on re-entry (a second watcher would leak the first). const gate = new BackendGate(async () => { console.error('[MCP] Initializing backend...'); try { @@ -174,7 +177,10 @@ async function main() { deps: { logger: { info: (msg: string) => console.error(`[MCP] ${msg}`) } }, }); - await indexer.initialize(); + if (!indexerInitialized) { + await indexer.initialize(); + indexerInitialized = true; + } await saveMetadata(storagePath, repositoryPath); // Startup catchup: index or update since last snapshot @@ -186,29 +192,31 @@ async function main() { ); // Start file watcher only once the backend is usable - const incrementalIndexer = createIncrementalIndexer({ - repositoryIndexer: indexer, - repositoryPath, - graphPath: filePaths.dependencyGraph, - logger: { - info: console.error.bind(console), - warn: console.error.bind(console), - error: console.error.bind(console), - }, - }); - - watcherHandle = await startFileWatcher({ - repositoryPath, - snapshotPath: filePaths.watcherSnapshot, - onChanges: async (changed, deleted) => { - await incrementalIndexer.onChanges(changed, deleted); - // Write snapshot after each successful incremental update - await watcherHandle?.writeSnapshot(); - }, - onError: (err) => { - console.error('[MCP] File watcher error:', err); - }, - }); + if (!watcherHandle) { + const incrementalIndexer = createIncrementalIndexer({ + repositoryIndexer: indexer, + repositoryPath, + graphPath: filePaths.dependencyGraph, + logger: { + info: console.error.bind(console), + warn: console.error.bind(console), + error: console.error.bind(console), + }, + }); + + watcherHandle = await startFileWatcher({ + repositoryPath, + snapshotPath: filePaths.watcherSnapshot, + onChanges: async (changed, deleted) => { + await incrementalIndexer.onChanges(changed, deleted); + // Write snapshot after each successful incremental update + await watcherHandle?.writeSnapshot(); + }, + onError: (err) => { + console.error('[MCP] File watcher error:', err); + }, + }); + } console.error('[MCP] Backend ready; file watcher started'); } catch (error) { @@ -286,6 +294,9 @@ async function main() { // Handle graceful shutdown const shutdown = async () => { + // Let an in-flight backend init settle first — closing the indexer + // underneath it is a use-after-close race. + await gate.waitForIdle(); if (watcherHandle) { await watcherHandle.unsubscribe().catch(() => {}); } diff --git a/packages/mcp-server/src/server/__tests__/backend-gate.test.ts b/packages/mcp-server/src/server/__tests__/backend-gate.test.ts index 3193886..6f03ce9 100644 --- a/packages/mcp-server/src/server/__tests__/backend-gate.test.ts +++ b/packages/mcp-server/src/server/__tests__/backend-gate.test.ts @@ -114,6 +114,44 @@ describe('BackendGate', () => { await new Promise((r) => setTimeout(r, 10)); }); + it('waitForIdle awaits an in-flight init without triggering a new one', async () => { + let calls = 0; + let release: () => void = () => {}; + const gate = new BackendGate(() => { + calls++; + return new Promise((r) => { + release = r; + }); + }); + gate.start(); + expect(calls).toBe(1); + + let idle = false; + const waiting = gate.waitForIdle().then(() => { + idle = true; + }); + await new Promise((r) => setTimeout(r, 10)); + expect(idle).toBe(false); + + release(); + await waiting; + expect(idle).toBe(true); + expect(calls).toBe(1); + }); + + it('waitForIdle resolves immediately after a failure without retrying init', async () => { + let calls = 0; + const gate = new BackendGate(async () => { + calls++; + throw new Error('nope'); + }); + gate.start(); + await new Promise((r) => setTimeout(r, 10)); + + await expect(gate.waitForIdle()).resolves.toBeUndefined(); + expect(calls).toBe(1); + }); + it('delegates tool definition and metadata to the inner adapter', () => { const gate = new BackendGate(async () => {}); const inner = new MockAdapter('my_tool'); diff --git a/packages/mcp-server/src/server/backend-gate.ts b/packages/mcp-server/src/server/backend-gate.ts index 99cf87d..43d109a 100644 --- a/packages/mcp-server/src/server/backend-gate.ts +++ b/packages/mcp-server/src/server/backend-gate.ts @@ -36,6 +36,17 @@ export class BackendGate { }); } + /** + * Resolve once no initialization attempt is in flight, without ever + * starting one. For shutdown: closing the indexer while init is still + * using it is a use-after-close race. + */ + async waitForIdle(): Promise { + if (this.current) { + await this.current.catch(() => {}); + } + } + /** Resolve when the backend is ready; reject with the init error if not. */ async ensureReady(): Promise { if (this.ready) return; @@ -71,6 +82,7 @@ class GatedToolAdapter extends ToolAdapter { this.metadata = inner.metadata; if (inner.validate) this.validate = inner.validate.bind(inner); if (inner.estimateTokens) this.estimateTokens = inner.estimateTokens.bind(inner); + if (inner.healthCheck) this.healthCheck = inner.healthCheck.bind(inner); } getToolDefinition(): ToolDefinition {