diff --git a/package.json b/package.json index 7356a9dc..e4b0cdd3 100644 --- a/package.json +++ b/package.json @@ -116,6 +116,7 @@ "geist": "^1.7.0", "gsap": "^3.14.2", "input-otp": "1.4.2", + "jsonrepair": "^3.13.2", "katex": "^0.16.33", "lucide-react": "^0.575.0", "mathlive": "^0.108.2", diff --git a/src/components/assistant-ui/thread.tsx b/src/components/assistant-ui/thread.tsx index 2fc33e01..48802c84 100644 --- a/src/components/assistant-ui/thread.tsx +++ b/src/components/assistant-ui/thread.tsx @@ -1240,36 +1240,29 @@ const AssistantMessage: FC = () => { ); }; -const CONTINUE_RESPONSE_CHAR_THRESHOLD = 150; - -/** Sentence-ending punctuation that indicates an LLM likely finished naturally. */ -const ENDS_WITH_SENTENCE_PUNCTUATION = /[.!?]$/; - -/** Trailing citation tags to strip before punctuation check (e.g. Source). */ -const TRAILING_CITATION_TAGS = /(?:\s*[\s\S]*?<\/citation>\s*)+$/; +/** + * Max characters above which we never show the Continue button. + * Long responses (citations, tables, mermaid, multi-paragraph) are assumed complete. + * We avoid punctuation-based heuristics since the agent often ends with + * , tables, mermaid blocks, etc. + */ +const CONTINUE_MAX_CHARS = 600; const AssistantActionBar: FC = () => { const { createCard, isCreating } = useCreateCardFromMessage({ debounceMs: 300 }); const message = useMessage(); const api = useAssistantApi(); - const { textLength, textContent, endsWithPunctuation } = useMemo(() => { + const { textLength, textContent } = useMemo(() => { const textParts = message.content.filter( (part): part is { type: "text"; text: string } => part.type === "text" ); const length = textParts.reduce((sum, part) => sum + (part.text?.length ?? 0), 0); const content = textParts.map((part) => part.text ?? "").join("\n\n"); - const trimmed = content.trim(); - const withoutTrailingCitations = trimmed.replace(TRAILING_CITATION_TAGS, "").trim(); - const endsWithPunctuation = - withoutTrailingCitations.length > 0 && - ENDS_WITH_SENTENCE_PUNCTUATION.test(withoutTrailingCitations); - return { textLength: length, textContent: content, endsWithPunctuation }; + return { textLength: length, textContent: content }; }, [message.content]); - const showContinueButton = - textLength > 0 && - (textLength < CONTINUE_RESPONSE_CHAR_THRESHOLD || !endsWithPunctuation); + const showContinueButton = textLength > 0 && textLength < CONTINUE_MAX_CHARS; const handleContinueClick = useCallback(() => { api?.thread().append({ diff --git a/src/lib/ai/tools/__tests__/edit-item-tool.test.ts b/src/lib/ai/tools/__tests__/edit-item-tool.test.ts new file mode 100644 index 00000000..f6527f4b --- /dev/null +++ b/src/lib/ai/tools/__tests__/edit-item-tool.test.ts @@ -0,0 +1,148 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const mockWorkspaceWorker = vi.fn(); +const mockLoadStateForTool = vi.fn(); +const mockResolveItem = vi.fn(); +const mockGetVirtualPath = vi.fn(); + +vi.mock("@/lib/ai/workers", () => ({ + workspaceWorker: (...args: unknown[]) => mockWorkspaceWorker(...args), +})); + +vi.mock("@/lib/ai/tools/tool-utils", () => ({ + loadStateForTool: (...args: unknown[]) => mockLoadStateForTool(...args), + resolveItem: (...args: unknown[]) => mockResolveItem(...args), +})); + +vi.mock("@/lib/utils/virtual-workspace-fs", () => ({ + getVirtualPath: (...args: unknown[]) => mockGetVirtualPath(...args), +})); + +describe("createEditItemTool", () => { + let createEditItemTool: (ctx: { + workspaceId: string; + userId: string; + activeFolderId?: string; + threadId?: string | null; + }) => any; + + const ctx = { + workspaceId: "ws-1", + userId: "user-1", + activeFolderId: undefined, + threadId: "thread-1", + }; + + const noteItem = { + id: "note-1", + type: "note", + name: "My Note", + data: {}, + }; + + beforeAll(async () => { + const mod = await import("@/lib/ai/tools/edit-item-tool"); + createEditItemTool = mod.createEditItemTool; + }); + + beforeEach(() => { + vi.clearAllMocks(); + mockLoadStateForTool.mockResolvedValue({ success: true, state: { items: [noteItem] } }); + mockResolveItem.mockReturnValue(noteItem); + mockWorkspaceWorker.mockResolvedValue({ success: true, itemId: "note-1", message: "ok" }); + mockGetVirtualPath.mockReturnValue("notes/My Note.md"); + }); + + it("fails fast when itemName is missing", async () => { + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ itemName: "", oldString: "a", newString: "b" }); + expect(result.success).toBe(false); + expect(result.message).toMatch(/itemName is required/i); + }); + + it("returns loadStateForTool failure", async () => { + mockLoadStateForTool.mockResolvedValueOnce({ success: false, message: "No workspace context available" }); + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ itemName: "My Note", oldString: "a", newString: "b" }); + expect(result.success).toBe(false); + expect(result.message).toMatch(/No workspace context available/); + expect(mockWorkspaceWorker).not.toHaveBeenCalled(); + }); + + it("returns item not found message when resolveItem misses", async () => { + mockResolveItem.mockReturnValueOnce(undefined); + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ itemName: "Unknown", oldString: "a", newString: "b" }); + expect(result.success).toBe(false); + expect(result.message).toMatch(/Could not find item "Unknown"/); + }); + + it("requires disambiguation when multiple exact-name candidates exist", async () => { + const duplicate = { ...noteItem, id: "note-2" }; + mockLoadStateForTool.mockResolvedValueOnce({ + success: true, + state: { items: [noteItem, duplicate] }, + }); + mockResolveItem.mockReturnValueOnce(noteItem); + mockGetVirtualPath + .mockReturnValueOnce("notes/My Note.md") + .mockReturnValueOnce("folder/My Note.md"); + + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ itemName: "My Note", oldString: "a", newString: "b" }); + expect(result.success).toBe(false); + expect(result.message).toMatch(/Multiple items named/); + expect(result.message).toMatch(/Disambiguate using path/); + }); + + it("rejects non-editable item types", async () => { + const imageItem = { id: "img-1", type: "image", name: "Figure", data: {} }; + mockResolveItem.mockReturnValueOnce(imageItem); + mockLoadStateForTool.mockResolvedValueOnce({ success: true, state: { items: [imageItem] } }); + + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ itemName: "Figure", oldString: "a", newString: "b" }); + expect(result.success).toBe(false); + expect(result.message).toMatch(/not editable/); + }); + + it("calls workspaceWorker edit and returns renamed itemName", async () => { + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ + itemName: "My Note", + oldString: "old", + newString: "new", + newName: "Renamed Note", + replaceAll: true, + sources: [{ title: "Ref", url: "https://example.com" }], + }); + + expect(mockWorkspaceWorker).toHaveBeenCalledWith( + "edit", + expect.objectContaining({ + workspaceId: "ws-1", + itemId: "note-1", + itemType: "note", + oldString: "old", + newString: "new", + replaceAll: true, + newName: "Renamed Note", + }) + ); + expect(result.success).toBe(true); + expect(result.itemName).toBe("Renamed Note"); + }); + + it("passes through worker failures", async () => { + mockWorkspaceWorker.mockResolvedValueOnce({ + success: false, + message: "Found multiple matches for oldString", + }); + const tool: any = createEditItemTool(ctx); + const result = await tool.execute({ itemName: "My Note", oldString: "a", newString: "b" }); + + expect(result.success).toBe(false); + expect(result.message).toMatch(/multiple matches/i); + }); +}); + diff --git a/src/lib/ai/tools/read-workspace.ts b/src/lib/ai/tools/read-workspace.ts index 0e13c33a..52a826be 100644 --- a/src/lib/ai/tools/read-workspace.ts +++ b/src/lib/ai/tools/read-workspace.ts @@ -4,8 +4,10 @@ import { loadStateForTool } from "./tool-utils"; import { fuzzyMatchItem } from "./tool-utils"; import { resolveItemByPath } from "./workspace-search-utils"; import { formatItemContent } from "@/lib/utils/format-workspace-context"; +import { getNoteContentAsMarkdown } from "@/lib/utils/format-workspace-context"; import { getVirtualPath } from "@/lib/utils/virtual-workspace-fs"; import type { WorkspaceToolContext } from "./workspace-tools"; +import type { NoteData } from "@/lib/workspace-state/types"; const DEFAULT_LIMIT = 500; const MAX_LIMIT = 2000; @@ -122,7 +124,13 @@ export function createReadWorkspaceTool(ctx: WorkspaceToolContext) { ? { pageStart, pageEnd } : undefined; - const fullContent = formatItemContent(item, pdfPageRange); + // Keep note reads aligned with editItem matching source. + // editItem for notes matches against serialized markdown content only + // (without appended metadata blocks like Sources). + const fullContent = + item.type === "note" + ? getNoteContentAsMarkdown(item.data as NoteData) + : formatItemContent(item, pdfPageRange); const allLines = fullContent.split(/\r?\n/); const totalLines = allLines.length; const startIdx = Math.max(0, lineStart - 1); diff --git a/src/lib/ai/workers/__tests__/workspace-worker.edit.test.ts b/src/lib/ai/workers/__tests__/workspace-worker.edit.test.ts new file mode 100644 index 00000000..05c959e3 --- /dev/null +++ b/src/lib/ai/workers/__tests__/workspace-worker.edit.test.ts @@ -0,0 +1,319 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const mockGetSession = vi.fn(); +const mockHeaders = vi.fn(); +const mockLoadWorkspaceState = vi.fn(); +const mockMarkdownToBlocks = vi.fn(); +const mockCreateEvent = vi.fn(); +const mockExecute = vi.fn(); + +const mockLimit = vi.fn(); +const mockWhere = vi.fn(() => ({ limit: mockLimit })); +const mockFrom = vi.fn(() => ({ where: mockWhere })); +const mockSelect = vi.fn(() => ({ from: mockFrom })); + +vi.mock("next/headers", () => ({ + headers: () => mockHeaders(), +})); + +vi.mock("@/lib/auth", () => ({ + auth: { + api: { + getSession: (...args: unknown[]) => mockGetSession(...args), + }, + }, +})); + +vi.mock("@/lib/db/client", () => ({ + db: { + select: (...args: unknown[]) => mockSelect(...args), + execute: (...args: unknown[]) => mockExecute(...args), + }, + workspaces: { + id: "id", + userId: "userId", + }, +})); + +vi.mock("@/lib/db/schema", () => ({ + workspaceCollaborators: { + workspaceId: "workspaceId", + userId: "userId", + permissionLevel: "permissionLevel", + }, +})); + +vi.mock("@/lib/workspace/state-loader", () => ({ + loadWorkspaceState: (...args: unknown[]) => mockLoadWorkspaceState(...args), +})); + +vi.mock("@/lib/editor/markdown-to-blocks", () => ({ + markdownToBlocks: (...args: unknown[]) => mockMarkdownToBlocks(...args), + fixLLMDoubleEscaping: (s: string) => s, +})); + +vi.mock("@/lib/workspace/events", () => ({ + createEvent: (...args: unknown[]) => mockCreateEvent(...args), +})); + +vi.mock("@/lib/workspace/unique-name", () => ({ + hasDuplicateName: () => false, +})); + +describe("workspaceWorker edit end-to-end paths", () => { + let workspaceWorker: typeof import("@/lib/ai/workers/workspace-worker").workspaceWorker; + let workspaceOperationQueues: Map>; + + beforeAll(async () => { + const workerMod = await import("@/lib/ai/workers/workspace-worker"); + workspaceWorker = workerMod.workspaceWorker; + const commonMod = await import("@/lib/ai/workers/common"); + workspaceOperationQueues = commonMod.workspaceOperationQueues; + }); + + beforeEach(() => { + vi.clearAllMocks(); + workspaceOperationQueues.clear(); + + mockHeaders.mockResolvedValue(new Headers()); + mockGetSession.mockResolvedValue({ user: { id: "user-1" } }); + mockLimit.mockResolvedValue([{ userId: "user-1" }]); // workspace owner path + mockMarkdownToBlocks.mockResolvedValue([]); + mockCreateEvent.mockImplementation((type: string, payload: unknown, userId: string) => ({ + id: "evt-1", + type, + payload, + timestamp: 123, + userId, + })); + }); + + it("edits quiz JSON and appends question near end", async () => { + mockLoadWorkspaceState.mockResolvedValue({ + items: [ + { + id: "quiz-1", + type: "quiz", + name: "Quiz 1", + subtitle: "", + data: { + questions: [ + { + id: "q1", + type: "multiple_choice", + questionText: "Q1", + options: ["A", "B", "C", "D"], + correctIndex: 0, + explanation: "e1", + }, + ], + }, + }, + ], + globalTitle: "", + workspaceId: "ws-1", + }); + + mockExecute + .mockResolvedValueOnce([{ version: 1 }]) // get_workspace_version + .mockResolvedValueOnce([{ result: "(2,f)" }]); // append_workspace_event + + const result = await workspaceWorker("edit", { + workspaceId: "ws-1", + itemId: "quiz-1", + itemType: "quiz", + itemName: "Quiz 1", + oldString: " ]\n}", + newString: [ + " ,", + " {", + ' "id": "q2",', + ' "type": "multiple_choice",', + ' "questionText": "Q2",', + ' "options": ["A", "B", "C", "D"],', + ' "correctIndex": 1,', + ' "explanation": "e2"', + " }", + " ]", + "}", + ].join("\n"), + }); + + expect(result.success).toBe(true); + expect(result.message).toMatch(/Updated quiz/); + expect((result as any).questionCount).toBe(2); + }); + + it("repairs malformed appended flashcard JSON and succeeds", async () => { + mockLoadWorkspaceState.mockResolvedValue({ + items: [ + { + id: "deck-1", + type: "flashcard", + name: "Deck 1", + subtitle: "", + data: { + cards: [{ id: "c1", front: "f1", back: "b1" }], + }, + }, + ], + globalTitle: "", + workspaceId: "ws-1", + }); + + mockExecute + .mockResolvedValueOnce([{ version: 4 }]) // get_workspace_version + .mockResolvedValueOnce([{ result: "(5,f)" }]); // append_workspace_event + + const result = await workspaceWorker("edit", { + workspaceId: "ws-1", + itemId: "deck-1", + itemType: "flashcard", + itemName: "Deck 1", + oldString: " ]\n}", + // intentionally malformed but repairable (single quotes + trailing comma) + newString: [ + " ,", + " {'id':'c2','front':'f2','back':'b2'},", + " ]", + "}", + ].join("\n"), + }); + + expect(result.success).toBe(true); + expect(result.message).toMatch(/Updated flashcard deck/); + expect((result as any).cardCount).toBe(2); + }); + + it("fails quiz edit when repaired JSON violates schema", async () => { + const initialQuestions = [ + { + id: "q1", + type: "multiple_choice", + questionText: "Q1", + options: ["A", "B", "C", "D"], + correctIndex: 0, + explanation: "e1", + }, + ]; + + mockLoadWorkspaceState.mockResolvedValue({ + items: [ + { + id: "quiz-1", + type: "quiz", + name: "Quiz 1", + subtitle: "", + data: { + questions: initialQuestions, + }, + }, + ], + globalTitle: "", + workspaceId: "ws-1", + }); + + const result = await workspaceWorker("edit", { + workspaceId: "ws-1", + itemId: "quiz-1", + itemType: "quiz", + itemName: "Quiz 1", + oldString: JSON.stringify({ questions: initialQuestions }, null, 2), + // repair may parse this, but schema should reject 3 options for MC + newString: JSON.stringify( + { + questions: [ + { + ...initialQuestions[0], + options: ["A", "B", "C"], + }, + ], + }, + null, + 2 + ), + }); + + expect(result.success).toBe(false); + expect(result.message).toMatch(/multiple_choice needs 4 options/i); + }); + + it("fails flashcard edit on unrecoverable invalid JSON", async () => { + mockLoadWorkspaceState.mockResolvedValue({ + items: [ + { + id: "deck-1", + type: "flashcard", + name: "Deck 1", + subtitle: "", + data: { + cards: [{ id: "c1", front: "f1", back: "b1" }], + }, + }, + ], + globalTitle: "", + workspaceId: "ws-1", + }); + + const result = await workspaceWorker("edit", { + workspaceId: "ws-1", + itemId: "deck-1", + itemType: "flashcard", + itemName: "Deck 1", + oldString: "{", + newString: "{ ???", + }); + + expect(result.success).toBe(false); + expect(result.message).toMatch(/Invalid JSON after edit/i); + }); + + it("fails with clear message when oldString is ambiguous", async () => { + mockLoadWorkspaceState.mockResolvedValue({ + items: [ + { + id: "quiz-1", + type: "quiz", + name: "Quiz 1", + subtitle: "", + data: { + questions: [ + { + id: "q1", + type: "multiple_choice", + questionText: "Same", + options: ["A", "B", "C", "D"], + correctIndex: 0, + explanation: "dup", + }, + { + id: "q2", + type: "multiple_choice", + questionText: "Same", + options: ["A", "B", "C", "D"], + correctIndex: 1, + explanation: "dup", + }, + ], + }, + }, + ], + globalTitle: "", + workspaceId: "ws-1", + }); + + const result = await workspaceWorker("edit", { + workspaceId: "ws-1", + itemId: "quiz-1", + itemType: "quiz", + itemName: "Quiz 1", + oldString: '"questionText": "Same"', + newString: '"questionText": "Updated"', + replaceAll: false, + }); + + expect(result.success).toBe(false); + expect(result.message).toMatch(/multiple matches/i); + }); +}); + diff --git a/src/lib/ai/workers/workspace-worker.ts b/src/lib/ai/workers/workspace-worker.ts index 5c2a3bcc..1a1e51b2 100644 --- a/src/lib/ai/workers/workspace-worker.ts +++ b/src/lib/ai/workers/workspace-worker.ts @@ -15,6 +15,7 @@ import { loadWorkspaceState } from "@/lib/workspace/state-loader"; import { hasDuplicateName } from "@/lib/workspace/unique-name"; import type { WorkspaceEvent } from "@/lib/workspace/events"; import { replace as applyReplace, trimDiff, normalizeLineEndings } from "@/lib/utils/edit-replace"; +import { parseJsonWithRepair } from "@/lib/utils/json-repair"; import { getNoteContentAsMarkdown } from "@/lib/utils/format-workspace-context"; import { serializeBlockNote } from "@/lib/utils/serialize-blocknote"; import type { Block } from "@/components/editor/BlockNoteEditor"; @@ -1088,10 +1089,17 @@ export async function workspaceWorker( let parsed: { cards?: Array<{ id?: string; front?: string; back?: string }> }; try { - parsed = JSON.parse(serialized); + const parsedResult = parseJsonWithRepair<{ cards?: Array<{ id?: string; front?: string; back?: string }> }>( + serialized + ); + parsed = parsedResult.value; } catch (e) { - const detail = e instanceof SyntaxError ? e.message : String(e); - throw new Error(`Invalid JSON after edit: ${detail}. Ensure oldString matches exactly from readWorkspace.`); + const detail = e instanceof Error ? e.message : String(e); + throw new Error( + `Invalid JSON after edit. The edited flashcard content is not valid JSON (even after repair). ` + + `Try replacing a larger unique block, or use full rewrite with oldString='' and a complete {"cards":[...]} JSON object. ` + + `Details: ${detail}` + ); } if (!Array.isArray(parsed.cards)) { throw new Error("Invalid structure: cards must be an array."); @@ -1157,10 +1165,15 @@ export async function workspaceWorker( let parsed: { questions?: QuizQuestion[] }; try { - parsed = JSON.parse(serialized); + const parsedResult = parseJsonWithRepair<{ questions?: QuizQuestion[] }>(serialized); + parsed = parsedResult.value; } catch (e) { - const detail = e instanceof SyntaxError ? e.message : String(e); - throw new Error(`Invalid JSON after edit: ${detail}. Ensure oldString matches exactly from readWorkspace.`); + const detail = e instanceof Error ? e.message : String(e); + throw new Error( + `Invalid JSON after edit. The edited quiz content is not valid JSON (even after repair). ` + + `Only edit the {"questions":[...]} JSON, and replace a larger unique block (or do full rewrite with oldString=''). ` + + `Details: ${detail}` + ); } if (!Array.isArray(parsed.questions)) { throw new Error("Invalid structure: questions must be an array."); diff --git a/src/lib/utils/__tests__/edit-replace.test.ts b/src/lib/utils/__tests__/edit-replace.test.ts index 31b370b4..9ae2ad08 100644 --- a/src/lib/utils/__tests__/edit-replace.test.ts +++ b/src/lib/utils/__tests__/edit-replace.test.ts @@ -75,6 +75,121 @@ describe("replace (edit/replace for oldString/newString)", () => { "# Note\n\n$$\n2x\n$$\n\nMore text" ); }); + + it("handles oldString copied with readWorkspace line prefixes", () => { + const content = "{\n \"questions\": [\n {\"questionText\":\"Q1\"}\n ]\n}"; + const oldWithPrefixes = [ + "1: {", + "2: \"questions\": [", + "3: {\"questionText\":\"Q1\"}", + "4: ]", + "5: }", + ].join("\n"); + const result = replace(content, oldWithPrefixes, "{\"questions\":[]}"); + expect(result).toBe("{\"questions\":[]}"); + }); + + it("handles oldString wrapped in code fences", () => { + const content = "{\n \"cards\": []\n}"; + const oldFenced = "```json\n{\n \"cards\": []\n}\n```"; + const result = replace(content, oldFenced, "{\"cards\":[{\"front\":\"A\",\"back\":\"B\"}]}"); + expect(result).toBe("{\"cards\":[{\"front\":\"A\",\"back\":\"B\"}]}"); + }); + + it("normalizes fenced JSON newString in JSON edit contexts", () => { + const content = "{\n \"questions\": []\n}"; + const oldTail = "[]\n}"; + const fencedNew = "```json\n[\n {\n \"id\": \"q2\"\n }\n]\n```"; + const result = replace(content, oldTail, fencedNew); + expect(result).toContain('"id": "q2"'); + expect(result).not.toContain("```"); + }); + + it("normalizes line-prefixed newString in JSON edit contexts", () => { + const content = "{\n \"questions\": []\n}"; + const oldTail = "[]\n}"; + const prefixedNew = [ + "1: [", + "2: {", + "3: \"id\": \"q2\"", + "4: }", + "5: ]", + ].join("\n"); + const result = replace(content, oldTail, prefixedNew); + expect(result).toContain('"id": "q2"'); + expect(result).not.toContain("1: ["); + }); + + it("supports appending a quiz question near end of JSON", () => { + const content = [ + "{", + ' "questions": [', + " {", + ' "id": "q1",', + ' "type": "multiple_choice",', + ' "questionText": "Q1",', + ' "options": ["A", "B", "C", "D"],', + ' "correctIndex": 0,', + ' "explanation": "e1"', + " }", + " ]", + "}", + ].join("\n"); + + const oldTail = " ]\n}"; + const newTail = [ + " ,", + " {", + ' "id": "q2",', + ' "type": "multiple_choice",', + ' "questionText": "Q2",', + ' "options": ["A", "B", "C", "D"],', + ' "correctIndex": 1,', + ' "explanation": "e2"', + " }", + " ]", + "}", + ].join("\n"); + + const result = replace(content, oldTail, newTail); + expect(result).toContain('"id": "q2"'); + expect(result).toContain('"questionText": "Q2"'); + }); + + it("supports appending a flashcard near end of JSON", () => { + const content = [ + "{", + ' "cards": [', + " {", + ' "id": "c1",', + ' "front": "f1",', + ' "back": "b1"', + " }", + " ]", + "}", + ].join("\n"); + + const oldTail = " ]\n}"; + const newTail = [ + " ,", + " {", + ' "id": "c2",', + ' "front": "f2",', + ' "back": "b2"', + " }", + " ]", + "}", + ].join("\n"); + + const result = replace(content, oldTail, newTail); + expect(result).toContain('"id": "c2"'); + expect(result).toContain('"front": "f2"'); + }); + + it("preserves ambiguity error when multiple matches remain", () => { + const content = "foo\nfoo\nfoo"; + expect(() => replace(content, "foo", "bar")).toThrow("multiple matches"); + }); }); describe("unescapeString", () => { diff --git a/src/lib/utils/__tests__/json-repair.test.ts b/src/lib/utils/__tests__/json-repair.test.ts new file mode 100644 index 00000000..d44e6b1b --- /dev/null +++ b/src/lib/utils/__tests__/json-repair.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { parseJsonWithRepair } from "@/lib/utils/json-repair"; + +describe("parseJsonWithRepair", () => { + it("parses valid JSON without repair", () => { + const input = '{"name":"test","items":[1,2,3]}'; + const result = parseJsonWithRepair<{ name: string; items: number[] }>(input); + expect(result.repaired).toBe(false); + expect(result.value.name).toBe("test"); + expect(result.value.items).toEqual([1, 2, 3]); + }); + + it("repairs missing closing bracket", () => { + const result = parseJsonWithRepair<{ name: string }>('{"name":"test"'); + expect(result.repaired).toBe(true); + expect(result.value.name).toBe("test"); + }); + + it("repairs single quotes and unquoted keys", () => { + const result = parseJsonWithRepair<{ name: string; score: number }>( + "{name: 'test', score: 42}" + ); + expect(result.repaired).toBe(true); + expect(result.value).toEqual({ name: "test", score: 42 }); + }); + + it("repairs trailing commas", () => { + const result = parseJsonWithRepair<{ items: number[] }>('{"items":[1,2,3,],}'); + expect(result.repaired).toBe(true); + expect(result.value.items).toEqual([1, 2, 3]); + }); + + it("repairs malformed appended quiz JSON with trailing comma", () => { + const malformed = ` + { + "questions": [ + {"id":"q1","type":"multiple_choice","questionText":"Q1","options":["A","B","C","D"],"correctIndex":0,"explanation":"ok"}, + ], + } + `; + const result = parseJsonWithRepair<{ questions: unknown[] }>(malformed); + expect(Array.isArray(result.value.questions)).toBe(true); + expect(result.value.questions).toHaveLength(1); + }); + + it("repairs malformed appended flashcard JSON with single quotes", () => { + const malformed = "{cards: [{'id':'c1','front':'f1','back':'b1'}],}"; + const result = parseJsonWithRepair<{ cards: Array<{ id: string }> }>(malformed); + expect(result.value.cards[0].id).toBe("c1"); + }); + + it("repairs JSON wrapped in code fences", () => { + const result = parseJsonWithRepair<{ questions: unknown[] }>( + "```json\n{\"questions\":[]}\n```" + ); + expect(result.value.questions).toEqual([]); + }); + + it("repairs fenced JSON without newline before closing fence", () => { + const result = parseJsonWithRepair<{ questions: unknown[] }>( + "```json\n{\"questions\":[]}```" + ); + expect(result.value.questions).toEqual([]); + }); + + it("removes basic comments", () => { + const result = parseJsonWithRepair<{ name: string; items: number[] }>(` + { + // comment + "name": "test", + /* block */ + "items": [1, 2] + } + `); + expect(result.value).toEqual({ name: "test", items: [1, 2] }); + }); + + it("repairs token placeholders as best-effort JSON", () => { + const result = parseJsonWithRepair<{ name?: unknown }>('{"name": ??? }'); + expect(result.repaired).toBe(true); + expect(typeof result.value).toBe("object"); + expect(result.value).not.toBeNull(); + }); + + it("throws for unrecoverable empty input", () => { + expect(() => parseJsonWithRepair("")).toThrow( + /Invalid JSON after repair attempt/ + ); + }); +}); + diff --git a/src/lib/utils/edit-replace.ts b/src/lib/utils/edit-replace.ts index 5c3c286f..0e18bd07 100644 --- a/src/lib/utils/edit-replace.ts +++ b/src/lib/utils/edit-replace.ts @@ -443,41 +443,127 @@ export function trimDiff(diff: string): string { return trimmedLines.join("\n"); } +function stripReadWorkspaceLinePrefixes(text: string): string { + const lines = text.split("\n"); + if (lines.length === 0) return text; + + let prefixedCount = 0; + let nonEmptyCount = 0; + for (const line of lines) { + if (!line.trim()) continue; + nonEmptyCount++; + if (/^\d+:\s/.test(line)) prefixedCount++; + } + + // Only strip when this strongly looks like readWorkspace output. + const shouldStrip = + prefixedCount >= 2 && nonEmptyCount > 0 && prefixedCount / nonEmptyCount >= 0.6; + if (!shouldStrip) return text; + + return lines.map((line) => line.replace(/^\d+:\s/, "")).join("\n"); +} + +function unwrapSingleCodeFence(text: string): string { + const trimmed = text.trim(); + const match = trimmed.match(/^```[a-zA-Z0-9_-]*\n([\s\S]*?)\n?```$/); + if (!match) return text; + return match[1]; +} + +function normalizeReplacementText(content: string, newString: string): string { + let normalized = newString; + const contentLooksJson = /"questions"\s*:|"cards"\s*:|^\s*[\[{]/.test(content); + + // Only unwrap full-value code fences for JSON-like content to avoid + // stripping intentional markdown code blocks in note edits. + if (contentLooksJson) { + const unwrapped = unwrapSingleCodeFence(normalized); + const unwrappedLooksJson = /^\s*[\[{]/.test(unwrapped.trim()); + if (unwrapped !== normalized && unwrappedLooksJson) { + normalized = unwrapped; + } + + // In JSON edit flows, model outputs may include readWorkspace line prefixes. + normalized = stripReadWorkspaceLinePrefixes(normalized); + } + + return normalized; +} + +function buildSearchCandidates(oldString: string): string[] { + const candidates = new Set(); + const base = oldString; + const noFence = unwrapSingleCodeFence(base); + const noLines = stripReadWorkspaceLinePrefixes(base); + const noFenceNoLines = stripReadWorkspaceLinePrefixes(noFence); + + for (const c of [base, noFence, noLines, noFenceNoLines]) { + if (c.length > 0) candidates.add(c); + } + return Array.from(candidates); +} + export function replace(content: string, oldString: string, newString: string, replaceAll = false): string { if (oldString === newString) { throw new Error("No changes to apply: oldString and newString are identical."); } + const replacementText = normalizeReplacementText(content, newString); let notFound = true; - for (const replacer of [ - SimpleReplacer, - LineTrimmedReplacer, - BlockAnchorReplacer, - WhitespaceNormalizedReplacer, - IndentationFlexibleReplacer, - EscapeNormalizedReplacer, - TrimmedBoundaryReplacer, - ContextAwareReplacer, - MultiOccurrenceReplacer, - ]) { - for (const search of replacer(content, oldString)) { - const index = content.indexOf(search); - if (index === -1) continue; - notFound = false; - if (replaceAll) { - return content.replaceAll(search, newString); + const searchCandidates = buildSearchCandidates(oldString); + + for (const candidate of searchCandidates) { + for (const replacer of [ + SimpleReplacer, + LineTrimmedReplacer, + BlockAnchorReplacer, + WhitespaceNormalizedReplacer, + IndentationFlexibleReplacer, + EscapeNormalizedReplacer, + TrimmedBoundaryReplacer, + ContextAwareReplacer, + MultiOccurrenceReplacer, + ]) { + for (const search of replacer(content, candidate)) { + const index = content.indexOf(search); + if (index === -1) continue; + notFound = false; + if (replaceAll) { + return content.replaceAll(search, replacementText); + } + const lastIndex = content.lastIndexOf(search); + if (index !== lastIndex) continue; + return content.substring(0, index) + replacementText + content.substring(index + search.length); } - const lastIndex = content.lastIndexOf(search); - if (index !== lastIndex) continue; - return content.substring(0, index) + newString + content.substring(index + search.length); } } if (notFound) { + const compactOld = oldString.replace(/\s+/g, " ").trim(); + const isLikelyTooShort = compactOld.length > 0 && compactOld.length <= 6; + const looksLikeTailToken = ["]", "}", "],", "},"].includes(compactOld); + const jsonLike = /"questions"\s*:|"cards"\s*:/.test(content); + + const hints: string[] = []; + if (isLikelyTooShort || looksLikeTailToken) { + hints.push("oldString appears too short/ambiguous (for example: ']' or '}')."); + } + if (jsonLike) { + hints.push( + "For quiz/flashcard JSON edits, include a larger unique block (3-8 lines) with field names near the target section." + ); + hints.push("If you're appending, replace the final JSON tail (for example, ' ]\\n}') with a longer unique tail."); + } + hints.push("Do not include readWorkspace line prefixes like '12: '."); + throw new Error( - "Could not find oldString in the file. It must match exactly, including whitespace, indentation, and line endings." + `Could not find oldString in the file. ${ + hints.join(" ") + } Ensure the snippet is copied exactly from readWorkspace and is not truncated.` ); } - throw new Error("Found multiple matches for oldString. Provide more surrounding context to make the match unique."); + throw new Error( + "Found multiple matches for oldString. oldString is likely too generic. Use a longer, unique snippet (3-8 lines with nearby field names/structure), or set replaceAll=true only when changing every occurrence is intentional." + ); } diff --git a/src/lib/utils/json-repair.ts b/src/lib/utils/json-repair.ts new file mode 100644 index 00000000..4b17c753 --- /dev/null +++ b/src/lib/utils/json-repair.ts @@ -0,0 +1,43 @@ +import { jsonrepair } from "jsonrepair"; + +type ParsedJsonResult = { + value: T; + repaired: boolean; + repairedText?: string; +}; + +function unwrapCodeFence(text: string): string { + const trimmed = text.trim(); + const match = trimmed.match(/^```[a-zA-Z0-9_-]*\n([\s\S]*?)\n?```$/); + return match ? match[1] : text; +} + +function normalizeSmartQuotes(text: string): string { + return text + .replaceAll("\u2018", "'") + .replaceAll("\u2019", "'") + .replaceAll("\u201C", '"') + .replaceAll("\u201D", '"'); +} + +export function parseJsonWithRepair(raw: string): ParsedJsonResult { + const base = unwrapCodeFence(normalizeSmartQuotes(raw)).trim(); + try { + return { value: JSON.parse(base) as T, repaired: false }; + } catch { + // Fall through to jsonrepair. + } + + try { + const candidate = jsonrepair(base); + return { + value: JSON.parse(candidate) as T, + repaired: true, + repairedText: candidate, + }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid JSON after repair attempt: ${detail}`); + } +} +