From da4d214c051152880e2450555c8f867013b80e03 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 11:51:24 +0530 Subject: [PATCH 01/12] [ENG-1855] Add Roam shared-node import discovery --- .../components/DiscoverSharedNodesDialog.tsx | 263 ++++++++++++++ .../__tests__/discoverSharedNodes.test.ts | 163 +++++++++ apps/roam/src/utils/discoverSharedNodes.ts | 333 ++++++++++++++++++ .../utils/registerCommandPaletteCommands.ts | 7 + 4 files changed, 766 insertions(+) create mode 100644 apps/roam/src/components/DiscoverSharedNodesDialog.tsx create mode 100644 apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts create mode 100644 apps/roam/src/utils/discoverSharedNodes.ts diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx new file mode 100644 index 000000000..a60962ba3 --- /dev/null +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -0,0 +1,263 @@ +import { + Button, + Callout, + Classes, + Dialog, + HTMLTable, + InputGroup, + Intent, + NonIdealState, + Spinner, + Tag, + Tooltip, +} from "@blueprintjs/core"; +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import createOverlayRender from "roamjs-components/util/createOverlayRender"; +import { + discoverSharedNodes, + type DiscoveredSharedNode, +} from "~/utils/discoverSharedNodes"; +import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; + +const formatModifiedAt = (modifiedAt: string): string => + new Date(modifiedAt).toLocaleString(); + +const SharedNodeRow = ({ node }: { node: DiscoveredSharedNode }) => ( + + + {node.sourceApp} + + +
+ {node.sourceSpaceName} +
+
+ {node.sourceSpaceId} +
+ + +
+ {node.title} +
+ + + {node.sourceNodeId ? ( +
+ {node.sourceNodeId} +
+ ) : ( + Not provided + )} + + + {formatModifiedAt(node.modifiedAt)} + + + {node.alreadyImported ? ( + + Imported + + ) : ( + Available + )} + + +); + +const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { + const [nodes, setNodes] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(""); + const [search, setSearch] = useState(""); + + const loadNodes = useCallback(async (): Promise => { + setLoading(true); + setError(""); + try { + const [client, context] = await Promise.all([ + getLoggedInClient(), + getSupabaseContext(), + ]); + if (!client || !context) + throw new Error("Could not connect to shared persistence."); + setNodes( + await discoverSharedNodes({ + client, + currentSpaceId: context.spaceId, + }), + ); + } catch (loadError) { + console.error("Failed to discover shared nodes:", loadError); + setError( + loadError instanceof Error + ? loadError.message + : "Could not load shared nodes.", + ); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void loadNodes(); + }, [loadNodes]); + + const visibleNodes = useMemo(() => { + const normalizedSearch = search.trim().toLocaleLowerCase(); + if (!normalizedSearch) return nodes; + return nodes.filter((node) => + [ + node.sourceApp, + node.sourceSpaceName, + node.sourceSpaceId, + node.title, + node.sourceNodeId, + ].some((value) => value?.toLocaleLowerCase().includes(normalizedSearch)), + ); + }, [nodes, search]); + + return ( + +
+
+ ) => + setSearch(event.target.value) + } + placeholder="Search shared nodes" + value={search} + /> + +
+ + {loading ? ( +
+ +
+ ) : error ? ( + +
{error}
+ + + ) : visibleNodes.length === 0 ? ( +
+ +
+ ) : ( +
+ + + + Source app + Source space + Title + Source ID + Modified + Status + + + + {visibleNodes.map((node) => ( + + ))} + + +
+ )} +
+
+
+ + {loading || error + ? "" + : `${visibleNodes.length} of ${nodes.length} nodes`} + + +
+
+ + ); +}; + +type Props = Record; + +export const renderDiscoverSharedNodesDialog = createOverlayRender( + "discourse-discover-shared-nodes", + DiscoverSharedNodesDialog, +); + +export default DiscoverSharedNodesDialog; diff --git a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts new file mode 100644 index 000000000..ca007246f --- /dev/null +++ b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import { buildDiscoveredSharedNodes } from "~/utils/discoverSharedNodes"; + +type BuildArgs = Parameters[0]; + +const resources: BuildArgs["resources"] = [ + { space_id: 20, source_local_id: "node-1" }, + { space_id: 20, source_local_id: "schema-1" }, +]; +const spaces: BuildArgs["spaces"] = [ + { + id: 20, + name: "Research vault", + platform: "Obsidian", + url: "obsidian:vault-a", + }, +]; +const concepts: BuildArgs["concepts"] = [ + { + is_schema: false, + last_modified: "2026-06-14T12:00:00.000Z", + schema_id: 200, + source_local_id: "node-1", + space_id: 20, + }, +]; +const contents: BuildArgs["contents"] = [ + { + content_type: "text/plain", + last_modified: "2026-06-14T13:00:00.000Z", + source_local_id: "node-1", + space_id: 20, + text: "EVD - REM sleep and recall", + variant: "direct", + }, + { + content_type: "text/markdown", + last_modified: "2026-06-14T15:00:00.000Z", + source_local_id: "node-1", + space_id: 20, + text: "# EVD - REM sleep and recall", + variant: "full", + }, +]; +const sourceNodeRid = "orn:obsidian.note:vault-a/node-1"; + +const build = ({ + conceptsOverride = concepts, + contentsOverride = contents, + currentSpaceId = 10, + importedSourceRids = new Set(), + resourcesOverride = resources, + spacesOverride = spaces, +}: { + conceptsOverride?: typeof concepts; + contentsOverride?: typeof contents; + currentSpaceId?: number; + importedSourceRids?: ReadonlySet; + resourcesOverride?: typeof resources; + spacesOverride?: typeof spaces; +} = {}) => + buildDiscoveredSharedNodes({ + concepts: conceptsOverride, + contents: contentsOverride, + currentSpaceId, + importedSourceRids, + resources: resourcesOverride, + spaces: spacesOverride, + }); + +describe("buildDiscoveredSharedNodes", () => { + it("builds a group-shared contract node with stable source identity", () => { + expect(build({ importedSourceRids: new Set([sourceNodeRid]) })).toEqual([ + { + alreadyImported: true, + modifiedAt: "2026-06-14T15:00:00.000Z", + sourceApp: "Obsidian", + sourceNodeId: "node-1", + sourceNodeRid, + sourceSpaceId: "obsidian:vault-a", + sourceSpaceName: "Research vault", + title: "EVD - REM sleep and recall", + }, + ]); + }); + + it("does not discover shared resources from the current space", () => { + expect(build({ currentSpaceId: 20 })).toEqual([]); + }); + + it("requires the exact shared resource identity", () => { + expect( + build({ + resourcesOverride: [{ space_id: 21, source_local_id: "node-1" }], + }), + ).toEqual([]); + }); + + it.each([ + { + name: "schema concept", + conceptsOverride: [{ ...concepts[0], is_schema: true }], + contentsOverride: contents, + }, + { + name: "missing node type", + conceptsOverride: [{ ...concepts[0], schema_id: null }], + contentsOverride: contents, + }, + { + name: "missing direct content", + conceptsOverride: concepts, + contentsOverride: [contents[1]], + }, + { + name: "missing full content", + conceptsOverride: concepts, + contentsOverride: [contents[0]], + }, + { + name: "untyped full content", + conceptsOverride: concepts, + contentsOverride: [contents[0], { ...contents[1], content_type: null }], + }, + ])("filters a node with $name", ({ conceptsOverride, contentsOverride }) => { + expect(build({ conceptsOverride, contentsOverride })).toEqual([]); + }); + + it("matches imports by RID rather than source-local ID alone", () => { + expect( + build({ + importedSourceRids: new Set(["orn:obsidian.note:another-vault/node-1"]), + })[0]?.alreadyImported, + ).toBe(false); + }); + + it("sorts newest nodes first", () => { + const olderConcept = { + ...concepts[0], + last_modified: "2026-06-10T12:00:00.000Z", + source_local_id: "node-2", + }; + const olderContents = contents.map((content) => ({ + ...content, + last_modified: "2026-06-10T12:00:00.000Z", + source_local_id: "node-2", + text: + content.variant === "direct" + ? "Older shared node" + : "# Older shared node", + })); + expect( + build({ + conceptsOverride: [olderConcept, concepts[0]], + contentsOverride: [...olderContents, ...contents], + resourcesOverride: [ + ...resources, + { space_id: 20, source_local_id: "node-2" }, + ], + }).map((node) => node.sourceNodeId), + ).toEqual(["node-1", "node-2"]); + }); +}); diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts new file mode 100644 index 000000000..ae78fded5 --- /dev/null +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -0,0 +1,333 @@ +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { getAvailableGroupIds } from "@repo/database/lib/groups"; +import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; +import type { Tables } from "@repo/database/dbTypes"; +import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; + +const IMPORTED_FROM_PROP_KEY = "importedFrom"; +const PAGE_SIZE = 1000; +const RESOURCE_ID_CHUNK_SIZE = 100; + +type ResourceAccess = Pick< + Tables<"ResourceAccess">, + "space_id" | "source_local_id" +>; +type SharedConcept = Pick< + Tables<"my_concepts">, + "is_schema" | "last_modified" | "schema_id" | "source_local_id" | "space_id" +>; +type SharedContent = Pick< + Tables<"my_contents">, + | "content_type" + | "last_modified" + | "source_local_id" + | "space_id" + | "text" + | "variant" +>; +type SharedSpace = Pick< + Tables<"my_spaces">, + "id" | "name" | "platform" | "url" +>; +type ValidSharedSpace = { + id: number; + name: string; + platform: "Roam" | "Obsidian"; + url: string; +}; + +export type DiscoveredSharedNode = { + alreadyImported: boolean; + modifiedAt: string; + sourceApp: "Roam" | "Obsidian"; + sourceNodeId?: string; + sourceNodeRid: string; + sourceSpaceId: string; + sourceSpaceName: string; + title: string; +}; + +type SharedNodeRows = { + concepts: SharedConcept[]; + contents: SharedContent[]; + resources: ResourceAccess[]; + spaces: SharedSpace[]; +}; + +const getResourceKey = ({ + sourceLocalId, + spaceId, +}: { + sourceLocalId: string; + spaceId: number; +}): string => `${spaceId}:${sourceLocalId}`; + +const getLatestTimestamp = (timestamps: (string | null)[]): string | null => { + const validTimestamps = timestamps.filter( + (timestamp): timestamp is string => + typeof timestamp === "string" && !Number.isNaN(Date.parse(timestamp)), + ); + if (validTimestamps.length === 0) return null; + return validTimestamps.reduce((latest, timestamp) => + Date.parse(timestamp) > Date.parse(latest) ? timestamp : latest, + ); +}; + +export const buildDiscoveredSharedNodes = ({ + concepts, + contents, + currentSpaceId, + importedSourceRids, + resources, + spaces, +}: SharedNodeRows & { + currentSpaceId: number; + importedSourceRids: ReadonlySet; +}): DiscoveredSharedNode[] => { + const sharedResourceKeys = new Set( + resources + .filter((resource) => resource.space_id !== currentSpaceId) + .map((resource) => + getResourceKey({ + sourceLocalId: resource.source_local_id, + spaceId: resource.space_id, + }), + ), + ); + const spacesById = new Map( + spaces.flatMap((space): [number, ValidSharedSpace][] => { + if ( + typeof space.id !== "number" || + typeof space.name !== "string" || + (space.platform !== "Roam" && space.platform !== "Obsidian") || + typeof space.url !== "string" + ) + return []; + return [ + [ + space.id, + { + id: space.id, + name: space.name, + platform: space.platform, + url: space.url, + }, + ], + ]; + }), + ); + const contentByResource = new Map< + string, + Partial> + >(); + + contents.forEach((content) => { + if ( + typeof content.space_id !== "number" || + typeof content.source_local_id !== "string" || + (content.variant !== "direct" && content.variant !== "full") + ) + return; + const key = getResourceKey({ + sourceLocalId: content.source_local_id, + spaceId: content.space_id, + }); + const variants = contentByResource.get(key) ?? {}; + variants[content.variant] = content; + contentByResource.set(key, variants); + }); + + return concepts + .flatMap((concept): DiscoveredSharedNode[] => { + if ( + concept.is_schema !== false || + concept.schema_id === null || + typeof concept.space_id !== "number" || + typeof concept.source_local_id !== "string" + ) + return []; + + const resourceKey = getResourceKey({ + sourceLocalId: concept.source_local_id, + spaceId: concept.space_id, + }); + if (!sharedResourceKeys.has(resourceKey)) return []; + + const space = spacesById.get(concept.space_id); + const variants = contentByResource.get(resourceKey); + const direct = variants?.direct; + const full = variants?.full; + if ( + !space || + typeof direct?.text !== "string" || + typeof full?.text !== "string" || + typeof full.content_type !== "string" + ) + return []; + + const modifiedAt = getLatestTimestamp([ + concept.last_modified, + direct.last_modified, + full.last_modified, + ]); + if (!modifiedAt) return []; + + let sourceNodeRid: string; + try { + sourceNodeRid = spaceUriAndLocalIdToRid( + space.url, + concept.source_local_id, + space.platform === "Obsidian" ? "note" : undefined, + ); + } catch { + return []; + } + + return [ + { + alreadyImported: importedSourceRids.has(sourceNodeRid), + modifiedAt, + sourceApp: space.platform, + sourceNodeId: concept.source_local_id || undefined, + sourceNodeRid, + sourceSpaceId: space.url, + sourceSpaceName: space.name, + title: direct.text, + }, + ]; + }) + .sort( + (left, right) => + Date.parse(right.modifiedAt) - Date.parse(left.modifiedAt) || + left.title.localeCompare(right.title), + ); +}; + +const getGroupSharedResources = async ( + client: DGSupabaseClient, +): Promise => { + const groupIds = await getAvailableGroupIds(client); + if (groupIds.length === 0) return []; + + const resources: ResourceAccess[] = []; + for (let from = 0; ; from += PAGE_SIZE) { + const response = await client + .from("ResourceAccess") + .select("space_id, source_local_id") + .in("account_uid", groupIds) + .order("space_id") + .order("source_local_id") + .range(from, from + PAGE_SIZE - 1); + if (response.error) throw response.error; + resources.push(...response.data); + if (response.data.length < PAGE_SIZE) break; + } + + return [ + ...new Map( + resources.map((resource) => [ + getResourceKey({ + sourceLocalId: resource.source_local_id, + spaceId: resource.space_id, + }), + resource, + ]), + ).values(), + ]; +}; + +const chunk = (values: T[], size: number): T[][] => + Array.from({ length: Math.ceil(values.length / size) }, (_, index) => + values.slice(index * size, (index + 1) * size), + ); + +const getSharedNodeRows = async ({ + client, + currentSpaceId, +}: { + client: DGSupabaseClient; + currentSpaceId: number; +}): Promise => { + const resources = (await getGroupSharedResources(client)).filter( + (resource) => resource.space_id !== currentSpaceId, + ); + if (resources.length === 0) + return { concepts: [], contents: [], resources, spaces: [] }; + + const spaceIds = [...new Set(resources.map((resource) => resource.space_id))]; + const spacesResponse = await client + .from("my_spaces") + .select("id, name, platform, url") + .in("id", spaceIds); + if (spacesResponse.error) throw spacesResponse.error; + + const concepts: SharedConcept[] = []; + const contents: SharedContent[] = []; + for (const spaceId of spaceIds) { + const sourceLocalIds = resources + .filter((resource) => resource.space_id === spaceId) + .map((resource) => resource.source_local_id); + for (const ids of chunk(sourceLocalIds, RESOURCE_ID_CHUNK_SIZE)) { + const [conceptsResponse, contentsResponse] = await Promise.all([ + client + .from("my_concepts") + .select( + "is_schema, last_modified, schema_id, source_local_id, space_id", + ) + .eq("space_id", spaceId) + .eq("is_schema", false) + .in("source_local_id", ids), + client + .from("my_contents") + .select( + "content_type, last_modified, source_local_id, space_id, text, variant", + ) + .eq("space_id", spaceId) + .in("source_local_id", ids) + .in("variant", ["direct", "full"]), + ]); + if (conceptsResponse.error) throw conceptsResponse.error; + if (contentsResponse.error) throw contentsResponse.error; + concepts.push(...conceptsResponse.data); + contents.push(...contentsResponse.data); + } + } + + return { + concepts, + contents, + resources, + spaces: spacesResponse.data, + }; +}; + +export const getImportedSourceRids = async (): Promise> => { + const query = `[:find [?rid ...] + :where + [?page :block/props ?props] + [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] + [(get ?dgData :${IMPORTED_FROM_PROP_KEY}) ?imported] + [(get ?imported :sourceNodeRid) ?rid]]`; + const result = (await window.roamAlphaAPI.data.async.q(query)) as unknown[]; + return new Set( + result.filter((rid): rid is string => typeof rid === "string"), + ); +}; + +export const discoverSharedNodes = async ({ + client, + currentSpaceId, +}: { + client: DGSupabaseClient; + currentSpaceId: number; +}): Promise => { + const [rows, importedSourceRids] = await Promise.all([ + getSharedNodeRows({ client, currentSpaceId }), + getImportedSourceRids(), + ]); + return buildDiscoveredSharedNodes({ + ...rows, + currentSpaceId, + importedSourceRids, + }); +}; diff --git a/apps/roam/src/utils/registerCommandPaletteCommands.ts b/apps/roam/src/utils/registerCommandPaletteCommands.ts index 00c7ff076..11d55ba38 100644 --- a/apps/roam/src/utils/registerCommandPaletteCommands.ts +++ b/apps/roam/src/utils/registerCommandPaletteCommands.ts @@ -53,6 +53,7 @@ import { getBlockSelection, insertPageRefAtRange, } from "./advancedSearchFooterUtils"; +import { renderDiscoverSharedNodesDialog } from "~/components/DiscoverSharedNodesDialog"; export const createDiscourseNodeFromCommand = ( extensionAPI: OnloadArgs["extensionAPI"], @@ -341,6 +342,11 @@ export const registerCommandPaletteCommands = (onloadArgs: OnloadArgs) => { renderSettings({ onloadArgs }); }; + const discoverSharedNodes = () => { + posthog.capture("Shared Nodes: Discover Command Triggered"); + renderDiscoverSharedNodesDialog({}); + }; + const toggleDiscourseContextOverlay = async () => { const currentValue = getPersonalSetting([ PERSONAL_KEYS.discourseContextOverlay, @@ -415,6 +421,7 @@ export const registerCommandPaletteCommands = (onloadArgs: OnloadArgs) => { void addCommand("DG: Export - Discourse graph", exportDiscourseGraph); void addCommand("DG: Open - Discourse settings", renderSettingsPopup); if (isSyncEnabled()) { + void addCommand("DG: Discover shared nodes", discoverSharedNodes); void addCommand("DG: Share current node", shareCurrentNode); } if (getFeatureFlag("Advanced node search enabled")) { From b805ddf2c534959da3d9540db13bb5bd9fddfa5a Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 15:53:25 +0530 Subject: [PATCH 02/12] [ENG-1855] Keep discovery dialog open --- apps/roam/src/components/DiscoverSharedNodesDialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx index a60962ba3..49da7b85c 100644 --- a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -143,7 +143,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { return ( Date: Fri, 10 Jul 2026 17:40:29 +0530 Subject: [PATCH 03/12] [ENG-1855] Fix discovery dialog dismissing on keyboard launch Roam's command palette runs commands on Enter keyup, which landed as a click on the dialog's autofocused close button and dismissed it instantly. Disable autoFocus/enforceFocus like AdvancedSearchDialog and restore outside-click close. Also from review: use Tailwind classes instead of inline styles, drop the unused ValidSharedSpace.id field, and unexport the file-internal getImportedSourceRids. --- .../components/DiscoverSharedNodesDialog.tsx | 86 ++++++------------- apps/roam/src/utils/discoverSharedNodes.ts | 4 +- 2 files changed, 27 insertions(+), 63 deletions(-) diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx index 49da7b85c..a45a52556 100644 --- a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -28,43 +28,31 @@ const SharedNodeRow = ({ node }: { node: DiscoveredSharedNode }) => ( {node.sourceApp} -
+
{node.sourceSpaceName}
{node.sourceSpaceId}
-
+
{node.title}
{node.sourceNodeId ? (
{node.sourceNodeId} @@ -73,7 +61,7 @@ const SharedNodeRow = ({ node }: { node: DiscoveredSharedNode }) => ( Not provided )} - + {formatModifiedAt(node.modifiedAt)} @@ -142,25 +130,23 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { return (
-
+
) => setSearch(event.target.value) @@ -180,40 +166,26 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => {
{loading ? ( -
+
) : error ? ( -
{error}
+
{error}
 ) : visibleNodes.length === 0 ? ( -
+
) : ( -
- +
+ Source app @@ -234,14 +206,8 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { )}
-
- +
+ {loading || error ? "" : `${visibleNodes.length} of ${nodes.length} nodes`} diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts index ae78fded5..89045db80 100644 --- a/apps/roam/src/utils/discoverSharedNodes.ts +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -30,7 +30,6 @@ type SharedSpace = Pick< "id" | "name" | "platform" | "url" >; type ValidSharedSpace = { - id: number; name: string; platform: "Roam" | "Obsidian"; url: string; @@ -107,7 +106,6 @@ export const buildDiscoveredSharedNodes = ({ [ space.id, { - id: space.id, name: space.name, platform: space.platform, url: space.url, @@ -301,7 +299,7 @@ const getSharedNodeRows = async ({ }; }; -export const getImportedSourceRids = async (): Promise> => { +const getImportedSourceRids = async (): Promise> => { const query = `[:find [?rid ...] :where [?page :block/props ?props] From aba903dad2ed8c2814d3efdf58a4f2220428d0ec Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 18:00:17 +0530 Subject: [PATCH 04/12] [ENG-1855] Use getAllPages for ResourceAccess pagination Replaces the hand-rolled paging loop with the existing @repo/database/lib/pagination helper, matching how syncDgNodesToSupabase already reads ResourceAccess. --- apps/roam/src/utils/discoverSharedNodes.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts index 89045db80..0b5463601 100644 --- a/apps/roam/src/utils/discoverSharedNodes.ts +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -1,5 +1,6 @@ import type { DGSupabaseClient } from "@repo/database/lib/client"; import { getAvailableGroupIds } from "@repo/database/lib/groups"; +import { getAllPages } from "@repo/database/lib/pagination"; import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; import type { Tables } from "@repo/database/dbTypes"; import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; @@ -207,19 +208,16 @@ const getGroupSharedResources = async ( const groupIds = await getAvailableGroupIds(client); if (groupIds.length === 0) return []; - const resources: ResourceAccess[] = []; - for (let from = 0; ; from += PAGE_SIZE) { - const response = await client + const resources = await getAllPages( + client .from("ResourceAccess") .select("space_id, source_local_id") .in("account_uid", groupIds) .order("space_id") - .order("source_local_id") - .range(from, from + PAGE_SIZE - 1); - if (response.error) throw response.error; - resources.push(...response.data); - if (response.data.length < PAGE_SIZE) break; - } + .order("source_local_id"), + PAGE_SIZE, + ); + if (!Array.isArray(resources)) throw resources; return [ ...new Map( From 3b91d1bf7789120bc2a9da14eabc73c2949bc638 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 19:08:57 +0530 Subject: [PATCH 05/12] [ENG-2019] Add shared cross-space node discovery module to packages/database --- .github/workflows/ci.yaml | 3 + packages/database/package.json | 4 +- .../src/lib/__tests__/sharedNodes.test.ts | 161 +++++++++ packages/database/src/lib/sharedNodes.ts | 319 ++++++++++++++++++ pnpm-lock.yaml | 3 + turbo.json | 3 + 6 files changed, 492 insertions(+), 1 deletion(-) create mode 100644 packages/database/src/lib/__tests__/sharedNodes.test.ts create mode 100644 packages/database/src/lib/sharedNodes.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48b0f4180..bf68f193d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,6 +37,9 @@ jobs: - name: Run Tests run: npx turbo run test --filter=roam --ui stream + - name: Run Package Unit Tests + run: npx turbo run test:unit --ui stream + lint-changed-files: if: github.event_name == 'pull_request' runs-on: ubuntu-latest diff --git a/packages/database/package.json b/packages/database/package.json index 5653b1eda..879730574 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -34,6 +34,7 @@ "lint:fix": "eslint --fix . && tsx scripts/lintSchemas.ts -f && tsx scripts/lintFunctions.ts", "migrate": "tsx scripts/migrate.ts", "test": "pnpm run build && cucumber-js", + "test:unit": "vitest run", "test:withserve": "pnpm run build && tsx scripts/serveAndTest.ts", "genenv": "tsx scripts/createEnv.mts", "gentypes": "tsx scripts/genTypes.ts", @@ -65,7 +66,8 @@ "ts-node-maintained": "^10.9.5", "tsx": "4.20.6", "typescript": "5.9.2", - "vercel": "53.1.0" + "vercel": "53.1.0", + "vitest": "catalog:" }, "prettier": { "plugins": [ diff --git a/packages/database/src/lib/__tests__/sharedNodes.test.ts b/packages/database/src/lib/__tests__/sharedNodes.test.ts new file mode 100644 index 000000000..138a80dba --- /dev/null +++ b/packages/database/src/lib/__tests__/sharedNodes.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { buildSharedNodeCandidates } from "../sharedNodes"; + +type BuildArgs = Parameters[0]; + +const resources: BuildArgs["resources"] = [ + { space_id: 20, source_local_id: "node-1" }, + { space_id: 20, source_local_id: "schema-1" }, +]; +const spaces: BuildArgs["spaces"] = [ + { + id: 20, + name: "Research vault", + platform: "Obsidian", + url: "obsidian:vault-a", + }, +]; +const concepts: BuildArgs["concepts"] = [ + { + is_schema: false, + last_modified: "2026-06-14T12:00:00.000Z", + schema_id: 200, + source_local_id: "node-1", + space_id: 20, + }, +]; +const contents: BuildArgs["contents"] = [ + { + author_id: 42, + content_type: "text/plain", + created: "2026-06-14T11:00:00.000Z", + last_modified: "2026-06-14T13:00:00.000Z", + metadata: { source: "obsidian" }, + source_local_id: "node-1", + space_id: 20, + text: "EVD - REM sleep and recall", + variant: "direct", + }, + { + author_id: null, + content_type: "text/markdown", + created: null, + last_modified: "2026-06-14T15:00:00.000Z", + metadata: null, + source_local_id: "node-1", + space_id: 20, + text: "# EVD - REM sleep and recall", + variant: "full", + }, +]; +const rid = "orn:obsidian.note:vault-a/node-1"; + +const build = ({ + conceptsOverride = concepts, + contentsOverride = contents, + currentSpaceId = 10, + resourcesOverride = resources, + spacesOverride = spaces, +}: { + conceptsOverride?: typeof concepts; + contentsOverride?: typeof contents; + currentSpaceId?: number; + resourcesOverride?: typeof resources; + spacesOverride?: typeof spaces; +} = {}) => + buildSharedNodeCandidates({ + concepts: conceptsOverride, + contents: contentsOverride, + currentSpaceId, + resources: resourcesOverride, + spaces: spacesOverride, + }); + +describe("buildSharedNodeCandidates", () => { + it("builds a group-shared contract node with stable source identity", () => { + expect(build()).toEqual([ + { + rid, + sourceLocalId: "node-1", + spaceId: 20, + spaceName: "Research vault", + spaceUri: "obsidian:vault-a", + platform: "Obsidian", + title: "EVD - REM sleep and recall", + created: "2026-06-14T11:00:00.000Z", + lastModified: "2026-06-14T15:00:00.000Z", + authorId: 42, + directMetadata: { source: "obsidian" }, + }, + ]); + }); + + it("does not discover shared resources from the current space", () => { + expect(build({ currentSpaceId: 20 })).toEqual([]); + }); + + it("requires the exact shared resource identity", () => { + expect( + build({ + resourcesOverride: [{ space_id: 21, source_local_id: "node-1" }], + }), + ).toEqual([]); + }); + + it.each([ + { + name: "schema concept", + conceptsOverride: [{ ...concepts[0]!, is_schema: true }], + contentsOverride: contents, + }, + { + name: "missing node type", + conceptsOverride: [{ ...concepts[0]!, schema_id: null }], + contentsOverride: contents, + }, + { + name: "missing direct content", + conceptsOverride: concepts, + contentsOverride: [contents[1]!], + }, + { + name: "missing full content", + conceptsOverride: concepts, + contentsOverride: [contents[0]!], + }, + { + name: "untyped full content", + conceptsOverride: concepts, + contentsOverride: [contents[0]!, { ...contents[1]!, content_type: null }], + }, + ])("filters a node with $name", ({ conceptsOverride, contentsOverride }) => { + expect(build({ conceptsOverride, contentsOverride })).toEqual([]); + }); + + it("sorts newest nodes first", () => { + const olderConcept = { + ...concepts[0]!, + last_modified: "2026-06-10T12:00:00.000Z", + source_local_id: "node-2", + }; + const olderContents = contents.map((content) => ({ + ...content, + last_modified: "2026-06-10T12:00:00.000Z", + source_local_id: "node-2", + text: + content.variant === "direct" + ? "Older shared node" + : "# Older shared node", + })); + expect( + build({ + conceptsOverride: [olderConcept, concepts[0]!], + contentsOverride: [...olderContents, ...contents], + resourcesOverride: [ + ...resources, + { space_id: 20, source_local_id: "node-2" }, + ], + }).map((node) => node.sourceLocalId), + ).toEqual(["node-1", "node-2"]); + }); +}); diff --git a/packages/database/src/lib/sharedNodes.ts b/packages/database/src/lib/sharedNodes.ts new file mode 100644 index 000000000..eb0405f24 --- /dev/null +++ b/packages/database/src/lib/sharedNodes.ts @@ -0,0 +1,319 @@ +import type { DGSupabaseClient } from "./client"; +import { getAvailableGroupIds } from "./groups"; +import { getAllPages } from "./pagination"; +import { spaceUriAndLocalIdToRid } from "./rid"; +import type { Json, Tables } from "../dbTypes"; + +const PAGE_SIZE = 1000; +const RESOURCE_ID_CHUNK_SIZE = 100; + +type ResourceAccess = Pick< + Tables<"ResourceAccess">, + "space_id" | "source_local_id" +>; +type SharedConcept = Pick< + Tables<"my_concepts">, + "is_schema" | "last_modified" | "schema_id" | "source_local_id" | "space_id" +>; +type SharedContent = Pick< + Tables<"my_contents">, + | "author_id" + | "content_type" + | "created" + | "last_modified" + | "metadata" + | "source_local_id" + | "space_id" + | "text" + | "variant" +>; +type SharedSpace = Pick< + Tables<"my_spaces">, + "id" | "name" | "platform" | "url" +>; +type ValidSharedSpace = { + name: string; + platform: "Roam" | "Obsidian"; + url: string; +}; + +export type SharedNodeCandidate = { + rid: string; + sourceLocalId: string; + spaceId: number; + spaceName: string; + spaceUri: string; + platform: "Roam" | "Obsidian"; + title: string; + created: string | null; + lastModified: string; + authorId?: number; + directMetadata: Json; +}; + +export type SharedNodeRows = { + concepts: SharedConcept[]; + contents: SharedContent[]; + resources: ResourceAccess[]; + spaces: SharedSpace[]; +}; + +const getResourceKey = ({ + sourceLocalId, + spaceId, +}: { + sourceLocalId: string; + spaceId: number; +}): string => `${spaceId}:${sourceLocalId}`; + +const getLatestTimestamp = (timestamps: (string | null)[]): string | null => { + const validTimestamps = timestamps.filter( + (timestamp): timestamp is string => + typeof timestamp === "string" && !Number.isNaN(Date.parse(timestamp)), + ); + if (validTimestamps.length === 0) return null; + return validTimestamps.reduce((latest, timestamp) => + Date.parse(timestamp) > Date.parse(latest) ? timestamp : latest, + ); +}; + +export const buildSharedNodeCandidates = ({ + concepts, + contents, + currentSpaceId, + resources, + spaces, +}: SharedNodeRows & { + currentSpaceId: number; +}): SharedNodeCandidate[] => { + const sharedResourceKeys = new Set( + resources + .filter((resource) => resource.space_id !== currentSpaceId) + .map((resource) => + getResourceKey({ + sourceLocalId: resource.source_local_id, + spaceId: resource.space_id, + }), + ), + ); + const spacesById = new Map( + spaces.flatMap((space): [number, ValidSharedSpace][] => { + if ( + typeof space.id !== "number" || + typeof space.name !== "string" || + (space.platform !== "Roam" && space.platform !== "Obsidian") || + typeof space.url !== "string" + ) + return []; + return [ + [ + space.id, + { + name: space.name, + platform: space.platform, + url: space.url, + }, + ], + ]; + }), + ); + const contentByResource = new Map< + string, + Partial> + >(); + + contents.forEach((content) => { + if ( + typeof content.space_id !== "number" || + typeof content.source_local_id !== "string" || + (content.variant !== "direct" && content.variant !== "full") + ) + return; + const key = getResourceKey({ + sourceLocalId: content.source_local_id, + spaceId: content.space_id, + }); + const variants = contentByResource.get(key) ?? {}; + variants[content.variant] = content; + contentByResource.set(key, variants); + }); + + return concepts + .flatMap((concept): SharedNodeCandidate[] => { + if ( + concept.is_schema !== false || + concept.schema_id === null || + typeof concept.space_id !== "number" || + typeof concept.source_local_id !== "string" + ) + return []; + + const resourceKey = getResourceKey({ + sourceLocalId: concept.source_local_id, + spaceId: concept.space_id, + }); + if (!sharedResourceKeys.has(resourceKey)) return []; + + const space = spacesById.get(concept.space_id); + const variants = contentByResource.get(resourceKey); + const direct = variants?.direct; + const full = variants?.full; + if ( + !space || + typeof direct?.text !== "string" || + typeof full?.text !== "string" || + typeof full.content_type !== "string" + ) + return []; + + const lastModified = getLatestTimestamp([ + concept.last_modified, + direct.last_modified, + full.last_modified, + ]); + if (!lastModified) return []; + + let rid: string; + try { + rid = spaceUriAndLocalIdToRid( + space.url, + concept.source_local_id, + space.platform === "Obsidian" ? "note" : undefined, + ); + } catch { + return []; + } + + return [ + { + rid, + sourceLocalId: concept.source_local_id, + spaceId: concept.space_id, + spaceName: space.name, + spaceUri: space.url, + platform: space.platform, + title: direct.text, + created: direct.created, + lastModified, + authorId: direct.author_id ?? undefined, + directMetadata: direct.metadata, + }, + ]; + }) + .sort( + (left, right) => + Date.parse(right.lastModified) - Date.parse(left.lastModified) || + left.title.localeCompare(right.title), + ); +}; + +const getGroupSharedResources = async ( + client: DGSupabaseClient, + groupIds?: string[], +): Promise => { + const availableGroupIds = groupIds ?? (await getAvailableGroupIds(client)); + if (availableGroupIds.length === 0) return []; + + const resources = await getAllPages( + client + .from("ResourceAccess") + .select("space_id, source_local_id") + .in("account_uid", availableGroupIds) + .order("space_id") + .order("source_local_id"), + PAGE_SIZE, + ); + if (!Array.isArray(resources)) throw resources; + + return [ + ...new Map( + resources.map((resource) => [ + getResourceKey({ + sourceLocalId: resource.source_local_id, + spaceId: resource.space_id, + }), + resource, + ]), + ).values(), + ]; +}; + +const chunk = (values: T[], size: number): T[][] => + Array.from({ length: Math.ceil(values.length / size) }, (_, index) => + values.slice(index * size, (index + 1) * size), + ); + +const getSharedNodeRows = async ({ + client, + currentSpaceId, + groupIds, +}: { + client: DGSupabaseClient; + currentSpaceId: number; + groupIds?: string[]; +}): Promise => { + const resources = (await getGroupSharedResources(client, groupIds)).filter( + (resource) => resource.space_id !== currentSpaceId, + ); + if (resources.length === 0) + return { concepts: [], contents: [], resources, spaces: [] }; + + const spaceIds = [...new Set(resources.map((resource) => resource.space_id))]; + const spacesResponse = await client + .from("my_spaces") + .select("id, name, platform, url") + .in("id", spaceIds); + if (spacesResponse.error) throw spacesResponse.error; + + const concepts: SharedConcept[] = []; + const contents: SharedContent[] = []; + for (const spaceId of spaceIds) { + const sourceLocalIds = resources + .filter((resource) => resource.space_id === spaceId) + .map((resource) => resource.source_local_id); + for (const ids of chunk(sourceLocalIds, RESOURCE_ID_CHUNK_SIZE)) { + const [conceptsResponse, contentsResponse] = await Promise.all([ + client + .from("my_concepts") + .select( + "is_schema, last_modified, schema_id, source_local_id, space_id", + ) + .eq("space_id", spaceId) + .eq("is_schema", false) + .in("source_local_id", ids), + client + .from("my_contents") + .select( + "author_id, content_type, created, last_modified, metadata, source_local_id, space_id, text, variant", + ) + .eq("space_id", spaceId) + .in("source_local_id", ids) + .in("variant", ["direct", "full"]), + ]); + if (conceptsResponse.error) throw conceptsResponse.error; + if (contentsResponse.error) throw contentsResponse.error; + concepts.push(...conceptsResponse.data); + contents.push(...contentsResponse.data); + } + } + + return { + concepts, + contents, + resources, + spaces: spacesResponse.data, + }; +}; + +export const listGroupSharedNodes = async ({ + client, + currentSpaceId, + groupIds, +}: { + client: DGSupabaseClient; + currentSpaceId: number; + groupIds?: string[]; +}): Promise => { + const rows = await getSharedNodeRows({ client, currentSpaceId, groupIds }); + return buildSharedNodeCandidates({ ...rows, currentSpaceId }); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3d0aeba4..ba8f12ba6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -642,6 +642,9 @@ importers: vercel: specifier: 53.1.0 version: 53.1.0(rollup@4.60.3)(typescript@5.9.2) + vitest: + specifier: 'catalog:' + version: 4.1.6(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.0)(@types/node@22.20.0)(jsdom@20.0.3)(msw@2.11.1(@types/node@22.20.0)(typescript@5.5.4))(vite@7.3.3(@types/node@22.20.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) packages/eslint-config: devDependencies: diff --git a/turbo.json b/turbo.json index 28a36e4ad..b96290138 100644 --- a/turbo.json +++ b/turbo.json @@ -75,6 +75,9 @@ "test": { "outputs": [] }, + "test:unit": { + "outputs": [] + }, "dev": { "passThroughEnv": ["OBSIDIAN_PLUGIN_PATH"], "cache": false, From 6dccc48e8cc5044b4d6f3e357244fbe625cbcf17 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 19:08:57 +0530 Subject: [PATCH 06/12] [ENG-2019] Swap Roam shared-node discovery onto the shared module --- .../__tests__/discoverSharedNodes.test.ts | 171 ++-------- apps/roam/src/utils/discoverSharedNodes.ts | 309 ++---------------- 2 files changed, 50 insertions(+), 430 deletions(-) diff --git a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts index ca007246f..4a3a9637e 100644 --- a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts +++ b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts @@ -1,82 +1,35 @@ import { describe, expect, it } from "vitest"; -import { buildDiscoveredSharedNodes } from "~/utils/discoverSharedNodes"; - -type BuildArgs = Parameters[0]; - -const resources: BuildArgs["resources"] = [ - { space_id: 20, source_local_id: "node-1" }, - { space_id: 20, source_local_id: "schema-1" }, -]; -const spaces: BuildArgs["spaces"] = [ - { - id: 20, - name: "Research vault", - platform: "Obsidian", - url: "obsidian:vault-a", - }, -]; -const concepts: BuildArgs["concepts"] = [ - { - is_schema: false, - last_modified: "2026-06-14T12:00:00.000Z", - schema_id: 200, - source_local_id: "node-1", - space_id: 20, - }, -]; -const contents: BuildArgs["contents"] = [ - { - content_type: "text/plain", - last_modified: "2026-06-14T13:00:00.000Z", - source_local_id: "node-1", - space_id: 20, - text: "EVD - REM sleep and recall", - variant: "direct", - }, - { - content_type: "text/markdown", - last_modified: "2026-06-14T15:00:00.000Z", - source_local_id: "node-1", - space_id: 20, - text: "# EVD - REM sleep and recall", - variant: "full", - }, -]; -const sourceNodeRid = "orn:obsidian.note:vault-a/node-1"; - -const build = ({ - conceptsOverride = concepts, - contentsOverride = contents, - currentSpaceId = 10, - importedSourceRids = new Set(), - resourcesOverride = resources, - spacesOverride = spaces, -}: { - conceptsOverride?: typeof concepts; - contentsOverride?: typeof contents; - currentSpaceId?: number; - importedSourceRids?: ReadonlySet; - resourcesOverride?: typeof resources; - spacesOverride?: typeof spaces; -} = {}) => - buildDiscoveredSharedNodes({ - concepts: conceptsOverride, - contents: contentsOverride, - currentSpaceId, - importedSourceRids, - resources: resourcesOverride, - spaces: spacesOverride, - }); - -describe("buildDiscoveredSharedNodes", () => { - it("builds a group-shared contract node with stable source identity", () => { - expect(build({ importedSourceRids: new Set([sourceNodeRid]) })).toEqual([ +import { toDiscoveredSharedNodes } from "~/utils/discoverSharedNodes"; +import type { SharedNodeCandidate } from "@repo/database/lib/sharedNodes"; + +const candidate: SharedNodeCandidate = { + rid: "orn:obsidian.note:vault-a/node-1", + sourceLocalId: "node-1", + spaceId: 20, + spaceName: "Research vault", + spaceUri: "obsidian:vault-a", + platform: "Obsidian", + title: "EVD - REM sleep and recall", + created: "2026-06-14T12:30:00.000Z", + lastModified: "2026-06-14T15:00:00.000Z", + authorId: 7, + directMetadata: null, +}; + +describe("toDiscoveredSharedNodes", () => { + it("maps a candidate to the exact discovered shared node shape", () => { + expect( + toDiscoveredSharedNodes({ + candidates: [candidate], + importedSourceRids: new Set([candidate.rid]), + }), + ).toEqual([ { alreadyImported: true, modifiedAt: "2026-06-14T15:00:00.000Z", sourceApp: "Obsidian", sourceNodeId: "node-1", - sourceNodeRid, + sourceNodeRid: "orn:obsidian.note:vault-a/node-1", sourceSpaceId: "obsidian:vault-a", sourceSpaceName: "Research vault", title: "EVD - REM sleep and recall", @@ -84,80 +37,12 @@ describe("buildDiscoveredSharedNodes", () => { ]); }); - it("does not discover shared resources from the current space", () => { - expect(build({ currentSpaceId: 20 })).toEqual([]); - }); - - it("requires the exact shared resource identity", () => { - expect( - build({ - resourcesOverride: [{ space_id: 21, source_local_id: "node-1" }], - }), - ).toEqual([]); - }); - - it.each([ - { - name: "schema concept", - conceptsOverride: [{ ...concepts[0], is_schema: true }], - contentsOverride: contents, - }, - { - name: "missing node type", - conceptsOverride: [{ ...concepts[0], schema_id: null }], - contentsOverride: contents, - }, - { - name: "missing direct content", - conceptsOverride: concepts, - contentsOverride: [contents[1]], - }, - { - name: "missing full content", - conceptsOverride: concepts, - contentsOverride: [contents[0]], - }, - { - name: "untyped full content", - conceptsOverride: concepts, - contentsOverride: [contents[0], { ...contents[1], content_type: null }], - }, - ])("filters a node with $name", ({ conceptsOverride, contentsOverride }) => { - expect(build({ conceptsOverride, contentsOverride })).toEqual([]); - }); - it("matches imports by RID rather than source-local ID alone", () => { expect( - build({ + toDiscoveredSharedNodes({ + candidates: [candidate], importedSourceRids: new Set(["orn:obsidian.note:another-vault/node-1"]), })[0]?.alreadyImported, ).toBe(false); }); - - it("sorts newest nodes first", () => { - const olderConcept = { - ...concepts[0], - last_modified: "2026-06-10T12:00:00.000Z", - source_local_id: "node-2", - }; - const olderContents = contents.map((content) => ({ - ...content, - last_modified: "2026-06-10T12:00:00.000Z", - source_local_id: "node-2", - text: - content.variant === "direct" - ? "Older shared node" - : "# Older shared node", - })); - expect( - build({ - conceptsOverride: [olderConcept, concepts[0]], - contentsOverride: [...olderContents, ...contents], - resourcesOverride: [ - ...resources, - { space_id: 20, source_local_id: "node-2" }, - ], - }).map((node) => node.sourceNodeId), - ).toEqual(["node-1", "node-2"]); - }); }); diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts index 0b5463601..80d4a8d3d 100644 --- a/apps/roam/src/utils/discoverSharedNodes.ts +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -1,40 +1,11 @@ import type { DGSupabaseClient } from "@repo/database/lib/client"; -import { getAvailableGroupIds } from "@repo/database/lib/groups"; -import { getAllPages } from "@repo/database/lib/pagination"; -import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; -import type { Tables } from "@repo/database/dbTypes"; +import { + listGroupSharedNodes, + type SharedNodeCandidate, +} from "@repo/database/lib/sharedNodes"; import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; const IMPORTED_FROM_PROP_KEY = "importedFrom"; -const PAGE_SIZE = 1000; -const RESOURCE_ID_CHUNK_SIZE = 100; - -type ResourceAccess = Pick< - Tables<"ResourceAccess">, - "space_id" | "source_local_id" ->; -type SharedConcept = Pick< - Tables<"my_concepts">, - "is_schema" | "last_modified" | "schema_id" | "source_local_id" | "space_id" ->; -type SharedContent = Pick< - Tables<"my_contents">, - | "content_type" - | "last_modified" - | "source_local_id" - | "space_id" - | "text" - | "variant" ->; -type SharedSpace = Pick< - Tables<"my_spaces">, - "id" | "name" | "platform" | "url" ->; -type ValidSharedSpace = { - name: string; - platform: "Roam" | "Obsidian"; - url: string; -}; export type DiscoveredSharedNode = { alreadyImported: boolean; @@ -47,255 +18,23 @@ export type DiscoveredSharedNode = { title: string; }; -type SharedNodeRows = { - concepts: SharedConcept[]; - contents: SharedContent[]; - resources: ResourceAccess[]; - spaces: SharedSpace[]; -}; - -const getResourceKey = ({ - sourceLocalId, - spaceId, -}: { - sourceLocalId: string; - spaceId: number; -}): string => `${spaceId}:${sourceLocalId}`; - -const getLatestTimestamp = (timestamps: (string | null)[]): string | null => { - const validTimestamps = timestamps.filter( - (timestamp): timestamp is string => - typeof timestamp === "string" && !Number.isNaN(Date.parse(timestamp)), - ); - if (validTimestamps.length === 0) return null; - return validTimestamps.reduce((latest, timestamp) => - Date.parse(timestamp) > Date.parse(latest) ? timestamp : latest, - ); -}; - -export const buildDiscoveredSharedNodes = ({ - concepts, - contents, - currentSpaceId, +export const toDiscoveredSharedNodes = ({ + candidates, importedSourceRids, - resources, - spaces, -}: SharedNodeRows & { - currentSpaceId: number; - importedSourceRids: ReadonlySet; -}): DiscoveredSharedNode[] => { - const sharedResourceKeys = new Set( - resources - .filter((resource) => resource.space_id !== currentSpaceId) - .map((resource) => - getResourceKey({ - sourceLocalId: resource.source_local_id, - spaceId: resource.space_id, - }), - ), - ); - const spacesById = new Map( - spaces.flatMap((space): [number, ValidSharedSpace][] => { - if ( - typeof space.id !== "number" || - typeof space.name !== "string" || - (space.platform !== "Roam" && space.platform !== "Obsidian") || - typeof space.url !== "string" - ) - return []; - return [ - [ - space.id, - { - name: space.name, - platform: space.platform, - url: space.url, - }, - ], - ]; - }), - ); - const contentByResource = new Map< - string, - Partial> - >(); - - contents.forEach((content) => { - if ( - typeof content.space_id !== "number" || - typeof content.source_local_id !== "string" || - (content.variant !== "direct" && content.variant !== "full") - ) - return; - const key = getResourceKey({ - sourceLocalId: content.source_local_id, - spaceId: content.space_id, - }); - const variants = contentByResource.get(key) ?? {}; - variants[content.variant] = content; - contentByResource.set(key, variants); - }); - - return concepts - .flatMap((concept): DiscoveredSharedNode[] => { - if ( - concept.is_schema !== false || - concept.schema_id === null || - typeof concept.space_id !== "number" || - typeof concept.source_local_id !== "string" - ) - return []; - - const resourceKey = getResourceKey({ - sourceLocalId: concept.source_local_id, - spaceId: concept.space_id, - }); - if (!sharedResourceKeys.has(resourceKey)) return []; - - const space = spacesById.get(concept.space_id); - const variants = contentByResource.get(resourceKey); - const direct = variants?.direct; - const full = variants?.full; - if ( - !space || - typeof direct?.text !== "string" || - typeof full?.text !== "string" || - typeof full.content_type !== "string" - ) - return []; - - const modifiedAt = getLatestTimestamp([ - concept.last_modified, - direct.last_modified, - full.last_modified, - ]); - if (!modifiedAt) return []; - - let sourceNodeRid: string; - try { - sourceNodeRid = spaceUriAndLocalIdToRid( - space.url, - concept.source_local_id, - space.platform === "Obsidian" ? "note" : undefined, - ); - } catch { - return []; - } - - return [ - { - alreadyImported: importedSourceRids.has(sourceNodeRid), - modifiedAt, - sourceApp: space.platform, - sourceNodeId: concept.source_local_id || undefined, - sourceNodeRid, - sourceSpaceId: space.url, - sourceSpaceName: space.name, - title: direct.text, - }, - ]; - }) - .sort( - (left, right) => - Date.parse(right.modifiedAt) - Date.parse(left.modifiedAt) || - left.title.localeCompare(right.title), - ); -}; - -const getGroupSharedResources = async ( - client: DGSupabaseClient, -): Promise => { - const groupIds = await getAvailableGroupIds(client); - if (groupIds.length === 0) return []; - - const resources = await getAllPages( - client - .from("ResourceAccess") - .select("space_id, source_local_id") - .in("account_uid", groupIds) - .order("space_id") - .order("source_local_id"), - PAGE_SIZE, - ); - if (!Array.isArray(resources)) throw resources; - - return [ - ...new Map( - resources.map((resource) => [ - getResourceKey({ - sourceLocalId: resource.source_local_id, - spaceId: resource.space_id, - }), - resource, - ]), - ).values(), - ]; -}; - -const chunk = (values: T[], size: number): T[][] => - Array.from({ length: Math.ceil(values.length / size) }, (_, index) => - values.slice(index * size, (index + 1) * size), - ); - -const getSharedNodeRows = async ({ - client, - currentSpaceId, }: { - client: DGSupabaseClient; - currentSpaceId: number; -}): Promise => { - const resources = (await getGroupSharedResources(client)).filter( - (resource) => resource.space_id !== currentSpaceId, - ); - if (resources.length === 0) - return { concepts: [], contents: [], resources, spaces: [] }; - - const spaceIds = [...new Set(resources.map((resource) => resource.space_id))]; - const spacesResponse = await client - .from("my_spaces") - .select("id, name, platform, url") - .in("id", spaceIds); - if (spacesResponse.error) throw spacesResponse.error; - - const concepts: SharedConcept[] = []; - const contents: SharedContent[] = []; - for (const spaceId of spaceIds) { - const sourceLocalIds = resources - .filter((resource) => resource.space_id === spaceId) - .map((resource) => resource.source_local_id); - for (const ids of chunk(sourceLocalIds, RESOURCE_ID_CHUNK_SIZE)) { - const [conceptsResponse, contentsResponse] = await Promise.all([ - client - .from("my_concepts") - .select( - "is_schema, last_modified, schema_id, source_local_id, space_id", - ) - .eq("space_id", spaceId) - .eq("is_schema", false) - .in("source_local_id", ids), - client - .from("my_contents") - .select( - "content_type, last_modified, source_local_id, space_id, text, variant", - ) - .eq("space_id", spaceId) - .in("source_local_id", ids) - .in("variant", ["direct", "full"]), - ]); - if (conceptsResponse.error) throw conceptsResponse.error; - if (contentsResponse.error) throw contentsResponse.error; - concepts.push(...conceptsResponse.data); - contents.push(...contentsResponse.data); - } - } - - return { - concepts, - contents, - resources, - spaces: spacesResponse.data, - }; -}; + candidates: SharedNodeCandidate[]; + importedSourceRids: ReadonlySet; +}): DiscoveredSharedNode[] => + candidates.map((candidate) => ({ + alreadyImported: importedSourceRids.has(candidate.rid), + modifiedAt: candidate.lastModified, + sourceApp: candidate.platform, + sourceNodeId: candidate.sourceLocalId || undefined, + sourceNodeRid: candidate.rid, + sourceSpaceId: candidate.spaceUri, + sourceSpaceName: candidate.spaceName, + title: candidate.title, + })); const getImportedSourceRids = async (): Promise> => { const query = `[:find [?rid ...] @@ -317,13 +56,9 @@ export const discoverSharedNodes = async ({ client: DGSupabaseClient; currentSpaceId: number; }): Promise => { - const [rows, importedSourceRids] = await Promise.all([ - getSharedNodeRows({ client, currentSpaceId }), + const [candidates, importedSourceRids] = await Promise.all([ + listGroupSharedNodes({ client, currentSpaceId }), getImportedSourceRids(), ]); - return buildDiscoveredSharedNodes({ - ...rows, - currentSpaceId, - importedSourceRids, - }); + return toDiscoveredSharedNodes({ candidates, importedSourceRids }); }; From 69fa2317b97d7928f7441511d7b1e5bc269a0cdb Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 19:08:57 +0530 Subject: [PATCH 07/12] [ENG-2019] Swap Obsidian import listing onto the shared module --- apps/obsidian/src/utils/importNodes.ts | 96 +++++++------------------- 1 file changed, 24 insertions(+), 72 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 06ba93edd..5cea3dd6f 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -2,6 +2,7 @@ import type { Json } from "@repo/database/dbTypes"; import matter from "gray-matter"; import { App, Notice, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { listGroupSharedNodes } from "@repo/database/lib/sharedNodes"; import type DiscourseGraphPlugin from "~/index"; import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; import type { DiscourseNode, ImportableNode } from "~/types"; @@ -40,84 +41,35 @@ export const getPublishedNodesForGroups = async ({ groupIds: string[]; currentSpaceId: number; }): Promise> => { - if (groupIds.length === 0) { - return []; - } - - // Query my_contents (RLS applied); exclude current space. Get both variants so we can use - // the latest last_modified per node and prefer "direct" for text (title). - const { data, error } = await client - .from("my_contents") - .select( - "source_local_id, space_id, text, created, last_modified, variant, metadata, author_id", - ) - .neq("space_id", currentSpaceId); - - if (error) { + const candidates = await listGroupSharedNodes({ + client, + currentSpaceId, + groupIds, + }).catch((error: { message?: string }) => { console.error("Error fetching published nodes:", error); throw new Error(`Failed to fetch published nodes: ${error.message}`); - } - - if (!data || data.length === 0) { - return []; - } - - type Row = { - source_local_id: string | null; - space_id: number | null; - text: string | null; - created: string | null; - last_modified: string | null; - variant: string | null; - author_id: number | null; - metadata: Json; - }; - - const key = (r: Row) => `${r.space_id ?? ""}\t${r.source_local_id ?? ""}`; - const groups = new Map(); - for (const row of data as Row[]) { - if (row.source_local_id == null || row.space_id == null) continue; - const k = key(row); - if (!groups.has(k)) groups.set(k, []); - groups.get(k)!.push(row); - } - - const nodes: Array = []; + }); - for (const rows of groups.values()) { - const withDate = rows.filter( - (r) => r.last_modified != null && r.text != null, - ); - if (withDate.length === 0) continue; - const latest = withDate.reduce((a, b) => - (a.last_modified ?? "") >= (b.last_modified ?? "") ? a : b, - ); - const direct = rows.find((r) => r.variant === "direct"); - const text = direct?.text ?? latest.text ?? ""; - const createdAt = latest.created - ? new Date(latest.created + "Z").valueOf() - : 0; - const modifiedAt = latest.last_modified - ? new Date(latest.last_modified + "Z").valueOf() - : 0; + return candidates.map((candidate) => { + const metadata = candidate.directMetadata; const filePath: string | undefined = - direct && - typeof direct.metadata === "object" && - typeof (direct.metadata as Record).filePath === "string" - ? (direct.metadata as Record).filePath + metadata !== null && + typeof metadata === "object" && + typeof (metadata as Record).filePath === "string" + ? (metadata as Record).filePath : undefined; - nodes.push({ - source_local_id: latest.source_local_id!, - space_id: latest.space_id!, - text, - createdAt, - modifiedAt, + return { + source_local_id: candidate.sourceLocalId, + space_id: candidate.spaceId, + text: candidate.title, + createdAt: candidate.created + ? new Date(candidate.created + "Z").valueOf() + : 0, + modifiedAt: new Date(candidate.lastModified + "Z").valueOf(), filePath, - authorId: latest.author_id ?? undefined, - }); - } - - return nodes; + authorId: candidate.authorId, + }; + }); }; export const getLocalNodeInstanceIds = ( From 0f0adf986720cbc3f0c5b084507e5b23dd95430d Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 19:20:33 +0530 Subject: [PATCH 08/12] [ENG-2019] Narrow Json metadata without any-casts in Obsidian mapper --- apps/obsidian/src/utils/importNodes.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 5cea3dd6f..03eecab0a 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -55,8 +55,9 @@ export const getPublishedNodesForGroups = async ({ const filePath: string | undefined = metadata !== null && typeof metadata === "object" && - typeof (metadata as Record).filePath === "string" - ? (metadata as Record).filePath + !Array.isArray(metadata) && + typeof metadata.filePath === "string" + ? metadata.filePath : undefined; return { source_local_id: candidate.sourceLocalId, From 39dc8cbfe7b38a68b71a6476b31450ca00f0c1d5 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 10 Jul 2026 22:57:57 +0530 Subject: [PATCH 09/12] [ENG-1856] Store source identity metadata for Roam imported nodes --- .../__tests__/importedSourceIdentity.test.ts | 140 ++++++++++++++++++ apps/roam/src/utils/discoverSharedNodes.ts | 17 +-- apps/roam/src/utils/importedSourceIdentity.ts | 93 ++++++++++++ 3 files changed, 234 insertions(+), 16 deletions(-) create mode 100644 apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts create mode 100644 apps/roam/src/utils/importedSourceIdentity.ts diff --git a/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts b/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts new file mode 100644 index 000000000..4d161e54c --- /dev/null +++ b/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts @@ -0,0 +1,140 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DISCOURSE_GRAPH_PROP_NAME } from "~/utils/createReifiedBlock"; +import { + findImportedNodeUidBySourceRid, + getImportedSourceRids, + IMPORTED_FROM_PROP_KEY, + parseImportedSourceIdentity, + readImportedSourceIdentity, + writeImportedSourceIdentity, +} from "~/utils/importedSourceIdentity"; +import type { json } from "~/utils/getBlockProps"; + +const SOURCE_NODE_RID = "orn:obsidian.note:vault-a/node-1"; +const SOURCE_MODIFIED_AT = "2026-06-14T15:00:00.000Z"; +const PAGE_UID = "page-uid"; + +const propsByUid = new Map>(); +const query = vi.fn(); + +const setRoamAlphaApi = (): void => { + (globalThis as { window: unknown }).window = { + roamAlphaAPI: { + data: { + async: { q: query }, + block: { + update: vi.fn( + ({ + block, + }: { + block: { props: Record; uid: string }; + }) => { + propsByUid.set(block.uid, block.props); + }, + ), + }, + }, + pull: (_pattern: string, [, uid]: [string, string]) => ({ + ":block/props": propsByUid.get(uid) ?? {}, + }), + }, + }; +}; + +beforeEach(() => { + propsByUid.clear(); + query.mockReset(); + setRoamAlphaApi(); +}); + +describe("imported source identity metadata", () => { + it("reads the source RID without depending on display metadata", () => { + const props = { + [DISCOURSE_GRAPH_PROP_NAME]: { + [IMPORTED_FROM_PROP_KEY]: { + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + sourceTitle: "Legacy title that may change", + }, + }, + }; + + expect(parseImportedSourceIdentity(props)).toEqual({ + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + }); + + it("returns undefined for missing or malformed source identity", () => { + expect(parseImportedSourceIdentity({})).toBeUndefined(); + expect( + parseImportedSourceIdentity({ + [DISCOURSE_GRAPH_PROP_NAME]: { + [IMPORTED_FROM_PROP_KEY]: { sourceNodeRid: 123 }, + }, + }), + ).toBeUndefined(); + }); + + it("writes the source RID and modified time while preserving sibling metadata", () => { + propsByUid.set(PAGE_UID, { + [DISCOURSE_GRAPH_PROP_NAME]: { + "relation-migration": { relationUid: 1718000000000 }, + }, + "other-extension": { enabled: true }, + }); + + writeImportedSourceIdentity({ + pageUid: PAGE_UID, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + + expect(readImportedSourceIdentity(PAGE_UID)).toEqual({ + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + expect(propsByUid.get(PAGE_UID)).toEqual({ + [DISCOURSE_GRAPH_PROP_NAME]: { + "relation-migration": { relationUid: 1718000000000 }, + [IMPORTED_FROM_PROP_KEY]: { + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }, + }, + "other-extension": { enabled: true }, + }); + }); +}); + +describe("imported source identity lookup", () => { + it("returns the stored RID set used for duplicate prevention", async () => { + query.mockResolvedValue([SOURCE_NODE_RID, 123, null]); + + await expect(getImportedSourceRids()).resolves.toEqual( + new Set([SOURCE_NODE_RID]), + ); + expect(query).toHaveBeenCalledOnce(); + expect(query.mock.calls[0]?.[0]).toContain(":sourceNodeRid"); + }); + + it("finds the imported Roam page by source RID", async () => { + query.mockResolvedValue([[PAGE_UID]]); + + await expect(findImportedNodeUidBySourceRid(SOURCE_NODE_RID)).resolves.toBe( + PAGE_UID, + ); + expect(query).toHaveBeenCalledWith( + expect.stringContaining(":sourceNodeRid"), + SOURCE_NODE_RID, + ); + }); + + it("returns null when the source RID has not been imported", async () => { + query.mockResolvedValue([]); + + await expect( + findImportedNodeUidBySourceRid(SOURCE_NODE_RID), + ).resolves.toBeNull(); + }); +}); diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts index 80d4a8d3d..f2bf162d8 100644 --- a/apps/roam/src/utils/discoverSharedNodes.ts +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -3,9 +3,7 @@ import { listGroupSharedNodes, type SharedNodeCandidate, } from "@repo/database/lib/sharedNodes"; -import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; - -const IMPORTED_FROM_PROP_KEY = "importedFrom"; +import { getImportedSourceRids } from "./importedSourceIdentity"; export type DiscoveredSharedNode = { alreadyImported: boolean; @@ -36,19 +34,6 @@ export const toDiscoveredSharedNodes = ({ title: candidate.title, })); -const getImportedSourceRids = async (): Promise> => { - const query = `[:find [?rid ...] - :where - [?page :block/props ?props] - [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] - [(get ?dgData :${IMPORTED_FROM_PROP_KEY}) ?imported] - [(get ?imported :sourceNodeRid) ?rid]]`; - const result = (await window.roamAlphaAPI.data.async.q(query)) as unknown[]; - return new Set( - result.filter((rid): rid is string => typeof rid === "string"), - ); -}; - export const discoverSharedNodes = async ({ client, currentSpaceId, diff --git a/apps/roam/src/utils/importedSourceIdentity.ts b/apps/roam/src/utils/importedSourceIdentity.ts new file mode 100644 index 000000000..ae5ba0a0f --- /dev/null +++ b/apps/roam/src/utils/importedSourceIdentity.ts @@ -0,0 +1,93 @@ +import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; +import getBlockProps, { type json } from "./getBlockProps"; +import setBlockProps from "./setBlockProps"; + +export type ImportedSourceIdentity = { + sourceModifiedAt: string; + sourceNodeRid: string; +}; + +export const IMPORTED_FROM_PROP_KEY = "importedFrom"; + +const isJsonObject = (value: json): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const parseImportedSourceIdentity = ( + props: Record, +): ImportedSourceIdentity | undefined => { + const discourseGraphProps = props[DISCOURSE_GRAPH_PROP_NAME]; + if (!isJsonObject(discourseGraphProps)) return undefined; + + const importedFrom = discourseGraphProps[IMPORTED_FROM_PROP_KEY]; + if (!isJsonObject(importedFrom)) return undefined; + + const { sourceModifiedAt, sourceNodeRid } = importedFrom; + if (typeof sourceModifiedAt !== "string" || typeof sourceNodeRid !== "string") + return undefined; + + return { sourceModifiedAt, sourceNodeRid }; +}; + +export const readImportedSourceIdentity = ( + pageUid: string, +): ImportedSourceIdentity | undefined => + parseImportedSourceIdentity(getBlockProps(pageUid)); + +export const writeImportedSourceIdentity = ({ + pageUid, + sourceModifiedAt, + sourceNodeRid, +}: { + pageUid: string; + sourceModifiedAt: string; + sourceNodeRid: string; +}): void => { + const existing = getBlockProps(pageUid)[DISCOURSE_GRAPH_PROP_NAME]; + const discourseGraphProps = isJsonObject(existing) ? existing : {}; + + setBlockProps(pageUid, { + [DISCOURSE_GRAPH_PROP_NAME]: { + ...discourseGraphProps, + [IMPORTED_FROM_PROP_KEY]: { sourceModifiedAt, sourceNodeRid }, + }, + }); +}; + +export const getImportedSourceRids = async (): Promise> => { + const query = `[:find [?rid ...] + :where + [?page :block/props ?props] + [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] + [(get ?dgData :${IMPORTED_FROM_PROP_KEY}) ?importedFrom] + [(get ?importedFrom :sourceNodeRid) ?rid]]`; + const result = (await window.roamAlphaAPI.data.async.q(query)) as unknown[]; + + return new Set( + result.filter((rid): rid is string => typeof rid === "string"), + ); +}; + +export const findImportedNodeUidBySourceRid = async ( + sourceNodeRid: string, +): Promise => { + const query = `[:find ?uid + :in $ ?sourceNodeRid + :where + [?page :block/uid ?uid] + [?page :block/props ?props] + [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] + [(get ?dgData :${IMPORTED_FROM_PROP_KEY}) ?importedFrom] + [(get ?importedFrom :sourceNodeRid) ?sourceNodeRid]]`; + const result = (await window.roamAlphaAPI.data.async.q( + query, + sourceNodeRid, + )) as [string][]; + + if (result.length > 1) { + console.warn( + `findImportedNodeUidBySourceRid: ${result.length} pages share source RID '${sourceNodeRid}'`, + ); + } + + return result[0]?.[0] ?? null; +}; From b977291b14fd5463cd4d67c03f5cd263e567ccdb Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 12 Jul 2026 23:48:16 +0530 Subject: [PATCH 10/12] [ENG-1858] Materialize Obsidian-origin markdown into Roam --- .../__tests__/importedSourceIdentity.test.ts | 4 +- .../__tests__/materializeObsidianNode.test.ts | 221 +++++++++++++++ apps/roam/src/utils/importedSourceIdentity.ts | 20 +- .../roam/src/utils/materializeObsidianNode.ts | 261 ++++++++++++++++++ 4 files changed, 497 insertions(+), 9 deletions(-) create mode 100644 apps/roam/src/utils/__tests__/materializeObsidianNode.test.ts create mode 100644 apps/roam/src/utils/materializeObsidianNode.ts diff --git a/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts b/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts index 4d161e54c..3c9826eda 100644 --- a/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts +++ b/apps/roam/src/utils/__tests__/importedSourceIdentity.test.ts @@ -76,7 +76,7 @@ describe("imported source identity metadata", () => { ).toBeUndefined(); }); - it("writes the source RID and modified time while preserving sibling metadata", () => { + it("writes the source RID and modified time while preserving sibling metadata", async () => { propsByUid.set(PAGE_UID, { [DISCOURSE_GRAPH_PROP_NAME]: { "relation-migration": { relationUid: 1718000000000 }, @@ -84,7 +84,7 @@ describe("imported source identity metadata", () => { "other-extension": { enabled: true }, }); - writeImportedSourceIdentity({ + await writeImportedSourceIdentity({ pageUid: PAGE_UID, sourceModifiedAt: SOURCE_MODIFIED_AT, sourceNodeRid: SOURCE_NODE_RID, diff --git a/apps/roam/src/utils/__tests__/materializeObsidianNode.test.ts b/apps/roam/src/utils/__tests__/materializeObsidianNode.test.ts new file mode 100644 index 000000000..08171d44a --- /dev/null +++ b/apps/roam/src/utils/__tests__/materializeObsidianNode.test.ts @@ -0,0 +1,221 @@ +import { contentTypes } from "@repo/content-model"; +import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { materializeObsidianNode } from "~/utils/materializeObsidianNode"; + +const mocks = vi.hoisted(() => ({ + deleteBlock: vi.fn(), + findImportedNodeUidBySourceRid: vi.fn(), + getShallowTreeByParentUid: vi.fn(), + writeImportedSourceIdentity: vi.fn(), +})); + +vi.mock("roamjs-components/queries/getShallowTreeByParentUid", () => ({ + default: mocks.getShallowTreeByParentUid, +})); + +vi.mock("roamjs-components/writes/deleteBlock", () => ({ + default: mocks.deleteBlock, +})); + +vi.mock("~/utils/importedSourceIdentity", () => ({ + findImportedNodeUidBySourceRid: mocks.findImportedNodeUidBySourceRid, + writeImportedSourceIdentity: mocks.writeImportedSourceIdentity, +})); + +const SOURCE_NODE_RID = "orn:obsidian.note:vault-a/node-1"; +const SOURCE_MODIFIED_AT = "2026-06-14T15:00:00.000Z"; +const NEW_PAGE_UID = "new-page-uid"; +const EXISTING_PAGE_UID = "existing-page-uid"; +const MARKDOWN = "# REM sleep correlates with recall\n\nUpdated evidence."; + +const node: CrossAppNode = { + localId: "node-1", + nodeType: { localId: "evidence" }, + content: { + direct: { value: "EVD - REM sleep and recall" }, + full: { + contentType: contentTypes.obsidianMarkdown, + value: MARKDOWN, + }, + }, + createdAt: new Date("2026-06-14T10:30:00.000Z"), + modifiedAt: new Date(SOURCE_MODIFIED_AT), + author: { localId: "author" }, +}; + +const pageFromMarkdown = vi.fn(); +const blockFromMarkdown = vi.fn(); +const deletePage = vi.fn(); +const updatePage = vi.fn(); + +const setRoamAlphaApi = (): void => { + (globalThis as { window: unknown }).window = { + roamAlphaAPI: { + data: { + block: { fromMarkdown: blockFromMarkdown }, + page: { + delete: deletePage, + fromMarkdown: pageFromMarkdown, + update: updatePage, + }, + }, + util: { generateUID: () => NEW_PAGE_UID }, + }, + }; +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.deleteBlock.mockResolvedValue(undefined); + mocks.getShallowTreeByParentUid.mockReturnValue([]); + pageFromMarkdown.mockResolvedValue({ uid: NEW_PAGE_UID }); + blockFromMarkdown.mockResolvedValue({ uids: [] }); + deletePage.mockResolvedValue(undefined); + updatePage.mockResolvedValue(undefined); + setRoamAlphaApi(); +}); + +describe("materializeObsidianNode", () => { + it("creates a Roam page from Obsidian markdown and stores source identity", async () => { + mocks.findImportedNodeUidBySourceRid.mockResolvedValue(null); + + await expect( + materializeObsidianNode({ + node, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }), + ).resolves.toEqual({ + success: true, + action: "created", + pageUid: NEW_PAGE_UID, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + + expect(pageFromMarkdown).toHaveBeenCalledWith({ + page: { + title: "EVD - REM sleep and recall", + uid: NEW_PAGE_UID, + }, + "markdown-string": MARKDOWN, + }); + expect(mocks.writeImportedSourceIdentity).toHaveBeenCalledWith({ + pageUid: NEW_PAGE_UID, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + }); + + it("replaces the existing imported page instead of creating a duplicate", async () => { + mocks.findImportedNodeUidBySourceRid.mockResolvedValue(EXISTING_PAGE_UID); + mocks.getShallowTreeByParentUid.mockReturnValue([ + { uid: "old-child-1", text: "Old content" }, + { uid: "old-child-2", text: "More old content" }, + ]); + + await expect( + materializeObsidianNode({ + node, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }), + ).resolves.toMatchObject({ + success: true, + action: "updated", + pageUid: EXISTING_PAGE_UID, + }); + + expect(pageFromMarkdown).not.toHaveBeenCalled(); + expect(updatePage).toHaveBeenCalledWith({ + page: { + title: "EVD - REM sleep and recall", + uid: EXISTING_PAGE_UID, + }, + "merge-pages": false, + }); + expect(mocks.deleteBlock).toHaveBeenCalledTimes(2); + expect(mocks.deleteBlock).toHaveBeenCalledWith("old-child-1"); + expect(mocks.deleteBlock).toHaveBeenCalledWith("old-child-2"); + expect(blockFromMarkdown).toHaveBeenCalledWith({ + location: { "parent-uid": EXISTING_PAGE_UID, order: "last" }, + "markdown-string": MARKDOWN, + }); + expect(mocks.writeImportedSourceIdentity).toHaveBeenCalledWith({ + pageUid: EXISTING_PAGE_UID, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + }); + + it("returns the source identity and failed stage when replacement fails", async () => { + mocks.findImportedNodeUidBySourceRid.mockResolvedValue(EXISTING_PAGE_UID); + blockFromMarkdown.mockRejectedValue(new Error("markdown parser failed")); + + await expect( + materializeObsidianNode({ + node, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }), + ).resolves.toEqual({ + success: false, + pageUid: EXISTING_PAGE_UID, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + error: { + message: `Failed to replace Roam content for '${SOURCE_NODE_RID}': markdown parser failed`, + stage: "replace-page-content", + }, + }); + expect(mocks.writeImportedSourceIdentity).not.toHaveBeenCalled(); + expect(mocks.deleteBlock).not.toHaveBeenCalled(); + }); + + it("removes a new page if its source identity cannot be stored", async () => { + mocks.findImportedNodeUidBySourceRid.mockResolvedValue(null); + mocks.writeImportedSourceIdentity.mockRejectedValue( + new Error("props update failed"), + ); + + const result = await materializeObsidianNode({ + node, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + }); + + expect(result).toMatchObject({ + success: false, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid: SOURCE_NODE_RID, + error: { + stage: "write-source-identity", + }, + }); + expect(result).not.toHaveProperty("pageUid"); + expect(deletePage).toHaveBeenCalledWith({ page: { uid: NEW_PAGE_UID } }); + }); + + it("rejects non-Obsidian payload identity before writing to Roam", async () => { + const sourceNodeRid = "orn:roam:graph-a/node-1"; + + await expect( + materializeObsidianNode({ + node, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid, + }), + ).resolves.toEqual({ + success: false, + sourceModifiedAt: SOURCE_MODIFIED_AT, + sourceNodeRid, + error: { + message: `Source node RID '${sourceNodeRid}' is not Obsidian-origin`, + stage: "validate-input", + }, + }); + expect(mocks.findImportedNodeUidBySourceRid).not.toHaveBeenCalled(); + expect(pageFromMarkdown).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/roam/src/utils/importedSourceIdentity.ts b/apps/roam/src/utils/importedSourceIdentity.ts index ae5ba0a0f..841081d98 100644 --- a/apps/roam/src/utils/importedSourceIdentity.ts +++ b/apps/roam/src/utils/importedSourceIdentity.ts @@ -1,6 +1,5 @@ import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; import getBlockProps, { type json } from "./getBlockProps"; -import setBlockProps from "./setBlockProps"; export type ImportedSourceIdentity = { sourceModifiedAt: string; @@ -41,14 +40,21 @@ export const writeImportedSourceIdentity = ({ pageUid: string; sourceModifiedAt: string; sourceNodeRid: string; -}): void => { - const existing = getBlockProps(pageUid)[DISCOURSE_GRAPH_PROP_NAME]; +}): Promise => { + const props = getBlockProps(pageUid); + const existing = props[DISCOURSE_GRAPH_PROP_NAME]; const discourseGraphProps = isJsonObject(existing) ? existing : {}; - setBlockProps(pageUid, { - [DISCOURSE_GRAPH_PROP_NAME]: { - ...discourseGraphProps, - [IMPORTED_FROM_PROP_KEY]: { sourceModifiedAt, sourceNodeRid }, + return window.roamAlphaAPI.data.block.update({ + block: { + uid: pageUid, + props: { + ...props, + [DISCOURSE_GRAPH_PROP_NAME]: { + ...discourseGraphProps, + [IMPORTED_FROM_PROP_KEY]: { sourceModifiedAt, sourceNodeRid }, + }, + }, }, }); }; diff --git a/apps/roam/src/utils/materializeObsidianNode.ts b/apps/roam/src/utils/materializeObsidianNode.ts new file mode 100644 index 000000000..bcb4c8d64 --- /dev/null +++ b/apps/roam/src/utils/materializeObsidianNode.ts @@ -0,0 +1,261 @@ +import { contentTypes } from "@repo/content-model"; +import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import { ridToSpaceUriAndLocalId } from "@repo/database/lib/rid"; +import getShallowTreeByParentUid from "roamjs-components/queries/getShallowTreeByParentUid"; +import deleteBlock from "roamjs-components/writes/deleteBlock"; +import { + findImportedNodeUidBySourceRid, + type ImportedSourceIdentity, + writeImportedSourceIdentity, +} from "./importedSourceIdentity"; + +type MaterializationStage = + | "validate-input" + | "find-imported-node" + | "create-page" + | "update-page-title" + | "replace-page-content" + | "write-source-identity"; + +type MaterializationFailure = ImportedSourceIdentity & { + success: false; + pageUid?: string; + error: { + message: string; + stage: MaterializationStage; + }; +}; + +type MaterializationSuccess = ImportedSourceIdentity & { + success: true; + action: "created" | "updated"; + pageUid: string; +}; + +export type MaterializeObsidianNodeResult = + | MaterializationFailure + | MaterializationSuccess; + +type RoamFromMarkdownApi = { + data: { + block: { + fromMarkdown: (args: { + location: { "parent-uid": string; order: "last" }; + "markdown-string": string; + }) => Promise<{ uids: string[] }>; + }; + page: { + delete: (args: { page: { uid: string } }) => Promise; + fromMarkdown: (args: { + page: { title: string; uid: string }; + "markdown-string": string; + }) => Promise<{ uid: string }>; + update: (args: { + page: { title: string; uid: string }; + "merge-pages": false; + }) => Promise; + }; + }; + util: { + generateUID: () => string; + }; +}; + +const getRoamFromMarkdownApi = (): RoamFromMarkdownApi => + window.roamAlphaAPI as unknown as RoamFromMarkdownApi; + +const getErrorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +const failure = ({ + error, + identity, + message, + pageUid, + stage, +}: { + error?: unknown; + identity: ImportedSourceIdentity; + message: string; + pageUid?: string; + stage: MaterializationStage; +}): MaterializationFailure => ({ + ...identity, + success: false, + ...(pageUid ? { pageUid } : {}), + error: { + message: error ? `${message}: ${getErrorMessage(error)}` : message, + stage, + }, +}); + +const validateInput = ({ + node, + sourceModifiedAt, + sourceNodeRid, +}: { + node: CrossAppNode; +} & ImportedSourceIdentity): string | undefined => { + if (!sourceNodeRid.trim()) return "Source node RID is required"; + + const { spaceUri } = ridToSpaceUriAndLocalId(sourceNodeRid); + if (!spaceUri.startsWith("obsidian:")) + return `Source node RID '${sourceNodeRid}' is not Obsidian-origin`; + + if (Number.isNaN(Date.parse(sourceModifiedAt))) + return `Source modified time '${sourceModifiedAt}' is invalid`; + + if (!node.content.direct.value.trim()) return "Source node title is required"; + + const contentType = node.content.full.contentType; + if ( + contentType !== contentTypes.markdown && + contentType !== contentTypes.obsidianMarkdown + ) + return `Unsupported Obsidian full content type '${contentType}'`; + + return undefined; +}; + +const replacePageContent = async ({ + markdown, + pageUid, +}: { + markdown: string; + pageUid: string; +}): Promise => { + const children = getShallowTreeByParentUid(pageUid); + await getRoamFromMarkdownApi().data.block.fromMarkdown({ + location: { "parent-uid": pageUid, order: "last" }, + "markdown-string": markdown, + }); + await Promise.all(children.map(({ uid }) => deleteBlock(uid))); +}; + +export const materializeObsidianNode = async ({ + node, + sourceModifiedAt, + sourceNodeRid, +}: { + node: CrossAppNode; +} & ImportedSourceIdentity): Promise => { + const identity = { sourceModifiedAt, sourceNodeRid }; + const validationError = validateInput({ node, ...identity }); + if (validationError) + return failure({ + identity, + message: validationError, + stage: "validate-input", + }); + + const title = node.content.direct.value.trim(); + const markdown = node.content.full.value; + let existingPageUid: string | null; + + try { + existingPageUid = await findImportedNodeUidBySourceRid(sourceNodeRid); + } catch (error) { + return failure({ + error, + identity, + message: `Failed to look up imported Roam node for '${sourceNodeRid}'`, + stage: "find-imported-node", + }); + } + + if (existingPageUid) { + try { + await getRoamFromMarkdownApi().data.page.update({ + page: { title, uid: existingPageUid }, + "merge-pages": false, + }); + } catch (error) { + return failure({ + error, + identity, + message: `Failed to update the Roam page title for '${sourceNodeRid}'`, + pageUid: existingPageUid, + stage: "update-page-title", + }); + } + + try { + await replacePageContent({ markdown, pageUid: existingPageUid }); + } catch (error) { + return failure({ + error, + identity, + message: `Failed to replace Roam content for '${sourceNodeRid}'`, + pageUid: existingPageUid, + stage: "replace-page-content", + }); + } + + try { + await writeImportedSourceIdentity({ + pageUid: existingPageUid, + ...identity, + }); + } catch (error) { + return failure({ + error, + identity, + message: `Content was updated, but source identity could not be refreshed for '${sourceNodeRid}'`, + pageUid: existingPageUid, + stage: "write-source-identity", + }); + } + + return { + ...identity, + success: true, + action: "updated", + pageUid: existingPageUid, + }; + } + + const pageUid = getRoamFromMarkdownApi().util.generateUID(); + try { + await getRoamFromMarkdownApi().data.page.fromMarkdown({ + page: { title, uid: pageUid }, + "markdown-string": markdown, + }); + } catch (error) { + return failure({ + error, + identity, + message: `Failed to create a Roam page for '${sourceNodeRid}'`, + stage: "create-page", + }); + } + + try { + await writeImportedSourceIdentity({ pageUid, ...identity }); + } catch (error) { + let cleanupError: unknown; + try { + await getRoamFromMarkdownApi().data.page.delete({ + page: { uid: pageUid }, + }); + } catch (caughtCleanupError) { + cleanupError = caughtCleanupError; + } + + const cleanupMessage = cleanupError + ? ` Cleanup also failed: ${getErrorMessage(cleanupError)}` + : " The newly created page was removed."; + return failure({ + identity, + message: `Roam content was created, but source identity could not be stored for '${sourceNodeRid}': ${getErrorMessage(error)}.${cleanupMessage}`, + ...(cleanupError ? { pageUid } : {}), + stage: "write-source-identity", + }); + } + + return { + ...identity, + success: true, + action: "created", + pageUid, + }; +}; From 2cb31690d4cbb6aff50e6239cdaee692ca348185 Mon Sep 17 00:00:00 2001 From: sid597 Date: Mon, 13 Jul 2026 01:29:58 +0530 Subject: [PATCH 11/12] [ENG-1859] Add Roam import action for selected shared nodes --- .../components/DiscoverSharedNodesDialog.tsx | 204 +++++++++++++++++- .../__tests__/discoverSharedNodes.test.ts | 1 + .../importDiscoveredSharedNode.test.ts | 90 ++++++++ .../importSelectedSharedNodes.test.ts | 105 +++++++++ apps/roam/src/utils/discoverSharedNodes.ts | 2 + .../src/utils/importDiscoveredSharedNode.ts | 31 +++ .../src/utils/importSelectedSharedNodes.ts | 56 +++++ .../src/lib/__tests__/sharedNodes.test.ts | 53 ++++- packages/database/src/lib/sharedNodes.ts | 121 +++++++++++ 9 files changed, 655 insertions(+), 8 deletions(-) create mode 100644 apps/roam/src/utils/__tests__/importDiscoveredSharedNode.test.ts create mode 100644 apps/roam/src/utils/__tests__/importSelectedSharedNodes.test.ts create mode 100644 apps/roam/src/utils/importDiscoveredSharedNode.ts create mode 100644 apps/roam/src/utils/importSelectedSharedNodes.ts diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx index a45a52556..f5d0b57d1 100644 --- a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -1,6 +1,7 @@ import { Button, Callout, + Checkbox, Classes, Dialog, HTMLTable, @@ -17,13 +18,41 @@ import { discoverSharedNodes, type DiscoveredSharedNode, } from "~/utils/discoverSharedNodes"; +import { importDiscoveredSharedNode } from "~/utils/importDiscoveredSharedNode"; +import { + importSelectedSharedNodes, + type SelectedSharedNodeImportResult, +} from "~/utils/importSelectedSharedNodes"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; const formatModifiedAt = (modifiedAt: string): string => new Date(modifiedAt).toLocaleString(); -const SharedNodeRow = ({ node }: { node: DiscoveredSharedNode }) => ( +const SharedNodeRow = ({ + disabled, + node, + onToggle, + selected, +}: { + disabled: boolean; + node: DiscoveredSharedNode; + onToggle: (node: DiscoveredSharedNode) => void; + selected: boolean; +}) => ( + + onToggle(node)} + title={ + node.sourceApp === "Obsidian" + ? undefined + : "Only Obsidian-origin nodes can be imported into Roam" + } + /> + {node.sourceApp} @@ -76,15 +105,62 @@ const SharedNodeRow = ({ node }: { node: DiscoveredSharedNode }) => ( ); +const ImportResultCallout = ({ + result, +}: { + result: SelectedSharedNodeImportResult; +}) => { + const failedCount = result.failed.length; + const intent = + failedCount === 0 + ? Intent.SUCCESS + : result.imported > 0 || result.skipped > 0 + ? Intent.WARNING + : Intent.DANGER; + + return ( + + {result.updated > 0 && ( +
0 ? "mb-2" : undefined}> + {result.updated} previously imported{" "} + {result.updated === 1 ? "node was" : "nodes were"} updated. +
+ )} + {failedCount > 0 && ( +
    + {result.failed.map(({ message, node }) => ( +
  • + {node.title}: {message} +
    • + ))} +
+ )} + + ); +}; + const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { const [nodes, setNodes] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(""); + const [importError, setImportError] = useState(""); + const [importing, setImporting] = useState(false); + const [importResult, setImportResult] = + useState(); const [search, setSearch] = useState(""); + const [selectedSourceRids, setSelectedSourceRids] = useState>( + new Set(), + ); const loadNodes = useCallback(async (): Promise => { setLoading(true); setError(""); + setImportError(""); + setImportResult(undefined); + setSelectedSourceRids(new Set()); try { const [client, context] = await Promise.all([ getLoggedInClient(), @@ -128,11 +204,88 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { ); }, [nodes, search]); + const selectedNodes = useMemo( + () => nodes.filter((node) => selectedSourceRids.has(node.sourceNodeRid)), + [nodes, selectedSourceRids], + ); + const visibleImportableNodes = useMemo( + () => visibleNodes.filter((node) => node.sourceApp === "Obsidian"), + [visibleNodes], + ); + const allVisibleSelected = + visibleImportableNodes.length > 0 && + visibleImportableNodes.every((node) => + selectedSourceRids.has(node.sourceNodeRid), + ); + const someVisibleSelected = visibleImportableNodes.some((node) => + selectedSourceRids.has(node.sourceNodeRid), + ); + + const toggleNode = useCallback((node: DiscoveredSharedNode): void => { + setSelectedSourceRids((current) => { + const next = new Set(current); + if (next.has(node.sourceNodeRid)) next.delete(node.sourceNodeRid); + else next.add(node.sourceNodeRid); + return next; + }); + }, []); + + const toggleAllVisible = useCallback((): void => { + setSelectedSourceRids((current) => { + const next = new Set(current); + if (allVisibleSelected) { + visibleImportableNodes.forEach((node) => + next.delete(node.sourceNodeRid), + ); + } else { + visibleImportableNodes.forEach((node) => next.add(node.sourceNodeRid)); + } + return next; + }); + }, [allVisibleSelected, visibleImportableNodes]); + + const importSelected = useCallback(async (): Promise => { + setImporting(true); + setImportError(""); + setImportResult(undefined); + try { + const client = await getLoggedInClient(); + if (!client) throw new Error("Could not connect to shared persistence."); + + const result = await importSelectedSharedNodes({ + materializeNode: (node) => importDiscoveredSharedNode({ client, node }), + nodes: selectedNodes, + }); + const failedSourceRids = new Set( + result.failed.map(({ node }) => node.sourceNodeRid), + ); + setNodes((current) => + current.map((node) => + selectedSourceRids.has(node.sourceNodeRid) && + !failedSourceRids.has(node.sourceNodeRid) + ? { ...node, alreadyImported: true } + : node, + ), + ); + setSelectedSourceRids(failedSourceRids); + setImportResult(result); + } catch (importError) { + console.error("Failed to import selected shared nodes:", importError); + setImportError( + importError instanceof Error + ? importError.message + : "Could not import selected shared nodes.", + ); + } finally { + setImporting(false); + } + }, [selectedNodes, selectedSourceRids]); + return ( void }) => {
+ {importResult && } + {importError && ( + + {importError} + + )} + {loading ? (
@@ -188,6 +348,17 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { + + + Source app Source space Title @@ -198,7 +369,13 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { {visibleNodes.map((node) => ( - + ))} @@ -210,9 +387,22 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { {loading || error ? "" - : `${visibleNodes.length} of ${nodes.length} nodes`} + : `${visibleNodes.length} of ${nodes.length} nodes ยท ${selectedNodes.length} selected`} - +
+ + +
diff --git a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts index 4a3a9637e..3b852dbb2 100644 --- a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts +++ b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts @@ -30,6 +30,7 @@ describe("toDiscoveredSharedNodes", () => { sourceApp: "Obsidian", sourceNodeId: "node-1", sourceNodeRid: "orn:obsidian.note:vault-a/node-1", + sourceSpaceDatabaseId: 20, sourceSpaceId: "obsidian:vault-a", sourceSpaceName: "Research vault", title: "EVD - REM sleep and recall", diff --git a/apps/roam/src/utils/__tests__/importDiscoveredSharedNode.test.ts b/apps/roam/src/utils/__tests__/importDiscoveredSharedNode.test.ts new file mode 100644 index 000000000..e6c55f5ad --- /dev/null +++ b/apps/roam/src/utils/__tests__/importDiscoveredSharedNode.test.ts @@ -0,0 +1,90 @@ +import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DiscoveredSharedNode } from "~/utils/discoverSharedNodes"; +import { importDiscoveredSharedNode } from "~/utils/importDiscoveredSharedNode"; + +const mocks = vi.hoisted(() => ({ + getSharedNodePayload: vi.fn(), + materializeObsidianNode: vi.fn(), +})); + +vi.mock("@repo/database/lib/sharedNodes", () => ({ + getSharedNodePayload: mocks.getSharedNodePayload, +})); +vi.mock("~/utils/materializeObsidianNode", () => ({ + materializeObsidianNode: mocks.materializeObsidianNode, +})); + +const node: DiscoveredSharedNode = { + alreadyImported: false, + modifiedAt: "2026-06-14T15:00:00.000Z", + sourceApp: "Obsidian", + sourceNodeId: "node-1", + sourceNodeRid: "orn:obsidian.note:vault-a/node-1", + sourceSpaceDatabaseId: 20, + sourceSpaceId: "obsidian:vault-a", + sourceSpaceName: "Research vault", + title: "EVD - REM sleep and recall", +}; +const payload = { localId: "node-1" } as CrossAppNode; +const client = {} as Parameters[0]["client"]; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.getSharedNodePayload.mockResolvedValue(payload); +}); + +describe("importDiscoveredSharedNode", () => { + it.each([ + ["created", "imported"], + ["updated", "updated"], + ] as const)("maps a %s materialization to %s", async (action, status) => { + mocks.materializeObsidianNode.mockResolvedValue({ + action, + pageUid: "page-uid", + sourceModifiedAt: node.modifiedAt, + sourceNodeRid: node.sourceNodeRid, + success: true, + }); + + await expect(importDiscoveredSharedNode({ client, node })).resolves.toBe( + status, + ); + expect(mocks.getSharedNodePayload).toHaveBeenCalledWith({ + client, + sourceLocalId: "node-1", + spaceId: 20, + }); + expect(mocks.materializeObsidianNode).toHaveBeenCalledWith({ + node: payload, + sourceModifiedAt: node.modifiedAt, + sourceNodeRid: node.sourceNodeRid, + }); + }); + + it("surfaces the materializer's actionable failure message", async () => { + mocks.materializeObsidianNode.mockResolvedValue({ + error: { + message: "Markdown could not be imported", + stage: "create-page", + }, + sourceModifiedAt: node.modifiedAt, + sourceNodeRid: node.sourceNodeRid, + success: false, + }); + + await expect(importDiscoveredSharedNode({ client, node })).rejects.toThrow( + "Markdown could not be imported", + ); + }); + + it("skips non-Obsidian nodes without loading their payload", async () => { + await expect( + importDiscoveredSharedNode({ + client, + node: { ...node, sourceApp: "Roam" }, + }), + ).resolves.toBe("skipped"); + expect(mocks.getSharedNodePayload).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/roam/src/utils/__tests__/importSelectedSharedNodes.test.ts b/apps/roam/src/utils/__tests__/importSelectedSharedNodes.test.ts new file mode 100644 index 000000000..2e8e24b53 --- /dev/null +++ b/apps/roam/src/utils/__tests__/importSelectedSharedNodes.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi } from "vitest"; +import type { DiscoveredSharedNode } from "~/utils/discoverSharedNodes"; +import { importSelectedSharedNodes } from "~/utils/importSelectedSharedNodes"; + +const createNode = ({ + sourceNodeRid, + title, +}: { + sourceNodeRid: string; + title: string; +}): DiscoveredSharedNode => ({ + alreadyImported: false, + modifiedAt: "2026-06-14T15:00:00.000Z", + sourceApp: "Obsidian", + sourceNodeId: sourceNodeRid.split("/").at(-1), + sourceNodeRid, + sourceSpaceDatabaseId: 20, + sourceSpaceId: "obsidian:vault-a", + sourceSpaceName: "Research vault", + title, +}); + +const firstNode = createNode({ + sourceNodeRid: "orn:obsidian.note:vault-a/node-1", + title: "First node", +}); +const secondNode = createNode({ + sourceNodeRid: "orn:obsidian.note:vault-a/node-2", + title: "Second node", +}); +const thirdNode = createNode({ + sourceNodeRid: "orn:obsidian.note:vault-a/node-3", + title: "Third node", +}); + +describe("importSelectedSharedNodes", () => { + it("reports imported, updated, and skipped nodes", async () => { + const materializeNode = vi + .fn() + .mockResolvedValueOnce("imported") + .mockResolvedValueOnce("updated") + .mockResolvedValueOnce("skipped"); + + await expect( + importSelectedSharedNodes({ + materializeNode, + nodes: [firstNode, secondNode, thirdNode], + }), + ).resolves.toEqual({ + failed: [], + imported: 2, + skipped: 1, + updated: 1, + }); + expect(materializeNode.mock.calls).toEqual([ + [firstNode], + [secondNode], + [thirdNode], + ]); + }); + + it("continues importing after an individual node fails", async () => { + const materializeNode = vi + .fn() + .mockRejectedValueOnce(new Error("Markdown could not be imported")) + .mockResolvedValueOnce("imported") + .mockRejectedValueOnce("Source content is missing"); + + await expect( + importSelectedSharedNodes({ + materializeNode, + nodes: [firstNode, secondNode, thirdNode], + }), + ).resolves.toEqual({ + failed: [ + { + message: "Markdown could not be imported", + node: firstNode, + }, + { + message: "Source content is missing", + node: thirdNode, + }, + ], + imported: 1, + skipped: 0, + updated: 0, + }); + expect(materializeNode).toHaveBeenCalledTimes(3); + }); + + it("returns empty counts when no nodes are selected", async () => { + const materializeNode = vi.fn(); + + await expect( + importSelectedSharedNodes({ materializeNode, nodes: [] }), + ).resolves.toEqual({ + failed: [], + imported: 0, + skipped: 0, + updated: 0, + }); + expect(materializeNode).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts index f2bf162d8..6585fc26e 100644 --- a/apps/roam/src/utils/discoverSharedNodes.ts +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -11,6 +11,7 @@ export type DiscoveredSharedNode = { sourceApp: "Roam" | "Obsidian"; sourceNodeId?: string; sourceNodeRid: string; + sourceSpaceDatabaseId: number; sourceSpaceId: string; sourceSpaceName: string; title: string; @@ -29,6 +30,7 @@ export const toDiscoveredSharedNodes = ({ sourceApp: candidate.platform, sourceNodeId: candidate.sourceLocalId || undefined, sourceNodeRid: candidate.rid, + sourceSpaceDatabaseId: candidate.spaceId, sourceSpaceId: candidate.spaceUri, sourceSpaceName: candidate.spaceName, title: candidate.title, diff --git a/apps/roam/src/utils/importDiscoveredSharedNode.ts b/apps/roam/src/utils/importDiscoveredSharedNode.ts new file mode 100644 index 000000000..2242daab6 --- /dev/null +++ b/apps/roam/src/utils/importDiscoveredSharedNode.ts @@ -0,0 +1,31 @@ +import { getSharedNodePayload } from "@repo/database/lib/sharedNodes"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import type { DiscoveredSharedNode } from "./discoverSharedNodes"; +import type { SharedNodeImportStatus } from "./importSelectedSharedNodes"; +import { materializeObsidianNode } from "./materializeObsidianNode"; + +export const importDiscoveredSharedNode = async ({ + client, + node, +}: { + client: DGSupabaseClient; + node: DiscoveredSharedNode; +}): Promise => { + if (node.sourceApp !== "Obsidian") return "skipped"; + if (!node.sourceNodeId) + throw new Error(`Shared node '${node.sourceNodeRid}' has no source ID`); + + const payload = await getSharedNodePayload({ + client, + sourceLocalId: node.sourceNodeId, + spaceId: node.sourceSpaceDatabaseId, + }); + const result = await materializeObsidianNode({ + node: payload, + sourceModifiedAt: node.modifiedAt, + sourceNodeRid: node.sourceNodeRid, + }); + if (!result.success) throw new Error(result.error.message); + + return result.action === "updated" ? "updated" : "imported"; +}; diff --git a/apps/roam/src/utils/importSelectedSharedNodes.ts b/apps/roam/src/utils/importSelectedSharedNodes.ts new file mode 100644 index 000000000..f286d7795 --- /dev/null +++ b/apps/roam/src/utils/importSelectedSharedNodes.ts @@ -0,0 +1,56 @@ +import type { DiscoveredSharedNode } from "./discoverSharedNodes"; + +export type SharedNodeImportStatus = "imported" | "skipped" | "updated"; + +export type SharedNodeImportFailure = { + message: string; + node: DiscoveredSharedNode; +}; + +export type SelectedSharedNodeImportResult = { + failed: SharedNodeImportFailure[]; + imported: number; + skipped: number; + updated: number; +}; + +export type MaterializeSharedNode = ( + node: DiscoveredSharedNode, +) => Promise; + +const getErrorMessage = (error: unknown): string => { + if (error instanceof Error && error.message) return error.message; + if (typeof error === "string" && error) return error; + return "Unknown import failure"; +}; + +export const importSelectedSharedNodes = async ({ + materializeNode, + nodes, +}: { + materializeNode: MaterializeSharedNode; + nodes: DiscoveredSharedNode[]; +}): Promise => { + const result: SelectedSharedNodeImportResult = { + failed: [], + imported: 0, + skipped: 0, + updated: 0, + }; + + for (const node of nodes) { + try { + const status = await materializeNode(node); + if (status === "updated") { + result.imported += 1; + result.updated += 1; + } else { + result[status] += 1; + } + } catch (error) { + result.failed.push({ message: getErrorMessage(error), node }); + } + } + + return result; +}; diff --git a/packages/database/src/lib/__tests__/sharedNodes.test.ts b/packages/database/src/lib/__tests__/sharedNodes.test.ts index 138a80dba..208815ec5 100644 --- a/packages/database/src/lib/__tests__/sharedNodes.test.ts +++ b/packages/database/src/lib/__tests__/sharedNodes.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { buildSharedNodeCandidates } from "../sharedNodes"; +import { + buildSharedNodeCandidates, + buildSharedNodePayload, +} from "../sharedNodes"; type BuildArgs = Parameters[0]; @@ -159,3 +162,51 @@ describe("buildSharedNodeCandidates", () => { ).toEqual(["node-1", "node-2"]); }); }); + +describe("buildSharedNodePayload", () => { + const concept = { + author_id: 42, + created: "2026-06-14T10:00:00.000Z", + last_modified: "2026-06-14T12:00:00.000Z", + schema_id: 200, + source_local_id: "node-1", + }; + + it("builds the CrossAppNode consumed by the Roam materializer", () => { + expect(buildSharedNodePayload({ concept, contents })).toEqual({ + author: { dbId: 42 }, + content: { + direct: { + contentType: "text/plain", + value: "EVD - REM sleep and recall", + }, + full: { + contentType: "text/markdown", + value: "# EVD - REM sleep and recall", + }, + }, + createdAt: new Date("2026-06-14T10:00:00.000Z"), + localId: "node-1", + modifiedAt: new Date("2026-06-14T15:00:00.000Z"), + nodeType: { dbId: 200 }, + }); + }); + + it("rejects an incomplete payload before it reaches the materializer", () => { + expect(() => + buildSharedNodePayload({ + concept, + contents: contents.filter((content) => content.variant !== "full"), + }), + ).toThrow("Shared node is missing its full content"); + }); + + it("uses the modified time when the source has no created time", () => { + const payload = buildSharedNodePayload({ + concept: { ...concept, created: null }, + contents: contents.map((content) => ({ ...content, created: null })), + }); + + expect(payload.createdAt).toEqual(new Date("2026-06-14T15:00:00.000Z")); + }); +}); diff --git a/packages/database/src/lib/sharedNodes.ts b/packages/database/src/lib/sharedNodes.ts index eb0405f24..a16fd89d9 100644 --- a/packages/database/src/lib/sharedNodes.ts +++ b/packages/database/src/lib/sharedNodes.ts @@ -1,3 +1,5 @@ +import { isSupportedContentType } from "@repo/content-model"; +import type { CrossAppNode } from "../crossAppContracts"; import type { DGSupabaseClient } from "./client"; import { getAvailableGroupIds } from "./groups"; import { getAllPages } from "./pagination"; @@ -31,6 +33,19 @@ type SharedSpace = Pick< Tables<"my_spaces">, "id" | "name" | "platform" | "url" >; +type SharedNodePayloadConcept = Pick< + Tables<"my_concepts">, + "author_id" | "created" | "last_modified" | "schema_id" | "source_local_id" +>; +type SharedNodePayloadContent = Pick< + Tables<"my_contents">, + | "author_id" + | "content_type" + | "created" + | "last_modified" + | "text" + | "variant" +>; type ValidSharedSpace = { name: string; platform: "Roam" | "Obsidian"; @@ -58,6 +73,77 @@ export type SharedNodeRows = { spaces: SharedSpace[]; }; +const getValidDate = (values: (string | null)[]): Date | undefined => { + const value = values.find( + (candidate): candidate is string => + typeof candidate === "string" && !Number.isNaN(Date.parse(candidate)), + ); + return value ? new Date(value) : undefined; +}; + +export const buildSharedNodePayload = ({ + concept, + contents, +}: { + concept: SharedNodePayloadConcept; + contents: SharedNodePayloadContent[]; +}): CrossAppNode => { + const direct = contents.find((content) => content.variant === "direct"); + const full = contents.find((content) => content.variant === "full"); + const authorId = concept.author_id ?? direct?.author_id ?? full?.author_id; + const modifiedAt = getValidDate( + [concept.last_modified, direct?.last_modified, full?.last_modified] + .filter((value): value is string => typeof value === "string") + .sort((left, right) => Date.parse(right) - Date.parse(left)), + ); + const sourceCreatedAt = getValidDate([ + concept.created, + direct?.created ?? null, + full?.created ?? null, + ]); + + if (typeof concept.source_local_id !== "string") + throw new Error("Shared node is missing its source-local ID"); + if (typeof concept.schema_id !== "number") + throw new Error("Shared node is missing its node type"); + if (typeof authorId !== "number") + throw new Error("Shared node is missing its author"); + if (!modifiedAt) throw new Error("Shared node is missing its modified time"); + const createdAt = sourceCreatedAt ?? modifiedAt; + if (typeof direct?.text !== "string") + throw new Error("Shared node is missing its direct content"); + if (typeof full?.text !== "string") + throw new Error("Shared node is missing its full content"); + if ( + typeof full.content_type !== "string" || + !isSupportedContentType(full.content_type) + ) + throw new Error( + `Shared node has unsupported full content type '${String(full.content_type)}'`, + ); + + return { + author: { dbId: authorId }, + content: { + direct: { + value: direct.text, + ...(typeof direct.content_type === "string" && + isSupportedContentType(direct.content_type) + ? { contentType: direct.content_type } + : {}), + }, + full: { + contentType: full.content_type, + value: full.text, + }, + }, + createdAt, + localId: concept.source_local_id, + modifiedAt, + nodeType: { dbId: concept.schema_id }, + }; +}; + const getResourceKey = ({ sourceLocalId, spaceId, @@ -317,3 +403,38 @@ export const listGroupSharedNodes = async ({ const rows = await getSharedNodeRows({ client, currentSpaceId, groupIds }); return buildSharedNodeCandidates({ ...rows, currentSpaceId }); }; + +export const getSharedNodePayload = async ({ + client, + sourceLocalId, + spaceId, +}: { + client: DGSupabaseClient; + sourceLocalId: string; + spaceId: number; +}): Promise => { + const [conceptResponse, contentsResponse] = await Promise.all([ + client + .from("my_concepts") + .select("author_id, created, last_modified, schema_id, source_local_id") + .eq("space_id", spaceId) + .eq("source_local_id", sourceLocalId) + .eq("is_schema", false) + .maybeSingle(), + client + .from("my_contents") + .select("author_id, content_type, created, last_modified, text, variant") + .eq("space_id", spaceId) + .eq("source_local_id", sourceLocalId) + .in("variant", ["direct", "full"]), + ]); + if (conceptResponse.error) throw conceptResponse.error; + if (contentsResponse.error) throw contentsResponse.error; + if (!conceptResponse.data) + throw new Error(`Shared node '${sourceLocalId}' is no longer available`); + + return buildSharedNodePayload({ + concept: conceptResponse.data, + contents: contentsResponse.data, + }); +}; From 2774f48d637da8c49d1b201b611f925d01b83402 Mon Sep 17 00:00:00 2001 From: sid597 Date: Mon, 13 Jul 2026 02:16:22 +0530 Subject: [PATCH 12/12] [ENG-1859] Avoid content model barrel in database types --- packages/database/src/crossAppContracts.ts | 2 +- packages/database/src/lib/sharedNodes.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/database/src/crossAppContracts.ts b/packages/database/src/crossAppContracts.ts index c996332c3..4166528a1 100644 --- a/packages/database/src/crossAppContracts.ts +++ b/packages/database/src/crossAppContracts.ts @@ -1,4 +1,4 @@ -import type { ContentType } from "@repo/content-model"; +import type { ContentType } from "@repo/content-model/constants"; import { Enums } from "./dbTypes"; export type LocalRef = { diff --git a/packages/database/src/lib/sharedNodes.ts b/packages/database/src/lib/sharedNodes.ts index a16fd89d9..d06331d1a 100644 --- a/packages/database/src/lib/sharedNodes.ts +++ b/packages/database/src/lib/sharedNodes.ts @@ -1,4 +1,4 @@ -import { isSupportedContentType } from "@repo/content-model"; +import { isSupportedContentType } from "@repo/content-model/constants"; import type { CrossAppNode } from "../crossAppContracts"; import type { DGSupabaseClient } from "./client"; import { getAvailableGroupIds } from "./groups";