perf: virtualize charts/grids/editors, lazy-hydrate notebook cells

e2e/tests/console/notebookResultRestore.spec.js

@@ -0,0 +1,157 @@
+/// <reference types="cypress" />
+
+// E2E coverage for notebook result persistence: a run cell's grid must survive
+// a scroll far away and back (release → re-hydrate from IndexedDB) and a full
+// page reload (snapshot restore), never degrading into a permanent shimmer or
+// a collapsed cell.
+
+const contextPath = process.env.QDB_HTTP_CONTEXT_WEB_CONSOLE || ""
+const baseUrl = `http://localhost:9999${contextPath}`
+
+const {
+  installFakeWebSocket,
+  TEST_BRIDGE_TOKEN,
+  TEST_BRIDGE_URL,
+} = require("../../utils/mcpFakeWebSocket")
+const { seedNotebookOnboarding } = require("../../utils")
+
+// Computed in SQL so the sentinel appears ONLY in the result grid — asserting
+// on it can never match the cell editor's SQL text.
+const PROBE_SQL = "select concat('sen', 'tinel_val') as probe_col"
+const PROBE_VALUE = "sentinel_val"
+const NOTEBOOK_LABEL = "Restore NB"
+// Enough collapsed cells below the probe to push it past the ±3-viewport
+// retain band when scrolled to the bottom, so its result releases.
+const FILLER_CELL_COUNT = 20
+
+const deepLinkSuffix = () =>
+  `?mcp-pair=1&mcp-ws=${encodeURIComponent(TEST_BRIDGE_URL)}` +
+  `&mcp-token=${encodeURIComponent(TEST_BRIDGE_TOKEN)}`
+
+const loginAndVisitDeepLink = () => {
+  cy.visit(`${baseUrl}/${deepLinkSuffix()}`, {
+    onBeforeLoad: (win) => {
+      win.localStorage.clear()
+      win.sessionStorage.clear()
+      win.indexedDB.deleteDatabase("web-console")
+      seedNotebookOnboarding(win)
+      win.localStorage.setItem(
+        "mcp:permissions",
+        JSON.stringify({ grantSchemaAccess: true, read: true, write: true }),
+      )
+      installFakeWebSocket(win)
+    },
+  })
+  cy.loginWithUserAndPassword()
+}
+
+const waitForPaired = () => {
+  cy.window({ timeout: 10000 }).its("__mcpFakeWS").should("exist")
+  cy.window({ timeout: 10000 }).should((win) => {
+    expect(win.__mcpFakeWS.framesOfType("hello").length).to.be.greaterThan(0)
+  })
+  cy.window().then((win) => win.__mcpFakeWS.helloAck())
+  cy.getByDataHook("mcp-bridge-status-pill", { timeout: 10000 }).should(
+    "contain",
+    "MCP connected",
+  )
+}
+
+const toolResult = (win, requestId) =>
+  win.__mcpFakeWS
+    .framesOfType("tool_result")
+    .find((r) => r.requestId === requestId)
+
+const awaitToolResult = (id) => {
+  cy.window({ timeout: 10000 }).should((win) => {
+    expect(toolResult(win, id), `result for ${id}`).to.exist
+  })
+  return cy.window().then((win) => {
+    const result = toolResult(win, id)
+    expect(result.isError, `tool ${id} errored`).to.not.equal(true)
+    const text = result.content[0].text
+    const payloadLine = text
+      .split("\n")
+      .find((line) => line.trimStart().startsWith("{"))
+    expect(payloadLine, `JSON payload in result for ${id}`).to.exist
+    return JSON.parse(payloadLine)
+  })
+}
+
+const callTool = (name, args) =>
+  cy
+    .window()
+    .then((win) => awaitToolResult(win.__mcpFakeWS.toolCall(name, args)))
+
+const probeGridCell = (options = {}) =>
+  cy.contains('[data-hook="grid-cell"]', PROBE_VALUE, options)
+
+describe("notebook result restore (e2e)", () => {
+  it("restores a run cell's grid after a far scroll round-trip and after a reload", () => {
+    // Given the browser is paired to the bridge
+    loginAndVisitDeepLink()
+    cy.getByDataHook("mcp-pair-consent-connect").click()
+    waitForPaired()
+
+    // Given a notebook whose run cell (the probe) is followed by a tall stack
+    // of filler cells — enough to push the probe past the ±3-viewport retain
+    // band when scrolled to the bottom. Built and run in the background over
+    // the bridge.
+    callTool("create_notebook", { label: NOTEBOOK_LABEL }).then((created) => {
+      callTool("get_notebook_state", { buffer_id: created.bufferId })
+      callTool("add_cell", {
+        buffer_id: created.bufferId,
+        sql: PROBE_SQL,
+        after_cell_id: null,
+        run: true,
+        type: "sql",
+      })
+      for (let i = 0; i < FILLER_CELL_COUNT; i++) {
+        callTool("add_cell", {
+          buffer_id: created.bufferId,
+          sql: `select ${i}`,
+          after_cell_id: null,
+          run: false,
+          type: "sql",
+        })
+      }
+    })
+
+    // Given the notebook is opened — the open reveals the agent's LAST-touched
+    // cell (the bottom filler), so the probe starts beyond the retain band
+    cy.getByDataHook("agent-changes-view", { timeout: 10000 }).click()
+    cy.get(".chrome-tab[active]").should(
+      "have.attr",
+      "data-tab-title",
+      NOTEBOOK_LABEL,
+    )
+
+    // When the probe is scrolled into view
+    cy.get("[data-cell-id]", { timeout: 10000 }).first().scrollIntoView()
+    // Then its grid hydrates on demand from the persisted snapshot
+    probeGridCell({ timeout: 10000 }).should("exist")
+
+    // When the user scrolls far past the retain band
+    cy.get("[data-cell-id]").last().scrollIntoView()
+    // Then the grid unmounts (content drop) and its data releases to IndexedDB
+    probeGridCell({ timeout: 10000 }).should("not.exist")
+
+    // When the user scrolls back to the probe
+    cy.get("[data-cell-id]").first().scrollIntoView()
+    // Then the same grid re-hydrates — not a stuck shimmer
+    probeGridCell({ timeout: 10000 }).should("exist")
+
+    // When the page reloads (the notebook stays the active tab)
+    cy.reload()
+    // Then the notebook reopens and its run cell restores the grid from the
+    // snapshot once scrolled back into the band (the reload lands on the
+    // persisted focused cell)
+    cy.get(".chrome-tab[active]", { timeout: 10000 }).should(
+      "have.attr",
+      "data-tab-title",
+      NOTEBOOK_LABEL,
+    )
+    cy.get("[data-cell-id]", { timeout: 15000 }).first().scrollIntoView()
+    probeGridCell({ timeout: 15000 }).should("exist")
+  })
+})

src/components/ResultGrid/ResultGrid.tsx

@@ -20,13 +20,21 @@ import {
 
 import type { ColumnDefinition } from "../../utils/questdb/types"
 import { unescapeHtml } from "../../utils/escapeHtml"
-import type { CellValue, ResultGridDataSource, ResultGridRow } from "./types"
+import type {
+  CellValue,
+  ResultGridDataSource,
+  ResultGridRow,
+  ResultGridViewport,
+} from "./types"
 import {
   clampColumnWidths,
+  columnId,
+  COLUMN_ID_PREFIX,
   sampleColumnWidths,
   isLeftAligned,
   formatCellValue,
   formatColumnType,
+  WIDTH_SAMPLE_ROWS,
 } from "./inlineGridUtils"
 import { useGridKeyboardNav } from "./useGridKeyboardNav"
 import {
@@ -66,10 +74,7 @@ declare module "@tanstack/react-table" {
   }
 }
 
-const WIDTH_SAMPLE_ROWS = 1000
 const FREEZE_HANDLE_EDGE_INSET = 4
-const COLUMN_ID_PREFIX = "col_"
-const columnId = (dataIndex: number) => `${COLUMN_ID_PREFIX}${dataIndex}`
 
 type GridCellProps = {
   rowIndex: number
@@ -148,6 +153,8 @@ type Props = {
   onColumnOrderCommit?: (order: string[]) => void
   initialPinnedColumns?: string[]
   onPinnedColumnsCommit?: (pinnedLeft: string[]) => void
+  initialViewport?: ResultGridViewport
+  onViewportSave?: (viewport: ResultGridViewport) => void
   onYieldFocus?: () => void
   onResetLayout?: () => void
   onSelectionChange?: (hasSelection: boolean) => void
@@ -210,6 +217,8 @@ export const ResultGrid = forwardRef<ResultGridHandle, Props>(
       onColumnOrderCommit,
       initialPinnedColumns,
       onPinnedColumnsCommit,
+      initialViewport,
+      onViewportSave,
       onYieldFocus,
       onResetLayout,
       onSelectionChange,
@@ -230,6 +239,18 @@ export const ResultGrid = forwardRef<ResultGridHandle, Props>(
     const gridRef = useRef<HTMLDivElement>(null)
     const scrollRef = useRef<HTMLDivElement>(null)
 
+    useLayoutEffect(
+      () => () => {
+        const viewport = scrollRef.current
+        if (!viewport) return
+        onViewportSave?.({
+          scrollTop: viewport.scrollTop,
+          scrollLeft: viewport.scrollLeft,
+        })
+      },
+      [onViewportSave],
+    )
+
     const containerWidth = useContainerWidth(gridRef)
     const { scrolledDown, shadowLeft, handleScroll } =
       useScrollShadows(scrollRef)
@@ -526,6 +547,7 @@ export const ResultGrid = forwardRef<ResultGridHandle, Props>(
       count: virtualRowCount,
       getScrollElement: () => scrollRef.current,
       estimateSize: () => ROW_HEIGHT,
+      initialOffset: initialViewport?.scrollTop ?? 0,
       overscan: 3,
     })
 
@@ -534,6 +556,7 @@ export const ResultGrid = forwardRef<ResultGridHandle, Props>(
       count: headers.length,
       getScrollElement: () => scrollRef.current,
       estimateSize: (index) => headers[index]?.getSize() ?? 100,
+      initialOffset: initialViewport?.scrollLeft ?? 0,
       overscan: 2,
     })
 

src/components/ResultGrid/index.ts

@@ -6,6 +6,7 @@ export type {
   DqlQueryResult,
   ResultGridDataSource,
   ResultGridRow,
+  ResultGridViewport,
 } from "./types"
 export { inMemoryDataSource } from "./types"
 

src/components/ResultGrid/inlineGridUtils.ts

@@ -79,6 +79,12 @@ const formatArrayValue = (
 // constant ceiling keeps the sampling loop bounded for very long values.
 const MAX_SAMPLED_WIDTH_PX = 4000
 
+export const WIDTH_SAMPLE_ROWS = 1000
+
+// tanstack column ids — also the key contract of persisted column layouts.
+export const COLUMN_ID_PREFIX = "col_"
+export const columnId = (dataIndex: number) => `${COLUMN_ID_PREFIX}${dataIndex}`
+
 export const sampleColumnWidths = (
   columns: ColumnDefinition[],
   dataset: ResultGridRow[],

src/components/ResultGrid/types.ts

@@ -8,6 +8,11 @@ export type CellValue = boolean | string | number | null
 
 export type ResultGridRow = CellValue[]
 
+export type ResultGridViewport = {
+  scrollTop: number
+  scrollLeft: number
+}
+
 export type ColumnLayout = {
   columnSizing?: Record<string, number>
   columnOrder?: string[]

src/hooks/useAdaptivePoll.test.ts

@@ -1,4 +1,58 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"
+import { runAdaptivePollLoop } from "./useAdaptivePoll"
+
+describe("runAdaptivePollLoop", () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it("samples the duration the fetch reports instead of wall-clock time", async () => {
+    // Given a fetch that resolves instantly but reports a 1s query duration
+    // (a queued fetch — wall clock here would sample ~0ms)
+    const controller = new AbortController()
+    const intervals: number[] = []
+    const loop = runAdaptivePollLoop({
+      fetchFn: () => Promise.resolve(1000),
+      signal: controller.signal,
+      minIntervalMs: 100,
+      maxIntervalMs: 60_000,
+      onIntervalChange: (interval) => intervals.push(interval),
+    })
+
+    // When the first fetch completes
+    await vi.advanceTimersByTimeAsync(0)
+
+    // Then the next interval derives from the reported duration (1000 × 2)
+    expect(intervals[0]).toBe(2000)
+    controller.abort()
+    await loop
+  })
+
+  it("falls back to wall-clock timing when the fetch reports nothing", async () => {
+    // Given an instant fetch with no duration sample
+    const controller = new AbortController()
+    const intervals: number[] = []
+    const loop = runAdaptivePollLoop({
+      fetchFn: () => Promise.resolve(),
+      signal: controller.signal,
+      minIntervalMs: 100,
+      maxIntervalMs: 60_000,
+      onIntervalChange: (interval) => intervals.push(interval),
+    })
+
+    // When the first fetch completes
+    await vi.advanceTimersByTimeAsync(0)
+
+    // Then the near-zero wall clock clamps to the minimum interval
+    expect(intervals[0]).toBe(100)
+    controller.abort()
+    await loop
+  })
+})
 
 describe("useAdaptivePoll core logic", () => {
   describe("calculateInterval algorithm", () => {