diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..891fa567b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,27 @@ +## Summary + +- + +## Verification + +- [ ] `npm run verify:cheap` +- [ ] `npm run verify:ui` when UI, routing, styling, browser behavior, reduced-motion, or forced-colors behavior changed +- [ ] `npm run verify:release` before release or handoff confidence claims +- [ ] `npm run format:check` +- [ ] `npm run check:production-readiness` when clinical workflow, privacy, environment, Supabase, source governance, or deployment behavior changed + +## Clinical Governance Preflight + +Complete this section when the change touches ingestion, answer generation, search/ranking, source rendering, document access, privacy, production env, or clinical output. + +- [ ] Source-backed claims still require linked source verification before clinical use +- [ ] No patient-identifiable document workflow was introduced or expanded without explicit governance approval +- [ ] Supabase target remains `Clinical KB Database` (`sjrfecxgysukkwxsowpy`) +- [ ] Service-role keys and private document access remain server-only +- [ ] Demo/synthetic content remains clearly separated from real clinical sources +- [ ] Source metadata, review status, and outdated/unknown-source behavior remain conservative +- [ ] Deployment classification/TGA SaMD impact was checked when clinical decision-support behavior changed + +## Notes + +- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81ea43a77..3a57af1f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,9 @@ jobs: - name: Install dependencies run: npm ci + - name: Production readiness (CI-safe) + run: npm run check:production-readiness:ci + - name: Lint run: npm run lint @@ -38,3 +41,9 @@ jobs: - name: Build run: npm run build + + - name: Install Playwright Chromium + run: npx playwright install --with-deps chromium + + - name: Chromium UI smoke + run: npm run test:e2e:chromium diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 87e5c02e3..938e23717 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,11 +13,11 @@ name: "CodeQL Advanced" on: push: - branches: [ "main", "protection" ] + branches: ["main", "protection"] pull_request: - branches: [ "main", "protection" ] + branches: ["main", "protection"] schedule: - - cron: '20 12 * * 2' + - cron: "20 12 * * 2" workflow_dispatch: jobs: @@ -44,12 +44,12 @@ jobs: fail-fast: false matrix: include: - - language: actions - build-mode: none - - language: javascript-typescript - build-mode: none - - language: python - build-mode: none + - language: actions + build-mode: none + - language: javascript-typescript + build-mode: none + - language: python + build-mode: none # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both @@ -59,46 +59,46 @@ jobs: # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - # Add any setup steps before running the `github/codeql-action/init` action. - # This includes steps like installing compilers or runtimes (`actions/setup-node` - # or others). This is typically only required for manual builds. - # - name: Setup runtime (example) - # uses: actions/setup-example@v1 + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - name: Run manual build steps - if: matrix.build-mode == 'manual' - shell: bash - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - name: Run manual build steps + if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{matrix.language}}" diff --git a/.prettierignore b/.prettierignore index 608de078a..85d0be3a2 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,3 +9,5 @@ sample-documents/ dev-server*.log package-lock.json public/demo-documents/ +.tmp-visual/ +scratch/ diff --git a/.tmp-visual/capture.mjs b/.tmp-visual/capture.mjs new file mode 100644 index 000000000..bcb0354dc --- /dev/null +++ b/.tmp-visual/capture.mjs @@ -0,0 +1,13 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/desktop-home.png', fullPage: true }); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/mobile-home.png', fullPage: true }); + await browser.close(); +})(); diff --git a/.tmp-visual/check-search-api.mjs b/.tmp-visual/check-search-api.mjs new file mode 100644 index 000000000..57eec52ec --- /dev/null +++ b/.tmp-visual/check-search-api.mjs @@ -0,0 +1,53 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + const events = []; + page.on('response', async (resp) => { + const url = resp.url(); + if (url.includes('/api/')) { + try { + const text = await resp.text(); + events.push({ + url, + status: resp.status(), + ok: resp.ok(), + text: text.slice(0, 220), + }); + } catch { + events.push({ + url, + status: resp.status(), + ok: resp.ok(), + text: '', + }); + } + } + }); + + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + const input = page.locator('input[placeholder="Ask a question"]'); + await input.fill('What monitoring is required after starting lithium?'); + + const askButton = page.locator('button[aria-label="Generate source-backed answer"]').first(); + await askButton.click(); + + await page.waitForTimeout(12000); + await page.waitForLoadState('networkidle'); + + const state = await page.evaluate(() => ({ + body: (document.body.innerText || '').replace(/\s+/g, ' ').slice(0, 2200), + statusLine: document.querySelector('.text-sm, p, h2, h3')?.textContent || '', + hasResultsText: (document.body.innerText || '').includes('Retrieved passages appear after a question') || false, + headingText: Array.from(document.querySelectorAll('h2, h3')).map((el) => (el.textContent || '').trim()), + ctaText: Array.from(document.querySelectorAll('button, a')).map((el) => (el.textContent || '').trim()).filter((t) => /Open source|Source PDF|Add scope|Ask|Search|Citations|Sources|Gaps|Verify/.test(t)), + })); + + await browser.close(); + + console.log('RESULTS:' + JSON.stringify({ + api: events.filter((e) => e.url.includes('/api/search') || e.url.includes('/api/tools') || e.url.includes('/api/documents') || e.url.includes('/api/local-project-id')), + state, + }, null, 2)); +})(); diff --git a/.tmp-visual/desktop-home.png b/.tmp-visual/desktop-home.png new file mode 100644 index 000000000..20b9d878f Binary files /dev/null and b/.tmp-visual/desktop-home.png differ diff --git a/.tmp-visual/final-query-desktop.png b/.tmp-visual/final-query-desktop.png new file mode 100644 index 000000000..085258ed2 Binary files /dev/null and b/.tmp-visual/final-query-desktop.png differ diff --git a/.tmp-visual/final-query-mobile.png b/.tmp-visual/final-query-mobile.png new file mode 100644 index 000000000..7cccbc4d3 Binary files /dev/null and b/.tmp-visual/final-query-mobile.png differ diff --git a/.tmp-visual/inspect-ask-controls.mjs b/.tmp-visual/inspect-ask-controls.mjs new file mode 100644 index 000000000..76d2eca24 --- /dev/null +++ b/.tmp-visual/inspect-ask-controls.mjs @@ -0,0 +1,42 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + await page.goto('http://localhost:4298', { waitUntil: 'domcontentloaded' }); + + const input = page.locator('input[placeholder="Ask a question"]'); + await input.waitFor({ state: 'attached', timeout: 12000 }); + + const controls = await page.evaluate(() => { + const inp = Array.from(document.querySelectorAll('input')).find((el) => el.getAttribute('placeholder') === 'Ask a question'); + if (!inp) return null; + const form = inp.closest('form'); + const allButtons = Array.from(document.querySelectorAll('button, input[type="submit"]')).map((el) => ({ + text: (el.textContent || '').trim(), + type: el.getAttribute('type') || (el.tagName === 'BUTTON' ? 'button' : ''), + value: el.getAttribute('value') || '', + aria: el.getAttribute('aria-label') || '', + id: el.id || '', + })); + const formButtons = form ? Array.from(form.querySelectorAll('button, input[type="submit"]')).map((el) => ({ + text: (el.textContent || '').trim(), + type: el.getAttribute('type') || (el.tagName === 'BUTTON' ? 'button' : ''), + value: el.getAttribute('value') || '', + aria: el.getAttribute('aria-label') || '', + id: el.id || '', + })) : []; + + return { + inputCount: document.querySelectorAll('input[placeholder="Ask a question"]').length, + inputOuterHTML: inp.outerHTML.slice(0, 400), + formAction: form ? form.getAttribute('action') : null, + formMethod: form ? form.getAttribute('method') : null, + formButtons, + allButtons: allButtons.slice(0, 80), + }; + }); + + console.log(JSON.stringify(controls, null, 2)); + await browser.close(); +})(); diff --git a/.tmp-visual/inspect-home.mjs b/.tmp-visual/inspect-home.mjs new file mode 100644 index 000000000..11a37c4e4 --- /dev/null +++ b/.tmp-visual/inspect-home.mjs @@ -0,0 +1,41 @@ +import { chromium } from 'playwright'; + +(async () => { + const page = await chromium.launch({ headless: true }).then(async (browser) => { + const p = await browser.newPage(); + await p.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + const body = await p.evaluate(() => document.body.innerText || ''); + const links = await p.evaluate(() => { + return Array.from(document.querySelectorAll('a')).map((a) => ({ + text: (a.textContent || '').trim(), + href: a.getAttribute('href') || '', + })).filter((x) => x.text || x.href); + }); + + const buttons = await p.evaluate(() => { + return Array.from(document.querySelectorAll('button')).map((b) => ({ + text: (b.textContent || '').trim(), + type: b.getAttribute('type') || '', + })); + }); + + const inputs = await p.evaluate(() => Array.from(document.querySelectorAll('input, textarea')).map((el) => ({ + tag: el.tagName, + type: el.getAttribute('type') || '', + placeholder: el.getAttribute('placeholder') || '', + id: el.id || '', + name: el.getAttribute('name') || '', + }))); + + console.log('TITLE:' + (await p.title())); + console.log('URL:' + p.url()); + console.log('HREFS:' + JSON.stringify(links.slice(0, 80), null, 2)); + console.log('BUTTONS:' + JSON.stringify(buttons.slice(0, 80), null, 2)); + console.log('INPUTS:' + JSON.stringify(inputs, null, 2)); + console.log('TEXT_SNIPPET_START'); + console.log(body.slice(0, 1200)); + console.log('TEXT_SNIPPET_END'); + + await browser.close(); + }); +})(); diff --git a/.tmp-visual/mobile-home.png b/.tmp-visual/mobile-home.png new file mode 100644 index 000000000..9281eaef8 Binary files /dev/null and b/.tmp-visual/mobile-home.png differ diff --git a/.tmp-visual/query-answer-desktop.png b/.tmp-visual/query-answer-desktop.png new file mode 100644 index 000000000..fbe0f5276 Binary files /dev/null and b/.tmp-visual/query-answer-desktop.png differ diff --git a/.tmp-visual/query-answer-mobile.png b/.tmp-visual/query-answer-mobile.png new file mode 100644 index 000000000..adc32bb50 Binary files /dev/null and b/.tmp-visual/query-answer-mobile.png differ diff --git a/.tmp-visual/query-answer-stream.mjs b/.tmp-visual/query-answer-stream.mjs new file mode 100644 index 000000000..25d3cda0b --- /dev/null +++ b/.tmp-visual/query-answer-stream.mjs @@ -0,0 +1,49 @@ +import { chromium } from "playwright"; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + + let reqPayload = null; + let respSnippet = null; + + page.on('request', (req) => { + if (req.url().includes('/api/answer/stream')) { + reqPayload = { + method: req.method(), + postData: req.postData(), + }; + } + }); + + page.on('response', async (resp) => { + if (resp.url().includes('/api/answer/stream')) { + try { + respSnippet = await resp.text(); + } catch (error) { + respSnippet = "read-failed"; + } + } + }); + + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + await page.fill('input[placeholder="Ask a question"]', 'What monitoring is required after starting lithium?'); + await page.locator('button[aria-label="Generate source-backed answer"]').click(); + await page.waitForTimeout(5000); + + const statusText = await page.evaluate(() => (document.body.innerText || '').replace(/\s+/g, ' ')); + + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-stream-desktop.png', fullPage: true }); + await browser.close(); + + console.log('RESULTS:' + JSON.stringify({ + request: reqPayload, + responseSnippet: (respSnippet || '').slice(0, 1200), + statusText: statusText.slice(0, 2200), + hasNoDataMarkers: { + noPassages: statusText.includes('Retrieved passages appear after a question'), + searching: statusText.includes('Searching indexed documents'), + notReady: statusText.includes('Search setup is not ready') || statusText.includes('setup is not ready') + } + }, null, 2)); +})(); diff --git a/.tmp-visual/query-api-trace.mjs b/.tmp-visual/query-api-trace.mjs new file mode 100644 index 000000000..64d8591e9 --- /dev/null +++ b/.tmp-visual/query-api-trace.mjs @@ -0,0 +1,42 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + const events = []; + + page.on('response', async (resp) => { + const url = resp.url(); + if (url.includes('/api/')) { + events.push({ + url, + status: resp.status(), + ok: resp.ok(), + }); + } + }); + + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + const input = page.locator('input[placeholder="Ask a question"]'); + await input.fill('What monitoring is required after starting lithium?'); + + const askButton = page.locator('button[aria-label="Generate source-backed answer"]'); + await askButton.click(); + + await page.waitForTimeout(15000); + await page.waitForLoadState('networkidle'); + + const endText = await page.evaluate(() => ({ + text: (document.body.innerText || '').replace(/\s+/g, ' ').slice(0, 2200), + hasResultPhrase: (document.body.innerText || '').includes('Retrieved passages appear after a question') || (document.body.innerText || '').includes('The answer, quotes, source PDFs, and diagrams will appear here.'), + queryValue: (document.querySelector('input[placeholder="Ask a question"]')?.value || ''), + })); + + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-post-click-desktop.png', fullPage: true }); + + await browser.close(); + const interesting = events.filter((event) => event.url.includes('/api/search') || event.url.includes('/api/search/')); + const all = events.slice(-20); + + console.log('RESULTS:' + JSON.stringify({ endText, interesting, all }, null, 2)); +})(); diff --git a/.tmp-visual/query-ask-submit.mjs b/.tmp-visual/query-ask-submit.mjs new file mode 100644 index 000000000..23d99285a --- /dev/null +++ b/.tmp-visual/query-ask-submit.mjs @@ -0,0 +1,47 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + + const input = page.locator('input[placeholder="Ask a question"]'); + await input.fill('What monitoring is required after starting lithium?'); + + const askButton = page.locator('button[aria-label="Generate source-backed answer"]').first(); + await askButton.waitFor({ state: 'visible', timeout: 12000 }); + await askButton.click(); + + // Wait for first response indicator + await page.waitForTimeout(1200); + await page.waitForSelector('text=Checking indexed library before showing document status', { timeout: 1000 }).catch(() => {}); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(2500); + + const state = await page.evaluate(() => { + const text = (document.body.innerText || '').replace(/\s+/g, ' '); + const phrase = 'Sourced synthesis with quotes, PDFs, and indexed diagrams.'; + const beforeQ = text.indexOf(phrase); + + const panelHeadings = Array.from(document.querySelectorAll('h2, h3')).map((el) => (el.textContent || '').trim()).filter(Boolean); + + const actionHints = Array.from(document.querySelectorAll('button, a')).map((el) => (el.textContent || '').trim()).filter((t) => t && /Open source|Add scope|Source PDF|Citations|Sources|Gaps|No linked citations|No source provenance|No source passages|quote cards|source passages|verify/i.test(t)); + + const allLabels = Array.from(new Set(actionHints)).slice(0, 80); + + return { + currentUrl: location.href, + headingCount: document.querySelectorAll('h2, h3, h4').length, + panelHeadings: panelHeadings.slice(0, 40), + labels: allLabels, + bodyPreview: document.body.innerText?.slice(0, 2800) || '', + }; + }); + + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-answer-desktop.png', fullPage: true }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-answer-mobile.png', fullPage: true }); + + await browser.close(); + console.log('RESULTS:' + JSON.stringify(state, null, 2)); +})(); diff --git a/.tmp-visual/query-ask.mjs b/.tmp-visual/query-ask.mjs new file mode 100644 index 000000000..61c39cea3 --- /dev/null +++ b/.tmp-visual/query-ask.mjs @@ -0,0 +1,56 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + + const q = 'What monitoring is required after starting lithium for bipolar disorder?'; + const input = page.locator('input[placeholder="Ask a question"]'); + await input.waitFor({ state: 'visible', timeout: 12000 }); + await input.fill(q); + const askBtn = page.getByRole('button', { name: /^ask$/i }).first(); + await askBtn.click(); + + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(2500); + + await page.screenshot({ path: 'C:/Dev\Apps\Database/.tmp-visual/query-answer-desktop.png', fullPage: true }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: 'C:/Dev\Apps\Database/.tmp-visual/query-answer-mobile.png', fullPage: true }); + + const dump = await page.evaluate(() => { + const getText = (selector, limit = 500) => { + const el = document.querySelector(selector); + if (!el) return ''; + return (el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, limit); + }; + + const outputRoot = document.querySelector('[data-testid="generated-answer"]') || + document.querySelector('[data-testid="rag-output"]') || + document.querySelector('main'); + + const labels = Array.from(document.querySelectorAll('button, a')).map((el) => (el.textContent || '').trim()) + .filter((t) => /Open source|Source PDF|Add scope|Citations|Sources|Gaps|No linked citations|No source provenance|No indexed clinically useful tables|No quote|No quotes|No passages|Verified|Evidence map/i.test(t)); + + const sections = Array.from(document.querySelectorAll('section, article')).map((section) => { + const heading = section.querySelector('h2, h3, h4')?.textContent?.trim() || ''; + const txt = (section.textContent || '').replace(/\s+/g, ' ').trim(); + if (!heading) return null; + return { heading, text: txt.slice(0, 260) }; + }).filter(Boolean).slice(0, 30); + + return { + url: location.href, + title: document.title, + queryInInput: (document.querySelector('input[placeholder="Ask a question"]')?.value || ''), + outputSnippet: outputRoot ? (outputRoot.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 1800) : '', + relevantLabels: [...new Set(labels)].slice(0, 50), + sectionSamples: sections, + firstError: document.body ? ((document.body.textContent || '').match(/error|failed|unauth|401|429|403/ig)?.[0] || '') : '' + }; + }); + + await browser.close(); + console.log('RESULTS:' + JSON.stringify(dump, null, 2)); +})(); diff --git a/.tmp-visual/query-disabled-desktop.png b/.tmp-visual/query-disabled-desktop.png new file mode 100644 index 000000000..27a575ed9 Binary files /dev/null and b/.tmp-visual/query-disabled-desktop.png differ diff --git a/.tmp-visual/query-disabled-state.mjs b/.tmp-visual/query-disabled-state.mjs new file mode 100644 index 000000000..584d7e187 --- /dev/null +++ b/.tmp-visual/query-disabled-state.mjs @@ -0,0 +1,37 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + + const input = page.locator('input[placeholder="Ask a question"]'); + await input.fill('What monitoring is required after starting lithium?'); + + const askButton = page.locator('button[aria-label="Generate source-backed answer"]'); + const info = await askButton.evaluate((btn) => ({ + disabled: btn.disabled, + text: (btn.textContent || '').trim(), + title: btn.getAttribute('title') || '', + aria: btn.getAttribute('aria-label') || '', + classes: btn.className, + })); + + const summary = await page.evaluate(() => ({ + bodyText: (document.body.innerText || '').replace(/\s+/g, ' ').slice(0, 2200), + statusBanner: (document.querySelector('[role="status"], .status, .text-sm')?.textContent || '').replace(/\s+/g, ' ').trim(), + hasReadyText: (document.body.innerText || '').includes('setup is not ready'), + headings: Array.from(document.querySelectorAll('h2, h3')).map((el) => (el.textContent || '').trim()), + ctaCandidates: Array.from(document.querySelectorAll('button')).map((btn) => ({ + text: (btn.textContent || '').trim(), + title: btn.getAttribute('title') || '', + aria: btn.getAttribute('aria-label') || '', + disabled: !!btn.disabled, + })).filter((x) => /Generate source-backed answer|Open mobile document scope|Clear refine|Source passages|Ask/i.test(x.text) || /ready/.test(x.title)), + })); + + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-disabled-desktop.png', fullPage: true }); + await browser.close(); + + console.log('RESULTS:' + JSON.stringify({ button: info, summary }, null, 2)); +})(); diff --git a/.tmp-visual/query-final.mjs b/.tmp-visual/query-final.mjs new file mode 100644 index 000000000..4a7003052 --- /dev/null +++ b/.tmp-visual/query-final.mjs @@ -0,0 +1,29 @@ +import { chromium } from 'playwright'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1440, height: 1200 } }); + await page.goto('http://localhost:4298', { waitUntil: 'networkidle' }); + await page.fill('input[placeholder="Ask a question"]', 'What monitoring is required after starting lithium?'); + const ask = page.locator('button[aria-label="Generate source-backed answer"]'); + await ask.click(); + await page.waitForTimeout(5000); + + const output = await page.evaluate(() => { + const body = (document.body.innerText || '').replace(/\s+/g, ' '); + return { + hasSearching: body.includes('Searching indexed documents.'), + hasSearchReadyText: body.includes('Search setup not ready'), + hasNoPassagesText: body.includes('Retrieved passages appear after a question'), + query: document.querySelector('input[placeholder="Ask a question"]')?.value || '', + labels: Array.from(document.querySelectorAll('button, a')).map((el) => (el.textContent || '').trim()).filter((text) => /Open source|Source PDF|Add scope|Citations|Sources|Gaps|No linked citations|No source provenance|Search setup|Retrieved passages/.test(text)), + }; + }); + + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/final-query-desktop.png', fullPage: true }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/final-query-mobile.png', fullPage: true }); + await browser.close(); + + console.log('RESULTS:' + JSON.stringify(output, null, 2)); +})(); diff --git a/.tmp-visual/query-noinput-desktop.png b/.tmp-visual/query-noinput-desktop.png new file mode 100644 index 000000000..69675b89c Binary files /dev/null and b/.tmp-visual/query-noinput-desktop.png differ diff --git a/.tmp-visual/query-noinput-mobile.png b/.tmp-visual/query-noinput-mobile.png new file mode 100644 index 000000000..bb52352e3 Binary files /dev/null and b/.tmp-visual/query-noinput-mobile.png differ diff --git a/.tmp-visual/query-post-click-desktop.png b/.tmp-visual/query-post-click-desktop.png new file mode 100644 index 000000000..81f2c28c6 Binary files /dev/null and b/.tmp-visual/query-post-click-desktop.png differ diff --git a/.tmp-visual/query-smoke.mjs b/.tmp-visual/query-smoke.mjs new file mode 100644 index 000000000..0a7a50ac6 --- /dev/null +++ b/.tmp-visual/query-smoke.mjs @@ -0,0 +1,103 @@ +import { chromium } from 'playwright'; + +const APP_URL = 'http://localhost:4298'; + +(async () => { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + + await page.goto(APP_URL, { waitUntil: 'domcontentloaded' }); + await page.waitForTimeout(1200); + + const querySelectors = [ + 'input[type="text"]', + 'input[placeholder*="Search" i]', + 'textarea', + '[data-testid="clinical-search-input"]', + '[name="q"]', + 'input[role="searchbox"]' + ]; + + let input = null; + for (const selector of querySelectors) { + const el = page.locator(selector).first(); + if (await el.count()) { + input = el; + break; + } + } + + if (!input) { + const allInputs = page.locator('input, textarea'); + const cnt = await allInputs.count(); + console.log(JSON.stringify({ status: 'no-query-input-found', inputCount: cnt })); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-noinput-desktop.png', fullPage: true }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-noinput-mobile.png', fullPage: true }); + await browser.close(); + process.exit(0); + } + + await input.fill('acute mania first line pharmacologic treatment'); + + const submitSelectors = [ + 'button:has-text("Search")', + 'button[type="submit"]', + 'form button', + '[data-testid="clinical-search-submit"]' + ]; + + for (const selector of submitSelectors) { + const btn = page.locator(selector).first(); + if (await btn.count()) { + await btn.click({ timeout: 1500 }).catch(async () => { + await page.keyboard.press('Enter'); + }); + break; + } + } + + if (!page.url().includes('/search') && !page.url().includes('/results')) { + await page.keyboard.press('Enter'); + } + + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1500); + + const desktopText = await page.evaluate(() => { + const texts = Array.from(document.querySelectorAll('body *')) + .map((node) => (node.textContent || '').trim()) + .filter(Boolean); + + const compact = texts.filter((t) => + /Open source|Source PDF|Add scope|No linked citations|No source provenance|Citations|Sources|Gaps|Verify/.test(t), + ); + + const preview = compact.slice(0, 60); + const headings = Array.from(document.querySelectorAll('h1, h2, h3')).map((h) => h.textContent?.trim()).filter(Boolean); + + return { compact: preview, headings, hasSourceButtons: !!document.querySelector('button:has-text("Open source")') }; + }); + + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-result-desktop.png', fullPage: true }); + + const openSourceButtons = await page.locator('button:has-text("Open source"), a:has-text("Open source"), a:has-text("Source PDF"), button:has-text("Add scope")').count(); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: 'C:/Dev/Apps/Database/.tmp-visual/query-result-mobile.png', fullPage: true }); + + const actionButtonLabels = await page.evaluate(() => { + return Array.from(document.querySelectorAll('button, a')).map((el) => (el.textContent || '').trim()).filter((t) => /(Open source|Source PDF|Add scope|Citations|No linked citations|No source provenance)/.test(t)); + }); + + await browser.close(); + + console.log('RESULTS:' + JSON.stringify({ + homeURL: APP_URL, + currentURL: page.url(), + headings: desktopText.headings.slice(0, 30), + compactHints: desktopText.compact.slice(0, 60), + openSourceButtons, + actionButtonLabels: Array.from(new Set(actionButtonLabels)).slice(0, 30), + }, null, 2)); +})(); diff --git a/.tmp-visual/query-stream-desktop.png b/.tmp-visual/query-stream-desktop.png new file mode 100644 index 000000000..1d45b91e8 Binary files /dev/null and b/.tmp-visual/query-stream-desktop.png differ diff --git a/.tmp-visual/stream-capture.mjs b/.tmp-visual/stream-capture.mjs new file mode 100644 index 000000000..3ffc54bf9 --- /dev/null +++ b/.tmp-visual/stream-capture.mjs @@ -0,0 +1,44 @@ +const query = 'What monitoring is required after starting lithium?'; + +(async () => { + const ctrl = new AbortController(); + setTimeout(() => ctrl.abort(), 12000); + + try { + const res = await fetch('http://localhost:4298/api/answer/stream', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ query, filters: {}, queryMode: 'auto' }), + signal: ctrl.signal, + }); + + console.log('STATUS', res.status, res.ok); + if (!res.body) { + console.log('NO_BODY'); + return; + } + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let output = ''; + let done = false; + + while (!done && output.length < 20000) { + const { value, done: d } = await reader.read(); + done = d; + if (value) { + output += decoder.decode(value, { stream: true }); + } + } + + console.log('STREAM_PREVIEW_START'); + console.log(output.slice(0, 3000)); + console.log('STREAM_PREVIEW_END'); + } catch (error) { + if (String(error.name) === 'AbortError') { + console.log('ABORTED_OK'); + } else { + console.error('ERROR', error && error.message ? error.message : String(error)); + } + } +})(); diff --git a/.tmp-visual/stream-headers.mjs b/.tmp-visual/stream-headers.mjs new file mode 100644 index 000000000..1fd3096f3 --- /dev/null +++ b/.tmp-visual/stream-headers.mjs @@ -0,0 +1,25 @@ +(async () => { + const query = 'What monitoring is required after starting lithium?'; + const abortMs = 6000; + const ctrl = new AbortController(); + setTimeout(() => ctrl.abort(), abortMs); + + try { + const res = await fetch('http://localhost:4298/api/answer/stream', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ query, filters: {}, queryMode: 'auto' }), + signal: ctrl.signal, + }); + console.log('status', res.status); + console.log('ok', res.ok); + console.log('type', res.headers.get('content-type')); + console.log('cache-control', res.headers.get('cache-control')); + console.log('connection', res.headers.get('connection')); + console.log('transfer-encoding', res.headers.get('transfer-encoding')); + console.log('x-powered-by', res.headers.get('x-powered-by')); + console.log('server', res.headers.get('server')); + } catch (error) { + console.log('ERR', String(error.name), String(error.message)); + } +})(); diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..8db5fbcad --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,13 @@ +{ + "recommendations": [ + "supabase.postgres-meta", + "ms-azuretools.vscode-docker", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "bradlc.vscode-tailwindcss", + "vitest.explorer", + "wallabyjs.quokka-vscode", + "ms-vscode.live-server", + "mutianwang.vitest-runner" + ] +} diff --git a/AGENTS.md b/AGENTS.md index 591ff5948..6b9fd5fac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,6 +101,18 @@ After setup: - Do not run a permanent watcher. Only start or verify the server when the current chat task needs the app or the user asks to run it. + + +# Process hardening phases + +- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate. +- For UI, frontend, browser, routing, styling, reduced-motion, or forced-colors changes, run `npm run ensure` before browser work and use `npm run verify:ui` as the Chromium UI gate. +- For release or handoff confidence, use `npm run verify:release`; this includes the full Playwright project set. +- For clinical ingestion, answer generation, source governance, privacy, production-readiness, or environment changes, run the smallest relevant domain check plus `npm run check:production-readiness`. +- For pull requests that touch ingestion, answer generation, search/ranking, source rendering, document access, privacy, production env, or clinical output, complete the clinical governance preflight in `.github/pull_request_template.md`. +- Track known verification debts and staged process improvements in `docs/process-hardening.md` instead of relying on chat-only memory. + + # Supabase project safety @@ -210,3 +222,19 @@ Run the smallest relevant checks that are available and appropriate, such as tes After completing `upload`, summarize the current branch and worktree state, whether the worktree is clean, what changed, files committed, commit hash and message if created, whether anything was pushed, remote branch and likely PR target, checks run and results, checks not run and why, branch cleanup candidates, branch references found or updated, risky actions skipped, and exact confirmation needed for any recommended follow-up. + + + +## Codex productivity defaults + +- Treat terse prompts as workflow shortcuts when the intent is clear. If the user says `run`, execute `npm run ensure`, verify the project identity through that helper, and return the printed local URL without a long log dump. +- For non-trivial changes, start from concrete repo state: branch, `git status`, relevant package scripts, recent failures, and local logs such as `dev-server.log` when runtime behavior is involved. +- For UI, browser, styling, routing, accessibility, or screenshot work, run `npm run ensure` before opening the app, then use browser QA and the smallest relevant UI proof before broader gates. +- Prefer the smallest failing check first. For this repo, use focused Vitest or Playwright targets before widening to `npm run verify:cheap`, `npm run verify:ui`, or `npm run verify:release`. +- When the user says `safely`, preserve unrelated staged, unstaged, and untracked work; stop only clearly repo-owned transient processes; and verify the result instead of doing broad cleanup. +- After auth, Supabase, ingestion, answer generation, search/ranking, clinical output, or source-governance changes, run the smallest domain check plus `npm run check:production-readiness`. Run `npm run check:supabase-project` after Supabase env/config changes. +- For handoff, archive-safety, or upload-style requests, inspect branch/upstream/status first, run the appropriate verification gate, and only commit or push when the request explicitly asks for that workflow. +- For codebase appraisal exports, stage outside the repo, include `EXPORT_MANIFEST.md`, exclude secrets/dependencies/build outputs/local state, and verify the archive can be opened before handoff. +- When a repeated repo-specific workflow is discovered, update this file or ask the user whether it should be remembered. + + diff --git a/README.md b/README.md index 235a1cc3b..badd54324 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ npm run ensure # check/start this project's dev server in the background npm run start # production preview on the same safe port selection npm run worker # local ingestion worker npm run check:supabase-project +npm run check:production-readiness # run production readiness validation preflight +npm run check:production-readiness:ci # CI-safe readiness preflight (env-absent tolerant) npm run samples # generate synthetic upload corpus npm run samples:check npm run lint @@ -110,8 +112,13 @@ npm run typecheck npm run test npm run test:coverage npm run test:e2e +npm run test:e2e:all +npm run test:e2e:accessibility npm run test:e2e:chromium npm run test:e2e:visual +npm run verify:cheap +npm run verify:ui +npm run verify:release npm run format npm run format:check npm run build diff --git a/docs/clinical-governance.md b/docs/clinical-governance.md index 3d54c5828..b3592dce0 100644 --- a/docs/clinical-governance.md +++ b/docs/clinical-governance.md @@ -26,3 +26,13 @@ Clinical KB is currently a source-backed clinical reference prototype. Before pr - Generated answers and copied drafts must be verified against linked source text, local policy, and patient context before use. - Do not add dose calculators, diagnostic scores, patient-facing recommendations, or automated treatment recommendations without dedicated clinical validation. - Keep demo content clearly synthetic and separated from real clinical content. + +## Pull Request Preflight + +Use the `.github/pull_request_template.md` clinical governance section for any change that touches ingestion, answer generation, search/ranking, source rendering, document access, privacy, production environment behavior, or clinical output. + +- Confirm the Supabase target remains `Clinical KB Database` (`sjrfecxgysukkwxsowpy`). +- Confirm service-role credentials and private document access remain server-only. +- Confirm unknown or outdated source metadata is treated conservatively. +- Confirm demo/synthetic content remains separated from real clinical sources. +- Confirm clinical decision-support behavior changes have deployment classification and TGA SaMD impact reviewed before production use. diff --git a/docs/process-hardening.md b/docs/process-hardening.md new file mode 100644 index 000000000..2f6524ee8 --- /dev/null +++ b/docs/process-hardening.md @@ -0,0 +1,45 @@ +# Process Hardening Plan + +This document turns the current process review into phased, durable repo practice. It separates changes that already take effect from work that should stay explicit until it is implemented. + +## Phase 1 - Active now + +- `npm run verify:cheap` is the default broad local gate for source/config/test changes: lint, typecheck, and unit tests. +- `npm run verify:ui` is the default UI gate: Chromium Playwright smoke, stress, and accessibility media checks. +- `npm run verify:release` is the release-confidence gate: lint, typecheck, unit tests, build, and the full Playwright browser project set. +- CI now installs Chromium and runs the Chromium UI gate after build. +- `tests/ui-accessibility.spec.ts` covers reduced-motion and forced-colors dashboard usability so those modes are no longer only reviewed by inspection. +- `tests/ui-tools.spec.ts` covers the `/tools` launcher at mobile and desktop sizes. +- `AGENTS.md` now points future agents to these gates and to this document. + +## Phase 2 - Active now + +- Previous deterministic smoke failures are reclassified as resolved in the current Chromium UI gate: `npm run verify:ui` passed 26/26 on June 23, 2026. +- Local scratch and visual-capture output are excluded from Prettier through `.prettierignore` so generated investigation files do not block the format gate. +- Pull requests now include a clinical governance preflight for ingestion, answer generation, source rendering, privacy, production environment, and clinical-output changes. +- `/tools` now has dedicated Playwright coverage in the UI gate. + +## Phase 3 - Structural cleanup + +- [ ] Decompose `src/components/ClinicalDashboard.tsx` into the planned `src/components/clinical-dashboard/` modules. +- Preserve `data-testid`, `aria-label`, and AST-pinned `ClinicalOutputPanel` contracts during the move. +- After decomposition, run `npm run verify:cheap`, `npm run verify:ui`, and focused visual/browser checks against the dashboard and document viewer. + +### Phase 3 progress (started) + +- Added `src/components/clinical-dashboard/` as the module boundary. +- `src/app/page.tsx` now imports `ClinicalDashboard` from the module path (`@/components/clinical-dashboard`) while preserving + the legacy source declaration file for AST and merge-guard compatibility. + +## Phase 4 - Release maturity + +- Decide whether CI should run all Playwright browser projects on protected branches, release branches, or a scheduled workflow instead of every push. +- Add explicit review ownership for clinical source governance, outdated-source handling, incident review, and decommission decisions. +- Record production-readiness outcomes in release notes whenever clinical workflow, source governance, privacy, or deployment assumptions change. + +## Known limits + +- Chromium UI coverage is active in CI now; Firefox and WebKit remain available through `npm run test:e2e` and `npm run verify:release`. +- The new accessibility media smoke verifies usability and layout in reduced-motion and forced-colors modes; it is not a full WCAG audit. +- The format gate intentionally ignores `.tmp-visual/` and `scratch/`; those folders are local investigation output, not release source. +- Process scripts do not commit, push, deploy, mutate Supabase data, or run dependency updates. diff --git a/docs/production-readiness-checklist.md b/docs/production-readiness-checklist.md new file mode 100644 index 000000000..762c05a3e --- /dev/null +++ b/docs/production-readiness-checklist.md @@ -0,0 +1,58 @@ +# Clinical KB Production Readiness Checklist (Executable Today) + +This is the runbook to make the app publishable in one focused pass. + +- Branch: `codex/premium-redesign` (do not touch `.env` / secrets directly). +- Runtime target: Next.js 16.2.7, Node 22.x, npm >= 10. +- Supabase target: `sjrfecxgysukkwxsowpy` (`Clinical KB Database`). + +## Immediate completion targets + +- [x] Security headers are enforced at the framework layer (`next.config.ts`). + - `X-Content-Type-Options`, `X-Frame-Options`, `Referrer-Policy`, + `Cross-Origin-*`, `Permissions-Policy`, and `Origin-Agent-Cluster`. +- [x] Server-side headers exposure is reduced (`X-Powered-By` disabled). +- [x] Added one-command production preflight: + - `npm run check:production-readiness` + - runs env validation, Supabase target checks, lockfile/env-file presence checks, and placeholder checks. +- [x] Added README-visible command for readiness preflight. +- [x] Added CI-safe production preflight: + - `npm run check:production-readiness:ci` + - used in CI and non-blocking on local-only secret absence. + +## Remaining high-priority publish items (same day) + +- [ ] Run the readiness preflight with fully populated `.env.local`. +- [ ] Run `npm run lint`, `npm run typecheck`, `npm run test`, `npm run build`. +- [ ] Run browser smoke verification for auth + search + answer formatting after final changes. +- [ ] Confirm no production-only WIP flags (e.g., local no-auth modes) are enabled in deployment + environment variables. +- [ ] Add deployment/runbook checks for observability, alerts, and rollback. + +## Execution flow for a publish candidate + +1. `npm run ensure` (capture the published local URL if browser checks are needed). +2. `npm run check:production-readiness`. +3. `npm run check:supabase-project`. +4. `npm run lint`. +5. `npm run typecheck`. +6. `npm run test`. +7. `npm run build`. +8. `npm run check:production-readiness:ci` (CI context only). +9. Frontend browser smoke: + +- auth flow +- protected endpoint behavior +- search + answer render path +- mobile viewport + +9. Staging deployment smoke + rollback rehearsal. + +## Command outputs to record + +- `scripts/production-readiness.ts` result: PASS / WARN / FAIL. +- `npm run lint` output. +- `npm run typecheck` output. +- `npm run test` output. +- `npm run build` output. +- Any blocking warnings from readiness preflight should be cleared before publishing. diff --git a/docs/redesign/01-audit.md b/docs/redesign/01-audit.md index 69f3ae7b6..c1f60da4f 100644 --- a/docs/redesign/01-audit.md +++ b/docs/redesign/01-audit.md @@ -1,5 +1,21 @@ # Design Audit — Clinical KB (June 2026 redesign) +## June 20 scoped run — dashboard and document viewer only + +This run intentionally excludes `/tools`, `src/app/tools/page.tsx`, and `src/lib/tools.ts` by user request. Those files may already be dirty in the worktree, but they are not part of this redesign pass and remain deferred for the next run. + +| Location | Finding | Class | Planned action | Tier | +| ------------------------ | ---------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------- | ---- | +| `/` header | Compact and functional, but visually heavy near the mobile height budget. | Upgrade | Refine shell spacing, hierarchy, and controls while preserving smoke budgets. | 1 | +| Answer result | Evidence, warnings, modes, and follow-ups can compete after an answer. | Upgrade | Keep primary answer first; move dense verification into calmer progressive drawers/sheets. | 2 | +| Scope control | Mobile sheet direction is sound; test contract and density need protection. | Polish | Preserve stable test IDs on the visible mobile sheet and refine spacing/focus flow. | 2 | +| Documents sheet | Loading/empty states, counts, list search, and tag quality feel utilitarian. | Upgrade | Add clearer health summary, tighter list hierarchy, and polished empty states. | 1 | +| Upload/indexing sheet | Setup, upload, health, and job queue compete in one long mobile sheet. | Rebuild | Use mobile segmented sections: Setup, Upload, Jobs; keep desktop efficient. | 2 | +| Document viewer mobile | Header/action cluster and PDF controls consume too much vertical space. | Rebuild | Compact header; move admin actions into a mobile sheet/menu; simplify visible controls. | 2 | +| Document viewer evidence | Empty pinned evidence card appears before PDF when no chunk is selected. | Upgrade | Show a compact hint until a citation chunk exists; use full pinned card only for active chunks. | 1 | +| Baseline checks | Lint, unit, and smoke baselines were not green before redesign. | Polish | Fix missing JSX key state, provenance cleanup test, and stale mobile scope smoke assertion first. | 1 | +| Strong areas | Focus rings, reduced motion, guide sheet, local server guard, bottom nav. | Keep | Protect behavior while refining visuals only. | — | + ## 1. Application map (regression baseline) ### Routes diff --git a/docs/redesign/02-design-direction.md b/docs/redesign/02-design-direction.md index f7ec1e6c4..7cb72cd45 100644 --- a/docs/redesign/02-design-direction.md +++ b/docs/redesign/02-design-direction.md @@ -1,5 +1,9 @@ # Design Direction — Clinical KB +## June 20 scoped run + +The active direction for the dashboard and document viewer is a quiet clinical command instrument: neutral, precise, source-first. Teal remains reserved for primary action, evidence, and focus; dense operational details should collapse into progressive sheets/drawers on mobile rather than competing with the answer or PDF. + ## Point of view A precision clinical instrument: calm, quiet, and trustworthy. A teal-tinted neutral foundation carries almost everything; the single teal accent is spent only on primary actions, evidence highlights, and focus. Depth comes from hairline borders and small layered shadows — never from heavy blur — and typography does the hierarchy work: confident headings, a 16px reading body with a capped measure, and tabular numerals wherever data lives. diff --git a/docs/redesign/03-decision-log.md b/docs/redesign/03-decision-log.md index 637f0c2ea..2d8d6f117 100644 --- a/docs/redesign/03-decision-log.md +++ b/docs/redesign/03-decision-log.md @@ -56,3 +56,31 @@ Entries are appended as work lands. Format: what changed / why better / consider **Why better:** restores the ≤180 budget the project already enforces, keeps the newer command-style features, and clears merge debris. **Rejected:** raising the test budget (would bless an over-tall mobile header); deleting the FAB menu (other branch's feature). **Verification:** the 6 header-height tests + the `:778` doc-search test pass on a warm server; typecheck, lint (no warnings), prettier clean; full chromium smoke re-run. + +## D7 — Scoped run excludes Tools + +**What:** `/tools`, `src/app/tools/page.tsx`, and `src/lib/tools.ts` are explicitly excluded from this run and tracked in deferred items. +**Why better:** The user asked to leave launcher/tools IA and card changes for the next run; preserving that boundary prevents unrelated dirty worktree changes from being mixed into the dashboard/viewer redesign. +**Rejected:** Opportunistically polishing the tools launcher while touching the broader shell (out of scope for this run). +**Verification:** Git diff was reviewed with those paths excluded from the implemented patch set. + +## D8 — Upload and indexing mobile workspace + +**What:** Rebuilt the upload/indexing drawer content into mobile segmented sections: `Setup`, `Upload`, and `Jobs`. The single upload form remains mounted once; desktop keeps an efficient two-column operational layout. +**Why better:** Mobile no longer presents setup, upload, health, and worker queues as one long sheet. Users can move directly to the section they need, and health-strip targets open the right section. +**Rejected:** Rendering separate mobile and desktop copies (would duplicate form state and test IDs); moving upload/indexing to a new route (Tier 3 route/capability change). +**Verification:** Focused eslint and `npm run typecheck` pass; full technical and browser verification is recorded in `06-verification.md`. + +## D9 — Document viewer mobile actions sheet + +**What:** Replaced the mobile rename/delete icon pair in the sticky viewer header with a single `Document actions` trigger. The sheet contains provenance, summarise, and existing document management actions. +**Why better:** The primary header now preserves back + title + summarise without a crowded action cluster, while all original admin capabilities remain available. +**Rejected:** Hiding summarise inside the sheet (adds friction and risks smoke coverage); removing admin actions from mobile (capability loss, Tier 3). +**Verification:** Focused eslint and `npm run typecheck` pass; browser verification covers the mobile action sheet. + +## D10 — Compact no-citation evidence hint + +**What:** When no citation chunk is active, the viewer shows a compact source-evidence hint instead of a full pinned evidence card before the PDF. The full highlighted passage card still appears for active chunks. +**Why better:** The PDF regains priority on mobile when there is no citation to inspect, while source evidence remains discoverable and anchored for navigation. +**Rejected:** Removing the evidence anchor entirely (would weaken navigation and tests); always showing the full card (kept the original hierarchy problem). +**Verification:** Focused eslint and `npm run typecheck` pass; pinned evidence and PDF ordering are included in visual QA. diff --git a/docs/redesign/04-deferred.md b/docs/redesign/04-deferred.md index 903374c90..45f2fcbfa 100644 --- a/docs/redesign/04-deferred.md +++ b/docs/redesign/04-deferred.md @@ -1,5 +1,9 @@ # Deferred Items +## 0. Tools page redesign (resolved June 23, 2026) + +`/tools`, `src/app/tools/page.tsx`, and `src/lib/tools.ts` are no longer deferred. The launcher now has dedicated mobile and desktop Playwright coverage through `tests/ui-tools.spec.ts`, included in `npm run verify:ui`. + ## 1. ESLint 10 / eslint-plugin-react incompatibility (pre-existing, Tier 3) There is a lockfile/install mismatch around ESLint that predates and is independent of the redesign: @@ -18,6 +22,6 @@ The 4,655-line `src/components/ClinicalDashboard.tsx` was not split into `clinic - **Restart point:** the Plan-verified move map (lowest-coupling first) is in the approved plan — `use-theme.ts`, `badges.tsx`, `display-text.ts`, `guide-dialog.tsx`, `utility-drawer.tsx`, `setup-checklist.tsx`, `upload-drawer.tsx`, `master-search-header.tsx`, `answer-content.tsx`, `evidence-panels.tsx`, `output-panel.tsx`, `visual-evidence.tsx`, `document-results.tsx`, `auth-panel.tsx`. - **Constraint reminder:** moving `ClinicalOutputPanel` requires updating `dashboardPath` in `tests/clinical-dashboard-merge-artifacts.test.ts` (it AST-pins the panel's location); all `data-testid`/`aria-label` strings must move verbatim. -## 3. Pre-existing smoke failures (not redesign-caused) +## 3. Pre-existing smoke failures (resolved in current UI gate) -Three `ui-smoke` tests fail deterministically on a warm server independent of styling (answer "Structured details" heading, private-source pdf-preview state, duplicate-upload "Queue document" enable logic). See `01-audit.md` baseline. Out of scope to fix here; flagged so they are not mistaken for redesign regressions. +The previously flagged answer, private-source preview, and duplicate-upload smoke cases now pass in the current Chromium UI gate. `npm run verify:ui` passed 26/26 on June 23, 2026, including the accessibility media smoke and `/tools` launcher smoke coverage. diff --git a/docs/redesign/05-changelog.md b/docs/redesign/05-changelog.md index d5aef7e44..34b696d95 100644 --- a/docs/redesign/05-changelog.md +++ b/docs/redesign/05-changelog.md @@ -1,5 +1,37 @@ # Changelog — Premium Redesign +## June 23 process hardening pass + +### Upgrade + +- **Verification scripts:** added cheap, UI, and release verification commands so process gates are explicit and reusable. +- **Accessibility media smoke:** added reduced-motion and forced-colors Chromium coverage. +- **Tools launcher coverage:** added mobile and desktop Playwright coverage for `/tools`. +- **Clinical governance preflight:** added a pull request checklist for clinical-source, privacy, environment, and production-readiness changes. + +### Polish + +- **Format gate hygiene:** ignored local `.tmp-visual/` and `scratch/` investigation output so generated files do not block Prettier. +- **Deferred-item cleanup:** reclassified the prior deterministic smoke failures as resolved under the current UI gate. + +## June 20 scoped run — dashboard/viewer, no Tools + +### Rebuild + +- **Upload and indexing drawer:** mobile segmented workflow (`Setup`, `Upload`, `Jobs`) with health-strip deep links; desktop keeps the full two-column operational layout. +- **Document viewer mobile actions:** crowded mobile admin icons moved into a focused `Document actions` sheet with provenance, summarise, and existing rename/delete controls. + +### Upgrade + +- **Viewer source evidence:** no-citation state is now a compact hint, while active citation chunks still render the full highlighted passage card. +- **Upload workflow feedback:** successful uploads switch mobile users straight to `Jobs`, reducing the handoff from upload to worker progress. + +### Polish + +- **Library health focus state:** fixed the health-strip focus outline to use the existing `--focus` token. +- **Baseline repairs:** clinical-safety source cleanup removes provenance/chunk boilerplate; mobile scope smoke now targets the visible scope surface. +- **Scope boundary:** `/tools`, `src/app/tools/page.tsx`, and `src/lib/tools.ts` were left untouched by this run. + ## Rebuild - **Guide modal → responsive `Sheet`** (`ClinicalDashboard.tsx`, `ui/sheet.tsx`): bottom sheet on mobile, centred dialog on desktop, with enter/exit animation, drag grip, focus trap, focus return, and safe-area padding. Accessible name `Clinical KB guide` and `Close guide` label preserved. diff --git a/docs/redesign/06-verification.md b/docs/redesign/06-verification.md index 32a02cd3f..4e7f7df15 100644 --- a/docs/redesign/06-verification.md +++ b/docs/redesign/06-verification.md @@ -2,6 +2,45 @@ Scope: ultra-premium mobile-first redesign — token system, component layer, dashboard + document-viewer mobile surfaces, plus reconciliation of merge-integration regressions that landed in `main` from parallel branches. Checks were run in `C:\Dev\Apps\Database` on the reconciliation branch after isolating the final `main` fixes. +## June 20 scoped run — dashboard/viewer only, Tools deferred + +Branch: `codex/premium-redesign`. Local server: `npm run ensure` confirmed `http://localhost:4298`; `/api/local-project-id` confirmed `Clinical KB` with `safeLocalOrigin: true`. + +| Check | Command | Result | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Focused lint | `npx eslint src\components\ClinicalDashboard.tsx src\components\DocumentViewer.tsx src\lib\clinical-safety.ts tests\ui-smoke.spec.ts` | Pass | +| Repo lint | `npm run lint` | Pass exit code; 10 warnings remain in pre-existing `.tmp-visual`, `src/lib/rag.ts`, and `tests/deep-memory.test.ts` files | +| Typecheck | `npm run typecheck` | Pass | +| Unit tests | `npm run test` | Pass, 58 files / 412 tests | +| Production build | `npm run build` | Pass | +| Scoped format | `npx prettier --check` on touched files | Pass | +| Repo format | `npm run format:check` | Fail: 53 unrelated pre-existing files remain unformatted, mostly staged `.tmp-visual`, `scratch`, scripts, and existing library/test files | +| Chromium smoke | `npx playwright test tests/ui-smoke.spec.ts --project=chromium --reporter=line` | Pass, 22/22 | +| Chromium stress | `npx playwright test tests/ui-stress.spec.ts --project=chromium --reporter=line` | Pass, 2/2 | + +Browser QA: + +- Browser/IAB was attempted first per frontend validation policy. It opened the mobile upload sheet, but `Page.captureScreenshot` timed out; screenshots fell back to repo Playwright and this fallback is recorded here. +- Playwright screenshots captured outside the repo: + - `C:\Users\joshs\AppData\Local\Temp\clinical-kb-premium-redesign\desktop-dashboard.png` + - `C:\Users\joshs\AppData\Local\Temp\clinical-kb-premium-redesign\mobile-upload-sheet.png` + - `C:\Users\joshs\AppData\Local\Temp\clinical-kb-premium-redesign\mobile-document-actions.png` +- Screenshot states verified: desktop dashboard at 1280×900, mobile upload sheet at 390×820, mobile real document actions sheet at 390×820. No horizontal overflow in all three captures. +- Console health: final screenshot pass had no console warnings/errors. An earlier desktop capture produced a React hydration mismatch caused by Playwright screenshotting before hydration finished; recapturing after a longer hydration settle cleared it. + +Functional/regression notes: + +- Upload/indexing sheet: mobile trigger opens the sheet; `Setup`, `Upload`, and `Jobs` tabs are reachable; setup checklist and upload labels remain visible; duplicate-upload smoke still completes. +- Document viewer: mobile header keeps summarise visible and moves admin actions behind `Open document actions`; action sheet opens on a real `/documents/{id}` route. +- Active PDF/chunk evidence states are verified through mocked Chromium smoke. The live first 150 local documents did not include an indexed document with pages/chunks, so live viewer screenshot used a queued PDF record for shell/action verification. +- `/tools`, `src/app/tools/page.tsx`, and `src/lib/tools.ts` were not edited by this scoped run. Existing staged `/tools` changes remain in the dirty worktree and are deferred. + +Unverified or limited: + +- Full Firefox/WebKit smoke not run. +- Reduced-motion and forced-colors were verified by code/token review and existing smoke coverage, not by a dedicated browser emulation pass in this run. +- Repo-wide format is not green because of unrelated dirty/staged files; touched scoped files are Prettier-clean. + ## 1. Technical checks (main checkout) — all green | Check | Command | Result | diff --git a/eslint.config.mjs b/eslint.config.mjs index b89d97b95..e5dee5498 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -15,6 +15,7 @@ const eslintConfig = defineConfig([ "playwright-report/**", "test-results/**", "sample-documents/**", + "scratch/**", "next-env.d.ts", ]), ]); diff --git a/next.config.ts b/next.config.ts index ca6c9392a..0ce9f06ba 100644 --- a/next.config.ts +++ b/next.config.ts @@ -1,7 +1,47 @@ import type { NextConfig } from "next"; +const isDevelopment = process.env.NODE_ENV === "development"; +const scriptSrc = `script-src 'self' 'unsafe-inline'${isDevelopment ? " 'unsafe-eval'" : ""}; `; + +const securityHeaders = [ + { key: "X-Content-Type-Options", value: "nosniff" }, + { key: "X-Frame-Options", value: "DENY" }, + { key: "Referrer-Policy", value: "strict-origin-when-cross-origin" }, + { key: "Cross-Origin-Resource-Policy", value: "same-site" }, + { key: "Cross-Origin-Opener-Policy", value: "same-origin" }, + { key: "Cross-Origin-Embedder-Policy", value: "require-corp" }, + { key: "Permissions-Policy", value: "camera=(), microphone=(), geolocation=(), payment=()" }, + { key: "Origin-Agent-Cluster", value: "?1" }, + { + key: "Content-Security-Policy", + value: + "default-src 'self'; " + + "base-uri 'self'; " + + "object-src 'none'; " + + "frame-ancestors 'none'; " + + "form-action 'self'; " + + "upgrade-insecure-requests; " + + "img-src 'self' data: blob: https:; " + + "media-src 'self' https:; " + + "connect-src 'self' https://sjrfecxgysukkwxsowpy.supabase.co https://*.supabase.co https://api.openai.com; " + + scriptSrc + + "style-src 'self' 'unsafe-inline'", + }, + { key: "X-Permitted-Cross-Domain-Policies", value: "none" }, + { key: "Strict-Transport-Security", value: "max-age=63072000; includeSubDomains; preload" }, +]; + const nextConfig: NextConfig = { devIndicators: false, + poweredByHeader: false, + async headers() { + return [ + { + source: "/(.*)", + headers: securityHeaders, + }, + ]; + }, }; export default nextConfig; diff --git a/package.json b/package.json index f6b443be9..fecec2c20 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,15 @@ "test": "vitest run", "test:coverage": "vitest run --coverage", "test:e2e": "playwright test", + "test:e2e:all": "playwright test", + "test:e2e:accessibility": "playwright test tests/ui-accessibility.spec.ts --project=chromium", "test:e2e:chromium": "playwright test --project=chromium", "test:e2e:visual": "playwright test --config=playwright.visual.config.ts", + "verify:cheap": "npm run lint && npm run typecheck && npm run test", + "verify:ui": "npm run test:e2e:chromium", + "verify:release": "npm run lint && npm run typecheck && npm run test && npm run build && npm run test:e2e", + "check:production-readiness": "tsx scripts/production-readiness.ts", + "check:production-readiness:ci": "tsx scripts/production-readiness.ts --ci", "format": "prettier --write .", "format:check": "prettier --check .", "worker": "tsx worker/index.ts", @@ -42,7 +49,12 @@ "purge:query-logs": "tsx scripts/purge-query-logs.ts", "audit:tables": "tsx scripts/audit-tables.ts", "samples": "tsx scripts/generate-sample-documents.ts", - "samples:check": "tsx scripts/check-sample-extraction.ts" + "samples:check": "tsx scripts/check-sample-extraction.ts", + "workflow:run": "node ../.local-dev/workflow-run.mjs", + "workflow:status": "node ../.local-dev/workflow-status.mjs", + "workflow:verify": "node ../.local-dev/workflow-verify.mjs", + "workflow:deps": "node ../.local-dev/workflow-deps.mjs", + "workflow:clean-state": "node ../.local-dev/workflow-clean-state.mjs" }, "dependencies": { "@next/env": "^16.2.7", diff --git a/playwright.config.ts b/playwright.config.ts index 5fa5edd99..1d3a04371 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -5,8 +5,8 @@ const baseURL = getPlaywrightBaseUrl(); export default defineConfig({ testDir: "./tests", - testMatch: /.*ui-(smoke|stress)\.spec\.ts/, - timeout: 30_000, + testMatch: /.*ui-(smoke|stress|accessibility|tools)\.spec\.ts/, + timeout: 60_000, expect: { timeout: 10_000, }, diff --git a/scratch/apply-fast-schema.sql b/scratch/apply-fast-schema.sql new file mode 100644 index 000000000..e76dbc9a4 --- /dev/null +++ b/scratch/apply-fast-schema.sql @@ -0,0 +1,575 @@ +SET search_path = public, extensions; + +-- ============================================================ +-- 1. RAG RETRIEVAL LOGS TABLE +-- Per-query diagnostics: candidate scores, latency breakdown, +-- miss detection. +-- ============================================================ + +create table if not exists public.rag_retrieval_logs ( + id uuid primary key default gen_random_uuid(), + owner_id uuid references auth.users(id) on delete set null, + query text not null, + normalized_query text, + query_class text, + retrieval_strategy text, + + -- Candidate scores + candidate_count integer not null default 0, + top_similarity double precision, + top_text_rank double precision, + top_hybrid_score double precision, + top_rrf_score double precision, + mean_hybrid_score double precision, + + -- Selected citations + selected_chunk_ids uuid[] not null default '{}', + selected_document_ids uuid[] not null default '{}', + selected_count integer not null default 0, + + -- Latency breakdown + embedding_latency_ms integer, + rpc_latency_ms integer, + rerank_latency_ms integer, + total_latency_ms integer, + + -- Source counts + vector_candidate_count integer, + text_candidate_count integer, + memory_card_count integer, + index_unit_count integer, + embedding_field_count integer, + + -- Miss detection + is_miss boolean not null default false, + miss_reason text, + + -- Metadata + embedding_cache_hit boolean, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); + +create index if not exists rag_retrieval_logs_owner_created_idx + on public.rag_retrieval_logs(owner_id, created_at desc); +create index if not exists rag_retrieval_logs_miss_idx + on public.rag_retrieval_logs(is_miss, created_at desc) + where is_miss = true; +create index if not exists rag_retrieval_logs_strategy_idx + on public.rag_retrieval_logs(retrieval_strategy, created_at desc); + +alter table public.rag_retrieval_logs enable row level security; +grant select, insert, update, delete on table public.rag_retrieval_logs to service_role; +grant select on table public.rag_retrieval_logs to authenticated; + +create policy "rag retrieval logs owner read" on public.rag_retrieval_logs + for select to authenticated using (owner_id = (select auth.uid())); + +-- ============================================================ +-- 2. CONTENT_HASH DEDUP ON DOCUMENT_EMBEDDING_FIELDS +-- Replace the problematic unique(document_id, source_chunk_id, +-- field_type, content) constraint that indexes full text. +-- ============================================================ + +alter table public.document_embedding_fields + add column if not exists content_hash text; + +-- Populate for existing rows +update public.document_embedding_fields + set content_hash = md5(content) + where content_hash is null; + +-- Drop the old constraint. +alter table public.document_embedding_fields + drop constraint if exists document_embedding_fields_document_id_source_chunk_id_field_key; +alter table public.document_embedding_fields + drop constraint if exists document_embedding_fields_document_id_source_chunk_id_fiel_key; + +-- Create hash-based unique index +create unique index if not exists document_embedding_fields_dedup_idx + on public.document_embedding_fields(document_id, source_chunk_id, field_type, content_hash); + +-- ============================================================ +-- 3. DROP DUPLICATE/REDUNDANT INDEXES +-- These single-column indexes are covered by composite indexes +-- with the same leading column. +-- ============================================================ + +drop index if exists document_chunks_document_id_idx; +drop index if exists document_sections_document_id_idx; +drop index if exists document_memory_cards_document_id_idx; +drop index if exists document_images_document_id_idx; +drop index if exists document_labels_document_id_idx; +drop index if exists document_embedding_fields_document_id_idx; +drop index if exists document_table_facts_document_id_idx; +drop index if exists document_index_quality_document_id_idx; + +-- ============================================================ +-- 4. OTHER EXPECTED SCHEMA INDEXES FROM OTHER MIGRATIONS +-- ============================================================ +create index if not exists ingestion_jobs_status_next_run_idx + on public.ingestion_jobs(status, next_run_at asc) + where status in ('pending', 'processing'); + +create index if not exists ingestion_jobs_document_status_idx + on public.ingestion_jobs(document_id, status); + +create index if not exists import_batches_status_created_idx + on public.import_batches(status, created_at desc); + +create index if not exists storage_cleanup_jobs_status_created_idx + on public.storage_cleanup_jobs(status, created_at desc); + +-- ============================================================ +-- 5. ENHANCED HYBRID SCORING +-- ============================================================ + +create or replace function public.match_document_chunks_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 12, + min_similarity double precision default 0.12, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + vector_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + row_number() over (order by c.embedding <=> query_embedding) as vector_rank, + null::bigint as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' + and 1 - (c.embedding <=> query_embedding) >= min_similarity + order by c.embedding <=> query_embedding + limit greatest(match_count * 6, 48) + ), + text_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + null::bigint as vector_rank, + row_number() over ( + order by + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc, + c.embedding <=> query_embedding + ) as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' + and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit greatest(match_count * 6, 48) + ), + combined as ( + select * from vector_ranked + union all + select * from text_ranked + ), + scored as ( + select + id, + document_id, + page_number, + chunk_index, + section_heading, + content, + retrieval_synopsis, + image_ids, + max(similarity)::double precision as similarity, + max(text_rank)::double precision as text_rank, + min(vector_rank) as vector_rank, + min(text_match_rank) as text_match_rank, + max(quality_score)::double precision as quality_score, + bool_or(has_deep_index) as has_deep_index, + max(doc_updated_at) as doc_updated_at + from combined + group by id, document_id, page_number, chunk_index, section_heading, content, retrieval_synopsis, image_ids + ), + scored_metrics as ( + select + scored.*, + ( + (scored.similarity * 0.62) + + (least(scored.text_rank, 1) * 0.22) + + (scored.quality_score * 0.10) + + (case when scored.doc_updated_at > now() - interval '90 days' then 0.06 else 0 end) + )::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + scored.vector_rank), 0) + + coalesce(1.0 / (60 + scored.text_match_rank), 0) + )::double precision as rrf_score + from scored + ), + hybrid_candidates as ( + select id + from scored_metrics + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count + ), + vector_candidates as ( + select id + from scored_metrics + order by similarity desc, hybrid_score desc + limit match_count + ), + text_candidates as ( + select id + from scored_metrics + order by text_rank desc, hybrid_score desc + limit match_count + ), + rrf_candidates as ( + select id + from scored_metrics + order by rrf_score desc, hybrid_score desc + limit match_count + ), + candidate_ids as ( + select id from hybrid_candidates + union + select id from vector_candidates + union + select id from text_candidates + union + select id from rrf_candidates + ) + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + c.similarity, + c.text_rank, + c.hybrid_score, + c.rrf_score, + public.chunk_image_metadata(c.image_ids) as images + from scored_metrics c + join candidate_ids candidates on candidates.id = c.id + join public.documents d on d.id = c.document_id + order by c.hybrid_score desc, c.rrf_score desc, c.similarity desc, c.text_rank desc + limit match_count; +$$; + +create or replace function public.match_document_index_units_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + ranked as ( + select u.id, u.document_id, u.source_chunk_id, u.source_image_id, u.unit_type, u.title, u.content, u.page_start, + u.page_end, u.heading_path, u.normalized_terms, u.source_span, u.quality_score, u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + (ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact') then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 end + )::double precision as text_rank, + u.metadata + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and u.source_chunk_id is not null + and (u.search_tsv @@ query.tsq or u.normalized_terms && query.terms) + order by text_rank desc, similarity desc + limit greatest(match_count * 3, 48) + ) + select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, heading_path, + normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case when unit_type = 'askable_question' then 0.04 else 0 end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count; +$$; + +create or replace function public.analyze_rag_tables() +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + analyze public.document_chunks; + analyze public.document_memory_cards; + analyze public.document_index_units; + analyze public.document_embedding_fields; + analyze public.document_table_facts; + analyze public.documents; +end; +$$; + +revoke execute on function public.analyze_rag_tables() from public, anon, authenticated; +grant execute on function public.analyze_rag_tables() to service_role; + +create or replace function public.reset_document_index(p_document_id uuid) +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + perform set_config('statement_timeout', '180000', true); + delete from public.document_index_units where document_id = p_document_id; + delete from public.document_memory_cards where document_id = p_document_id; + delete from public.document_sections where document_id = p_document_id; + delete from public.document_table_facts where document_id = p_document_id; + delete from public.document_embedding_fields where document_id = p_document_id; + delete from public.document_index_quality where document_id = p_document_id; + delete from public.document_chunks where document_id = p_document_id; + delete from public.document_images where document_id = p_document_id; + delete from public.document_pages where document_id = p_document_id; +end; +$$; + +-- Update health check +create or replace function public.search_schema_health() +returns jsonb +language plpgsql +stable +set search_path = public, extensions, pg_catalog, pg_temp +as $$ +declare + missing text[] := array[]::text[]; + vector_type_oid oid; + vector_schema text; + index_name text; +begin + select t.oid, n.nspname + into vector_type_oid, vector_schema + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where t.typname = 'vector' + and n.nspname = 'extensions' + limit 1; + + if vector_type_oid is null then + missing := array_append(missing, 'extensions.vector_type'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_memory_cards_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_index_units_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); + end if; + + if to_regclass('public.document_index_units') is null then + missing := array_append(missing, 'document_index_units.table'); + end if; + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); + end if; + if not exists (select 1 from pg_class where relname = 'documents_title_trgm_idx') then + missing := array_append(missing, 'documents_title_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_chunks_content_trgm_idx') then + missing := array_append(missing, 'document_chunks_content_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_labels_label_trgm_idx') then + missing := array_append(missing, 'document_labels_label_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_summaries_summary_trgm_idx') then + missing := array_append(missing, 'document_summaries_summary_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_index_units_embedding_hnsw_idx') then + missing := array_append(missing, 'document_index_units_embedding_hnsw_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_embedding_fields_owner_idx', 'document_embedding_fields_owner_id_idx')) then + missing := array_append(missing, 'document_embedding_fields_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_table_facts_owner_idx', 'document_table_facts_owner_id_idx')) then + missing := array_append(missing, 'document_table_facts_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_table_facts_source_image_idx') then + missing := array_append(missing, 'document_table_facts_source_image_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_embedding_fields_dedup_idx') then + missing := array_append(missing, 'document_embedding_fields_dedup_idx'); + end if; + foreach index_name in array array[ + 'document_pages_document_idx', + 'document_images_document_idx', + 'document_sections_document_idx', + 'document_memory_cards_document_idx', + 'document_chunks_document_idx', + 'document_table_facts_document_idx', + 'document_embedding_fields_document_idx', + 'document_index_units_document_idx', + 'ingestion_jobs_status_next_run_idx', + 'ingestion_jobs_document_status_idx', + 'documents_owner_status_idx', + 'import_batches_status_created_idx', + 'storage_cleanup_jobs_status_created_idx' + ] loop + if not exists (select 1 from pg_class where relname = index_name) then + missing := array_append(missing, index_name); + end if; + end loop; + + return jsonb_build_object( + 'ok', cardinality(missing) = 0, + 'missing', missing, + 'vector_extension_schema', vector_schema, + 'checked_at', now() + ); +end; +$$; + +revoke execute on function public.search_schema_health() from public, anon, authenticated; +grant execute on function public.search_schema_health() to service_role; diff --git a/scratch/audit-guidelines.ts b/scratch/audit-guidelines.ts new file mode 100644 index 000000000..985389e68 --- /dev/null +++ b/scratch/audit-guidelines.ts @@ -0,0 +1,107 @@ +import { loadEnvConfig } from "@next/env"; +import * as fs from "fs"; +import * as path from "path"; + +loadEnvConfig(process.cwd()); + +function getFilesRecursively(dir: string): string[] { + let results: string[] = []; + if (!fs.existsSync(dir)) return []; + const list = fs.readdirSync(dir); + for (const file of list) { + const filePath = path.join(dir, file); + const stat = fs.statSync(filePath); + if (stat && stat.isDirectory()) { + results = results.concat(getFilesRecursively(filePath)); + } else { + if (/\.(pdf|docx|doc)$/i.test(file)) { + results.push(filePath); + } + } + } + return results; +} + +async function main() { + const { createAdminClient } = await import("../src/lib/supabase/admin"); + const supabase = createAdminClient(); + + // 1. Scan the local directory + const rootDir = "C:\\Users\\joshs\\OneDrive\\Medicine\\Guidelines"; + console.log(`Scanning local files in ${rootDir}...`); + const localFiles = getFilesRecursively(rootDir); + console.log(`Found ${localFiles.length} local guideline files (PDF/DOC/DOCX).`); + + // 2. Fetch all documents from the database + console.log("Fetching documents from Supabase..."); + const { data: dbDocs, error } = await supabase + .from("documents") + .select("id, title, status, metadata"); + + if (error) { + console.error("Error fetching database documents:", error); + return; + } + + console.log(`Found ${dbDocs?.length || 0} documents in Supabase.`); + + // Create a map of lowercased source path -> db doc + const dbDocsByPath = new Map(); + for (const doc of dbDocs || []) { + const meta = doc.metadata && typeof doc.metadata === "object" ? (doc.metadata as any) : {}; + if (meta.source_path) { + dbDocsByPath.set(meta.source_path.toLowerCase(), doc); + } + } + + // 3. Match them up + const categories = ["BMJ", "EMHS", "KEMH", "NMHS", "PHC", "RKPG", "SMHS"]; + const stats: Record = {}; + + for (const cat of categories) { + stats[cat] = { total: 0, indexed: 0, processing: 0, queued: 0, missing: 0, missingFiles: [] }; + } + + for (const file of localFiles) { + // Determine category from path + const relative = path.relative(rootDir, file); + const topFolder = relative.split(path.sep)[0]; + + if (stats[topFolder]) { + stats[topFolder].total++; + const matched = dbDocsByPath.get(file.toLowerCase()); + if (matched) { + if (matched.status === "indexed") { + stats[topFolder].indexed++; + } else if (matched.status === "processing") { + stats[topFolder].processing++; + } else if (matched.status === "queued") { + stats[topFolder].queued++; + } else { + stats[topFolder].missing++; + stats[topFolder].missingFiles.push(relative); + } + } else { + stats[topFolder].missing++; + stats[topFolder].missingFiles.push(relative); + } + } + } + + console.log("=== Auditing Summary by Directory ==="); + for (const [cat, info] of Object.entries(stats)) { + console.log(`\nDirectory: ${cat}`); + console.log(` Total Local Files: ${info.total}`); + console.log(` Indexed (Done): ${info.indexed}`); + console.log(` Processing: ${info.processing}`); + console.log(` Queued: ${info.queued}`); + console.log(` Not in Database: ${info.missing}`); + if (info.missing > 0 && info.missing <= 5) { + console.log(` Missing files:`, info.missingFiles); + } else if (info.missing > 5) { + console.log(` Missing files (first 5):`, info.missingFiles.slice(0, 5)); + } + } +} + +main().catch(console.error); diff --git a/scratch/check-indexes.ts b/scratch/check-indexes.ts new file mode 100644 index 000000000..b31ad49b0 --- /dev/null +++ b/scratch/check-indexes.ts @@ -0,0 +1,22 @@ +import { loadEnvConfig } from "@next/env"; +loadEnvConfig(process.cwd()); + +async function main() { + const { createAdminClient } = await import("../src/lib/supabase/admin"); + const supabase = createAdminClient(); + + // We can query pg_indexes view using Supabase client if it is exposed, or we can use RPC + // Wait, let's see if we can query from a system catalog or check what indexes exist. + // Actually, pg_indexes might not be directly exposed. Let's see if we can do a select from pg_indexes. + const { data, error } = await supabase + .from("pg_indexes") // this might fail if not in public schema, but system catalogs aren't exposed in PostgREST by default. + .select("*"); + + if (error) { + console.error("System query error (expected if pg_indexes is not exposed):", error.message); + } else { + console.log(data); + } +} + +main().catch(console.error); diff --git a/scratch/check-queued-docs.ts b/scratch/check-queued-docs.ts new file mode 100644 index 000000000..a0b962b9a --- /dev/null +++ b/scratch/check-queued-docs.ts @@ -0,0 +1,22 @@ +import { loadEnvConfig } from "@next/env"; +loadEnvConfig(process.cwd()); + +async function main() { + const { createAdminClient } = await import("../src/lib/supabase/admin"); + const supabase = createAdminClient(); + + const { data: jobs, error } = await supabase + .from("ingestion_jobs") + .select("id, status, error_message, attempt_count, locked_at, document_id, documents(title)") + .in("status", ["pending", "processing"]); + + if (error) { + console.error(error); + return; + } + + console.log("=== Active Ingestion Jobs ==="); + console.log(JSON.stringify(jobs, null, 2)); +} + +main().catch(console.error); diff --git a/scratch/count-files.ts b/scratch/count-files.ts new file mode 100644 index 000000000..a8a73dc94 --- /dev/null +++ b/scratch/count-files.ts @@ -0,0 +1,59 @@ +import { readdir, stat } from "node:fs/promises"; +import path from "node:path"; + +const paths = [ + 'C:\\Users\\joshs\\OneDrive\\Medicine\\Guidelines\\NMHS', + 'C:\\Users\\joshs\\OneDrive\\Medicine\\Guidelines\\PHC', + 'C:\\Users\\joshs\\OneDrive\\Medicine\\Guidelines\\RKPG', + 'C:\\Users\\joshs\\OneDrive\\Medicine\\Guidelines\\SMHS' +]; + +const extensions = [".pdf", ".docx", ".xlsx", ".txt"]; + +async function countFiles(dir: string): Promise<{ count: number; bytes: number }> { + let count = 0; + let bytes = 0; + + async function visit(current: string) { + let entries; + try { + entries = await readdir(current, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + await visit(fullPath); + } else if (entry.isFile()) { + const ext = path.extname(entry.name).toLowerCase(); + if (extensions.includes(ext)) { + count++; + try { + const s = await stat(fullPath); + bytes += s.size; + } catch {} + } + } + } + } + + await visit(dir); + return { count, bytes }; +} + +async function main() { + let grandTotalCount = 0; + let grandTotalBytes = 0; + for (const dir of paths) { + const { count, bytes } = await countFiles(dir); + const mb = (bytes / (1024 * 1024)).toFixed(2); + console.log(`${dir}: ${count} files (${mb} MB)`); + grandTotalCount += count; + grandTotalBytes += bytes; + } + const grandTotalMB = (grandTotalBytes / (1024 * 1024)).toFixed(2); + console.log(`Grand Total: ${grandTotalCount} files (${grandTotalMB} MB)`); +} + +main().catch(console.error); diff --git a/scratch/count-status.ts b/scratch/count-status.ts new file mode 100644 index 000000000..e4265b86c --- /dev/null +++ b/scratch/count-status.ts @@ -0,0 +1,54 @@ +import { loadEnvConfig } from "@next/env"; +loadEnvConfig(process.cwd()); + +async function main() { + const { createAdminClient } = await import("../src/lib/supabase/admin"); + const supabase = createAdminClient(); + + // Total in documents table + const { data: docCounts, error: docError } = await supabase + .from("documents") + .select("status, metadata"); + + if (docError) { + console.error("Error fetching documents:", docError); + return; + } + + const statusCounts: Record = {}; + let enrichmentPending = 0; + + for (const doc of docCounts || []) { + statusCounts[doc.status] = (statusCounts[doc.status] || 0) + 1; + const meta = doc.metadata && typeof doc.metadata === "object" ? (doc.metadata as any) : {}; + if (meta.enrichment_status === "pending") { + enrichmentPending++; + } + } + + console.log("=== Document Table Status ==="); + console.log("Total rows in documents table:", docCounts?.length); + console.log("Breakdown by status:", statusCounts); + console.log("Documents with metadata.enrichment_status === 'pending':", enrichmentPending); + + // Total in ingestion_jobs + const { data: jobCounts, error: jobError } = await supabase + .from("ingestion_jobs") + .select("status"); + + if (jobError) { + console.error("Error fetching jobs:", jobError); + return; + } + + const jobStatusCounts: Record = {}; + for (const job of jobCounts || []) { + jobStatusCounts[job.status] = (jobStatusCounts[job.status] || 0) + 1; + } + + console.log("=== Ingestion Jobs Status ==="); + console.log("Total jobs:", jobCounts?.length); + console.log("Breakdown by status:", jobStatusCounts); +} + +main().catch(console.error); diff --git a/scratch/create-hnsw-indexes.sql b/scratch/create-hnsw-indexes.sql new file mode 100644 index 000000000..0c478aa82 --- /dev/null +++ b/scratch/create-hnsw-indexes.sql @@ -0,0 +1,21 @@ +-- SQL script to recreate the missing HNSW vector indexes. +-- Run this script in the Supabase Dashboard SQL Editor for your project: +-- https://supabase.com/dashboard/project/sjrfecxgysukkwxsowpy/sql/new + +SET statement_timeout = 0; -- disable statement timeout for index creation + +-- 1. Create HNSW index on document_chunks (69,000+ rows) +-- This is a large build and must be run without session timeout. +CREATE INDEX IF NOT EXISTS document_chunks_embedding_hnsw_idx + ON public.document_chunks USING hnsw (embedding vector_cosine_ops) + WITH (m = 24, ef_construction = 128); + +-- 2. Create HNSW index on document_memory_cards (10,000+ rows) +CREATE INDEX IF NOT EXISTS document_memory_cards_embedding_hnsw_idx + ON public.document_memory_cards USING hnsw (embedding vector_cosine_ops) + WITH (m = 24, ef_construction = 128); + +-- 3. Create HNSW index on document_embedding_fields (3,000+ rows) +CREATE INDEX IF NOT EXISTS document_embedding_fields_embedding_hnsw_idx + ON public.document_embedding_fields USING hnsw (embedding vector_cosine_ops) + WITH (m = 24, ef_construction = 128); diff --git a/scratch/fix-indexes.ts b/scratch/fix-indexes.ts new file mode 100644 index 000000000..348f296cb --- /dev/null +++ b/scratch/fix-indexes.ts @@ -0,0 +1,15 @@ +import { loadEnvConfig } from "@next/env"; +loadEnvConfig(process.cwd()); + +async function main() { + const { createAdminClient } = await import("../src/lib/supabase/admin"); + const supabase = createAdminClient(); + + console.log("Applying missing schema indexes..."); + + // 1. Create document_embedding_fields_owner_idx + const { error: error1 } = await supabase.rpc("search_schema_health"); // Just a test, but we can execute raw SQL if we have a way. + // Wait, does Supabase JS client support raw SQL execution? No, unless we use RPC or run it via schema migration. + // Ah, wait! Is there a function we can use, or does the migration schema script apply them? + // Wait, let's see how the test suite applies migrations or runs SQL. +} diff --git a/scratch/queue-queued-docs.ts b/scratch/queue-queued-docs.ts new file mode 100644 index 000000000..3b389ca8c --- /dev/null +++ b/scratch/queue-queued-docs.ts @@ -0,0 +1,57 @@ +import { loadEnvConfig } from "@next/env"; +loadEnvConfig(process.cwd()); + +async function main() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + const { env } = await import("@/lib/env"); + const supabase = createAdminClient(); + + const { data: docs, error } = await supabase + .from("documents") + .select("id, metadata") + .eq("status", "queued"); + + if (error) { + console.error("Error fetching queued documents:", error); + return; + } + + if (!docs || docs.length === 0) { + console.log("No queued documents found to create jobs for."); + return; + } + + for (const doc of docs) { + const metadata = doc.metadata as Record; + const batchId = metadata?.import_batch_id || null; + + console.log(`Creating ingestion job for document ${doc.id} (batch: ${batchId})`); + + const { data: existingJobs } = await supabase + .from("ingestion_jobs") + .select("id") + .eq("document_id", doc.id); + + if (existingJobs && existingJobs.length > 0) { + console.log(`Job already exists for document ${doc.id}, skipping.`); + continue; + } + + const { error: insertError } = await supabase.from("ingestion_jobs").insert({ + document_id: doc.id, + batch_id: batchId, + status: "pending", + stage: "queued", + progress: 0, + max_attempts: env.WORKER_MAX_ATTEMPTS || 3, + }); + + if (insertError) { + console.error(`Error inserting job for document ${doc.id}:`, insertError); + } else { + console.log(`Successfully created ingestion job for document ${doc.id}`); + } + } +} + +main().catch(console.error); diff --git a/scratch/run-migrations-safe.sql b/scratch/run-migrations-safe.sql new file mode 100644 index 000000000..1096201c4 --- /dev/null +++ b/scratch/run-migrations-safe.sql @@ -0,0 +1,603 @@ +-- Set no timeout for the session +SET statement_timeout = 0; +SET search_path = public, extensions; + +-- ============================================================ +-- 1. HNSW INDEX TUNING +-- Rebuild with m=24 (graph connectivity) and ef_construction=128 +-- (build-time recall). Default was m=16, ef_construction=64. +-- ============================================================ + +drop index if exists document_chunks_embedding_hnsw_idx; +create index document_chunks_embedding_hnsw_idx + on public.document_chunks using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +drop index if exists document_memory_cards_embedding_hnsw_idx; +create index document_memory_cards_embedding_hnsw_idx + on public.document_memory_cards using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +drop index if exists document_index_units_embedding_hnsw_idx; +create index document_index_units_embedding_hnsw_idx + on public.document_index_units using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +drop index if exists document_embedding_fields_embedding_hnsw_idx; +create index document_embedding_fields_embedding_hnsw_idx + on public.document_embedding_fields using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +-- ============================================================ +-- 2. RAG RETRIEVAL LOGS TABLE +-- Per-query diagnostics: candidate scores, latency breakdown, +-- miss detection. +-- ============================================================ + +create table if not exists public.rag_retrieval_logs ( + id uuid primary key default gen_random_uuid(), + owner_id uuid references auth.users(id) on delete set null, + query text not null, + normalized_query text, + query_class text, + retrieval_strategy text, + + -- Candidate scores + candidate_count integer not null default 0, + top_similarity double precision, + top_text_rank double precision, + top_hybrid_score double precision, + top_rrf_score double precision, + mean_hybrid_score double precision, + + -- Selected citations + selected_chunk_ids uuid[] not null default '{}', + selected_document_ids uuid[] not null default '{}', + selected_count integer not null default 0, + + -- Latency breakdown + embedding_latency_ms integer, + rpc_latency_ms integer, + rerank_latency_ms integer, + total_latency_ms integer, + + -- Source counts + vector_candidate_count integer, + text_candidate_count integer, + memory_card_count integer, + index_unit_count integer, + embedding_field_count integer, + + -- Miss detection + is_miss boolean not null default false, + miss_reason text, + + -- Metadata + embedding_cache_hit boolean, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); + +create index if not exists rag_retrieval_logs_owner_created_idx + on public.rag_retrieval_logs(owner_id, created_at desc); +create index if not exists rag_retrieval_logs_miss_idx + on public.rag_retrieval_logs(is_miss, created_at desc) + where is_miss = true; +create index if not exists rag_retrieval_logs_strategy_idx + on public.rag_retrieval_logs(retrieval_strategy, created_at desc); + +alter table public.rag_retrieval_logs enable row level security; +grant select, insert, update, delete on table public.rag_retrieval_logs to service_role; +grant select on table public.rag_retrieval_logs to authenticated; + +create policy "rag retrieval logs owner read" on public.rag_retrieval_logs + for select to authenticated using (owner_id = (select auth.uid())); + +-- ============================================================ +-- 3. CONTENT_HASH DEDUP ON DOCUMENT_EMBEDDING_FIELDS +-- Replace the problematic unique(document_id, source_chunk_id, +-- field_type, content) constraint that indexes full text. +-- ============================================================ + +alter table public.document_embedding_fields + add column if not exists content_hash text; + +-- Populate for existing rows +update public.document_embedding_fields + set content_hash = md5(content) + where content_hash is null; + +-- Drop the old constraint. +alter table public.document_embedding_fields + drop constraint if exists document_embedding_fields_document_id_source_chunk_id_field_key; +alter table public.document_embedding_fields + drop constraint if exists document_embedding_fields_document_id_source_chunk_id_fiel_key; + +-- Create hash-based unique index +create unique index if not exists document_embedding_fields_dedup_idx + on public.document_embedding_fields(document_id, source_chunk_id, field_type, content_hash); + +-- ============================================================ +-- 4. DROP DUPLICATE/REDUNDANT INDEXES +-- These single-column indexes are covered by composite indexes +-- with the same leading column. +-- ============================================================ + +drop index if exists document_chunks_document_id_idx; +drop index if exists document_sections_document_id_idx; +drop index if exists document_memory_cards_document_id_idx; +drop index if exists document_images_document_id_idx; +drop index if exists document_labels_document_id_idx; +drop index if exists document_embedding_fields_document_id_idx; +drop index if exists document_table_facts_document_id_idx; +drop index if exists document_index_quality_document_id_idx; + +-- ============================================================ +-- 5. OTHER EXPECTED SCHEMA INDEXES FROM OTHER MIGRATIONS +-- ============================================================ +create index if not exists ingestion_jobs_status_next_run_idx + on public.ingestion_jobs(status, next_run_at asc) + where status in ('pending', 'processing'); + +create index if not exists ingestion_jobs_document_status_idx + on public.ingestion_jobs(document_id, status); + +create index if not exists import_batches_status_created_idx + on public.import_batches(status, created_at desc); + +create index if not exists storage_cleanup_jobs_status_created_idx + on public.storage_cleanup_jobs(status, created_at desc); + +-- ============================================================ +-- 6. ENHANCED HYBRID SCORING +-- ============================================================ + +create or replace function public.match_document_chunks_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 12, + min_similarity double precision default 0.12, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + vector_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + row_number() over (order by c.embedding <=> query_embedding) as vector_rank, + null::bigint as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' + and 1 - (c.embedding <=> query_embedding) >= min_similarity + order by c.embedding <=> query_embedding + limit greatest(match_count * 6, 48) + ), + text_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + null::bigint as vector_rank, + row_number() over ( + order by + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc, + c.embedding <=> query_embedding + ) as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' + and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit greatest(match_count * 6, 48) + ), + combined as ( + select * from vector_ranked + union all + select * from text_ranked + ), + scored as ( + select + id, + document_id, + page_number, + chunk_index, + section_heading, + content, + retrieval_synopsis, + image_ids, + max(similarity)::double precision as similarity, + max(text_rank)::double precision as text_rank, + min(vector_rank) as vector_rank, + min(text_match_rank) as text_match_rank, + max(quality_score)::double precision as quality_score, + bool_or(has_deep_index) as has_deep_index, + max(doc_updated_at) as doc_updated_at + from combined + group by id, document_id, page_number, chunk_index, section_heading, content, retrieval_synopsis, image_ids + ), + scored_metrics as ( + select + scored.*, + ( + (scored.similarity * 0.62) + + (least(scored.text_rank, 1) * 0.22) + + (scored.quality_score * 0.10) + + (case when scored.doc_updated_at > now() - interval '90 days' then 0.06 else 0 end) + )::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + scored.vector_rank), 0) + + coalesce(1.0 / (60 + scored.text_match_rank), 0) + )::double precision as rrf_score + from scored + ), + hybrid_candidates as ( + select id + from scored_metrics + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count + ), + vector_candidates as ( + select id + from scored_metrics + order by similarity desc, hybrid_score desc + limit match_count + ), + text_candidates as ( + select id + from scored_metrics + order by text_rank desc, hybrid_score desc + limit match_count + ), + rrf_candidates as ( + select id + from scored_metrics + order by rrf_score desc, hybrid_score desc + limit match_count + ), + candidate_ids as ( + select id from hybrid_candidates + union + select id from vector_candidates + union + select id from text_candidates + union + select id from rrf_candidates + ) + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + c.similarity, + c.text_rank, + c.hybrid_score, + c.rrf_score, + public.chunk_image_metadata(c.image_ids) as images + from scored_metrics c + join candidate_ids candidates on candidates.id = c.id + join public.documents d on d.id = c.document_id + order by c.hybrid_score desc, c.rrf_score desc, c.similarity desc, c.text_rank desc + limit match_count; +$$; + +create or replace function public.match_document_index_units_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + ranked as ( + select u.id, u.document_id, u.source_chunk_id, u.source_image_id, u.unit_type, u.title, u.content, u.page_start, + u.page_end, u.heading_path, u.normalized_terms, u.source_span, u.quality_score, u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + (ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact') then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 end + )::double precision as text_rank, + u.metadata + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and u.source_chunk_id is not null + and (u.search_tsv @@ query.tsq or u.normalized_terms && query.terms) + order by text_rank desc, similarity desc + limit greatest(match_count * 3, 48) + ) + select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, heading_path, + normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case when unit_type = 'askable_question' then 0.04 else 0 end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count; +$$; + +create or replace function public.analyze_rag_tables() +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + analyze public.document_chunks; + analyze public.document_memory_cards; + analyze public.document_index_units; + analyze public.document_embedding_fields; + analyze public.document_table_facts; + analyze public.documents; +end; +$$; + +revoke execute on function public.analyze_rag_tables() from public, anon, authenticated; +grant execute on function public.analyze_rag_tables() to service_role; + +create or replace function public.reset_document_index(p_document_id uuid) +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + perform set_config('statement_timeout', '180000', true); + delete from public.document_index_units where document_id = p_document_id; + delete from public.document_memory_cards where document_id = p_document_id; + delete from public.document_sections where document_id = p_document_id; + delete from public.document_table_facts where document_id = p_document_id; + delete from public.document_embedding_fields where document_id = p_document_id; + delete from public.document_index_quality where document_id = p_document_id; + delete from public.document_chunks where document_id = p_document_id; + delete from public.document_images where document_id = p_document_id; + delete from public.document_pages where document_id = p_document_id; +end; +$$; + +-- Update health check with correct indexes +create or replace function public.search_schema_health() +returns jsonb +language plpgsql +stable +set search_path = public, extensions, pg_catalog, pg_temp +as $$ +declare + missing text[] := array[]::text[]; + vector_type_oid oid; + vector_schema text; + index_name text; +begin + select t.oid, n.nspname + into vector_type_oid, vector_schema + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where t.typname = 'vector' + and n.nspname = 'extensions' + limit 1; + + if vector_type_oid is null then + missing := array_append(missing, 'extensions.vector_type'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_memory_cards_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_index_units_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); + end if; + + if to_regclass('public.document_index_units') is null then + missing := array_append(missing, 'document_index_units.table'); + end if; + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); + end if; + if not exists (select 1 from pg_class where relname = 'documents_title_trgm_idx') then + missing := array_append(missing, 'documents_title_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_chunks_content_trgm_idx') then + missing := array_append(missing, 'document_chunks_content_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_labels_label_trgm_idx') then + missing := array_append(missing, 'document_labels_label_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_summaries_summary_trgm_idx') then + missing := array_append(missing, 'document_summaries_summary_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_index_units_embedding_hnsw_idx') then + missing := array_append(missing, 'document_index_units_embedding_hnsw_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_embedding_fields_owner_idx', 'document_embedding_fields_owner_id_idx')) then + missing := array_append(missing, 'document_embedding_fields_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_table_facts_owner_idx', 'document_table_facts_owner_id_idx')) then + missing := array_append(missing, 'document_table_facts_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_table_facts_source_image_idx') then + missing := array_append(missing, 'document_table_facts_source_image_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_embedding_fields_dedup_idx') then + missing := array_append(missing, 'document_embedding_fields_dedup_idx'); + end if; + foreach index_name in array array[ + 'document_pages_document_idx', + 'document_images_document_idx', + 'document_sections_document_idx', + 'document_memory_cards_document_idx', + 'document_chunks_document_idx', + 'document_table_facts_document_idx', + 'document_embedding_fields_document_idx', + 'document_index_units_document_idx', + 'ingestion_jobs_status_next_run_idx', + 'ingestion_jobs_document_status_idx', + 'documents_owner_status_idx', + 'import_batches_status_created_idx', + 'storage_cleanup_jobs_status_created_idx' + ] loop + if not exists (select 1 from pg_class where relname = index_name) then + missing := array_append(missing, index_name); + end if; + end loop; + + return jsonb_build_object( + 'ok', cardinality(missing) = 0, + 'missing', missing, + 'vector_extension_schema', vector_schema, + 'checked_at', now() + ); +end; +$$; + +revoke execute on function public.search_schema_health() from public, anon, authenticated; +grant execute on function public.search_schema_health() to service_role; diff --git a/scratch/show-samples.ts b/scratch/show-samples.ts new file mode 100644 index 000000000..830a9bff5 --- /dev/null +++ b/scratch/show-samples.ts @@ -0,0 +1,22 @@ +import { loadEnvConfig } from "@next/env"; +loadEnvConfig(process.cwd()); + +async function main() { + const { createAdminClient } = await import("../src/lib/supabase/admin"); + const supabase = createAdminClient(); + + const { data: sampleDocs, error } = await supabase + .from("documents") + .select("id, title, status, metadata") + .limit(10); + + if (error) { + console.error(error); + return; + } + + console.log("=== Sample Documents ==="); + console.log(JSON.stringify(sampleDocs, null, 2)); +} + +main().catch(console.error); diff --git a/scratch/update-health-check.sql b/scratch/update-health-check.sql new file mode 100644 index 000000000..6da2830c0 --- /dev/null +++ b/scratch/update-health-check.sql @@ -0,0 +1,132 @@ +create or replace function public.search_schema_health() +returns jsonb +language plpgsql +stable +set search_path = public, extensions, pg_catalog, pg_temp +as $$ +declare + missing text[] := array[]::text[]; + vector_type_oid oid; + vector_schema text; + index_name text; +begin + select t.oid, n.nspname + into vector_type_oid, vector_schema + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where t.typname = 'vector' + and n.nspname = 'extensions' + limit 1; + + if vector_type_oid is null then + missing := array_append(missing, 'extensions.vector_type'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_memory_cards_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_index_units_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); + end if; + + if to_regclass('public.document_index_units') is null then + missing := array_append(missing, 'document_index_units.table'); + end if; + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); + end if; + if not exists (select 1 from pg_class where relname = 'documents_title_trgm_idx') then + missing := array_append(missing, 'documents_title_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_chunks_content_trgm_idx') then + missing := array_append(missing, 'document_chunks_content_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_labels_label_trgm_idx') then + missing := array_append(missing, 'document_labels_label_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_summaries_summary_trgm_idx') then + missing := array_append(missing, 'document_summaries_summary_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_index_units_embedding_hnsw_idx') then + missing := array_append(missing, 'document_index_units_embedding_hnsw_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_embedding_fields_owner_idx', 'document_embedding_fields_owner_id_idx')) then + missing := array_append(missing, 'document_embedding_fields_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_table_facts_owner_idx', 'document_table_facts_owner_id_idx')) then + missing := array_append(missing, 'document_table_facts_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_table_facts_source_image_idx') then + missing := array_append(missing, 'document_table_facts_source_image_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_embedding_fields_dedup_idx') then + missing := array_append(missing, 'document_embedding_fields_dedup_idx'); + end if; + foreach index_name in array array[ + 'document_pages_document_idx', + 'document_images_document_idx', + 'document_sections_document_idx', + 'document_memory_cards_document_idx', + 'document_chunks_document_idx', + 'document_table_facts_document_idx', + 'document_embedding_fields_document_idx', + 'document_index_units_document_idx', + 'ingestion_jobs_status_next_run_idx', + 'ingestion_jobs_document_status_idx', + 'documents_owner_status_idx', + 'import_batches_status_created_idx', + 'storage_cleanup_jobs_status_created_idx' + ] loop + if not exists (select 1 from pg_class where relname = index_name) then + missing := array_append(missing, index_name); + end if; + end loop; + + return jsonb_build_object( + 'ok', cardinality(missing) = 0, + 'missing', missing, + 'vector_extension_schema', vector_schema, + 'checked_at', now() + ); +end; +$$; + +revoke execute on function public.search_schema_health() from public, anon, authenticated; +grant execute on function public.search_schema_health() to service_role; diff --git a/scripts/audit-tables.ts b/scripts/audit-tables.ts index 48633545e..1fe7c42eb 100644 --- a/scripts/audit-tables.ts +++ b/scripts/audit-tables.ts @@ -87,9 +87,7 @@ async function main() { `id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`, ); } else { - documentQuery = documentQuery.or( - `file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`, - ); + documentQuery = documentQuery.or(`file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`); } } diff --git a/scripts/backfill-smart-index.ts b/scripts/backfill-smart-index.ts index 4e60f5389..652b7e548 100644 --- a/scripts/backfill-smart-index.ts +++ b/scripts/backfill-smart-index.ts @@ -78,6 +78,10 @@ function hashText(value: string) { return createHash("sha256").update(value).digest("hex"); } +function hashEmbeddingFieldText(value: string) { + return createHash("md5").update(value).digest("hex"); +} + function compactSearchText(value: unknown, limit = 900) { const clean = String(value ?? "") .replace(/\s+/g, " ") @@ -173,7 +177,13 @@ async function repairChunkIndexingMetadata(args: { ? documentMetadata.index_generation_id : randomUUID(); - const chunksToUpdate: { chunk: Record; contentHash: string; sectionPath: string[]; anchorId: string; metadata: Record }[] = []; + const chunksToUpdate: { + chunk: Record; + contentHash: string; + sectionPath: string[]; + anchorId: string; + metadata: Record; + }[] = []; for (const chunk of args.chunks) { const sectionPath = chunkSectionPath(chunk); @@ -182,7 +192,7 @@ async function repairChunkIndexingMetadata(args: { typeof chunk.content_hash === "string" && chunk.content_hash ? chunk.content_hash : hashText(`${chunk.section_heading ?? ""}\n${chunk.content ?? ""}`); - + const chunkMetadata = metadataRecord(chunk.metadata); const needsUpdate = chunk.content_hash !== contentHash || @@ -232,7 +242,7 @@ async function repairChunkIndexingMetadata(args: { }) .eq("id", item.chunk.id); if (error) throw new Error(error.message); - }) + }), ); } } @@ -313,6 +323,7 @@ async function upsertDocumentLevelEmbeddingFields(args: { source_chunk_id: sourceChunkId, field_type: input.field_type, content: input.content, + content_hash: hashEmbeddingFieldText(input.content), embedding: embeddings[index], metadata: { source: "backfill_document_level" }, })); diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index 3b1395202..127be9a10 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -36,12 +36,7 @@ function asNullableString(value: unknown) { function metadataRowFromProjection(row: MetadataProjectionRow): MetadataRow | null { if (typeof row.document_id !== "string" || row.document_id.length === 0) return null; - const { - rag_memory_version, - rag_indexing_version, - rag_enrichment_version, - ...rest - } = row; + const { rag_memory_version, rag_indexing_version, rag_enrichment_version, ...rest } = row; const nextMetadata: unknown = rag_memory_version !== undefined || rag_indexing_version !== undefined || rag_enrichment_version !== undefined @@ -89,6 +84,40 @@ function metadataRecord(metadata: unknown): Record { : {}; } +type DocumentHealthRow = { + id: string; + owner_id: string | null; + title: string | null; + content_hash: string | null; + status: string | null; + page_count: number | null; + chunk_count: number | null; + metadata: unknown; +}; + +async function loadAllDocuments(supabase: SupabaseLike, pageSize = 1000) { + const documents: DocumentHealthRow[] = []; + let cursor: string | null = null; + + for (;;) { + const query = supabase + .from("documents") + .select("id,owner_id,title,content_hash,status,page_count,chunk_count,metadata") + .order("id", { ascending: true }); + const pagedQuery: QueryBuilder = cursor ? query.gt("id", cursor) : query; + + const { data, error } = (await pagedQuery.limit(pageSize)) as QueryResponse; + if (error) throw new Error(error.message); + + documents.push(...(data ?? [])); + if (!data || data.length < pageSize) break; + cursor = data.at(-1)?.id ?? null; + if (!cursor) break; + } + + return documents; +} + function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) { return metadataRecord(metadata).rag_enrichment_version === expectedVersion; } @@ -117,7 +146,10 @@ function missingSchemaMessage(error: { message: string } | Error | null | undefi function readableReadinessError(error: unknown) { const message = error instanceof Error ? error.message : String(error); if (/]/i.test(message)) { - const title = message.match(/\s*([^<]+)\s*<\/title>/i)?.[1]?.replace(/\s+/g, " ").trim(); + const title = message + .match(/<title>\s*([^<]+)\s*<\/title>/i)?.[1] + ?.replace(/\s+/g, " ") + .trim(); const code = message.match(/\b5\d\d\b/)?.[0]; return `Supabase readiness query failed with an HTML gateway response${ title ? ` (${title})` : code ? ` (${code})` : "" @@ -136,10 +168,9 @@ async function loadEnrichmentRows(supabase: SupabaseLike, documentIds: string[]) for (let start = 0; start < documentIds.length; start += 5) { const ids = documentIds.slice(start, start + 5); const [summaryResult, labelResult] = await Promise.all([ - supabase.from("document_summaries") - .select("document_id,metadata->rag_enrichment_version") - .in("document_id", ids), - supabase.from("document_labels") + supabase.from("document_summaries").select("document_id,metadata->rag_enrichment_version").in("document_id", ids), + supabase + .from("document_labels") .select("document_id,source,metadata->rag_enrichment_version") .in("document_id", ids), ]); @@ -147,28 +178,28 @@ async function loadEnrichmentRows(supabase: SupabaseLike, documentIds: string[]) if (summaryResult.error) throw new Error(summaryResult.error.message); if (labelResult.error) throw new Error(labelResult.error.message); - const mappedSummaries: MetadataRow[] = []; - for (const row of summaryResult.data ?? []) { - const source = asMetadataProjection(row); - const documentId = typeof source.document_id === "string" ? source.document_id : null; - if (!documentId) continue; - mappedSummaries.push({ - document_id: documentId, - metadata: { rag_enrichment_version: asNullableString(source.rag_enrichment_version) }, - }); - } + const mappedSummaries: MetadataRow[] = []; + for (const row of summaryResult.data ?? []) { + const source = asMetadataProjection(row); + const documentId = typeof source.document_id === "string" ? source.document_id : null; + if (!documentId) continue; + mappedSummaries.push({ + document_id: documentId, + metadata: { rag_enrichment_version: asNullableString(source.rag_enrichment_version) }, + }); + } - const mappedLabels: MetadataRow[] = []; - for (const row of labelResult.data ?? []) { - const source = asMetadataProjection(row); - const documentId = typeof source.document_id === "string" ? source.document_id : null; - if (!documentId) continue; - mappedLabels.push({ - document_id: documentId, - source: typeof source.source === "string" ? source.source : null, - metadata: { rag_enrichment_version: asNullableString(source.rag_enrichment_version) }, - }); - } + const mappedLabels: MetadataRow[] = []; + for (const row of labelResult.data ?? []) { + const source = asMetadataProjection(row); + const documentId = typeof source.document_id === "string" ? source.document_id : null; + if (!documentId) continue; + mappedLabels.push({ + document_id: documentId, + source: typeof source.source === "string" ? source.source : null, + metadata: { rag_enrichment_version: asNullableString(source.rag_enrichment_version) }, + }); + } summaries.push(...mappedSummaries); labels.push(...mappedLabels); @@ -177,16 +208,22 @@ async function loadEnrichmentRows(supabase: SupabaseLike, documentIds: string[]) return { summaries, labels }; } -async function loadRowsForDocuments(supabase: SupabaseLike, table: string, select: string, documentIds: string[]) { +async function loadRowsForDocuments( + supabase: SupabaseLike, + table: string, + select: string, + documentIds: string[], + orderColumns = ["document_id", "id"], +) { const rows: MetadataRow[] = []; for (let start = 0; start < documentIds.length; start += 5) { const ids = documentIds.slice(start, start + 5); for (let rangeStart = 0; ; rangeStart += 1000) { - const { data, error } = await supabase - .from(table) - .select(select) - .in("document_id", ids) - .range(rangeStart, rangeStart + 999); + let query = supabase.from(table).select(select).in("document_id", ids); + for (const column of orderColumns) { + query = query.order(column, { ascending: true }); + } + const { data, error } = await query.range(rangeStart, rangeStart + 999); if (error) throw new Error(error.message); for (const row of data ?? []) { @@ -201,16 +238,16 @@ async function loadRowsForDocuments(supabase: SupabaseLike, table: string, selec async function loadDeepMemoryRows(supabase: SupabaseLike, documentIds: string[]) { const sections = await loadRowsForDocuments( - supabase, - "document_sections", - "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", - documentIds + supabase, + "document_sections", + "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", + documentIds, ); const memoryCards = await loadRowsForDocuments( - supabase, - "document_memory_cards", - "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", - documentIds + supabase, + "document_memory_cards", + "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", + documentIds, ); const chunks = await loadRowsForDocuments( supabase, @@ -219,31 +256,32 @@ async function loadDeepMemoryRows(supabase: SupabaseLike, documentIds: string[]) documentIds, ); const tableFacts = await loadRowsForDocuments( - supabase, - "document_table_facts", - "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", - documentIds + supabase, + "document_table_facts", + "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", + documentIds, ); const embeddingFields = await loadRowsForDocuments( - supabase, - "document_embedding_fields", - "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", - documentIds + supabase, + "document_embedding_fields", + "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", + documentIds, ); const qualityRows = await loadRowsForDocuments( - supabase, - "document_index_quality", - "document_id,quality_score,issues", - documentIds + supabase, + "document_index_quality", + "document_id,quality_score,issues", + documentIds, + ["document_id"], ); let indexUnits: MetadataRow[] = []; const missingSchema: string[] = []; try { indexUnits = await loadRowsForDocuments( - supabase, - "document_index_units", - "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", - documentIds + supabase, + "document_index_units", + "document_id,metadata->rag_memory_version,metadata->rag_indexing_version", + documentIds, ); } catch (error) { const message = missingSchemaMessage(error instanceof Error ? error : null); @@ -292,15 +330,14 @@ async function main() { ? ((schemaHealth as SchemaHealth).missing as unknown[] | undefined) : undefined; - const { data: documents, error: documentsError } = await supabase - .from("documents") - .select("id,owner_id,title,content_hash,status,page_count,chunk_count,metadata"); - if (documentsError) throw new Error(documentsError.message); - const supabaseForChecks = supabase as unknown as SupabaseLike; - const indexedDocuments = (documents ?? []).filter( + const documents = await loadAllDocuments(supabaseForChecks); + const indexedDocuments = documents.filter( (document) => document.status === "indexed" && metadataRecord(document.metadata).enrichment_status !== "pending", ); + const pendingIndexedDocuments = documents.filter( + (document) => document.status === "indexed" && metadataRecord(document.metadata).enrichment_status === "pending", + ); const enrichmentRows = await loadEnrichmentRows( supabaseForChecks, indexedDocuments.map((document) => document.id), @@ -337,7 +374,7 @@ async function main() { duplicateHashGroups.set(key, [...(duplicateHashGroups.get(key) ?? []), document.title ?? document.id]); } const duplicateGroups = Array.from(duplicateHashGroups.values()).filter((titles) => titles.length > 1); - const emptyIndexedDocuments = (documents ?? []).filter( + const emptyIndexedDocuments = indexedDocuments.filter( (document) => document.status === "indexed" && ((document.page_count ?? 0) === 0 || (document.chunk_count ?? 0) === 0), ); @@ -408,25 +445,34 @@ async function main() { }); const requireCurrentEnrichmentVersion = strictEnrichmentVersionRequired(); - const missingEmbeddingResult = await supabase.from("document_chunks").select("id", { count: "exact", head: true }).is("embedding", null); - const failedJobsResult = await supabase.from("ingestion_jobs").select("id,documents!inner(status,chunk_count)").eq("status", "failed"); + const missingEmbeddingResult = await supabase + .from("document_chunks") + .select("id", { count: "exact", head: true }) + .is("embedding", null); + const failedJobsResult = await supabase + .from("ingestion_jobs") + .select("id,documents!inner(status,chunk_count)") + .eq("status", "failed"); const activeJobsResult = await supabase - .from("ingestion_jobs") - .select("id", { count: "exact", head: true }) - .in("status", ["pending", "processing"]); + .from("ingestion_jobs") + .select("id", { count: "exact", head: true }) + .in("status", ["pending", "processing"]); const imageCountResult = await supabase.from("document_images").select("id", { count: "exact", head: true }); - const searchableImageResult = await supabase.from("document_images").select("id", { count: "exact", head: true }).eq("searchable", true); + const searchableImageResult = await supabase + .from("document_images") + .select("id", { count: "exact", head: true }) + .eq("searchable", true); const oversizedBatchesResult = await supabase - .from("import_batches") - .select("id,name,total_files,status") - .gt("total_files", 150) - .in("status", ["queued", "processing"]); + .from("import_batches") + .select("id,name,total_files,status") + .gt("total_files", 150) + .in("status", ["queued", "processing"]); const cleanupIssuesResult = await supabase - .from("storage_cleanup_jobs") - .select("id,status,last_error") - .in("status", ["pending", "failed"]) - .order("created_at", { ascending: true }) - .limit(5); + .from("storage_cleanup_jobs") + .select("id,status,last_error") + .in("status", ["pending", "failed"]) + .order("created_at", { ascending: true }) + .limit(5); for (const result of [ missingEmbeddingResult, @@ -491,9 +537,6 @@ async function main() { issues.push(`indexed documents missing current deep-memory version: ${documentsMissingCurrentDeepMemoryVersion}`); } if (actionableFailedJobs.length > 0) issues.push(`actionable failed ingestion jobs: ${actionableFailedJobs.length}`); - if ((oversizedBatchesResult.data ?? []).length > 0) { - issues.push(`active oversized import batches: ${(oversizedBatchesResult.data ?? []).length}`); - } if ((cleanupIssuesResult.data ?? []).length > 0) { issues.push(`pending or failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); } @@ -511,7 +554,15 @@ async function main() { ); console.log(`Embedding model: ${env.OPENAI_EMBEDDING_MODEL}`); console.log(`Worker concurrency: ${env.WORKER_CONCURRENCY}`); - console.log(`Documents: ${(documents ?? []).length}; empty indexed: ${emptyIndexedDocuments.length}`); + console.log( + `Documents: ${documents.length}; completed indexed: ${indexedDocuments.length}; pending indexed: ${pendingIndexedDocuments.length}`, + ); + console.log( + `Indexed documents: completed=${indexedDocuments.length}; pending=${pendingIndexedDocuments.length}; empty=${emptyIndexedDocuments.length}`, + ); + if (pendingIndexedDocuments.length > 0) { + console.log(`Pending enrichment queue: ${pendingIndexedDocuments.length}`); + } console.log(`Chunk-count mismatches: ${documentsWithChunkCountMismatch.length}`); console.log( `Chunk fingerprint coverage: missing hashes=${chunksMissingContentHash.length}; missing generations=${chunksMissingIndexGeneration.length}; mixed-generation documents=${documentsWithStaleIndexGeneration.length}`, @@ -537,10 +588,12 @@ async function main() { `Failed jobs: ${(failedJobsResult.data ?? []).length}; actionable failed: ${actionableFailedJobs.length}; pending/processing jobs: ${activeJobsResult.count ?? 0}`, ); console.log(`Images: ${imageCountResult.count ?? 0}; searchable: ${searchableImageResult.count ?? 0}`); - console.log(`Active oversized batches: ${(oversizedBatchesResult.data ?? []).length}`); + console.log(`Active oversized batches: ${(oversizedBatchesResult.data ?? []).length} (ignored for readiness)`); console.log(`Pending/failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); if (issues.length > 0) { + const completionSplit = `indexed completion split: completed=${indexedDocuments.length}; pending=${pendingIndexedDocuments.length}`; + issues.push(completionSplit); throw new Error(`Indexing readiness check failed: ${issues.join("; ")}`); } } diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts index 400d5dde2..d53101cee 100644 --- a/scripts/enrich-documents.ts +++ b/scripts/enrich-documents.ts @@ -88,13 +88,9 @@ async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId? if (args.document) { const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(args.document); if (isUuid) { - query = query.or( - `id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`, - ); + query = query.or(`id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`); } else { - query = query.or( - `file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`, - ); + query = query.or(`file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`); } } @@ -153,7 +149,12 @@ async function loadRowsForDocuments(supabase: SupabaseAdmin, table: string, sele async function loadDeepMemoryCoverage(supabase: SupabaseAdmin, documentIds: string[]) { const sections = await loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds); - const memoryCards = await loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds); + const memoryCards = await loadRowsForDocuments( + supabase, + "document_memory_cards", + "document_id,metadata", + documentIds, + ); const coverage = new Map<string, { sections: MetadataRow[]; memoryCards: MetadataRow[] }>(); for (const documentId of documentIds) coverage.set(documentId, { sections: [], memoryCards: [] }); @@ -536,80 +537,120 @@ async function main() { for (const document of documents) { const documentCoverage = coverage.get(document.id); const taskPromise = (async () => { - try { - if (args.mode === "metadata-stamp") { - if (!documentCoverage?.summary || documentCoverage.labels.every((label) => label.source !== "generated")) { - console.log(`SKIP cannot metadata-stamp missing enrichment: ${document.file_name}`); + let attempts = 0; + const maxAttempts = 6; + let success = false; + + while (attempts < maxAttempts && !success) { + try { + if (args.mode === "metadata-stamp") { + if (!documentCoverage?.summary || documentCoverage.labels.every((label) => label.source !== "generated")) { + console.log(`SKIP cannot metadata-stamp missing enrichment: ${document.file_name}`); + return; + } + await stampExistingEnrichmentVersion({ + supabase, + document, + coverage: documentCoverage, + ragEnrichmentVersion, + }); + completed += 1; + console.log(`STAMPED ${document.file_name}`); return; } - await stampExistingEnrichmentVersion({ - supabase, - document, - coverage: documentCoverage, - ragEnrichmentVersion, - }); - completed += 1; - console.log(`STAMPED ${document.file_name}`); - return; - } - let imageMetadata = metadataRecord(document.metadata); - let imageStats = { searchable: 0, skipped: 0, total: 0 }; - if (args.mode === "summaries-labels-images") { - imageStats = await classifyExistingImages(supabase, document.id); - imageMetadata = { - ...imageMetadata, - searchable_image_count: imageStats.searchable, - skipped_image_count: imageStats.skipped, - image_enriched_at: new Date().toISOString(), - }; - await supabase - .from("documents") - .update({ - image_count: imageStats.searchable, - metadata: imageMetadata, - }) - .eq("id", document.id); - } + let imageMetadata = metadataRecord(document.metadata); + let imageStats = { searchable: 0, skipped: 0, total: 0 }; + if (args.mode === "summaries-labels-images") { + imageStats = await classifyExistingImages(supabase, document.id); + imageMetadata = { + ...imageMetadata, + searchable_image_count: imageStats.searchable, + skipped_image_count: imageStats.skipped, + image_enriched_at: new Date().toISOString(), + }; + await supabase + .from("documents") + .update({ + image_count: imageStats.searchable, + metadata: imageMetadata, + }) + .eq("id", document.id); + } - const evidence = await loadEvidence(supabase, document.id); - if (evidence.chunks.length === 0) { - console.log(`SKIP no chunks: ${document.file_name}`); - return; - } + const evidence = await loadEvidence(supabase, document.id); + if (evidence.chunks.length === 0) { + console.log(`SKIP no chunks: ${document.file_name}`); + return; + } - let enrichmentSummary: string | null = null; - if (args.mode === "summaries-labels-images") { - const enrichment = await upsertDocumentEnrichment({ + let enrichmentSummary: string | null = null; + if (args.mode === "summaries-labels-images") { + const enrichment = await upsertDocumentEnrichment({ + supabase, + document: { ...document, metadata: imageMetadata }, + chunks: evidence.chunks, + images: evidence.images, + }); + enrichmentSummary = enrichment.summary.summary; + } else if (args.mode === "deep-memory") { + const { data: sumData } = await supabase + .from("document_summaries") + .select("summary") + .eq("document_id", document.id) + .maybeSingle(); + if (sumData?.summary) { + enrichmentSummary = sumData.summary; + } + } + const deepMemory = await upsertDocumentDeepMemory({ supabase, document: { ...document, metadata: imageMetadata }, chunks: evidence.chunks, images: evidence.images, + summary: enrichmentSummary, }); - enrichmentSummary = enrichment.summary.summary; - } else if (args.mode === "deep-memory") { - const { data: sumData } = await supabase - .from("document_summaries") - .select("summary") - .eq("document_id", document.id) - .maybeSingle(); - if (sumData?.summary) { - enrichmentSummary = sumData.summary; + const { data: latestDoc } = await supabase + .from("documents") + .select("metadata") + .eq("id", document.id) + .single(); + const latestMetadata = metadataRecord(latestDoc?.metadata); + await supabase + .from("documents") + .update({ + metadata: { + ...latestMetadata, + enrichment_status: "completed", + enrichment_error: null, + }, + }) + .eq("id", document.id); + + completed += 1; + console.log( + `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} sections=${deepMemory.sections.length} memory=${deepMemory.memoryCards.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`, + ); + success = true; + + // Small delay to space out OpenAI calls + await new Promise((resolve) => setTimeout(resolve, 1500)); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const isRateLimit = /rate limit|rate_limit|429/i.test(errorMessage); + + if (isRateLimit && attempts < maxAttempts - 1) { + attempts += 1; + const delayMs = Math.pow(2, attempts) * 8000 + Math.random() * 3000; + console.warn( + `Rate limit hit on ${document.file_name}. Waiting ${Math.round(delayMs / 1000)}s before retry ${attempts}/${maxAttempts}...`, + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } else { + console.error(`ERROR enriching ${document.file_name}:`, errorMessage); + break; // Break the retry loop for non-rate-limit errors } } - const deepMemory = await upsertDocumentDeepMemory({ - supabase, - document: { ...document, metadata: imageMetadata }, - chunks: evidence.chunks, - images: evidence.images, - summary: enrichmentSummary, - }); - completed += 1; - console.log( - `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} sections=${deepMemory.sections.length} memory=${deepMemory.memoryCards.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`, - ); - } catch (error) { - console.error(`ERROR enriching ${document.file_name}:`, error instanceof Error ? error.message : error); } })(); diff --git a/scripts/ensure-local-server.mjs b/scripts/ensure-local-server.mjs index c5f1156bd..f2b210d82 100644 --- a/scripts/ensure-local-server.mjs +++ b/scripts/ensure-local-server.mjs @@ -44,7 +44,7 @@ async function isPortBusy(port) { return false; } -function requestJson(url, timeoutMs = 900) { +function requestJson(url, timeoutMs = 3500) { return new Promise((resolve) => { const request = http.get(url, { timeout: timeoutMs }, (response) => { let body = ""; @@ -69,8 +69,12 @@ function requestJson(url, timeoutMs = 900) { } async function isThisProject(port) { - const payload = await requestJson(`http://localhost:${port}${identityPath}`); - return payload?.appName === appName && payload?.projectId === localProjectId(projectRoot); + for (let attempt = 0; attempt < 3; attempt += 1) { + const payload = await requestJson(`http://localhost:${port}${identityPath}`); + if (payload?.appName === appName && payload?.projectId === localProjectId(projectRoot)) return true; + if (attempt < 2) await sleep(250); + } + return false; } async function findExistingProjectServer(startPort) { diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index ebccfc8ea..724f63af8 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -326,9 +326,7 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[] content_recall_at_5: Number( (results.reduce((sum, result) => sum + result.contentRecallAt5, 0) / contentRecallDenominator).toFixed(4), ), - top_k_hit_rate: Number( - (results.filter((result) => result.hitAtK).length / Math.max(results.length, 1)).toFixed(4), - ), + top_k_hit_rate: Number((results.filter((result) => result.hitAtK).length / Math.max(results.length, 1)).toFixed(4)), mrr_at_10: Number( (results.reduce((sum, result) => sum + result.reciprocalRankAt10, 0) / Math.max(results.length, 1)).toFixed(4), ), diff --git a/scripts/eval-search.ts b/scripts/eval-search.ts index 486172128..c865131be 100644 --- a/scripts/eval-search.ts +++ b/scripts/eval-search.ts @@ -211,14 +211,43 @@ async function main() { if (!args.json) console.log(`Running ${cases.length} search eval case(s), scope=${scope}.`); for (const testCase of cases) { - const startedAt = Date.now(); - const search = await searchChunksWithTelemetry({ - query: testCase.question, - ownerId, - topK: 12, - minSimilarity: 0.12, - skipCache: true, - }); + let attempts = 0; + const maxAttempts = 6; + let search; + let startedAt = Date.now(); + + while (attempts < maxAttempts) { + try { + startedAt = Date.now(); + search = await searchChunksWithTelemetry({ + query: testCase.question, + ownerId, + topK: 12, + minSimilarity: 0.12, + skipCache: true, + }); + break; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const isRateLimit = /rate limit|rate_limit|429/i.test(errorMessage); + attempts += 1; + if (isRateLimit && attempts < maxAttempts) { + const delayMs = Math.pow(2, attempts) * 8000 + Math.random() * 3000; + console.warn( + `Rate limit hit on query: "${testCase.question}". Waiting ${Math.round(delayMs / 1000)}s before retry ${attempts}/${maxAttempts}...`, + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } else { + console.error(`ERROR running search eval for "${testCase.question}":`, errorMessage); + throw error; + } + } + } + + if (!search) { + throw new Error(`Failed to perform search for question: ${testCase.question}`); + } + const latencyMs = search.telemetry.supabase_rpc_latency_ms + search.telemetry.embedding_latency_ms + @@ -264,6 +293,9 @@ async function main() { console.log(` Q: ${testCase.question}`); console.log(` Top files: ${result.topFiles.join("; ") || "none"} payloadBytes=${result.payloadBytes}`); } + + // Small delay to space out OpenAI calls + await new Promise((resolve) => setTimeout(resolve, 1200)); } const thresholdFailures = summarizeFailures(results); diff --git a/scripts/import-documents.ts b/scripts/import-documents.ts index 899af4ad1..bd6d4c77b 100644 --- a/scripts/import-documents.ts +++ b/scripts/import-documents.ts @@ -163,9 +163,7 @@ async function main() { const exactCopyCount = files.filter((file) => existing.has(file.contentHash)).length; const dryRunBatches = chunkImportFiles(files, args.queueBatchSize); - console.log( - `DRY RUN queue plan: ${dryRunBatches.length} batch(es) of up to ${args.queueBatchSize} file(s).`, - ); + console.log(`DRY RUN queue plan: ${dryRunBatches.length} batch(es) of up to ${args.queueBatchSize} file(s).`); for (const file of files.slice(0, args.queueBatchSize)) { const duplicate = existing.get(file.contentHash); console.log( @@ -255,9 +253,7 @@ async function main() { let failed = 0; const importBatches = chunkImportFiles(files, args.queueBatchSize); - console.log( - `Queueing ${files.length} file(s) in ${importBatches.length} batch(es) of up to ${args.queueBatchSize}.`, - ); + console.log(`Queueing ${files.length} file(s) in ${importBatches.length} batch(es) of up to ${args.queueBatchSize}.`); for (const [batchIndex, fileBatch] of importBatches.entries()) { const start = batchIndex * args.queueBatchSize + 1; diff --git a/scripts/playwright-base-url.ts b/scripts/playwright-base-url.ts index 56e4d1913..53410c111 100644 --- a/scripts/playwright-base-url.ts +++ b/scripts/playwright-base-url.ts @@ -8,7 +8,7 @@ const localUrlPattern = /^http:\/\/localhost:\d+$/; const identityScript = ` const http = require("node:http"); const url = process.argv[1] + "/api/local-project-id"; -const request = http.get(url, { timeout: 1500 }, (response) => { +const request = http.get(url, { timeout: 5000 }, (response) => { let body = ""; response.setEncoding("utf8"); response.on("data", (chunk) => { body += chunk; }); diff --git a/scripts/production-readiness.ts b/scripts/production-readiness.ts new file mode 100644 index 000000000..142146390 --- /dev/null +++ b/scripts/production-readiness.ts @@ -0,0 +1,205 @@ +import { access, readFile } from "node:fs/promises"; +import { constants } from "node:fs"; +import path from "node:path"; +import { loadEnvConfig } from "@next/env"; + +import { checkSupabaseProjectConfig } from "@/lib/supabase/project"; + +loadEnvConfig(process.cwd()); + +const isCiMode = process.argv.includes("--ci"); + +type Result = { + failures: string[]; + warnings: string[]; + passes: string[]; +}; + +function isMissingEnvError(message: string) { + return message.startsWith("Missing server environment variables") || message.startsWith("Missing OPENAI_API_KEY."); +} + +function recordIssue(message: string, options: { downgradeToWarningInCi?: boolean } = {}) { + if (isCiMode && options.downgradeToWarningInCi) { + result.warnings.push(`${message} (CI)`); + return; + } + result.failures.push(message); +} + +const result: Result = { + failures: [], + warnings: [], + passes: [], +}; + +function placeholderLooksLikeExample(value: string) { + return /replace-with|your-|example|-example-|\{\w+\}|xxxx|todo|placeholder/i.test(value); +} + +async function checkRequiredFile(filePath: string, message: string) { + try { + await access(filePath, constants.F_OK); + return true; + } catch { + result.failures.push(message); + return false; + } +} + +async function checkOptionalFile(filePath: string, message: string) { + try { + await access(filePath, constants.F_OK); + result.passes.push(message); + return true; + } catch { + result.warnings.push(`${message} (missing)`); + return false; + } +} + +function checkNodeRuntime() { + const major = Number(process.versions.node.split(".")[0]); + if (major >= 22) { + result.passes.push(`Node runtime is ${process.versions.node}.`); + return; + } + result.failures.push(`Node ${process.versions.node} is too old. This project targets Node 22.x.`); +} + +function recordNoAuthProductionCheck() { + if ( + (process.env.NODE_ENV === "production" || process.env.VERCEL_ENV === "production") && + (process.env.NEXT_PUBLIC_LOCAL_NO_AUTH === "true" || process.env.LOCAL_NO_AUTH === "true") + ) { + result.failures.push("Local no-auth mode is enabled in production-like environment variables."); + } +} + +async function checkFileForServiceRoleExposure() { + const envFiles = [".env", ".env.local", ".env.production", ".env.development"]; + for (const fileName of envFiles) { + const filePath = path.join(process.cwd(), fileName); + try { + const content = await readFile(filePath, "utf8"); + const hasPlainServiceRole = /NEXT_PUBLIC_SERVICE_ROLE_KEY|SUPABASE_SERVICE_ROLE_KEY/.test(content); + if (!hasPlainServiceRole) { + continue; + } + result.warnings.push( + `${fileName} contains a service-role key marker. Keep these files out of source control and verify only server-side usage.`, + ); + } catch { + // file is optional in this repo shape + } + } +} + +async function main() { + checkNodeRuntime(); + recordNoAuthProductionCheck(); + await checkFileForServiceRoleExposure(); + + if (!(await checkRequiredFile(path.join(process.cwd(), "package-lock.json"), "package-lock.json is required"))) { + // keep going so we can show all diagnostics + } + await checkRequiredFile( + path.join(process.cwd(), ".env.example"), + ".env.example is required for documented environment contract.", + ); + + await checkOptionalFile(path.join(process.cwd(), ".env.local"), "Local override file .env.local is present"); + await checkOptionalFile(path.join(process.cwd(), ".env"), "Top-level .env exists"); + + let envModule: typeof import("@/lib/env") | null = null; + try { + envModule = await import("@/lib/env"); + } catch (error) { + result.failures.push( + `Environment schema validation failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + if (envModule) { + try { + envModule.requireServerEnv(); + result.passes.push("Server env includes required Supabase project values."); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (isMissingEnvError(message)) { + recordIssue(`Missing server env config: ${message}`, { downgradeToWarningInCi: true }); + } else { + result.failures.push(`Missing server env config: ${message}`); + } + } + + try { + envModule.requireOpenAIEnv(); + result.passes.push("OpenAI API key is configured."); + if (placeholderLooksLikeExample(envModule.env.OPENAI_API_KEY ?? "")) { + result.failures.push("OPENAI_API_KEY still looks like a placeholder."); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (isMissingEnvError(message)) { + recordIssue(`OpenAI configuration issue: ${message}`, { downgradeToWarningInCi: true }); + } else { + result.failures.push(`OpenAI configuration issue: ${message}`); + } + } + + if (placeholderLooksLikeExample(envModule.env.NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY ?? "")) { + result.warnings.push("NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY looks like a placeholder."); + } + if (placeholderLooksLikeExample(envModule.env.SUPABASE_SERVICE_ROLE_KEY ?? "")) { + result.failures.push("SUPABASE_SERVICE_ROLE_KEY looks like a placeholder."); + } + } + + const supabaseCheck = checkSupabaseProjectConfig({ + NEXT_PUBLIC_SUPABASE_URL: process.env.NEXT_PUBLIC_SUPABASE_URL, + SUPABASE_PROJECT_REF: process.env.SUPABASE_PROJECT_REF, + SUPABASE_PROJECT_NAME: process.env.SUPABASE_PROJECT_NAME, + }); + if (supabaseCheck.status === "ready") { + result.passes.push("Supabase project config points to Clinical KB Database."); + } else if (supabaseCheck.status === "warning") { + if (supabaseCheck.warnings.length) { + result.warnings.push(...supabaseCheck.warnings); + } + result.passes.push("Supabase URL is correct."); + } else if (supabaseCheck.status === "missing" && isCiMode) { + result.warnings.push("NEXT_PUBLIC_SUPABASE_URL is not set in this environment (CI)."); + } else { + result.failures.push(...supabaseCheck.problems); + } + + console.log("[Production Readiness]"); + console.log(`Project: ${supabaseCheck.expected.name} (${supabaseCheck.expected.ref})`); + if (supabaseCheck.observed.configuredName) { + console.log(`Configured name: ${supabaseCheck.observed.configuredName}`); + } + console.log(`Configured ref: ${supabaseCheck.observed.configuredRef ?? "not set"}`); + console.log(""); + + if (result.passes.length > 0) { + console.log(`PASS (${result.passes.length}):`); + for (const item of result.passes) console.log(` - ${item}`); + } + if (result.warnings.length > 0) { + console.log(`WARN (${result.warnings.length}):`); + for (const item of result.warnings) console.log(` - ${item}`); + } + if (result.failures.length > 0) { + console.log(`FAIL (${result.failures.length}):`); + for (const item of result.failures) console.log(` - ${item}`); + process.exitCode = 1; + } else { + console.log("READY: no blocking production-readiness failures."); + } +} + +main().catch((error) => { + result.failures.push(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/reindex-health.ts b/scripts/reindex-health.ts index 64c6f2367..316282663 100644 --- a/scripts/reindex-health.ts +++ b/scripts/reindex-health.ts @@ -60,19 +60,34 @@ async function main() { failedJobs, chunksWithSynopsis, ] = await Promise.all([ - safeCount("documents_indexed", supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "indexed")), - safeCount("documents_queued", supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "queued")), + safeCount( + "documents_indexed", + supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "indexed"), + ), + safeCount( + "documents_queued", + supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "queued"), + ), safeCount( "documents_processing", supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "processing"), ), - safeCount("documents_failed", supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "failed")), - safeCount("jobs_pending", supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "pending")), + safeCount( + "documents_failed", + supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "failed"), + ), + safeCount( + "jobs_pending", + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "pending"), + ), safeCount( "jobs_processing", supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "processing"), ), - safeCount("jobs_failed", supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed")), + safeCount( + "jobs_failed", + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed"), + ), safeCount( "chunks_with_retrieval_synopsis", supabase diff --git a/scripts/supabase-recovery-status.ts b/scripts/supabase-recovery-status.ts index 77b5c1e7e..34bf03a03 100644 --- a/scripts/supabase-recovery-status.ts +++ b/scripts/supabase-recovery-status.ts @@ -47,12 +47,18 @@ async function main() { const staleCutoff = new Date(Date.now() - env.WORKER_STALE_AFTER_MINUTES * 60_000).toISOString(); const [pendingJobs, processingJobs, failedJobs, staleJobs, queuedDocuments, failedDocuments] = await Promise.all([ - safeCount("jobs_pending", supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "pending")), + safeCount( + "jobs_pending", + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "pending"), + ), safeCount( "jobs_processing", supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "processing"), ), - safeCount("jobs_failed", supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed")), + safeCount( + "jobs_failed", + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed"), + ), safeCount( "jobs_stale_processing", supabase @@ -61,8 +67,14 @@ async function main() { .eq("status", "processing") .lt("locked_at", staleCutoff), ), - safeCount("documents_queued", supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "queued")), - safeCount("documents_failed", supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "failed")), + safeCount( + "documents_queued", + supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "queued"), + ), + safeCount( + "documents_failed", + supabase.from("documents").select("id", { count: "exact", head: true }).eq("status", "failed"), + ), ]); const counts = [pendingJobs, processingJobs, failedJobs, staleJobs, queuedDocuments, failedDocuments]; diff --git a/src/app/api/ingestion/jobs/[id]/retry/route.ts b/src/app/api/ingestion/jobs/[id]/retry/route.ts index 144eccd5b..5510fd1b2 100644 --- a/src/app/api/ingestion/jobs/[id]/retry/route.ts +++ b/src/app/api/ingestion/jobs/[id]/retry/route.ts @@ -59,8 +59,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: if (!data) { return NextResponse.json( { - error: - "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.", + error: "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.", }, { status: 409 }, ); diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 00128e2cf..710932770 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -21,6 +21,20 @@ import type { ChunkImage, ClinicalSourceMetadata, SearchResult } from "@/lib/typ export const runtime = "nodejs"; +type RetrievalLogWriteMetrics = { + attempts: number; + failures: number; + lastFailureAt: string | null; + lastFailureMessage: string | null; +}; + +const retrievalLogWriteMetrics: RetrievalLogWriteMetrics = { + attempts: 0, + failures: 0, + lastFailureAt: null, + lastFailureMessage: null, +}; + const searchSchema = z.object({ query: z.string().trim().min(1), topK: z.number().int().min(1).max(20).optional(), @@ -388,6 +402,88 @@ function latencyBucket(ms: number) { return "gte_5000ms"; } +function logRetrievalDiagnostics(args: { + supabase: ReturnType<typeof createAdminClient>; + ownerId: string; + query: string; + results: SearchResult[]; + telemetry: Record<string, unknown>; + relevance: { verdict: string; score: number }; +}) { + void (async () => { + retrievalLogWriteMetrics.attempts += 1; + try { + const topResult = args.results[0] ?? null; + const topScore = topResult ? (topResult.hybrid_score ?? topResult.similarity ?? 0) : 0; + const scores = args.results.slice(0, 20).map((r) => r.hybrid_score ?? r.similarity ?? 0); + const meanScore = scores.length ? scores.reduce((a, b) => a + b, 0) / scores.length : 0; + const isMiss = + args.results.length === 0 || + args.relevance.verdict === "none" || + args.relevance.verdict === "nearby" || + topScore < 0.48; + const latencyMs = telemetryLatencyMs(args.telemetry); + + await args.supabase.from("rag_retrieval_logs").insert({ + owner_id: args.ownerId, + query: queryTextForStorage(args.query), + normalized_query: normalizeQueryText(args.query), + query_class: (args.telemetry.query_class as string) ?? null, + retrieval_strategy: (args.telemetry.retrieval_strategy as string) ?? null, + candidate_count: args.results.length, + top_similarity: topResult?.similarity ?? null, + top_text_rank: topResult?.text_rank ?? null, + top_hybrid_score: Number.isFinite(topScore) ? topScore : null, + top_rrf_score: topResult?.rrf_score ?? null, + mean_hybrid_score: meanScore || null, + selected_chunk_ids: args.results + .slice(0, 12) + .map((r) => r.id) + .filter((id) => /^[0-9a-f-]{36}$/i.test(id)), + selected_document_ids: Array.from(new Set(args.results.slice(0, 12).map((r) => r.document_id))).filter((id) => + /^[0-9a-f-]{36}$/i.test(id), + ), + selected_count: Math.min(args.results.length, 12), + embedding_latency_ms: + typeof args.telemetry.embedding_latency_ms === "number" ? args.telemetry.embedding_latency_ms : null, + vector_candidate_count: + typeof args.telemetry.vector_candidate_count === "number" ? args.telemetry.vector_candidate_count : null, + text_candidate_count: + typeof args.telemetry.text_candidate_count === "number" ? args.telemetry.text_candidate_count : null, + embedding_field_count: + typeof args.telemetry.embedding_field_count === "number" ? args.telemetry.embedding_field_count : null, + rpc_latency_ms: + typeof args.telemetry.supabase_rpc_latency_ms === "number" ? args.telemetry.supabase_rpc_latency_ms : null, + rerank_latency_ms: + typeof args.telemetry.rerank_latency_ms === "number" ? args.telemetry.rerank_latency_ms : null, + total_latency_ms: latencyMs || null, + memory_card_count: + typeof args.telemetry.memory_card_count === "number" ? args.telemetry.memory_card_count : null, + index_unit_count: typeof args.telemetry.index_unit_count === "number" ? args.telemetry.index_unit_count : null, + is_miss: isMiss, + miss_reason: isMiss ? (args.results.length === 0 ? "no_results" : `weak_${args.relevance.verdict}`) : null, + embedding_cache_hit: + typeof args.telemetry.embedding_cache_hit === "boolean" ? args.telemetry.embedding_cache_hit : null, + metadata: { + relevance_score: args.relevance.score, + latency_bucket: latencyBucket(latencyMs), + }, + }); + } catch (error) { + retrievalLogWriteMetrics.failures += 1; + retrievalLogWriteMetrics.lastFailureAt = new Date().toISOString(); + retrievalLogWriteMetrics.lastFailureMessage = + error instanceof Error ? `${error.name}: ${error.message}` : "Unknown retrieval logging error"; + if (retrievalLogWriteMetrics.failures <= 3 || retrievalLogWriteMetrics.failures % 25 === 0) { + console.warn("rag_retrieval_logs insert failed", { + ...retrievalLogWriteMetrics, + ownerId: args.ownerId, + }); + } + } + })(); +} + function logSearchObservation(args: { supabase: ReturnType<typeof createAdminClient>; ownerId: string; @@ -577,6 +673,7 @@ async function buildScopedSearchPayload( rrf_top_score: search.telemetry.rrf_top_score, }, }; + logRetrievalDiagnostics({ supabase, ownerId, query: body.query, results, telemetry: search.telemetry, relevance }); logSearchObservation({ supabase, ownerId, query: body.query, results, payload }); return payload; } diff --git a/src/app/page.tsx b/src/app/page.tsx index 40e1d905f..bdb327a09 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,4 +1,5 @@ -import { ClinicalDashboard } from "@/components/ClinicalDashboard"; +import { ClinicalDashboard } from "@/components/clinical-dashboard"; + export default function Home() { return <ClinicalDashboard />; } diff --git a/src/app/tools/page.tsx b/src/app/tools/page.tsx index 31b80f04b..7e6d835f6 100644 --- a/src/app/tools/page.tsx +++ b/src/app/tools/page.tsx @@ -6,6 +6,7 @@ import { Brain, CheckCircle2, CircleDashed, + ClipboardCheck, ClipboardList, ExternalLink, FileImage, @@ -21,26 +22,29 @@ import { Sparkles, Target, UploadCloud, - ClipboardCheck, type LucideIcon, } from "lucide-react"; import { cn } from "@/components/ui-primitives"; -import { type ToolCategory, type ToolIconName, type ToolItem, toolCatalog } from "@/lib/tools"; +import { defaultFavoriteToolIds, type ToolCategory, type ToolIconName, type ToolItem, toolCatalog } from "@/lib/tools"; -type CardTheme = { - accent: string; +type ToneName = "primary" | "info" | "success" | "warning" | "danger"; + +type ToneTheme = { + aura: string; border: string; button: string; glow: string; icon: string; - pill: string; rail: string; surface: string; + text: string; }; -type ClinicalSignal = { - label: string; - value: string; +type ToolPresentation = { + cadence: string; + role: string; + shortLabel: string; + tone: ToneName; }; const iconRegistry = { @@ -64,210 +68,251 @@ const iconRegistry = { Clipboard: ClipboardCheck, } satisfies Record<ToolIconName, LucideIcon>; -const clinicalSignals: ClinicalSignal[] = [ - { label: "Project suite", value: "9 local clinical apps" }, - { label: "Clinical workflow", value: "Assessment to handover" }, - { label: "Fast path", value: "One click to each workspace" }, -]; +const toneTheme: Record<ToneName, ToneTheme> = { + primary: { + aura: "bg-[radial-gradient(circle_at_18%_0%,color-mix(in_srgb,var(--primary)_20%,transparent),transparent_18rem)]", + border: "border-[color:color-mix(in_srgb,var(--primary)_28%,transparent)]", + button: "bg-[color:var(--primary)] text-[color:var(--primary-contrast)] hover:bg-[color:var(--primary-strong)]", + glow: "shadow-[0_18px_52px_color-mix(in_srgb,var(--primary)_14%,transparent)]", + icon: "border-[color:color-mix(in_srgb,var(--primary)_34%,transparent)] bg-[color:color-mix(in_srgb,var(--primary)_14%,transparent)] text-[color:var(--primary-100)]", + rail: "bg-[color:var(--primary)]", + surface: + "bg-[linear-gradient(145deg,color-mix(in_srgb,var(--app-shell-muted)_82%,transparent),color-mix(in_srgb,var(--app-shell)_96%,black))]", + text: "text-[color:var(--primary-100)]", + }, + info: { + aura: "bg-[radial-gradient(circle_at_18%_0%,color-mix(in_srgb,var(--info)_20%,transparent),transparent_18rem)]", + border: "border-[color:color-mix(in_srgb,var(--info)_30%,transparent)]", + button: "bg-[color:var(--info-soft)] text-[color:var(--app-shell)] hover:bg-white", + glow: "shadow-[0_18px_52px_color-mix(in_srgb,var(--info)_14%,transparent)]", + icon: "border-[color:color-mix(in_srgb,var(--info)_34%,transparent)] bg-[color:color-mix(in_srgb,var(--info)_14%,transparent)] text-[color:var(--info-bg)]", + rail: "bg-[color:var(--info)]", + surface: + "bg-[linear-gradient(145deg,color-mix(in_srgb,var(--info)_16%,var(--app-shell-muted)),color-mix(in_srgb,var(--app-shell)_96%,black))]", + text: "text-[color:var(--info-bg)]", + }, + success: { + aura: "bg-[radial-gradient(circle_at_18%_0%,color-mix(in_srgb,var(--success)_20%,transparent),transparent_18rem)]", + border: "border-[color:color-mix(in_srgb,var(--success)_30%,transparent)]", + button: "bg-[color:var(--success-soft)] text-[color:var(--app-shell)] hover:bg-white", + glow: "shadow-[0_18px_52px_color-mix(in_srgb,var(--success)_14%,transparent)]", + icon: "border-[color:color-mix(in_srgb,var(--success)_34%,transparent)] bg-[color:color-mix(in_srgb,var(--success)_14%,transparent)] text-[color:var(--success-bg)]", + rail: "bg-[color:var(--success)]", + surface: + "bg-[linear-gradient(145deg,color-mix(in_srgb,var(--success)_16%,var(--app-shell-muted)),color-mix(in_srgb,var(--app-shell)_96%,black))]", + text: "text-[color:var(--success-bg)]", + }, + warning: { + aura: "bg-[radial-gradient(circle_at_18%_0%,color-mix(in_srgb,var(--warning)_20%,transparent),transparent_18rem)]", + border: "border-[color:color-mix(in_srgb,var(--warning)_30%,transparent)]", + button: "bg-[color:var(--warning-soft)] text-[color:var(--app-shell)] hover:bg-white", + glow: "shadow-[0_18px_52px_color-mix(in_srgb,var(--warning)_14%,transparent)]", + icon: "border-[color:color-mix(in_srgb,var(--warning)_34%,transparent)] bg-[color:color-mix(in_srgb,var(--warning)_14%,transparent)] text-[color:var(--warning-bg)]", + rail: "bg-[color:var(--warning)]", + surface: + "bg-[linear-gradient(145deg,color-mix(in_srgb,var(--warning)_16%,var(--app-shell-muted)),color-mix(in_srgb,var(--app-shell)_96%,black))]", + text: "text-[color:var(--warning-bg)]", + }, + danger: { + aura: "bg-[radial-gradient(circle_at_18%_0%,color-mix(in_srgb,var(--danger)_20%,transparent),transparent_18rem)]", + border: "border-[color:color-mix(in_srgb,var(--danger)_30%,transparent)]", + button: "bg-[color:var(--danger-soft)] text-[color:var(--app-shell)] hover:bg-white", + glow: "shadow-[0_18px_52px_color-mix(in_srgb,var(--danger)_14%,transparent)]", + icon: "border-[color:color-mix(in_srgb,var(--danger)_34%,transparent)] bg-[color:color-mix(in_srgb,var(--danger)_14%,transparent)] text-[color:var(--danger-bg)]", + rail: "bg-[color:var(--danger)]", + surface: + "bg-[linear-gradient(145deg,color-mix(in_srgb,var(--danger)_16%,var(--app-shell-muted)),color-mix(in_srgb,var(--app-shell)_96%,black))]", + text: "text-[color:var(--danger-bg)]", + }, +}; + +const categoryTone: Record<ToolCategory, ToneName> = { + Admin: "danger", + Clinical: "primary", + Docs: "info", + Operations: "warning", + Research: "success", +}; -const categoryTheme: Record<ToolCategory, CardTheme> = { - Clinical: { - accent: "text-cyan-100", - border: "border-cyan-200/24", - button: "bg-cyan-200 text-slate-950 hover:bg-white", - glow: "shadow-[0_20px_70px_rgb(34_211_238_/_11%)]", - icon: "border-cyan-200/30 bg-cyan-200/14 text-cyan-100", - pill: "border-cyan-100/20 bg-cyan-100/[0.08] text-cyan-100", - rail: "from-cyan-200 via-teal-200 to-emerald-200", - surface: "from-cyan-950/72 via-slate-950 to-slate-950", +const toolPresentation: Record<string, ToolPresentation> = { + differentials: { + cadence: "Rule-outs", + role: "Check likely rule-outs, red flags, and competing DSM-5 explanations.", + shortLabel: "Diffs", + tone: "success", + }, + "dsm-5-diagnoses": { + cadence: "DSM-5 criteria", + role: "Open criteria, symptom clusters, and diagnostic anchors.", + shortLabel: "DSM", + tone: "success", + }, + forms: { + cadence: "Capture", + role: "Start structured intake, review, and patient-facing form workflows.", + shortLabel: "Forms", + tone: "warning", + }, + formulation: { + cadence: "Case theory", + role: "Build formulation from problems, risks, maintaining factors, and treatment direction.", + shortLabel: "Form", + tone: "primary", + }, + medications: { + cadence: "Prescribing", + role: "Check prescribing context, monitoring, safety issues, and medication review.", + shortLabel: "Meds", + tone: "primary", }, - Operations: { - accent: "text-amber-100", - border: "border-amber-200/24", - button: "bg-amber-200 text-slate-950 hover:bg-white", - glow: "shadow-[0_20px_70px_rgb(251_191_36_/_11%)]", - icon: "border-amber-200/30 bg-amber-200/14 text-amber-100", - pill: "border-amber-100/20 bg-amber-100/[0.08] text-amber-100", - rail: "from-amber-200 via-orange-200 to-rose-200", - surface: "from-amber-950/62 via-slate-950 to-slate-950", + "psychiatry-notes": { + cadence: "Output", + role: "Open summaries, documentation flows, and review-ready note outputs.", + shortLabel: "Notes", + tone: "danger", }, - Docs: { - accent: "text-violet-100", - border: "border-violet-200/24", - button: "bg-violet-200 text-slate-950 hover:bg-white", - glow: "shadow-[0_20px_70px_rgb(196_181_253_/_11%)]", - icon: "border-violet-200/30 bg-violet-200/14 text-violet-100", - pill: "border-violet-100/20 bg-violet-100/[0.08] text-violet-100", - rail: "from-violet-200 via-fuchsia-200 to-cyan-100", - surface: "from-violet-950/66 via-slate-950 to-slate-950", + services: { + cadence: "Pathways", + role: "Find referral pathways, access points, and service-matching options.", + shortLabel: "Svc", + tone: "warning", }, - Research: { - accent: "text-emerald-100", - border: "border-emerald-200/24", - button: "bg-emerald-200 text-slate-950 hover:bg-white", - glow: "shadow-[0_20px_70px_rgb(110_231_183_/_11%)]", - icon: "border-emerald-200/30 bg-emerald-200/14 text-emerald-100", - pill: "border-emerald-100/20 bg-emerald-100/[0.08] text-emerald-100", - rail: "from-emerald-200 via-lime-100 to-cyan-100", - surface: "from-emerald-950/62 via-slate-950 to-slate-950", + specifiers: { + cadence: "Qualifiers", + role: "Review severity, course, and specifier language for a diagnosis.", + shortLabel: "Spec", + tone: "info", }, - Admin: { - accent: "text-fuchsia-100", - border: "border-fuchsia-200/24", - button: "bg-fuchsia-200 text-slate-950 hover:bg-white", - glow: "shadow-[0_20px_70px_rgb(244_114_182_/_11%)]", - icon: "border-fuchsia-200/30 bg-fuchsia-200/14 text-fuchsia-100", - pill: "border-fuchsia-100/20 bg-fuchsia-100/[0.08] text-fuchsia-100", - rail: "from-fuchsia-200 via-rose-200 to-amber-100", - surface: "from-fuchsia-950/62 via-slate-950 to-slate-950", + therapy: { + cadence: "Treatment", + role: "Open treatment planning, session structure, and intervention options.", + shortLabel: "Tx", + tone: "primary", }, }; +const favoriteTools = defaultFavoriteToolIds + .map((id) => toolCatalog.find((tool) => tool.id === id)) + .filter((tool): tool is ToolItem => Boolean(tool)); + function isInactive(tool: ToolItem) { return tool.status === "offline" || tool.status === "coming-soon"; } -function getClinicalTrack(tool: ToolItem) { - const trackById: Partial<Record<ToolItem["id"], string>> = { - differentials: "Differential", - "dsm-5-diagnoses": "Diagnosis", - forms: "Capture", - formulation: "Case theory", - medications: "Prescribing", - "psychiatry-notes": "Notes", - services: "Pathways", - specifiers: "Qualifiers", - therapy: "Treatment", - }; - - return trackById[tool.id] ?? "Clinical"; -} - -function getShortProjectLabel(tool: ToolItem) { - const shortLabelById: Partial<Record<ToolItem["id"], string>> = { - differentials: "Diffs", - "dsm-5-diagnoses": "DSM", - forms: "Forms", - formulation: "Form", - medications: "Meds", - "psychiatry-notes": "Notes", - services: "Svc", - specifiers: "Specs", - therapy: "Tx", - }; - - return shortLabelById[tool.id] ?? tool.title; +function getPresentation(tool: ToolItem) { + return ( + toolPresentation[tool.id] ?? { + cadence: tool.category, + role: tool.description, + shortLabel: tool.title, + tone: categoryTone[tool.category], + } + ); } function getLaunchContext(tool: ToolItem) { try { const url = new URL(tool.href); - const port = url.port ? `:${url.port}` : ""; - return `Local ${port}`; + return `${url.hostname}${url.port ? `:${url.port}` : ""}`; } catch { - return tool.target === "external" ? "External" : "Internal"; + return tool.target === "external" ? "External app" : "Internal route"; } } function getStatusLabel(tool: ToolItem) { - if (tool.status === "coming-soon") { - return "Soon"; - } - - if (tool.status === "offline") { - return "Paused"; - } - - if (tool.status === "beta") { - return "Preview"; - } - - return "Local"; + if (tool.status === "coming-soon") return "Soon"; + if (tool.status === "offline") return "Paused"; + if (tool.status === "beta") return "Preview"; + return "Live"; } function LaunchCard({ tool }: { tool: ToolItem }) { const Icon = iconRegistry[tool.icon]; const disabled = isInactive(tool); + const presentation = getPresentation(tool); + const theme = toneTheme[presentation.tone]; const target = tool.target === "external" && tool.openInNewTab ? "_blank" : undefined; const rel = tool.target === "external" ? "noopener noreferrer" : undefined; - const theme = categoryTheme[tool.category]; - const statusLabel = getStatusLabel(tool); - const track = getClinicalTrack(tool); - const launchContext = getLaunchContext(tool); return ( <article className={cn( - "group relative isolate overflow-hidden rounded-md border bg-gradient-to-br p-4 text-white transition duration-300 hover:-translate-y-1", + "group relative isolate overflow-hidden rounded-xl border p-4 text-[color:var(--primary-contrast)] shadow-[var(--shadow-tight)] motion-safe:transition motion-safe:duration-200 motion-safe:ease-[var(--ease-out-soft)] motion-safe:hover:-translate-y-1 motion-safe:hover:shadow-[var(--shadow-hover)] dark:text-[color:var(--text-heading)]", + "bg-[linear-gradient(145deg,color-mix(in_srgb,var(--app-shell-muted)_76%,transparent),color-mix(in_srgb,var(--app-shell)_96%,black))]", theme.border, - theme.glow, - theme.surface, - disabled && "opacity-70", + disabled && "opacity-60", )} > - <div className={cn("absolute inset-x-0 top-0 h-1 bg-gradient-to-r", theme.rail)} /> - <div className="pointer-events-none absolute inset-0 opacity-[0.11] [background-image:linear-gradient(rgba(255,255,255,.16)_1px,transparent_1px),linear-gradient(90deg,rgba(255,255,255,.12)_1px,transparent_1px)] [background-size:34px_34px]" /> - <div className="pointer-events-none absolute inset-0 bg-[linear-gradient(145deg,rgba(255,255,255,.1),transparent_34%,rgba(255,255,255,.04))]" /> - <div className="pointer-events-none absolute -right-10 -top-10 h-28 w-28 rounded-full bg-white/[0.04] blur-2xl transition duration-300 group-hover:bg-white/[0.08]" /> - - <div className="relative z-10 flex min-h-[16rem] flex-col"> - <div className="flex items-start justify-between gap-4"> - <span className={cn("grid h-12 w-12 shrink-0 place-items-center rounded-md border", theme.icon)}> - <Icon className="h-6 w-6" /> + <div className={cn("pointer-events-none absolute inset-0 opacity-45", theme.aura)} aria-hidden /> + <div + className="pointer-events-none absolute inset-0 opacity-[0.045] [background-image:linear-gradient(color-mix(in_srgb,white_22%,transparent)_1px,transparent_1px),linear-gradient(90deg,color-mix(in_srgb,white_18%,transparent)_1px,transparent_1px)] [background-size:32px_32px]" + aria-hidden + /> + <div className={cn("absolute inset-x-0 top-0 h-0.5 opacity-70", theme.rail)} aria-hidden /> + + <div className="relative z-10 flex min-h-52 flex-col"> + <div className="flex items-start justify-between gap-3"> + <span className={cn("grid h-11 w-11 shrink-0 place-items-center rounded-lg border", theme.icon)}> + <Icon className="h-5 w-5" aria-hidden /> </span> <span className={cn( - "inline-flex min-h-8 items-center gap-1.5 rounded-md border px-2.5 text-xs font-black", - disabled ? "border-white/14 bg-white/[0.06] text-white/54" : theme.pill, + "inline-flex min-h-8 items-center gap-1.5 rounded-md border px-2 text-xs font-semibold", + disabled + ? "border-white/10 bg-white/[0.045] text-white/48" + : "border-[color:color-mix(in_srgb,var(--success)_28%,transparent)] bg-[color:color-mix(in_srgb,var(--success)_10%,transparent)] text-[color:var(--success-bg)]", )} > - {disabled ? <CircleDashed className="h-3.5 w-3.5" /> : <CheckCircle2 className="h-3.5 w-3.5" />} - {statusLabel} + {disabled ? ( + <CircleDashed className="h-3.5 w-3.5" aria-hidden /> + ) : ( + <CheckCircle2 className="h-3.5 w-3.5" aria-hidden /> + )} + {getStatusLabel(tool)} </span> </div> - <div className="mt-6"> - <p className={cn("text-xs font-black uppercase tracking-[0.18em]", theme.accent)}>{track}</p> - <h2 className="mt-2 text-2xl font-black leading-none text-white">{tool.title}</h2> - <p className="mt-3 line-clamp-3 text-sm leading-6 text-white/68">{tool.description}</p> + <div className="mt-5"> + <p className={cn("text-xs font-semibold uppercase", theme.text)}>{presentation.cadence}</p> + <h2 className="mt-2 text-2xl font-black leading-none text-[color:var(--primary-contrast)] dark:text-[color:var(--text-heading)]"> + {tool.title} + </h2> + <p className="mt-3 text-sm leading-6 text-white/68">{presentation.role}</p> </div> - <div className="mt-auto flex items-end justify-between gap-3 pt-5"> - <span className="inline-flex min-h-9 items-center gap-2 rounded-md border border-white/10 bg-white/[0.045] px-3 text-xs font-bold text-white/48"> - <ExternalLink className="h-3.5 w-3.5" /> - {launchContext} - </span> - {disabled ? ( - <span className="inline-flex min-h-10 items-center justify-center gap-2 rounded-md border border-white/14 bg-white/[0.05] px-4 text-sm font-bold text-white/48"> - Unavailable - </span> - ) : ( - <a - href={tool.href} - target={target} - rel={rel} - className={cn( - "inline-flex min-h-10 items-center justify-center gap-2 rounded-md px-4 text-sm font-black transition", - theme.button, - )} - aria-label={`Launch ${tool.title}`} - > - Launch - <ArrowRight className="h-4 w-4" /> - </a> - )} + <div className="mt-auto pt-5"> + <div className="flex items-center justify-between gap-3 border-t border-white/10 pt-3"> + <span className="nums min-w-0 truncate text-xs font-semibold text-white/48">{getLaunchContext(tool)}</span> + {disabled ? ( + <span className="inline-flex min-h-[44px] items-center justify-center rounded-lg border border-white/10 bg-white/[0.045] px-3 text-sm font-semibold text-white/46"> + Unavailable + </span> + ) : ( + <a + href={tool.href} + target={target} + rel={rel} + className="inline-flex min-h-[44px] shrink-0 items-center justify-center gap-2 rounded-lg bg-[color:var(--primary)] px-3 text-sm font-black text-[color:var(--primary-contrast)] shadow-[var(--shadow-inset)] outline-none motion-safe:transition motion-safe:duration-150 motion-safe:ease-[var(--ease-out-soft)] motion-safe:hover:-translate-y-0.5 hover:bg-[color:var(--primary-strong)] focus-visible:ring-4 focus-visible:ring-[color:var(--focus)]/30" + aria-label={`Launch ${tool.title}`} + > + Launch + <ArrowRight className="h-4 w-4" aria-hidden /> + </a> + )} + </div> </div> </div> </article> ); } -function ProjectTabStrip() { +function AppDock({ tools }: { tools: ToolItem[] }) { return ( - <div className="mt-6 grid max-w-2xl grid-cols-2 gap-1.5 sm:flex sm:max-w-3xl sm:flex-wrap sm:gap-1"> - {toolCatalog.map((tool) => { + <div className="grid grid-cols-3 gap-2"> + {tools.map((tool) => { const Icon = iconRegistry[tool.icon]; - const theme = categoryTheme[tool.category]; + const presentation = getPresentation(tool); + const theme = toneTheme[presentation.tone]; return ( <a @@ -275,15 +320,18 @@ function ProjectTabStrip() { href={tool.href} target={tool.openInNewTab ? "_blank" : undefined} rel={tool.target === "external" ? "noopener noreferrer" : undefined} - aria-label={`Open ${tool.title}`} className={cn( - "group/tab relative inline-flex min-h-7 min-w-0 items-center gap-1.5 overflow-hidden rounded-md border bg-white/[0.045] px-2 text-[0.64rem] font-black text-white/68 shadow-[inset_0_1px_0_rgb(255_255_255_/_7%)] transition hover:-translate-y-0.5 hover:bg-white/[0.075] hover:text-white sm:px-1.5", + "group flex min-h-[56px] items-center gap-2 overflow-hidden rounded-lg border bg-white/[0.055] px-3 text-left shadow-[var(--shadow-inset)] outline-none motion-safe:transition motion-safe:duration-150 motion-safe:ease-[var(--ease-out-soft)] motion-safe:hover:-translate-y-0.5 hover:bg-white/[0.085] focus-visible:ring-4 focus-visible:ring-[color:var(--focus)]/30", theme.border, )} + aria-label={`Launch priority tool ${tool.title}`} > - <span className={cn("h-1.5 w-1.5 shrink-0 rounded-full bg-gradient-to-r", theme.rail)} /> - <Icon className={cn("h-3 w-3 shrink-0 transition group-hover/tab:scale-110", theme.accent)} /> - <span className="truncate">{getShortProjectLabel(tool)}</span> + <span className={cn("h-2 w-2 shrink-0 rounded-full", theme.rail)} aria-hidden /> + <Icon className={cn("h-4 w-4 shrink-0 transition group-hover:scale-110", theme.text)} aria-hidden /> + <span className="min-w-0"> + <span className="block truncate text-xs font-black leading-4 text-white">{presentation.shortLabel}</span> + <span className="sr-only">{presentation.cadence}</span> + </span> </a> ); })} @@ -291,119 +339,109 @@ function ProjectTabStrip() { ); } -function ClinicalSignalPanel() { +function HeroConsole() { return ( - <div className="relative hidden min-h-[31rem] lg:block"> - <div className="absolute left-6 top-0 h-52 w-52 rounded-md border border-cyan-200/20 bg-cyan-200/[0.08] shadow-[0_24px_90px_rgb(34_211_238_/_12%)]" /> - <div className="absolute right-0 top-24 h-52 w-52 rounded-md border border-violet-200/20 bg-violet-200/[0.08] shadow-[0_24px_90px_rgb(196_181_253_/_10%)]" /> - <div className="absolute bottom-0 left-20 h-52 w-52 rounded-md border border-emerald-200/20 bg-emerald-200/[0.08] shadow-[0_24px_90px_rgb(110_231_183_/_10%)]" /> - <div className="absolute left-1/2 top-[45%] w-80 -translate-x-1/2 -translate-y-1/2 rounded-md border border-white/16 bg-slate-950/84 p-5 shadow-[inset_0_1px_0_rgb(255_255_255_/_10%),0_30px_90px_rgb(0_0_0_/_36%)]"> - <div className="flex items-center justify-between gap-3 border-b border-white/10 pb-4"> - <span className="grid h-12 w-12 place-items-center rounded-md border border-cyan-100/24 bg-cyan-100/[0.1] text-cyan-100"> - <Sparkles className="h-6 w-6" /> - </span> - <div className="text-right"> - <p className="text-xs font-black text-white/44">Clinical launch room</p> - <p className="mt-1 text-sm font-black text-cyan-100">Project suite</p> - </div> - </div> - - <div className="mt-5 space-y-3"> - {clinicalSignals.map((signal) => ( - <div - key={signal.label} - className="flex items-center justify-between gap-4 rounded-md border border-white/10 bg-white/[0.045] px-3 py-3" - > - <div> - <p className="text-xs font-black uppercase tracking-[0.2em] text-white/34">{signal.label}</p> - <p className="mt-1 text-sm font-bold text-white/78">{signal.value}</p> - </div> - <CheckCircle2 className="h-5 w-5 shrink-0 text-emerald-100" /> + <aside className="relative overflow-hidden rounded-2xl border border-white/10 bg-white/[0.055] p-3 shadow-[var(--shadow-lux)] backdrop-blur-xl lg:p-4"> + <div + className="pointer-events-none absolute inset-0 bg-[radial-gradient(circle_at_26%_0%,color-mix(in_srgb,var(--primary)_20%,transparent),transparent_16rem),linear-gradient(180deg,color-mix(in_srgb,white_9%,transparent),transparent)]" + aria-hidden + /> + <div className="relative z-10"> + <div className="flex items-center justify-between gap-3 border-b border-white/10 pb-3"> + <div className="flex items-center gap-3"> + <span className="grid h-10 w-10 place-items-center rounded-lg border border-white/10 bg-white/[0.07] text-[color:var(--primary-100)]"> + <Sparkles className="h-5 w-5" aria-hidden /> + </span> + <div> + <p className="text-sm font-black text-white">Priority handoffs</p> + <p className="text-xs font-semibold text-white/46">Pinned clinical tasks</p> </div> - ))} + </div> + <span className="inline-flex min-h-8 items-center rounded-md border border-[color:color-mix(in_srgb,var(--success)_30%,transparent)] bg-[color:color-mix(in_srgb,var(--success)_12%,transparent)] px-2 text-xs font-semibold text-[color:var(--success-bg)]"> + {favoriteTools.length} pinned + </span> </div> - - <div className="mt-5 grid grid-cols-[1fr_auto_1fr] items-center gap-3 text-white/34"> - <div className="h-px bg-white/12" /> - <Target className="h-5 w-5 text-violet-100" /> - <div className="h-px bg-white/12" /> + <div className="mt-3"> + <AppDock tools={favoriteTools} /> </div> </div> - <Search className="absolute left-16 top-14 h-10 w-10 text-cyan-100" /> - <ShieldAlert className="absolute bottom-14 left-36 h-10 w-10 text-emerald-100" /> - <Target className="absolute right-14 top-40 h-10 w-10 text-violet-100" /> - </div> + </aside> ); } export default function ToolsLauncherPage() { return ( - <div className="min-h-full bg-[#050d10] text-white"> + <div className="min-h-full bg-[color:var(--app-shell)] text-[color:var(--primary-contrast)] dark:text-[color:var(--text-heading)]"> <div className="relative isolate min-h-full overflow-hidden"> - <div className="pointer-events-none absolute inset-0 bg-[linear-gradient(135deg,#07161b_0%,#071014_42%,#020608_100%)]" /> - <div className="pointer-events-none absolute inset-0 opacity-[0.12] [background-image:linear-gradient(rgba(255,255,255,.13)_1px,transparent_1px),linear-gradient(90deg,rgba(255,255,255,.11)_1px,transparent_1px)] [background-size:56px_56px]" /> - <div className="pointer-events-none absolute inset-x-0 top-0 h-64 bg-gradient-to-b from-cyan-200/[0.08] to-transparent" /> - <div className="pointer-events-none absolute inset-x-0 bottom-0 h-72 bg-gradient-to-t from-black/58 to-transparent" /> - - <section className="relative z-10 mx-auto flex min-h-svh max-w-7xl flex-col px-4 py-6 sm:px-6 sm:py-8"> - <nav className="flex items-center justify-between gap-3"> - <p className="text-sm font-black text-cyan-100">Clinical KB Tools Atelier</p> - <Link - href="/" - className="inline-flex min-h-10 items-center justify-center gap-2 rounded-md border border-white/14 bg-white/[0.06] px-3 text-sm font-bold text-white/84 transition hover:border-white/28 hover:bg-white/[0.1]" - > - <LayoutList className="h-4 w-4" /> - Dashboard - </Link> - </nav> - - <div className="grid flex-1 items-center gap-10 py-12 lg:grid-cols-[minmax(0,1fr)_26rem]"> - <div className="max-w-4xl"> - <p className="text-sm font-black text-white/54">Command studio</p> - <h1 className="mt-4 text-6xl font-black leading-[0.88] text-white sm:text-8xl lg:text-9xl"> - Launch the clinical stack. - </h1> - <p className="mt-7 max-w-2xl text-lg font-medium leading-8 text-white/68 sm:text-xl sm:leading-9"> - A refined command surface for formulation, diagnosis, therapy, medications, services, differentials, - forms, specifiers, and psychiatry notes. - </p> + <div + className="pointer-events-none absolute inset-0 bg-[radial-gradient(circle_at_18%_-8%,color-mix(in_srgb,var(--primary)_20%,transparent),transparent_34rem),radial-gradient(circle_at_86%_8%,color-mix(in_srgb,var(--info)_16%,transparent),transparent_30rem),linear-gradient(180deg,var(--app-shell-muted)_0%,var(--app-shell)_46%,color-mix(in_srgb,var(--app-shell)_70%,black)_100%)]" + aria-hidden + /> + <div + className="pointer-events-none absolute inset-0 opacity-[0.1] [background-image:linear-gradient(color-mix(in_srgb,white_18%,transparent)_1px,transparent_1px),linear-gradient(90deg,color-mix(in_srgb,white_14%,transparent)_1px,transparent_1px)] [background-size:48px_48px]" + aria-hidden + /> + <div + className="pointer-events-none absolute inset-x-0 bottom-0 h-80 bg-gradient-to-t from-black/70 to-transparent" + aria-hidden + /> + + <main id="main-content" className="relative z-10"> + <section className="mx-auto flex min-h-[68svh] max-w-7xl flex-col px-4 pb-6 pt-safe sm:px-6 lg:min-h-[64svh]"> + <nav className="flex items-center justify-between gap-3 py-4"> + <Link + href="/" + className="inline-flex min-h-[44px] min-w-[44px] items-center justify-center rounded-lg border border-white/10 bg-white/[0.06] px-3 text-white/76 shadow-[var(--shadow-inset)] outline-none transition hover:border-white/20 hover:bg-white/[0.1] hover:text-white focus-visible:ring-4 focus-visible:ring-[color:var(--focus)]/30" + aria-label="Return to Clinical KB dashboard" + > + <LayoutList className="h-5 w-5" aria-hidden /> + </Link> + <a + href="#launchers" + className="inline-flex min-h-[44px] items-center justify-center gap-2 rounded-lg border border-white/10 bg-white/[0.06] px-3 text-sm font-semibold text-white/76 shadow-[var(--shadow-inset)] outline-none transition hover:border-white/20 hover:bg-white/[0.1] hover:text-white focus-visible:ring-4 focus-visible:ring-[color:var(--focus)]/30" + > + Launchers + <ArrowDown className="h-4 w-4" aria-hidden /> + </a> + </nav> + + <div className="grid flex-1 items-center gap-5 py-4 sm:py-5 lg:grid-cols-[minmax(0,1fr)_23rem] lg:gap-8"> + <div className="max-w-4xl"> + <h1 className="max-w-3xl text-5xl font-black leading-[0.92] text-white sm:text-6xl lg:text-7xl xl:text-8xl"> + Open the right clinical tool. + </h1> + <p className="mt-5 max-w-2xl text-base font-semibold leading-7 text-white/68 sm:text-lg sm:leading-8"> + Jump to formulation, DSM-5 criteria, medications, differentials, notes, forms, therapy, specifiers, or + service pathways. Each launch opens the local app shown on the card. + </p> + </div> - <div className="mt-8 h-px max-w-xl bg-gradient-to-r from-cyan-100/42 via-white/16 to-transparent" /> - <ProjectTabStrip /> + <HeroConsole /> </div> - - <ClinicalSignalPanel /> - </div> - - <a - href="#launchers" - className="mb-2 inline-flex w-fit items-center gap-2 rounded-md border border-white/14 bg-white/[0.06] px-4 py-3 text-sm font-black text-white/84 transition hover:border-white/28 hover:bg-white/[0.1]" - > - Open launchers - <ArrowDown className="h-4 w-4" /> - </a> - </section> - - <main id="launchers" className="relative z-10 mx-auto max-w-7xl px-4 pb-10 sm:px-6"> - <div className="border-t border-white/10 pt-8"> - <div className="mb-6 flex flex-col justify-between gap-4 sm:flex-row sm:items-end"> - <div> - <h2 className="text-3xl font-black leading-none text-white sm:text-4xl">Clinical app launchers</h2> - <p className="mt-3 max-w-2xl text-sm leading-6 text-white/58"> - Direct routes into the main local clinical applications in this workspace. - </p> + </section> + + <section id="launchers" className="scroll-mt-8 px-4 pb-10 sm:px-6"> + <div className="mx-auto max-w-7xl border-t border-white/10 pt-6"> + <div className="mb-5 flex flex-col gap-3 sm:flex-row sm:items-end sm:justify-between"> + <div> + <h2 className="text-3xl font-black leading-none text-white sm:text-4xl">All clinical tools</h2> + <p className="mt-3 max-w-2xl text-sm leading-6 text-white/56"> + Use the host label to confirm the local app, then open the tool in a new tab. + </p> + </div> + <span className="inline-flex min-h-9 w-fit items-center gap-2 rounded-lg border border-white/10 bg-white/[0.05] px-3 text-xs font-semibold text-white/54 shadow-[var(--shadow-inset)]"> + <ExternalLink className="h-4 w-4 text-[color:var(--primary-100)]" aria-hidden /> + Opens in a new tab + </span> </div> - <div className="flex items-center gap-2 text-sm font-bold text-white/48"> - <Sparkles className="h-4 w-4 text-cyan-100" /> - Launch only, no dashboard noise + + <div className="grid gap-3 sm:grid-cols-2 xl:grid-cols-3"> + {toolCatalog.map((tool) => ( + <LaunchCard key={tool.id} tool={tool} /> + ))} </div> </div> - <div className="grid gap-4 sm:grid-cols-2 xl:grid-cols-3"> - {toolCatalog.map((tool) => ( - <LaunchCard key={tool.id} tool={tool} /> - ))} - </div> - </div> + </section> </main> </div> </div> diff --git a/src/components/AccessibleTable.tsx b/src/components/AccessibleTable.tsx index cb9fd09e6..624300c2b 100644 --- a/src/components/AccessibleTable.tsx +++ b/src/components/AccessibleTable.tsx @@ -1,7 +1,17 @@ "use client"; import { Maximize2, X } from "lucide-react"; -import { useCallback, useEffect, useId, useMemo, useRef, useState, useSyncExternalStore } from "react"; +import { + type KeyboardEvent, + type ReactNode, + useCallback, + useEffect, + useId, + useMemo, + useRef, + useState, + useSyncExternalStore, +} from "react"; import { cn, textMuted } from "@/components/ui-primitives"; import { normalizeAccessibleTable, type NormalizedAccessibleTable } from "@/lib/accessible-table-normalization"; @@ -60,40 +70,28 @@ function clinicalOnlyTable(table: NormalizedAccessibleTable) { return { header, body, lowConfidence: table.lowConfidence, lowConfidenceReason: table.lowConfidenceReason }; } -function useMobileTableExpansion(enabled: boolean) { - const subscribe = useCallback( - (callback: () => void) => { - if (!enabled || typeof window === "undefined") return () => undefined; - const media = window.matchMedia(tableExpandMediaQuery); - media.addEventListener("change", callback); - return () => media.removeEventListener("change", callback); - }, - [enabled], - ); - - const getSnapshot = useCallback(() => { - return enabled && typeof window !== "undefined" && window.matchMedia(tableExpandMediaQuery).matches; - }, [enabled]); - - return useSyncExternalStore(subscribe, getSnapshot, () => false); -} - function AccessibleTableMarkup({ caption, header, body, compact, expanded = false, + rowActions, + actionsHeader = "Actions", }: { caption?: string | null; header: string[]; body: string[][]; compact: boolean; expanded?: boolean; + rowActions?: Array<ReactNode | null>; + actionsHeader?: string; }) { const visibleBody = expanded ? body : body.slice(0, compact ? 6 : 20); - const columnCount = Math.max(header.length, 1); + const hasActions = Boolean(rowActions?.some(Boolean)); + const columnCount = Math.max(header.length + (hasActions ? 1 : 0), 1); const displayRows = visibleBody.map((row) => header.map((_, index) => row[index] ?? "")); + const displayActions = visibleBody.map((_, index) => rowActions?.[index] ?? null); return ( <div @@ -112,49 +110,21 @@ function AccessibleTableMarkup({ {caption} </div> ) : null} - {!expanded ? ( - <div className="grid gap-2 p-2 md:hidden"> - {displayRows.map((row, rowIndex) => { - const pairs = header - .map((label, index) => ({ label, value: row[index] ?? "" })) - .filter((pair) => pair.value || !compact); - return ( - <dl - key={`${rowIndex}:${row.join("|")}`} - className="grid gap-2 rounded-md border border-[color:var(--border)]/75 bg-[color:var(--surface-raised)] p-3 shadow-[var(--shadow-inset)]" - > - {pairs.map((pair, pairIndex) => ( - <div - key={`${rowIndex}:${pair.label}:${pairIndex}`} - className="grid gap-1 border-b border-[color:var(--border)]/60 pb-2 last:border-b-0 last:pb-0" - > - <dt className={cn("text-[10px] font-bold uppercase tracking-[0.08em]", textMuted)}> - {pair.label || `Column ${pairIndex + 1}`} - </dt> - <dd className="nums min-w-0 whitespace-pre-wrap break-words text-sm leading-6 text-[color:var(--text)] [overflow-wrap:anywhere]"> - {pair.value || <span className={textMuted}>-</span>} - </dd> - </div> - ))} - </dl> - ); - })} - </div> - ) : null} - <div className={cn("overflow-x-auto", !expanded && "hidden md:block")}> + <div className="overflow-x-auto"> <table aria-label={caption ?? undefined} className={cn( - "w-full table-fixed border-separate border-spacing-0 text-left", + "w-full border-separate border-spacing-0 text-left md:table-fixed", expanded ? "text-[15px]" : "text-sm", )} > - <colgroup> + <colgroup className="hidden md:table-column-group"> {header.map((cell, index) => ( <col key={`${cell}:col:${index}`} style={{ width: `${100 / columnCount}%` }} /> ))} + {hasActions ? <col style={{ width: `${100 / columnCount}%` }} /> : null} </colgroup> - <thead> + <thead className="sr-only md:not-sr-only md:table-header-group"> <tr className="bg-[color:var(--surface-subtle)]"> {header.map((cell, index) => ( <th @@ -170,14 +140,29 @@ function AccessibleTableMarkup({ {cell} </th> ))} + {hasActions ? ( + <th + scope="col" + className={cn( + "nums border-b border-l border-[color:var(--border)]/70 align-top font-semibold leading-5 text-[color:var(--text)]", + "whitespace-normal break-words [overflow-wrap:anywhere]", + expanded ? "px-4 py-3 text-sm" : "px-3 py-2 text-xs", + )} + > + {actionsHeader} + </th> + ) : null} </tr> </thead> - <tbody> + <tbody className="block space-y-2 p-2 md:table-row-group md:space-y-0 md:p-0"> {displayRows.map((row, rowIndex) => { return ( <tr key={`${rowIndex}:${row.join("|")}`} - className="border-t border-[color:var(--border)]/70 even:bg-[color:var(--surface-subtle)]/35" + className={cn( + "block rounded-md border border-[color:var(--border)]/75 bg-[color:var(--surface-raised)] p-3 shadow-[var(--shadow-inset)]", + "md:table-row md:rounded-none md:border-0 md:bg-transparent md:p-0 md:shadow-none md:even:bg-[color:var(--surface-subtle)]/35", + )} > {header.map((_, cellIndex) => { const cell = row[cellIndex] ?? ""; @@ -185,16 +170,47 @@ function AccessibleTableMarkup({ <td key={`${rowIndex}:${cellIndex}`} className={cn( - "nums align-top text-[color:var(--text)]", + "nums block align-top text-[color:var(--text)] md:table-cell", "whitespace-pre-wrap break-words [overflow-wrap:anywhere]", - cellIndex > 0 && "border-l border-[color:var(--border)]/60", - expanded ? "px-4 py-3 leading-6" : "px-3 py-2 leading-5", + "border-b border-[color:var(--border)]/60 pb-2 last:border-b-0 md:border-b-0 md:border-t md:border-[color:var(--border)]/70 md:last:border-b-0", + cellIndex > 0 && "md:border-l md:border-[color:var(--border)]/60", + expanded ? "md:px-4 md:py-3 md:leading-6" : "md:px-3 md:py-2 md:leading-5", + cellIndex > 0 && "pt-2 md:pt-0", )} > - {cell || <span className={textMuted}>-</span>} + <span + className={cn( + "mb-1 block text-[10px] font-bold uppercase tracking-[0.08em] md:hidden", + textMuted, + )} + > + {header[cellIndex] || `Column ${cellIndex + 1}`} + </span> + <span className="block min-w-0 text-sm leading-6 md:text-inherit md:leading-inherit"> + {cell || <span className={textMuted}>-</span>} + </span> </td> ); })} + {hasActions ? ( + <td + className={cn( + "block pt-2 align-top md:table-cell md:border-l md:border-t md:border-[color:var(--border)]/60", + expanded ? "md:px-4 md:py-3" : "md:px-3 md:py-2", + )} + onClick={(event) => event.stopPropagation()} + > + <span + className={cn( + "mb-1 block text-[10px] font-bold uppercase tracking-[0.08em] md:hidden", + textMuted, + )} + > + {actionsHeader} + </span> + {displayActions[rowIndex] || <span className={textMuted}>-</span>} + </td> + ) : null} </tr> ); })} @@ -210,6 +226,27 @@ function AccessibleTableMarkup({ ); } +function useMobileTableExpansion(enabledByDefault: boolean) { + const subscribe = useCallback( + (callback: () => void) => { + if (!enabledByDefault || typeof window === "undefined" || typeof window.matchMedia !== "function") + return () => {}; + const media = window.matchMedia(tableExpandMediaQuery); + media.addEventListener("change", callback); + return () => media.removeEventListener("change", callback); + }, + [enabledByDefault], + ); + + const getSnapshot = useCallback(() => { + if (!enabledByDefault || typeof window === "undefined" || typeof window.matchMedia !== "function") return false; + return window.matchMedia(tableExpandMediaQuery).matches; + }, [enabledByDefault]); + + const isMobile = useSyncExternalStore(subscribe, getSnapshot, () => false); + return enabledByDefault && isMobile; +} + export function AccessibleTable({ caption, markdown, @@ -219,6 +256,8 @@ export function AccessibleTable({ expandOnMobile = false, dialogTitle, clinicalOnly = false, + rowActions, + actionsHeader, }: { caption?: string | null; markdown?: string | null; @@ -228,6 +267,8 @@ export function AccessibleTable({ expandOnMobile?: boolean; dialogTitle?: string | null; clinicalOnly?: boolean; + rowActions?: Array<ReactNode | null>; + actionsHeader?: string; }) { const dialogId = useId(); const closeButtonRef = useRef<HTMLButtonElement>(null); @@ -249,7 +290,7 @@ export function AccessibleTable({ const previousOverflow = document.body.style.overflow; document.body.style.overflow = "hidden"; const focusTimer = window.setTimeout(() => closeButtonRef.current?.focus(), 0); - const handleKeyDown = (event: KeyboardEvent) => { + const handleKeyDown = (event: globalThis.KeyboardEvent) => { if (event.key === "Escape") setOpen(false); }; document.addEventListener("keydown", handleKeyDown); @@ -270,14 +311,18 @@ export function AccessibleTable({ const table = ( <> {lowConfidence ? ( - <p - data-testid="table-low-confidence-note" - className={cn("mb-1 text-xs", textMuted)} - > + <p data-testid="table-low-confidence-note" className={cn("mb-1 text-xs", textMuted)}> Table structure could not be confidently reconstructed — verify values against the source document. </p> ) : null} - <AccessibleTableMarkup caption={displayCaption} header={header} body={body} compact={compact} /> + <AccessibleTableMarkup + caption={displayCaption} + header={header} + body={body} + compact={compact} + rowActions={rowActions} + actionsHeader={actionsHeader} + /> </> ); @@ -287,12 +332,26 @@ export function AccessibleTable({ setOpen(true); } + function handleSurfaceKeyDown(event: KeyboardEvent<HTMLDivElement>) { + if (!canExpand) return; + if (event.key !== "Enter" && event.key !== " ") return; + event.preventDefault(); + openDialog(event.currentTarget); + } + return ( <> <div className="relative min-w-0"> <div data-testid="accessible-table-surface" onClick={(event) => openDialog(event.currentTarget)} + onKeyDown={handleSurfaceKeyDown} + role={canExpand ? "button" : undefined} + tabIndex={canExpand ? 0 : -1} + aria-label={canExpand ? `Open ${title} full table` : undefined} + aria-haspopup={canExpand ? "dialog" : undefined} + aria-expanded={canExpand ? dialogOpen : undefined} + aria-controls={dialogOpen ? dialogId : undefined} className={cn( "min-w-0", canExpand && @@ -314,6 +373,7 @@ export function AccessibleTable({ }} className="absolute right-2 top-2 inline-flex min-h-11 min-w-11 items-center justify-center rounded-lg border border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] text-[color:var(--text)] shadow-[var(--shadow-tight)] transition hover:border-[color:var(--border-strong)] focus-visible:outline-none focus-visible:ring-4 focus-visible:ring-[color:var(--focus)]/25" > + <span className="sr-only">Open {title} full table</span> <Maximize2 className="h-4 w-4" /> </button> ) : null} @@ -349,7 +409,15 @@ export function AccessibleTable({ </button> </div> <div className="min-h-0 flex-1 overflow-auto p-3 pb-[max(1rem,env(safe-area-inset-bottom))]"> - <AccessibleTableMarkup caption={displayCaption} header={header} body={body} compact={false} expanded /> + <AccessibleTableMarkup + caption={displayCaption} + header={header} + body={body} + compact={false} + expanded + rowActions={rowActions} + actionsHeader={actionsHeader} + /> </div> </div> </div> diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 7930b2cc5..9da31cfd2 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -8,7 +8,6 @@ import { BookOpen, CheckCircle2, ChevronDown, - Clipboard, ClipboardCheck, ExternalLink, FileImage, @@ -19,14 +18,12 @@ import { LogIn, LogOut, Mail, - Moon, Quote, RefreshCw, Search, ShieldAlert, SlidersHorizontal, Sparkles, - Sun, Tag, Target, UploadCloud, @@ -48,26 +45,21 @@ import { answerSurface, clinicalDivider, cn, - commandInput, evidenceSurface, EmptyState, - eyebrowText, fieldControlPlain, fieldControlWithIcon, fieldIcon, floatingControl, iconTilePremium, fieldLabel, - LoadingPanel, metadataPill, navPill, panel, panelSubtle, - premiumHeaderSurface, primaryControl, proseMeasure, raisedCard, - shellChip, SourceProvenance, SourceStatusBadge, sourceCard, @@ -81,8 +73,23 @@ import { } from "@/components/ui-primitives"; import { Sheet } from "@/components/ui/sheet"; import { AUTH_EMAIL_STORAGE_KEY, useAuthSession } from "@/lib/supabase/client"; -import { nextTheme, resolveThemePreference, type ResolvedTheme } from "@/lib/theme"; import { SafeBoldText } from "@/components/SafeBoldText"; +import { AnswerEmptyState, AnswerSkeleton, CopyButton } from "@/components/clinical-dashboard/answer-status"; +import { useTheme } from "@/components/clinical-dashboard/use-theme"; +import { StatusBadge, StrengthBadge } from "@/components/clinical-dashboard/badges"; +import { MasterSearchHeader } from "@/components/clinical-dashboard/master-search-header"; +import { + DocumentSearchResultsPanel, + MatchExplanationChips, + type SearchFacets, +} from "@/components/clinical-dashboard/document-search-results"; +import { + hasStrongRelevanceIcon, + isWeakRelevance, + QueryCoverageChips, + relevanceChipLabel, + RelevanceBadge, +} from "@/components/clinical-dashboard/relevance"; import { answerPayloadIsUsable, isRetryableError, @@ -97,6 +104,12 @@ import { type AnswerPayload, type SearchError, } from "@/components/clinical-dashboard/search-utils"; +import { + logSourceOpen, + SourceActionRow, + SourcePassageLinks, + sourceResultHref, +} from "@/components/clinical-dashboard/source-actions"; import { parseAnswerDisplayContent, type AnswerDisplayLine, @@ -112,14 +125,10 @@ import { import { groupSourceGovernanceWarnings, type SourceGovernanceWarning } from "@/lib/source-governance"; import { smartEvidenceTags } from "@/lib/evidence-tags"; import { - buildSmartDocumentTagFacets, - filterDocumentsBySmartTagFacets, reviewDocumentTagQuality, - smartDocumentFacetGroups, tagSearchText, type SmartDocumentTag, type SmartDocumentTagFacet, - type SmartDocumentTagGroup, type SmartDocumentTagQualityIssueKind, } from "@/lib/document-tags"; import type { @@ -136,7 +145,6 @@ import type { RelatedDocument, EvidenceSummary, SearchResult, - SourceEvidenceRelevance, SearchScopeSummary, VisualEvidenceCard, ClinicalQueryMode, @@ -153,30 +161,8 @@ import { shouldPollForUpdates, } from "@/lib/ward-output"; -const sampleQueries = [ - { - label: "Monitoring overview", - query: "What monitoring and escalation issues should I consider across these documents?", - }, - { - label: "Lithium safety-net", - query: "What toxicity safety-net symptoms should be reviewed for lithium?", - }, - { - label: "Clozapine table", - query: "What clozapine monitoring items are shown in the table image?", - }, - { - label: "Risk escalation", - query: "When should acute risk be escalated for senior review?", - }, -] as const; - const navigationHashes = ["#search", "#quotes", "#images", "#sources"] as const; const mobileSectionFabMediaQuery = "(max-width: 768px), ((max-width: 1023px) and (hover: none) and (pointer: coarse))"; - -const themeStorageKey = "clinical-kb-theme"; -const themeChangeEvent = "clinical-kb-theme-change"; const authEmailChangeEvent = "clinical-kb-auth-email-change"; const recentQueryStorageKey = "clinical-kb-recent-queries"; const documentPageSize = 150; @@ -241,17 +227,6 @@ type BatchesPayload = { pollAfterMs?: number | null; }; -type SearchFacet = { value: string; count: number }; -type SearchFacets = { - status?: SearchFacet[]; - validation?: SearchFacet[]; - extractionQuality?: SearchFacet[]; - sections?: SearchFacet[]; - labels?: SearchFacet[]; - documentTypes?: SearchFacet[]; - evidence?: SearchFacet[]; -}; - const clinicalQueryModeOptions: Array<{ value: ClinicalQueryMode; label: string }> = [ { value: "auto", label: "Auto" }, { value: "monitoring_schedule", label: "Monitoring" }, @@ -262,17 +237,6 @@ const clinicalQueryModeOptions: Array<{ value: ClinicalQueryMode; label: string { value: "compare_guidance", label: "Compare" }, ]; -function splitFilterText(value: string) { - return value - .split(",") - .map((item) => item.trim()) - .filter(Boolean); -} - -function filterText(values?: string[]) { - return (values ?? []).join(", "); -} - function compactScopeFilters(filters: SearchScopeFilters) { const next: SearchScopeFilters = {}; if (filters.medications?.length) next.medications = filters.medications; @@ -398,33 +362,6 @@ function normalizeNavigationHash(hash: string) { return navigationHashes.includes(hash as (typeof navigationHashes)[number]) ? hash : "#search"; } -function getThemeSnapshot(): ResolvedTheme { - if (typeof window === "undefined") return "light"; - const storedTheme = window.localStorage.getItem(themeStorageKey); - const prefersDark = window.matchMedia("(prefers-color-scheme: dark)").matches; - return resolveThemePreference(storedTheme, prefersDark); -} - -function getServerThemeSnapshot(): ResolvedTheme { - return "light"; -} - -function subscribeTheme(onStoreChange: () => void) { - if (typeof window === "undefined") return () => undefined; - const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); - const notify = () => onStoreChange(); - - window.addEventListener("storage", notify); - window.addEventListener(themeChangeEvent, notify); - mediaQuery.addEventListener("change", notify); - - return () => { - window.removeEventListener("storage", notify); - window.removeEventListener(themeChangeEvent, notify); - mediaQuery.removeEventListener("change", notify); - }; -} - function getAuthEmailSnapshot() { if (typeof window === "undefined") return ""; try { @@ -451,79 +388,6 @@ function subscribeAuthEmail(onStoreChange: () => void) { }; } -function useTheme() { - const theme = useSyncExternalStore(subscribeTheme, getThemeSnapshot, getServerThemeSnapshot); - - useEffect(() => { - document.documentElement.classList.toggle("dark", theme === "dark"); - }, [theme]); - - function toggleTheme() { - const resolved = nextTheme(theme); - window.localStorage.setItem(themeStorageKey, resolved); - document.documentElement.classList.toggle("dark", resolved === "dark"); - window.dispatchEvent(new Event(themeChangeEvent)); - } - - return { theme, toggleTheme }; -} - -function statusTone(status: string) { - if (status === "indexed" || status === "completed") { - return { - icon: CheckCircle2, - className: toneSuccess, - }; - } - if (status === "failed") { - return { - icon: AlertCircle, - className: toneDanger, - }; - } - if (status === "processing") { - return { - icon: Loader2, - className: toneInfo, - }; - } - return { - icon: FileText, - className: toneNeutral, - }; -} - -function StatusBadge({ status }: { status: string }) { - const tone = statusTone(status); - const Icon = tone.icon; - - return ( - <span - className={cn( - "inline-flex min-h-7 items-center gap-1.5 rounded-md border px-2.5 text-xs font-semibold", - tone.className, - )} - > - <Icon className={cn("h-3.5 w-3.5", status === "processing" && "animate-spin")} /> - {status} - </span> - ); -} - -function StrengthBadge({ strength }: { strength?: string }) { - const label = strength ?? "source"; - const className = strength === "strong" ? toneSuccess : strength === "limited" ? toneWarning : toneInfo; - - return ( - <span - className={cn("inline-flex min-h-7 items-center gap-1.5 rounded-md border px-2 text-xs font-semibold", className)} - > - <CheckCircle2 className="h-3.5 w-3.5" /> - {label} - </span> - ); -} - function SourceImage({ endpoint, caption, @@ -674,97 +538,6 @@ function SectionHeading({ ); } -function relevanceChipLabel( - relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, - grounded = false, -) { - if (!relevance) return grounded ? "Source-backed" : "No direct support"; - if (relevance.verdict === "direct") return "Source-backed"; - if (relevance.verdict === "partial") return "Partial support"; - if (relevance.verdict === "nearby") return "Nearby only"; - return "No direct support"; -} - -function relevanceChipClasses( - relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, - grounded = false, -) { - const verdict = relevance?.verdict ?? (grounded ? "direct" : "none"); - if (verdict === "direct") { - return "border-[color:var(--success)]/20 bg-[color:var(--success-soft)]/45 text-[color:var(--success)]"; - } - if (verdict === "partial") { - return "border-[color:var(--primary)]/25 bg-[color:var(--primary-soft)]/45 text-[color:var(--primary)]"; - } - return "border-[color:var(--warning)]/25 bg-[color:var(--warning-soft)]/45 text-[color:var(--warning)]"; -} - -function hasStrongRelevanceIcon( - relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, - grounded = false, -) { - const verdict = relevance?.verdict ?? (grounded ? "direct" : "none"); - return verdict === "direct" || verdict === "partial"; -} - -function isWeakRelevance(relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined) { - return !relevance?.isSourceBacked || relevance.verdict === "nearby" || relevance.verdict === "none"; -} - -function RelevanceBadge({ - relevance, - grounded = false, - testId, -}: { - relevance?: EvidenceRelevance | SourceEvidenceRelevance | null; - grounded?: boolean; - testId?: string; -}) { - const showStrongIcon = hasStrongRelevanceIcon(relevance, grounded); - const label = relevanceChipLabel(relevance, grounded); - return ( - <span - data-testid={testId} - className={cn( - "inline-flex min-h-7 items-center gap-1 rounded-md border px-2 text-[11px] font-semibold leading-none sm:min-h-8 sm:gap-1.5 sm:px-2.5 sm:text-xs", - relevanceChipClasses(relevance, grounded), - )} - aria-label={label} - title={relevance?.supportReason ?? label} - > - {showStrongIcon ? <CheckCircle2 className="h-3.5 w-3.5" /> : <AlertCircle className="h-3.5 w-3.5" />} - <span>{label}</span> - </span> - ); -} - -function QueryCoverageChips({ - relevance, - limit = 4, -}: { - relevance?: SourceEvidenceRelevance | EvidenceRelevance | null; - limit?: number; -}) { - if (!relevance) return null; - const chips = - "chips" in relevance && relevance.chips.length - ? relevance.chips - : [ - relevance.matchedTerms.length ? `matched: ${relevance.matchedTerms.slice(0, 3).join(", ")}` : "", - relevance.missingTerms.length ? `missing: ${relevance.missingTerms.slice(0, 3).join(", ")}` : "", - relevanceChipLabel(relevance), - ].filter(Boolean); - return ( - <div className="flex flex-wrap gap-1.5"> - {chips.slice(0, limit).map((chip) => ( - <span key={chip} className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> - {chip} - </span> - ))} - </div> - ); -} - function ScopeAndGovernanceNotice({ scope, warnings, @@ -798,922 +571,24 @@ function ScopeAndGovernanceNotice({ {groupedWarnings.map((warning) => ( <li key={warning.code}> {warning.message} - {warning.titles.length ? ( - <details className="mt-1 font-medium text-[color:var(--text-muted)]"> - <summary className="cursor-pointer">Sources affected</summary> - <span className="mt-1 block">{warning.titles.slice(0, 5).join(", ")}</span> - </details> - ) : null} - </li> - ))} - </ul> - ) : null} - </div> - ); -} - -function MasterSearchHeader({ - documents, - query, - searchMode, - loading, - selectedDocumentIds, - queryMode, - scopeFilters, - hasAnswer, - demoMode, - realDataReady, - theme, - onQueryChange, - onSearchModeChange, - onAsk, - onClearQuery, - onClearScope, - onQueryModeChange, - onScopeFiltersChange, - onToggleScope, - onScopeOpenChange, - onOpenGuide, - onToggleTheme, -}: { - documents: ClinicalDocument[]; - query: string; - searchMode: SearchMode; - loading: boolean; - selectedDocumentIds: string[]; - queryMode: ClinicalQueryMode; - scopeFilters: SearchScopeFilters; - hasAnswer: boolean; - demoMode: boolean; - realDataReady: boolean; - theme: ResolvedTheme; - onQueryChange: (query: string) => void; - onSearchModeChange: (mode: SearchMode) => void; - onAsk: () => void; - onClearQuery: () => void; - onClearScope: () => void; - onQueryModeChange: (mode: ClinicalQueryMode) => void; - onScopeFiltersChange: (filters: SearchScopeFilters) => void; - onToggleScope: (documentId: string) => void; - onScopeOpenChange?: (open: boolean) => void; - onOpenGuide: () => void; - onToggleTheme: () => void; -}) { - const trimmedQuery = query.trim(); - const canAsk = trimmedQuery.length >= 1 && !loading && realDataReady; - const compactMobile = hasAnswer; - const [scopeFilter, setScopeFilter] = useState(""); - const [scopeOpen, setScopeOpen] = useState(false); - const [scopeSheetOpen, setScopeSheetOpen] = useState(false); - const scopeDetailsRef = useRef<HTMLDetailsElement | null>(null); - const scopeSummaryRef = useRef<HTMLElement | null>(null); - const scopeFilterInputRef = useRef<HTMLInputElement | null>(null); - const selectedDocuments = selectedDocumentIds - .map((id) => documents.find((document) => document.id === id)) - .filter((document): document is ClinicalDocument => Boolean(document)); - const scopeSummary = selectedDocumentIds.length === 0 ? "All documents" : `${selectedDocumentIds.length} scoped`; - const scopePreview = selectedDocuments - .slice(0, 2) - .map((document) => document?.title.replace(/^Synthetic /, "")) - .filter(Boolean) - .join(", "); - const normalizedScopeFilter = scopeFilter.trim().toLowerCase(); - const recentlyUpdatedDocuments = [...documents].sort((a, b) => { - const bTime = Date.parse(b.updated_at || b.created_at || ""); - const aTime = Date.parse(a.updated_at || a.created_at || ""); - return (Number.isNaN(bTime) ? 0 : bTime) - (Number.isNaN(aTime) ? 0 : aTime); - }); - const matchingDocuments = normalizedScopeFilter - ? recentlyUpdatedDocuments.filter((document) => - [document.title, document.file_name, document.description, tagSearchText(document)] - .filter(Boolean) - .some((value) => value?.toLowerCase().includes(normalizedScopeFilter)), - ) - : recentlyUpdatedDocuments; - const largeScopeSet = documents.length > 12; - const requireScopeFilter = largeScopeSet && !normalizedScopeFilter; - const visibleScopeDocuments = [ - ...selectedDocuments, - ...(requireScopeFilter ? [] : matchingDocuments.filter((document) => !selectedDocumentIds.includes(document.id))), - ].slice(0, 12); - const hiddenScopeMatchCount = requireScopeFilter - ? Math.max(0, selectedDocuments.length ? documents.length - selectedDocuments.length : documents.length) - : Math.max(0, matchingDocuments.length - visibleScopeDocuments.length); - const submitLabel = searchMode === "answer" ? (trimmedQuery ? "Answer" : "Ask") : "Docs"; - const collectionOptions = useMemo(() => { - const values = new Set<string>(); - for (const document of documents) { - const metadata = - document.metadata && typeof document.metadata === "object" - ? (document.metadata as Record<string, unknown>) - : {}; - const collection = metadata.collection; - if (typeof collection === "string" && collection.trim()) values.add(collection.trim()); - } - return Array.from(values).sort((a, b) => a.localeCompare(b)); - }, [documents]); - - const closeScope = useCallback((restoreFocus = false) => { - const details = scopeDetailsRef.current; - if (!details?.open) return; - details.open = false; - setScopeOpen(false); - if (restoreFocus) scopeSummaryRef.current?.focus(); - }, []); - - useEffect(() => { - onScopeOpenChange?.(scopeOpen || scopeSheetOpen); - }, [onScopeOpenChange, scopeOpen, scopeSheetOpen]); - - useEffect(() => { - const details = scopeDetailsRef.current; - if (!scopeOpen || !details?.open) return undefined; - - function handlePointerDown(event: PointerEvent) { - const target = event.target; - if (!(target instanceof Node)) return; - if (!scopeDetailsRef.current?.contains(target)) closeScope(false); - } - - function handleKeyDown(event: KeyboardEvent) { - if (event.key === "Escape") { - event.preventDefault(); - closeScope(true); - } - } - - document.addEventListener("pointerdown", handlePointerDown); - document.addEventListener("keydown", handleKeyDown); - return () => { - document.removeEventListener("pointerdown", handlePointerDown); - document.removeEventListener("keydown", handleKeyDown); - }; - }, [closeScope, scopeOpen]); - - function submit(event: FormEvent<HTMLFormElement>) { - event.preventDefault(); - onAsk(); - } - - function documentScopeTitle(document: ClinicalDocument) { - return document.title.replace(/^Synthetic /, "").replace(/\.pdf$/i, ""); - } - - function documentScopeMeta(document: ClinicalDocument) { - const title = documentScopeTitle(document).toLowerCase(); - const fileName = document.file_name; - const fileBase = fileName.replace(/\.pdf$/i, "").toLowerCase(); - if (fileBase === title || fileBase.startsWith(title)) return `${document.page_count ?? "?"} pages`; - return `${fileName} ¡ ${document.page_count ?? "?"} pages`; - } - - function renderScopeRows() { - return ( - <div className="grid gap-3"> - <section className="min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)] p-2.5 sm:hidden"> - <div className="mb-2 flex min-h-7 items-center justify-between gap-2 px-0.5"> - <p className={eyebrowText}>Refine search</p> - <span className="text-[11px] font-semibold text-[color:var(--text-soft)]">Mode, status, topics</span> - </div> - <div className="grid gap-2"> - <select - value={queryMode} - onChange={(event) => onQueryModeChange(event.target.value as ClinicalQueryMode)} - aria-label="Clinical query mode" - className="h-10 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - > - {clinicalQueryModeOptions.map((option) => ( - <option key={option.value} value={option.value}> - {option.label} - </option> - ))} - </select> - <div className="grid grid-cols-2 gap-2"> - <input - value={filterText(scopeFilters.medications)} - onChange={(event) => - onScopeFiltersChange({ ...scopeFilters, medications: splitFilterText(event.target.value) }) - } - placeholder="Medication" - className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - /> - <input - value={filterText(scopeFilters.topics)} - onChange={(event) => - onScopeFiltersChange({ ...scopeFilters, topics: splitFilterText(event.target.value) }) - } - placeholder="Topic" - className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - /> - <select - value={scopeFilters.sourceStatuses?.[0] ?? ""} - onChange={(event) => - onScopeFiltersChange({ - ...scopeFilters, - sourceStatuses: event.target.value - ? [event.target.value as NonNullable<SearchScopeFilters["sourceStatuses"]>[number]] - : [], - }) - } - className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - > - <option value="">Any status</option> - <option value="current">Current</option> - <option value="review_due">Review due</option> - <option value="outdated">Outdated</option> - <option value="unknown">Unknown</option> - </select> - <select - value={scopeFilters.locality ?? ""} - onChange={(event) => - onScopeFiltersChange({ - ...scopeFilters, - locality: event.target.value ? (event.target.value as SearchScopeFilters["locality"]) : undefined, - }) - } - className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - > - <option value="">Any locality</option> - <option value="local">Local only</option> - <option value="non_local">Non-local only</option> - </select> - </div> - <input - value={filterText(scopeFilters.collections)} - onChange={(event) => - onScopeFiltersChange({ ...scopeFilters, collections: splitFilterText(event.target.value) }) - } - placeholder={collectionOptions.length ? `Collection: ${collectionOptions[0]}` : "Collection"} - className="h-10 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - /> - <button - type="button" - onClick={() => onScopeFiltersChange({})} - className={cn(floatingControl, "min-h-9 px-3 text-xs")} - > - Clear refine filters - </button> - </div> - </section> - <section className="min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-2.5"> - <div className="mb-2 flex min-h-7 items-center justify-between gap-2 px-0.5"> - <p className={eyebrowText}>Document scope</p> - <span className="nums shrink-0 text-[11px] font-semibold text-[color:var(--text-soft)]"> - {selectedDocumentIds.length ? `${selectedDocumentIds.length} selected` : `${documents.length} available`} - </span> - </div> - <div className="space-y-2"> - <label className="relative block"> - <Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-[color:var(--text-soft)]" /> - <input - ref={scopeFilterInputRef} - value={scopeFilter} - onChange={(event) => setScopeFilter(event.target.value)} - data-testid="document-scope-filter" - aria-label="Filter document scope" - placeholder="Filter documents by title or file" - className="h-10 w-full rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] pl-9 pr-3 text-sm font-semibold text-[color:var(--text)] shadow-[var(--shadow-inset)] outline-none transition placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" - /> - </label> - <div className="flex flex-wrap items-center gap-2"> - <button - type="button" - onClick={onClearScope} - className={cn( - shellChip, - selectedDocumentIds.length === 0 - ? "border-[color:var(--primary)]/40 bg-[color:var(--primary-soft)] text-[color:var(--primary-strong)]" - : "border-[color:var(--border)] bg-[color:var(--surface-raised)] text-[color:var(--text-muted)] hover:bg-[color:var(--surface-subtle)]", - )} - > - All documents - </button> - {scopeFilter ? ( - <span className="nums rounded-md bg-[color:var(--surface-raised)] px-2 py-1 text-[11px] font-semibold text-[color:var(--text-muted)]"> - {matchingDocuments.length} match{matchingDocuments.length === 1 ? "" : "es"} - </span> - ) : ( - <span className="rounded-md bg-[color:var(--surface-raised)] px-2 py-1 text-[11px] font-semibold text-[color:var(--text-muted)]"> - Recently updated first - </span> - )} - </div> - <div className="max-h-72 overflow-y-auto pr-1 polished-scroll"> - <div className="grid gap-1.5"> - {requireScopeFilter && visibleScopeDocuments.length === 0 ? ( - <p className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-3 py-2 text-sm font-medium text-[color:var(--text-muted)]"> - Type to filter {documents.length} documents. Selected documents stay pinned here. - </p> - ) : null} - {visibleScopeDocuments.map((document) => { - const selected = selectedDocumentIds.includes(document.id); - return ( - <button - key={document.id} - type="button" - onClick={() => onToggleScope(document.id)} - title={document.title} - className={cn( - "grid min-h-[44px] w-full grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2 rounded-lg border px-2.5 py-2 text-left transition motion-safe:duration-150", - selected - ? "border-[color:var(--primary)]/40 bg-[color:var(--primary-soft)] text-[color:var(--primary-strong)]" - : "border-[color:var(--border)] bg-[color:var(--surface-raised)] text-[color:var(--text)] hover:border-[color:var(--border-strong)] hover:bg-[color:var(--surface-subtle)]", - )} - > - <span - className={cn( - "grid h-5 w-5 place-items-center rounded-md border", - selected - ? "border-[color:var(--primary)]/50 bg-[color:var(--primary-soft)] text-[color:var(--primary)]" - : "border-[color:var(--border-strong)] bg-[color:var(--surface-subtle)]", - )} - aria-hidden - > - {selected ? <CheckCircle2 className="h-3.5 w-3.5" /> : null} - </span> - <span className="min-w-0"> - <span className="block truncate text-sm font-semibold">{documentScopeTitle(document)}</span> - <span className="nums block truncate text-[11px] font-medium text-[color:var(--text-soft)]"> - {documentScopeMeta(document)} - </span> - <DocumentTagCloud - labels={document.labels} - query={scopeFilter} - limit={2} - compact - expandable={false} - className="mt-1" - /> - </span> - {selected ? ( - <span className="rounded-md bg-[color:var(--primary-soft)] px-2 py-1 text-[11px] font-bold text-[color:var(--primary-strong)]"> - In scope - </span> - ) : null} - </button> - ); - })} - {!requireScopeFilter && visibleScopeDocuments.length === 0 ? ( - <p className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-3 py-2 text-sm font-medium text-[color:var(--text-muted)]"> - No documents match that filter. Clear the filter or search by file name. - </p> - ) : null} - </div> - </div> - {hiddenScopeMatchCount > 0 ? ( - <p className="nums px-1 text-xs font-medium text-[color:var(--text-soft)]"> - {requireScopeFilter - ? `${documents.length} documents available. Type a title or file name to narrow the list.` - : `Showing ${visibleScopeDocuments.length} of ${matchingDocuments.length}. Keep typing to narrow the list.`} - </p> - ) : null} - </div> - </section> - </div> - ); - } - - return ( - <header - id="search" - className={cn( - "sticky top-0 z-30 px-3 pb-2 pt-[max(0.5rem,env(safe-area-inset-top))] sm:px-4 lg:px-8", - premiumHeaderSurface, - "sm:py-2.5", - )} - style={{ backgroundColor: "var(--app-shell)" }} - > - <div className="mx-auto max-w-7xl space-y-2"> - <div className="flex items-center justify-between gap-3"> - <div className="flex min-w-0 items-center gap-3"> - <div - className={cn( - "grid shrink-0 place-items-center rounded-lg border border-white/20 bg-[linear-gradient(135deg,var(--primary),var(--primary-strong))] text-[color:var(--primary-contrast)] shadow-[var(--glow-soft)]", - compactMobile ? "h-9 w-9 sm:h-[44px] sm:w-[44px]" : "h-[44px] w-[44px]", - )} - > - <BookOpen className="h-5 w-5" /> - </div> - <div className="min-w-0"> - <div className="flex min-w-0 items-center gap-2"> - <h1 className="truncate text-base font-semibold">Clinical Guide</h1> - {demoMode && ( - <span className="hidden shrink-0 rounded-md border border-amber-300/25 bg-amber-300/12 px-2 py-0.5 text-[11px] font-bold uppercase tracking-[0.08em] text-amber-100 sm:inline-flex"> - Demo data - </span> - )} - </div> - <p className={cn("truncate text-xs font-medium text-slate-300", compactMobile && "hidden sm:block")}> - {demoMode ? "Synthetic data only" : "Ask indexed guidelines"} - </p> - </div> - </div> - <div className="flex shrink-0 items-center gap-2"> - {demoMode && ( - <span className="inline-flex rounded-md border border-amber-300/25 bg-amber-300/12 px-2 py-1 text-[11px] font-bold uppercase tracking-[0.08em] text-amber-100 sm:hidden"> - Demo - </span> - )} - <Link - href="/tools" - aria-label="Open clinical tools" - title="Open clinical tools" - className="group relative grid h-[44px] w-[44px] shrink-0 place-items-center overflow-hidden rounded-lg border border-cyan-100/20 bg-cyan-100/[0.08] text-cyan-50 shadow-[inset_0_1px_0_rgb(255_255_255_/_12%),var(--shadow-tight)] transition hover:-translate-y-0.5 hover:border-cyan-100/35 hover:bg-cyan-100/[0.13] hover:text-white" - > - <span className="pointer-events-none absolute inset-x-2 top-0 h-px bg-gradient-to-r from-transparent via-cyan-100/70 to-transparent" /> - <Sparkles className="relative h-4.5 w-4.5" /> - <span className="sr-only">Clinical tools</span> - </Link> - <button - type="button" - onClick={onOpenGuide} - className="hidden h-[44px] shrink-0 items-center gap-2 rounded-lg border border-white/15 bg-white/7 px-3 text-xs font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition hover:border-white/25 hover:bg-white/12 sm:inline-flex" - aria-label="Open user guide" - > - <BookOpen className="h-4 w-4" /> - Guide - </button> - <button - type="button" - onClick={onToggleTheme} - className="grid h-[44px] w-[44px] shrink-0 place-items-center rounded-lg border border-white/15 bg-white/7 text-slate-100 shadow-[var(--shadow-tight)] transition hover:border-white/25 hover:bg-white/12" - aria-label={`Switch to ${theme === "dark" ? "light" : "dark"} mode`} - > - {theme === "dark" ? <Sun className="h-4.5 w-4.5" /> : <Moon className="h-4.5 w-4.5" />} - </button> - </div> - </div> - - <div className="grid gap-2 rounded-[var(--radius-lg)] border border-white/10 bg-white/6 p-1 shadow-[var(--shadow-inset)] sm:flex sm:flex-wrap sm:items-center sm:justify-between"> - <div role="group" aria-label="Search mode" className="grid grid-cols-2 gap-1 sm:min-w-[14rem]"> - {[ - { mode: "answer" as const, label: "Answer", icon: Sparkles }, - { mode: "documents" as const, label: "Documents", icon: FileText }, - ].map((item) => { - const active = searchMode === item.mode; - const Icon = item.icon; - return ( - <button - key={item.mode} - type="button" - onClick={() => onSearchModeChange(item.mode)} - className={cn( - "inline-flex min-h-[44px] items-center justify-center gap-2 rounded-[var(--radius-md)] px-3 text-sm font-semibold transition", - active - ? "bg-white text-slate-950 shadow-[var(--shadow-tight)]" - : "text-slate-200 hover:bg-white/10 hover:text-white", - )} - aria-pressed={active} - aria-label={item.mode === "answer" ? "Switch to answer mode" : "Switch to document search mode"} - > - <Icon className="h-4 w-4" /> - {item.label} - </button> - ); - })} - </div> - <span className="hidden px-2 text-xs font-medium text-slate-300 lg:inline"> - {searchMode === "answer" ? "Synthesize cited clinical guidance" : "List matching source documents"} - </span> - <div className="ml-auto hidden min-w-0 items-center gap-2 sm:flex"> - <details className="group relative"> - <summary className="flex h-9 cursor-pointer list-none items-center justify-between gap-2 rounded-md border border-white/15 bg-white/7 px-3 text-xs font-semibold text-slate-100"> - <SlidersHorizontal className="h-4 w-4 shrink-0" /> - Refine - <ChevronDown className="h-4 w-4 shrink-0 transition group-open:rotate-180" /> - </summary> - <div className="absolute right-0 top-[calc(100%+0.5rem)] z-40 grid w-[min(42rem,calc(100vw-2rem))] gap-2 rounded-lg border border-white/15 bg-[color:var(--surface-glass)] p-3 shadow-[var(--shadow-elevated)] backdrop-blur-xl sm:grid-cols-2 lg:grid-cols-3"> - <select - value={queryMode} - onChange={(event) => onQueryModeChange(event.target.value as ClinicalQueryMode)} - aria-label="Clinical query mode" - className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" - > - {clinicalQueryModeOptions.map((option) => ( - <option key={option.value} value={option.value}> - {option.label} - </option> - ))} - </select> - <input - value={filterText(scopeFilters.medications)} - onChange={(event) => - onScopeFiltersChange({ ...scopeFilters, medications: splitFilterText(event.target.value) }) - } - placeholder="Medication labels" - className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" - /> - <input - value={filterText(scopeFilters.topics)} - onChange={(event) => - onScopeFiltersChange({ ...scopeFilters, topics: splitFilterText(event.target.value) }) - } - placeholder="Topic labels" - className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" - /> - <input - value={filterText(scopeFilters.collections)} - onChange={(event) => - onScopeFiltersChange({ ...scopeFilters, collections: splitFilterText(event.target.value) }) - } - placeholder={collectionOptions.length ? `Collection: ${collectionOptions[0]}` : "Collection"} - className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" - /> - <select - value={scopeFilters.sourceStatuses?.[0] ?? ""} - onChange={(event) => - onScopeFiltersChange({ - ...scopeFilters, - sourceStatuses: event.target.value - ? [event.target.value as NonNullable<SearchScopeFilters["sourceStatuses"]>[number]] - : [], - }) - } - className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" - > - <option value="">Any status</option> - <option value="current">Current</option> - <option value="review_due">Review due</option> - <option value="outdated">Outdated</option> - <option value="unknown">Unknown</option> - </select> - <select - value={scopeFilters.locality ?? ""} - onChange={(event) => - onScopeFiltersChange({ - ...scopeFilters, - locality: event.target.value ? (event.target.value as SearchScopeFilters["locality"]) : undefined, - }) - } - className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" - > - <option value="">Any locality</option> - <option value="local">Local only</option> - <option value="non_local">Non-local only</option> - </select> - <button - type="button" - onClick={() => onScopeFiltersChange({})} - className="h-9 rounded-md border border-white/15 bg-white/7 px-2 text-xs font-semibold text-slate-100 hover:bg-white/12" - > - Clear filters - </button> - </div> - </details> - </div> - </div> - - <form - onSubmit={submit} - className="grid grid-cols-[minmax(0,1fr)_52px_52px] gap-2 sm:grid-cols-[minmax(0,1fr)_136px_108px] lg:grid-cols-[minmax(0,1fr)_148px_116px]" - > - <label className="relative min-w-0"> - <Search className="pointer-events-none absolute left-4 top-1/2 h-5 w-5 -translate-y-1/2 text-slate-400" /> - <input - value={query} - onChange={(event) => onQueryChange(event.target.value)} - onKeyDown={(event) => { - if ((event.metaKey || event.ctrlKey) && event.key === "Enter") onAsk(); - }} - aria-label="Search indexed guidelines by question or keyword" - placeholder="Ask a question" - className={commandInput} - /> - {query && ( - <button - type="button" - onClick={onClearQuery} - className="absolute right-2 top-1/2 grid h-[44px] w-[44px] -translate-y-1/2 place-items-center rounded-lg text-slate-500 transition hover:bg-slate-100 hover:text-slate-950 dark:hover:bg-slate-800 dark:hover:text-slate-50" - aria-label="Clear search question" - > - <X className="h-4 w-4" /> - </button> - )} - </label> - <button - type="submit" - disabled={!canAsk} - title={ - !realDataReady - ? "Search setup is not ready" - : trimmedQuery.length < 1 - ? searchMode === "answer" - ? "Enter a clinical question" - : "Enter a document search term" - : searchMode === "answer" - ? "Generate a source-backed answer" - : "Find matching documents" - } - className={cn( - primaryControl, - "min-h-[44px] whitespace-nowrap rounded-[var(--radius-lg)] px-0 text-sm sm:px-5", - )} - aria-label={searchMode === "answer" ? "Generate source-backed answer" : "Find matching documents"} - > - {loading ? <Loader2 className="h-4 w-4 animate-spin" /> : <Search className="h-4 w-4" />} - <span className="sr-only sm:not-sr-only">{submitLabel}</span> - </button> - <> - <button - type="button" - data-testid="scope-trigger" - onClick={() => setScopeSheetOpen(true)} - className="flex min-h-[44px] cursor-pointer list-none items-center justify-center gap-1.5 whitespace-nowrap rounded-[var(--radius-lg)] border border-white/15 bg-white/7 px-0 text-sm font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition motion-safe:duration-150 hover:border-white/25 hover:bg-white/12 sm:hidden" - aria-label="Open document scope" - aria-expanded={scopeSheetOpen} - > - <Filter className="h-4 w-4" /> - <span className="sr-only">Scope</span> - {selectedDocumentIds.length ? ( - <span className="rounded-md bg-teal-200/15 px-1.5 py-0.5 text-[10px] font-bold text-teal-50"> - {selectedDocumentIds.length} - </span> - ) : null} - </button> - - <details - ref={scopeDetailsRef} - onToggle={(event) => { - const open = event.currentTarget.open; - setScopeOpen(open); - if (open) window.setTimeout(() => scopeFilterInputRef.current?.focus(), 0); - }} - className="group relative hidden sm:block" - > - <summary - ref={scopeSummaryRef} - data-testid="scope-trigger" - className="flex min-h-[44px] cursor-pointer list-none items-center justify-center gap-1.5 whitespace-nowrap rounded-[var(--radius-lg)] border border-white/15 bg-white/7 px-0 text-sm font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition motion-safe:duration-150 hover:border-white/25 hover:bg-white/12 sm:gap-2 sm:px-3 sm:text-xs" - aria-label="Toggle document scope" - aria-expanded={scopeOpen} - > - <Filter className="h-4 w-4" /> - <span className="sr-only sm:not-sr-only">Scope</span> - {selectedDocumentIds.length ? ( - <span className="rounded-md bg-teal-200/15 px-1.5 py-0.5 text-[10px] font-bold text-teal-50"> - {selectedDocumentIds.length} - </span> - ) : null} - </summary> - <div - data-testid="scope-command-popover" - className="polished-scroll absolute right-0 top-[calc(100%+0.5rem)] z-40 max-h-[min(70dvh,28rem)] w-[28rem] overflow-y-auto overscroll-contain rounded-xl border border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] p-2.5 pb-2.5 text-[color:var(--text)] shadow-[var(--shadow-elevated)] backdrop-blur-xl motion-safe:animate-pop-in" - > - <div className="mb-2 flex min-h-8 items-center justify-between px-1 text-xs font-semibold text-[color:var(--text-muted)]"> - <span>Document scope</span> - <span className="nums">{scopeSummary}</span> - </div> - {scopePreview ? ( - <p className="mb-2 truncate px-1 text-xs text-[color:var(--text-soft)]">{scopePreview}</p> - ) : null} - {renderScopeRows()} - </div> - </details> - - <Sheet - open={scopeSheetOpen} - onClose={() => setScopeSheetOpen(false)} - title="Document scope" - description="Choose documents and filters for the next search." - closeLabel="Close document scope" - initialFocusRef={scopeFilterInputRef} - contentClassName="sm:hidden" - > - <div className="mb-2 flex min-h-8 items-center justify-between px-1 text-xs font-semibold text-[color:var(--text-muted)]"> - <span>Document scope</span> - <span className="nums">{scopeSummary}</span> - </div> - {scopePreview ? ( - <p className="mb-2 truncate px-1 text-xs text-[color:var(--text-soft)]">{scopePreview}</p> - ) : null} - {renderScopeRows()} - </Sheet> - </> - </form> - </div> - </header> - ); -} - -function CopyButton({ - label, - shortLabel, - ariaLabel, - copied, - onClick, -}: { - label: string; - shortLabel?: string; - ariaLabel?: string; - copied: boolean; - onClick: () => void; -}) { - return ( - <button - type="button" - onClick={onClick} - aria-label={ariaLabel ?? label} - className={cn(floatingControl, "px-3 text-xs")} - > - {copied ? <ClipboardCheck className="h-4 w-4" /> : <Clipboard className="h-4 w-4" />} - <span className="sm:hidden">{copied ? "Copied" : (shortLabel ?? label)}</span> - <span className="hidden sm:inline">{copied ? "Copied" : label}</span> - </button> - ); -} - -function AnswerEmptyState({ - onPickSample, - recentQueries = [], - documentsLoading = false, -}: { - onPickSample: (sample: string) => void; - recentQueries?: string[]; - documentsLoading?: boolean; -}) { - return ( - <div className="space-y-3"> - <EmptyState - icon={Search} - title="Ask indexed guidelines" - body="The answer, quotes, source PDFs, and diagrams will appear here." - /> - {documentsLoading ? ( - <LoadingPanel label="Checking indexed library before showing document status" variant="skeleton" lines={2} /> - ) : null} - {recentQueries.length > 0 ? ( - <section - aria-label="Recent questions" - className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)] p-3 shadow-[var(--shadow-inset)]" - > - <div className="mb-2 flex min-h-7 items-center justify-between gap-2"> - <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> - Recent questions - </p> - <span className={cn("text-[11px] font-semibold", textMuted)}>Resume</span> - </div> - <div className="grid gap-2"> - {recentQueries.map((recent) => ( - <button - key={recent} - type="button" - onClick={() => onPickSample(recent)} - title={recent} - className={cn( - floatingControl, - "min-h-10 justify-start px-3 text-left text-xs font-semibold sm:text-sm", - )} - > - <Search className="h-3.5 w-3.5 shrink-0" /> - <span className="min-w-0 truncate">{recent}</span> - </button> - ))} - </div> - </section> - ) : null} - <section - aria-label="Example questions" - className={cn( - "rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-3 shadow-[var(--shadow-inset)]", - )} - > - <div className="mb-2 flex min-h-7 items-center justify-between gap-2"> - <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> - Starter actions - </p> - <span className={cn("text-[11px] font-semibold", textMuted)}>Set question</span> - </div> - <div className="grid gap-2 sm:grid-cols-2"> - {sampleQueries.map((sample) => ( - <button - key={sample.query} - type="button" - onClick={() => onPickSample(sample.query)} - title={sample.query} - aria-label={`Use sample question: ${sample.query}`} - className={cn( - floatingControl, - "min-h-10 justify-start px-3 text-left text-xs motion-safe:transition-colors motion-safe:duration-150", - )} - > - <Sparkles className="h-3.5 w-3.5" /> - <span className="min-w-0 truncate">{sample.label}</span> - </button> - ))} - </div> - </section> - </div> - ); -} - -function SourceActionRow({ - viewerHref, - sourceTitle, - documentId, - onScopeDocument, - onFollowUp, - imageCount = 0, - divider = true, -}: { - viewerHref: string; - sourceTitle: string; - documentId: string; - onScopeDocument: (documentId: string) => void; - onFollowUp?: () => void; - imageCount?: number; - divider?: boolean; -}) { - return ( - <div className={cn("flex flex-wrap gap-2", divider && "border-t border-[color:var(--border)] pt-3")}> - <Link href={viewerHref} className={cn(primaryControl, "min-h-[44px] px-4 text-xs")}> - <FileText className="h-4 w-4" /> - Open page - </Link> - {onFollowUp && ( - <button - type="button" - onClick={onFollowUp} - className={cn(floatingControl, "px-3 text-xs")} - aria-label={`Ask a follow-up from ${sourceTitle}`} - > - <Search className="h-4 w-4" /> - <span className="sm:hidden">Follow-up</span> - <span className="hidden sm:inline">Ask follow-up</span> - </button> - )} - <button - type="button" - onClick={() => onScopeDocument(documentId)} - className={cn(floatingControl, "px-3 text-xs")} - aria-label={`Search only ${sourceTitle}`} - > - <Filter className="h-4 w-4" /> - <span className="sm:hidden">Scope</span> - <span className="hidden sm:inline">Use as scope</span> - </button> - {imageCount > 0 && ( - <span className={cn(metadataPill, "min-h-[44px] rounded-lg px-3")}> - {imageCount} indexed image{imageCount === 1 ? "" : "s"} - </span> - )} - </div> - ); -} - -function sourceResultHref(source: SearchResult) { - return `/documents/${source.document_id}?page=${source.page_number ?? 1}&chunk=${source.id}`; -} - -function logSourceOpen(query: string, source: SearchResult) { - if (!query.trim()) return; - void fetch("/api/search/interaction", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - query, - documentId: source.document_id, - chunkId: source.id, - fileName: source.file_name, - title: source.title, - }), - keepalive: true, - }).catch(() => undefined); -} - -function SourcePassageLinks({ - heading, - sources, - compact = false, -}: { - heading: string; - sources: SearchResult[]; - compact?: boolean; -}) { - if (sources.length === 0) return null; - - return ( - <div className="flex flex-wrap items-center gap-1.5"> - {sources.slice(0, compact ? 2 : 3).map((source, index) => ( - <Link - key={`${heading}:${source.id}:${index}`} - href={sourceResultHref(source)} - className={cn( - compact ? metadataPill : floatingControl, - "min-h-8 gap-1.5 px-2.5 text-[11px] sm:min-h-9 sm:px-3", - )} - title={`${source.title} ¡ page ${source.page_number ?? "n/a"} ¡ chunk ${source.chunk_index}`} - aria-label={`Open source passage ${index + 1} for ${heading}`} - > - <ExternalLink className="h-3.5 w-3.5" /> - <span>p.{source.page_number ?? "n/a"}</span> - <span className="hidden sm:inline">chunk {source.chunk_index}</span> - {source.source_strength ? <span className="hidden sm:inline">¡ {source.source_strength}</span> : null} - </Link> - ))} + {warning.titles.length ? ( + <details className="mt-1 font-medium text-[color:var(--text-muted)]"> + <summary className="cursor-pointer">Sources affected</summary> + <span className="mt-1 block">{warning.titles.slice(0, 5).join(", ")}</span> + </details> + ) : null} + </li> + ))} + </ul> + ) : null} </div> ); } +const ragTextFallback = "No usable answer text."; +const sectionTextFallback = "No usable section text."; +const sourceExcerptFallback = "No excerpt available."; + function SourceLinkedAnswer({ sections, fallbackText, @@ -1727,9 +602,7 @@ function SourceLinkedAnswer({ if (sections.length === 0) { return ( - <FormattedAnswerContent - content={parseAnswerDisplayContent(fallbackText || "No usable answer text for this result.", responseMode)} - /> + <FormattedAnswerContent content={parseAnswerDisplayContent(fallbackText || ragTextFallback, responseMode)} /> ); } @@ -2011,7 +884,7 @@ function FormattedAnswerContent({ content }: { content: ParsedAnswerDisplay }) { <div className="min-w-0 space-y-1 font-medium text-[color:var(--text-heading)]"> {line ? <AnswerLineLabel line={line} /> : null} <p className="min-w-0 break-words [overflow-wrap:anywhere]"> - <SafeBoldText text={line?.text ?? "No usable answer text for this result."} /> + <SafeBoldText text={line?.text ?? ragTextFallback} /> </p> </div> </div> @@ -2064,7 +937,7 @@ function SafetyFindingsPanel({ findings }: { findings: ReturnType<typeof extract <SectionHeading icon={ShieldAlert} title="Safety-critical source findings" - description="Items are extracted only from retrieved source text. Verify the linked source before clinical use." + description="Items come from source text. Verify before clinical use." hideDescriptionOnMobile compactMobile /> @@ -2084,7 +957,7 @@ function SafetyFindingsPanel({ findings }: { findings: ReturnType<typeof extract raisedCard, "inline-flex min-h-[44px] items-center gap-1.5 px-3 text-xs font-semibold text-[color:var(--primary)]", )} - aria-label={`Open source for ${formatSafetyFindingLabel(finding)}`} + aria-label={`Open source ${formatSafetyFindingLabel(finding)}`} > <ExternalLink className="h-4 w-4" /> Source @@ -2347,7 +1220,7 @@ function EvidenceSummaryCard({ className={cn(primaryControl, "min-h-[44px] px-3 text-xs")} aria-label={`Open ${sourceLabel.toLowerCase()}: ${formatCitationLabel(bestSource)}`} > - Open page + Open source <ExternalLink className="h-4 w-4" /> </Link> <button @@ -2357,7 +1230,7 @@ function EvidenceSummaryCard({ aria-label={`Search only ${bestSource.title}`} > <Filter className="h-4 w-4" /> - Use as scope + Add scope </button> </div> </article> @@ -2385,7 +1258,7 @@ function EvidenceSummaryCard({ <QueryCoverageChips relevance={relevance} limit={compact ? 3 : 5} /> <p className={cn("text-xs leading-5", textMuted)}> - {supportLabel}. Open the source before using copied text or clinical drafts. + {supportLabel}. Verify source before copying, including any clinical draft text. </p> </aside> ); @@ -2511,12 +1384,12 @@ function EvidenceVerificationStrip({ const retrievalGateBlocked = answer.retrievalDiagnostics?.gateStatus === "blocked"; const checks = [ { - label: "Bottom line cited", - value: citationCount ? `${citationCount} citation${citationCount === 1 ? "" : "s"}` : "No citations", + label: "Citations", + value: citationCount ? `${citationCount} citation${citationCount === 1 ? "" : "s"}` : "None", ready: citationCount > 0, }, { - label: "Sources retrieved", + label: "Sources", value: `${sourceCount} source${sourceCount === 1 ? "" : "s"}`, ready: sourceCount > 0, }, @@ -2531,12 +1404,12 @@ function EvidenceVerificationStrip({ ready: !retrievalGateBlocked, }, { - label: "Gaps reviewed", + label: "Gaps", value: governanceWarningCount ? `${governanceWarningCount} status note${governanceWarningCount === 1 ? "" : "s"}` : gapCount ? `${gapCount} gap${gapCount === 1 ? "" : "s"}` - : "No gaps flagged", + : "None", ready: !weakEvidence && !gapCount && !governanceWarningCount, }, ]; @@ -2591,10 +1464,10 @@ function AnswerViewModeControl({ value: AnswerViewMode; onChange: (mode: AnswerViewMode) => void; }) { - const modes: Array<{ value: AnswerViewMode; label: string; icon: typeof Search }> = [ - { value: "standard", label: "Standard", icon: ListChecks }, - { value: "high_yield", label: "High-yield", icon: Target }, - { value: "evidence_map", label: "Evidence map", icon: BookOpen }, + const modes: Array<{ value: AnswerViewMode; label: string; shortLabel: string; icon: typeof Search }> = [ + { value: "standard", label: "Standard", shortLabel: "All", icon: ListChecks }, + { value: "high_yield", label: "High-yield", shortLabel: "Key", icon: Target }, + { value: "evidence_map", label: "Evidence map", shortLabel: "Map", icon: BookOpen }, ]; return ( @@ -2613,15 +1486,18 @@ function AnswerViewModeControl({ type="button" onClick={() => onChange(mode.value)} aria-pressed={active} + aria-label={`Show ${mode.label.toLowerCase()} answer view`} + title={mode.label} className={cn( - "inline-flex min-h-9 min-w-0 flex-1 basis-[7rem] items-center justify-center gap-1.5 rounded-md px-2.5 text-xs font-semibold transition sm:flex-none sm:basis-auto", + "inline-flex min-h-9 min-w-0 flex-1 basis-[4.75rem] items-center justify-center gap-1.5 rounded-md px-2 text-xs font-semibold transition sm:flex-none sm:basis-auto sm:px-2.5", active ? "bg-[color:var(--primary)] text-white shadow-sm" : "text-[color:var(--text-muted)] hover:bg-[color:var(--surface-subtle)] hover:text-[color:var(--text)]", )} > <Icon className="h-3.5 w-3.5 shrink-0" /> - <span className="truncate">{mode.label}</span> + <span className="truncate sm:hidden">{mode.shortLabel}</span> + <span className="hidden truncate sm:inline">{mode.label}</span> </button> ); })} @@ -2629,6 +1505,16 @@ function AnswerViewModeControl({ ); } +const simpleClinicalTableProps = { + compact: false, + expandOnMobile: true, +} as const; + +function compactEvidenceCell(value: string | null | undefined, max = 140) { + const text = value ? value.replace(/\s+/g, " ").trim() : ""; + return text.length > max ? `${text.slice(0, max - 1).trim()}â€Ļ` : text; +} + function EvidenceMapTable({ rows }: { rows: AnswerEvidenceMapRow[] }) { if (rows.length === 0) { return ( @@ -2640,58 +1526,24 @@ function EvidenceMapTable({ rows }: { rows: AnswerEvidenceMapRow[] }) { ); } + const tableRows = rows.map((row) => [ + compactEvidenceCell(row.section), + row.supportLevel, + String(row.citationCount), + compactEvidenceCell(row.sourceStatus), + compactEvidenceCell(row.bestSourceLabel, 72), + row.bestLinkedPassage || "Open source passage.", + ]); + return ( - <div - data-testid="answer-evidence-map" - className="overflow-x-auto rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)]" - > - <table className="min-w-full border-collapse text-left text-sm"> - <caption className={cn("caption-top px-3 py-2 text-left text-xs font-semibold", textMuted)}> - Source support by answer section - </caption> - <thead> - <tr className="bg-[color:var(--surface-subtle)]"> - {["Answer section", "Support", "Citations", "Source status", "Best linked passage"].map((heading) => ( - <th - key={heading} - scope="col" - className="border-b border-[color:var(--border)] px-3 py-2 align-top text-xs font-semibold text-[color:var(--text)]" - > - {heading} - </th> - ))} - </tr> - </thead> - <tbody> - {rows.map((row) => ( - <tr key={row.id} className="border-t border-[color:var(--border)]/70"> - <td className="max-w-56 px-3 py-2 align-top font-semibold text-[color:var(--text-heading)]"> - {row.section} - </td> - <td className="px-3 py-2 align-top"> - <span className={metadataPill}>{row.supportLevel}</span> - </td> - <td className="px-3 py-2 align-top text-[color:var(--text)]">{row.citationCount}</td> - <td className={cn("max-w-56 px-3 py-2 align-top text-xs leading-5", textMuted)}>{row.sourceStatus}</td> - <td className="min-w-72 px-3 py-2 align-top"> - <p className="text-[13px] font-semibold text-[color:var(--text)]">{row.bestSourceLabel}</p> - <p className={cn("mt-1 line-clamp-3 text-xs leading-5", textMuted)}> - <SafeBoldText text={row.bestLinkedPassage} /> - </p> - {row.href ? ( - <Link - href={row.href} - className="mt-2 inline-flex min-h-8 items-center gap-1.5 rounded-md border border-[color:var(--border)] px-2 text-xs font-semibold text-[color:var(--primary)] hover:bg-[color:var(--primary-soft)]" - > - Open passage - <ExternalLink className="h-3.5 w-3.5" /> - </Link> - ) : null} - </td> - </tr> - ))} - </tbody> - </table> + <div data-testid="answer-evidence-map"> + <AccessibleTable + caption="Source support by answer section" + columns={["Section", "Support level", "Citations", "Evidence status", "Top source", "Passage sample"]} + rows={tableRows} + dialogTitle="Source support by answer section" + {...simpleClinicalTableProps} + /> </div> ); } @@ -2767,7 +1619,7 @@ function QuoteCards({ <EmptyState icon={Quote} title="No exact quotes returned" - body="This answer did not include separate quote cards. Use the answer citations and source passages to verify the source text." + body="No separate quote cards. Verify linked citations and source passages before use." /> ) : ( <div className="grid gap-3 md:grid-cols-2"> @@ -2842,7 +1694,6 @@ function ClinicalOutputPanel({ .filter((section) => section.items.length > 0 || Boolean(section.tables?.length)); const orderedDetailSections = sortClinicalDetailSections(detailSections); const summaryItems = clinicalDetailSummaryItems(orderedDetailSections); - const verifySection = sections.find((section) => section.id === "verify-source"); const title = viewMode === "evidence_map" ? "Evidence map" @@ -2857,7 +1708,7 @@ function ClinicalOutputPanel({ : viewMode === "high_yield" ? "Actions, thresholds, cautions, escalation triggers, monitoring, and dose details." : showLead - ? "Dense source-backed structure for review before clinical use." + ? "Dense source-backed structure for review." : "Adaptive source-backed support below the concise answer."; const content = ( @@ -2965,10 +1816,9 @@ function ClinicalOutputPanel({ markdown={table.markdown} rows={table.rows} columns={table.columns} - compact - expandOnMobile + {...simpleClinicalTableProps} clinicalOnly - dialogTitle={table.caption} + dialogTitle={table.caption || "Clinical table"} /> </div> ))} @@ -2979,7 +1829,7 @@ function ClinicalOutputPanel({ {section.items.map((item, index) => ( <li key={`${section.id}:${index}:${item.slice(0, 48)}`} - className="grid min-h-12 min-w-0 grid-cols-[auto_minmax(0,1fr)] gap-2 rounded-md border border-[color:var(--border)]/70 bg-[color:var(--surface-raised)] px-3 py-2 shadow-[var(--shadow-inset)]" + className="grid min-h-10 min-w-0 grid-cols-[auto_minmax(0,1fr)] gap-2 rounded-md border border-[color:var(--border)]/70 bg-[color:var(--surface-raised)] px-3 py-2 shadow-[var(--shadow-inset)] sm:min-h-9" > <span className={cn("mt-1 h-4 w-1 shrink-0 rounded-full", meta.accentClassName)} @@ -2997,19 +1847,6 @@ function ClinicalOutputPanel({ })} </div> ) : null} - {verifySection ? ( - <div className={cn("mt-3 border-t border-[color:var(--border)] pt-3", textMuted)}> - <div className="flex items-start gap-2.5"> - <Target className="mt-1 h-4 w-4 shrink-0 text-[color:var(--primary)]" /> - <div className="min-w-0"> - <p className="text-xs font-bold uppercase tracking-[0.08em]">Verify source</p> - <p className="mt-1 text-sm leading-6"> - <SafeBoldText text={verifySection.items[0] ?? "Open the cited source passage before clinical use."} /> - </p> - </div> - </div> - </div> - ) : null} </section> ); @@ -3059,9 +1896,16 @@ function SmartFollowUpChips({ answer.queryClass === "comparison" || (answer.documentBreakdown?.length ?? 0) >= 2 || (answer.sources?.length ?? 0) >= 2; - const chips: Array<{ label: string; icon: typeof Search; onClick: () => void; hidden?: boolean }> = [ + const chips: Array<{ + label: string; + shortLabel: string; + icon: typeof Search; + onClick: () => void; + hidden?: boolean; + }> = [ { label: "Show thresholds only", + shortLabel: "Thresholds", icon: Target, hidden: !hasThresholdEvidence, onClick: () => { @@ -3071,6 +1915,7 @@ function SmartFollowUpChips({ }, { label: "Compare sources", + shortLabel: "Compare", icon: BookOpen, hidden: !hasComparisonEvidence, onClick: () => { @@ -3080,11 +1925,13 @@ function SmartFollowUpChips({ }, { label: "Limit to local/current sources", + shortLabel: "Local/current", icon: Filter, onClick: onLimitToLocalCurrent, }, { label: "Search this document only", + shortLabel: "This doc", icon: FileText, hidden: !bestSource, onClick: () => { @@ -3093,12 +1940,14 @@ function SmartFollowUpChips({ }, { label: "Show exact quotes", + shortLabel: "Quotes", icon: Quote, hidden: !answer.quoteCards?.length, onClick: onShowQuotes, }, { label: "Try broader search", + shortLabel: "Broader", icon: SlidersHorizontal, hidden: !weakEvidence, onClick: onTryBroaderSearch, @@ -3120,10 +1969,13 @@ function SmartFollowUpChips({ key={chip.label} type="button" onClick={chip.onClick} + aria-label={chip.label} + title={chip.label} className={cn(floatingControl, "min-h-9 px-2.5 text-xs")} > <Icon className="h-3.5 w-3.5" /> - {chip.label} + <span className="sm:hidden">{chip.shortLabel}</span> + <span className="hidden sm:inline">{chip.label}</span> </button> ); })} @@ -3194,6 +2046,32 @@ function WhyThisMatchedPanel({ sources }: { sources: SearchResult[] }) { ); } +function compactClinicalTableCaption(item: VisualEvidenceCard) { + const raw = item.tableTitle || item.tableLabel || item.caption || "Clinical table"; + const cleaned = sourceTextForCompactDisplay(raw) + .replace(/\btable\s+\d+\s*[:.-]?\s*/i, "") + .replace(/\b(?:page|p\.)\s*\d+\b/gi, "") + .replace(/\s{2,}/g, " ") + .trim(); + const caption = cleaned || "Clinical table"; + return caption.length <= 72 ? caption : `${caption.slice(0, 69).trim()}...`; +} + +function visualEvidenceHeader(item: VisualEvidenceCard) { + const titleSource = [item.tableLabel, item.tableTitle].filter(Boolean).join(" ¡ "); + const titleText = sourceTextForCompactDisplay(titleSource).trim(); + const captionText = sourceTextForCompactDisplay(item.caption ?? "").trim(); + const normalizedTitle = titleText.toLowerCase(); + const normalizedCaption = captionText.toLowerCase(); + const isDuplicateCaption = + Boolean(normalizedCaption) && + (normalizedCaption.startsWith(normalizedTitle) || normalizedCaption === normalizedTitle); + return { + title: titleText || captionText || "Visual evidence", + caption: isDuplicateCaption ? null : captionText, + }; +} + function VisualEvidenceStrip({ evidence, collapsed = false, @@ -3213,7 +2091,7 @@ function VisualEvidenceStrip({ <UtilityDrawer icon={FileImage} title="Nearby visual evidence" - summary="Collapsed because only nearby source support was found." + summary="Nearby source support only." mobileSummary={`${evidence.length} visuals`} > <VisualEvidenceStrip evidence={evidence} embedded /> @@ -3227,16 +2105,12 @@ function VisualEvidenceStrip({ <SectionHeading icon={FileImage} title="Tables and diagrams" - description="Clinical tables, diagrams, and images extracted from indexed source documents." + description="Clinical tables, diagrams, and images from indexed documents." hideDescriptionOnMobile compactMobile /> {evidence.length === 0 ? ( - <EmptyState - icon={FileImage} - title="No indexed images cited" - body="This answer did not cite extracted diagrams or image captions." - /> + <EmptyState icon={FileImage} title="No indexed visuals" body="This answer did not cite any indexed images." /> ) : ( <div className="grid gap-3 md:grid-cols-2"> {evidence.map((item) => { @@ -3246,7 +2120,8 @@ function VisualEvidenceStrip({ ? item.tableTextSnippet : null; const hasStructuredTable = Boolean(tableMarkdown || item.tableRows?.length || item.tableColumns?.length); - const tableCaption = [item.tableTitle, item.caption].filter(Boolean)[0] ?? "Clinical table"; + const tableCaption = compactClinicalTableCaption(item); + const sourceHeader = visualEvidenceHeader(item); const displayLabels = smartEvidenceTags( item.labels, [[item.tableLabel, item.tableTitle].filter(Boolean).join(": "), item.caption, item.tableTextSnippet] @@ -3259,17 +2134,16 @@ function VisualEvidenceStrip({ <SourceImage endpoint={item.signed_url_endpoint} caption={item.caption} /> </div> <figcaption className="mt-2 space-y-1.5 text-[15px] leading-6 text-[color:var(--text)] sm:mt-3"> - {item.tableTitle ? <p className="font-semibold">{item.tableTitle}</p> : null} - {!hasStructuredTable ? <p>{item.caption}</p> : null} + {!hasStructuredTable ? <p className="font-semibold">{sourceHeader.title}</p> : null} + {!hasStructuredTable && sourceHeader.caption ? <p>{sourceHeader.caption}</p> : null} <AccessibleTable caption={tableCaption} markdown={tableMarkdown} rows={item.tableRows} columns={item.tableColumns} - compact - expandOnMobile + {...simpleClinicalTableProps} clinicalOnly - dialogTitle={tableCaption} + dialogTitle={tableCaption || "Clinical table"} /> {!hasStructuredTable && item.tableTextSnippet ? ( <p className={cn("line-clamp-3 text-sm leading-6", textMuted)}>{item.tableTextSnippet}</p> @@ -3290,17 +2164,13 @@ function VisualEvidenceStrip({ clinicalDivider, )} > - {!hasStructuredTable ? ( - <> - <span className={cn("text-[15px] font-semibold leading-6 sm:hidden", textMuted)}> - {formatCompactCitationLabel(item)} - </span> - <span className={cn("hidden text-xs font-semibold leading-5 sm:inline", textMuted)}> - {item.title}, page {item.page_number ?? "n/a"} - </span> - </> - ) : null} - {!hasStructuredTable && item.image_type && ( + <span className={cn("text-[15px] font-semibold leading-6 sm:hidden", textMuted)}> + {formatCompactCitationLabel(item)} + </span> + <span className={cn("hidden text-xs font-semibold leading-5 sm:inline", textMuted)}> + {item.title}, page {item.page_number ?? "n/a"} + </span> + {item.image_type && ( <span className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> {item.image_type.replaceAll("_", " ")} </span> @@ -3308,7 +2178,7 @@ function VisualEvidenceStrip({ {!hasStructuredTable ? <QueryCoverageChips relevance={item.relevance} limit={2} /> : null} <Link href={item.viewer_href} className={cn(floatingControl, "min-h-[44px] px-4 text-xs")}> <ExternalLink className="h-4 w-4" /> - Open page + Open source </Link> </div> </figure> @@ -3383,356 +2253,6 @@ function RelatedDocumentsPanel({ ); } -function SearchFacetDisclosure({ facets }: { facets?: SearchFacets | null }) { - const [expanded, setExpanded] = useState(false); - if (!facets) return null; - const chips = [ - ...(facets.status ?? []).map((facet) => ({ ...facet, prefix: "status" })), - ...(facets.documentTypes ?? []).map((facet) => ({ ...facet, prefix: "type" })), - ...(facets.sections ?? []).map((facet) => ({ ...facet, prefix: "section" })), - ...(facets.evidence ?? []).map((facet) => ({ ...facet, prefix: "evidence" })), - ].slice(0, 14); - if (chips.length === 0) return null; - return ( - <div className="w-fit max-w-full"> - <button - type="button" - onClick={() => setExpanded((current) => !current)} - aria-expanded={expanded} - className={cn( - metadataPill, - "min-h-8 cursor-pointer list-none gap-1.5 px-2.5 text-[11px] transition hover:border-[color:var(--border-strong)] hover:text-[color:var(--text)]", - )} - > - <Filter className="h-3.5 w-3.5" /> - Result filters - <span className="text-[color:var(--text-soft)]">({chips.length})</span> - <ChevronDown className={cn("h-3.5 w-3.5 transition", expanded && "rotate-180")} /> - </button> - {expanded ? ( - <div className="mt-2 flex max-w-3xl flex-wrap gap-1.5 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-2"> - {chips.map((facet) => ( - <span key={`${facet.prefix}:${facet.value}`} className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> - {facet.prefix}: {facet.value} ({facet.count}) - </span> - ))} - </div> - ) : null} - </div> - ); -} - -const documentFacetIcons: Record<SmartDocumentTagGroup, typeof Tag> = { - Medication: Target, - Risk: ShieldAlert, - Workflow: ListChecks, - Topic: Tag, - Population: FileText, - Setting: FileText, - Service: Sparkles, - "Document type": FileText, - Manual: Sparkles, -}; - -function DocumentTagFacetRail({ - groups, - activeKeys, - onToggle, - onClear, -}: { - groups: Array<{ group: SmartDocumentTagGroup; facets: SmartDocumentTagFacet[] }>; - activeKeys: string[]; - onToggle: (facet: SmartDocumentTagFacet) => void; - onClear: () => void; -}) { - if (groups.length === 0) return null; - const active = new Set(activeKeys); - - return ( - <aside - aria-label="Document tag filters" - className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-3" - > - <div className="flex flex-wrap items-center justify-between gap-2"> - <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]">Tag facets</p> - {activeKeys.length > 0 ? ( - <button type="button" onClick={onClear} className={cn(floatingControl, "min-h-8 px-2 text-[11px]")}> - <X className="h-3.5 w-3.5" /> - Clear - </button> - ) : null} - </div> - <div className="mt-3 grid gap-3 lg:grid-cols-2 xl:grid-cols-3"> - {smartDocumentFacetGroups - .map((group) => groups.find((item) => item.group === group)) - .filter((group): group is { group: SmartDocumentTagGroup; facets: SmartDocumentTagFacet[] } => Boolean(group)) - .map(({ group, facets }) => { - const Icon = documentFacetIcons[group]; - return ( - <section key={group} className="min-w-0"> - <h3 className="flex items-center gap-1.5 text-[11px] font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> - <Icon className="h-3.5 w-3.5 text-[color:var(--primary)]" /> - {group} - </h3> - <div className="mt-2 flex flex-wrap gap-1.5"> - {facets.map((facet) => { - const selected = active.has(facet.key); - return ( - <button - key={facet.key} - type="button" - onClick={() => onToggle(facet)} - aria-pressed={selected} - title={`Filter to ${facet.label}`} - className={cn( - "inline-flex min-h-7 max-w-full items-center gap-1 rounded-md border px-2 text-[11px] font-semibold shadow-[var(--shadow-inset)] transition", - selected - ? "border-[color:var(--primary)]/35 bg-[color:var(--primary-soft)] text-[color:var(--primary)]" - : "border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] text-[color:var(--text-muted)] hover:border-[color:var(--border-strong)] hover:text-[color:var(--text)]", - )} - > - <span className="truncate">{facet.label}</span> - <span className="rounded bg-[color:var(--surface)] px-1 text-[10px] text-[color:var(--text-soft)]"> - {facet.count} - </span> - </button> - ); - })} - </div> - </section> - ); - })} - </div> - </aside> - ); -} - -function MatchExplanationChips({ source }: { source: SearchResult }) { - const explanation = source.match_explanation; - const reasons = explanation?.reasons?.length - ? explanation.reasons - : [ - source.score_explanation?.titleBoost ? "title" : "", - source.score_explanation?.textRank ? "text" : "", - source.score_explanation?.vectorScore ? "vector" : "", - source.source_metadata?.document_status ? `status:${source.source_metadata.document_status}` : "", - ].filter(Boolean); - const score = source.score_explanation?.finalScore ?? source.hybrid_score ?? source.similarity; - const chips = [ - ...reasons.slice(0, 5), - Number.isFinite(score) ? `score:${Number(score).toFixed(2)}` : "", - explanation?.indexQualityScore !== undefined && explanation.indexQualityScore !== null - ? `index:${Number(explanation.indexQualityScore).toFixed(2)}` - : "", - explanation?.indexQualityIssues?.length ? "index warning" : "", - explanation?.tableHit ? "table fact" : "", - explanation?.indexUnitType ? `unit:${explanation.indexUnitType.replaceAll("_", " ")}` : "", - ].filter(Boolean); - if (chips.length === 0) return null; - return ( - <div className="mt-2 flex flex-wrap gap-1.5"> - {chips.slice(0, 7).map((chip) => ( - <span key={chip} className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> - {chip} - </span> - ))} - </div> - ); -} - -function DocumentSearchResultsPanel({ - matches, - query, - loading, - documentCount, - realDataReady, - authUnavailable, - apiUnavailable, - setupWarning, - relevance, - facets, - onScopeDocument, - onAnswerFromDocument, - onTagSearch, -}: { - matches: DocumentMatch[]; - query: string; - loading: boolean; - documentCount: number; - realDataReady: boolean; - authUnavailable: boolean; - apiUnavailable: boolean; - setupWarning: string | null; - relevance?: EvidenceRelevance | null; - facets?: SearchFacets | null; - onScopeDocument: (documentId: string) => void; - onAnswerFromDocument: (documentId: string) => void; - onTagSearch: (tag: SmartDocumentTag | SmartDocumentTagFacet) => void; -}) { - const trimmedQuery = query.trim(); - const [activeFacetState, setActiveFacetState] = useState<{ query: string; keys: string[] }>({ query: "", keys: [] }); - const activeFacetKeys = useMemo( - () => (activeFacetState.query === query ? activeFacetState.keys : []), - [activeFacetState, query], - ); - const tagFacetGroups = useMemo(() => buildSmartDocumentTagFacets(matches, { query }), [matches, query]); - const visibleMatches = useMemo( - () => filterDocumentsBySmartTagFacets(matches, activeFacetKeys), - [matches, activeFacetKeys], - ); - - function toggleTagFacet(facet: SmartDocumentTagFacet) { - setActiveFacetState((current) => { - const keys = current.query === query ? current.keys : []; - return { - query, - keys: keys.includes(facet.key) ? keys.filter((key) => key !== facet.key) : [...keys, facet.key], - }; - }); - } - - if (loading) return <LoadingPanel label="Finding matching documents" />; - - if (apiUnavailable || !realDataReady || authUnavailable) { - return ( - <EmptyState - icon={AlertCircle} - title="Document search unavailable" - body={ - apiUnavailable - ? "The local API is unavailable. Check the app server before searching documents." - : authUnavailable - ? "Sign in or enable local no-auth mode before listing private indexed documents." - : setupWarning || "Complete the search setup before using Documents mode." - } - /> - ); - } - - if (matches.length === 0) { - if (documentCount === 0) { - return ( - <EmptyState - icon={FileText} - title="No indexed documents" - body="Upload and index source documents before using Documents mode." - /> - ); - } - - if (!trimmedQuery) { - return ( - <EmptyState - icon={FileText} - title="Search documents" - body="Enter a clinical topic, medication, workflow, or policy name to list matching source documents." - /> - ); - } - - return ( - <EmptyState - icon={FileText} - title="No matching documents" - body={`No indexed documents matched "${trimmedQuery}". Try a medication, acronym, policy name, or workflow term.`} - /> - ); - } - - return ( - <div className="space-y-3"> - <div className="flex flex-wrap items-center gap-2"> - <div className={cn(metadataPill, "nums inline-flex min-h-8 w-fit max-w-full flex-wrap gap-x-1.5 leading-5")}> - {matches.length} document match{matches.length === 1 ? "" : "es"} for "{query.trim()}" - </div> - {relevance ? <RelevanceBadge relevance={relevance} /> : null} - </div> - <SearchFacetDisclosure facets={facets} /> - <DocumentTagFacetRail - groups={tagFacetGroups} - activeKeys={activeFacetKeys} - onToggle={toggleTagFacet} - onClear={() => setActiveFacetState({ query, keys: [] })} - /> - {activeFacetKeys.length > 0 ? ( - <div className={cn(metadataPill, "min-h-8 w-fit max-w-full text-[11px]")}> - {visibleMatches.length} result{visibleMatches.length === 1 ? "" : "s"} after tag filters - </div> - ) : null} - <div className="grid gap-3"> - {visibleMatches.length === 0 ? ( - <div className={cn(panelSubtle, "p-4 text-sm font-semibold text-[color:var(--text-muted)]")}> - No document matches include all selected tag facets. - </div> - ) : null} - {visibleMatches.map((document) => ( - <article key={document.document_id} className={cn(sourceCard, "p-3 sm:p-4")}> - <div className="grid gap-3 lg:grid-cols-[minmax(0,1fr)_auto] lg:items-start"> - <div className="min-w-0"> - <Link - href={`/documents/${document.document_id}?page=${document.bestPages[0] ?? 1}&chunk=${document.bestChunkIds[0] ?? ""}`} - className="inline-flex min-h-[44px] items-center text-base font-semibold text-[color:var(--text-heading)] transition hover:text-[color:var(--primary)]" - > - <span className="line-clamp-2">{document.title}</span> - </Link> - <p className={cn("text-xs leading-5", textMuted)}> - {document.file_name} ¡ pages {document.bestPages.join(", ") || "n/a"} ¡ {document.tableCount} tables ¡{" "} - {document.imageCount} images - </p> - <p className={cn("mt-1 text-xs leading-5", textMuted)}>{document.matchReason}</p> - <div className="mt-2"> - <QueryCoverageChips relevance={document.relevance} /> - </div> - </div> - <div className="flex flex-wrap items-center gap-2"> - <RelevanceBadge relevance={document.relevance} /> - <Link - href={`/documents/${document.document_id}?page=${document.bestPages[0] ?? 1}&chunk=${document.bestChunkIds[0] ?? ""}`} - className={cn(floatingControl, "min-h-[44px] px-3 text-xs")} - aria-label={`Open ${document.title}`} - > - <ExternalLink className="h-4 w-4" /> - Open - </Link> - <button - type="button" - onClick={() => onScopeDocument(document.document_id)} - className={cn(floatingControl, "min-h-[44px] px-3 text-xs")} - aria-label={`Scope search to ${document.title}`} - > - <Filter className="h-4 w-4" /> - Scope - </button> - <button - type="button" - onClick={() => onAnswerFromDocument(document.document_id)} - className={cn(primaryControl, "min-h-[44px] rounded-lg px-3 text-xs")} - aria-label={`Answer from ${document.title}`} - > - <Sparkles className="h-4 w-4" /> - Answer - </button> - </div> - </div> - {document.summarySnippet && ( - <p className={cn("mt-2 line-clamp-3 text-[15px] leading-6", textMuted)}> - <SafeBoldText text={document.summarySnippet} /> - </p> - )} - <DocumentTagCloud - labels={document.labels} - query={query} - limit={4} - className="mt-3" - onTagClick={onTagSearch} - /> - </article> - ))} - </div> - </div> - ); -} - const displayJsonArtifactPattern = /"?(answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)"?\s*:\s*/i; @@ -3858,13 +2378,29 @@ function sectionBodyContent(body: string, responseMode?: RagAnswer["responseMode const normalized = sanitizeAnswerDisplayText(body, { minLength: 8, minTokens: 2 }); if (!normalized) { return { - ...parseAnswerDisplayContent("No usable section text available.", responseMode), + ...parseAnswerDisplayContent(sectionTextFallback, responseMode), safe: false, }; } return { ...parseAnswerDisplayContent(normalized, responseMode), safe: true }; } +function sourceDisplayTitle(source: SearchResult) { + return source.title + .replace(/^Synthetic /, "") + .replace(/\.pdf$/i, "") + .trim(); +} + +function sourceDisplayMeta(source: SearchResult, title: string) { + const fileBase = source.file_name?.replace(/\.pdf$/i, "").trim(); + const titleBase = title.toLowerCase(); + const fileBaseNormalized = (fileBase ?? "").toLowerCase(); + const includeFile = + Boolean(fileBase) && fileBaseNormalized !== titleBase && !fileBaseNormalized.startsWith(titleBase); + return [includeFile ? source.file_name : null, `page ${source.page_number ?? "n/a"}`].filter(Boolean).join(" ¡ "); +} + function SourceList({ sources, query, @@ -3886,14 +2422,9 @@ function SourceList({ <article key={source.id} className={cn(sourceCard, "overflow-hidden p-0")}> {(() => { const snippet = compactSourceSnippet(source.content); - const fallback = "No high-yield excerpt available for this passage."; - const sourceTitle = source.title.replace(/^Synthetic /, "").replace(/\.pdf$/i, ""); - const fileBase = source.file_name.replace(/\.pdf$/i, "").toLowerCase(); - const titleBase = sourceTitle.toLowerCase(); - const showFileName = fileBase !== titleBase && !fileBase.startsWith(titleBase); - const sourceMeta = [showFileName ? source.file_name : null, `page ${source.page_number ?? "n/a"}`] - .filter(Boolean) - .join(" ¡ "); + const fallback = sourceExcerptFallback; + const sourceTitle = sourceDisplayTitle(source); + const sourceMeta = sourceDisplayMeta(source, sourceTitle); const tableFacts = (source.table_facts ?? []) .slice(0, 3) .map(compactTableFact) @@ -3925,10 +2456,10 @@ function SourceList({ href={sourceResultHref(source)} onClick={() => logSourceOpen(query, source)} className={cn(floatingControl, "min-h-[44px] px-3 text-xs")} - aria-label={`Open source page for ${source.title}`} + aria-label={`Open source for ${source.title}`} > <ExternalLink className="h-4 w-4" /> - Open page + Open source </Link> <button type="button" @@ -3937,7 +2468,7 @@ function SourceList({ aria-label={`Scope search to ${source.title}`} > <Filter className="h-4 w-4" /> - Use as scope + Add scope </button> </div> </div> @@ -3946,14 +2477,12 @@ function SourceList({ <span className="grid h-7 w-7 place-items-center rounded-lg bg-[color:var(--surface)] text-[color:var(--primary)] ring-1 ring-[color:var(--primary)]/20"> <Quote className="h-4 w-4" /> </span> - <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--primary)]"> - Relevant excerpt - </p> + <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--primary)]">Excerpt</p> </div> <p className={cn( proseMeasure, - "border-l-4 border-[color:var(--primary)] pl-3 break-words [overflow-wrap:anywhere]", + "line-clamp-3 border-l-4 border-[color:var(--primary)] pl-3 break-words [overflow-wrap:anywhere]", )} > {snippet ? <SafeBoldText text={snippet} /> : <span className="italic">{fallback}</span>} @@ -3962,7 +2491,7 @@ function SourceList({ {tableFacts.length ? ( <div className="border-t border-[color:var(--border)] px-3 py-3 sm:px-4"> <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> - Structured table matches + Structured matches </p> <div className="mt-2 grid gap-2 sm:grid-cols-2"> {tableFacts.map((fact) => ( @@ -3990,33 +2519,6 @@ function SourceList({ ); } -function AnswerSkeleton() { - return ( - <div className="space-y-4" aria-label="Loading answer"> - <div className="space-y-3 rounded-lg border border-[color:var(--primary)]/20 bg-[color:var(--primary-soft)]/45 p-4"> - <div className="h-4 w-10/12 animate-pulse rounded bg-[color:var(--surface-subtle)]" /> - <div className="h-4 w-full animate-pulse rounded bg-[color:var(--surface-subtle)]" /> - <div className="h-4 w-8/12 animate-pulse rounded bg-[color:var(--surface-subtle)]" /> - <div className={cn(sourceCard, "mt-4 flex min-h-[60px] items-center justify-between gap-3 p-3")}> - <div className="min-w-0 flex-1 space-y-2"> - <div className="h-3 w-24 animate-pulse rounded bg-[color:var(--surface-subtle)]" /> - <div className="h-4 w-48 max-w-full animate-pulse rounded bg-[color:var(--surface-subtle)]" /> - </div> - <div className="h-[44px] w-20 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> - </div> - </div> - <div className="flex flex-wrap gap-2"> - <div className="h-[44px] w-48 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> - <div className="h-[44px] w-40 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> - </div> - <div className="grid gap-3 sm:grid-cols-2"> - <div className="h-28 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> - <div className="hidden h-28 animate-pulse rounded-lg bg-[color:var(--surface-subtle)] sm:block" /> - </div> - </div> - ); -} - function StagedAnswerResultSurface({ answer, query, @@ -4082,20 +2584,6 @@ function StagedAnswerResultSurface({ <div className="min-w-0 space-y-4 motion-safe:animate-fade-up sm:space-y-5" data-dashboard-stage="answer-surface"> <div className={cn(answerSurface, "space-y-3 p-2.5 sm:p-3")}> <NaturalLanguageAnswer text={safeAnswerText || answer.answer} /> - <AnswerInsightBar - answer={answer} - bestSource={bestSource} - relevance={currentRelevance} - queryMode={queryMode} - sourceGovernanceWarnings={sourceGovernanceWarnings} - /> - <EvidenceVerificationStrip - answer={answer} - bestSource={bestSource} - sourceSummary={sourceSummary} - weakEvidence={weakEvidence} - governanceWarningCount={groupedGovernanceWarningCount} - /> <ClinicalOutputPanel answer={answer} @@ -4121,8 +2609,23 @@ function StagedAnswerResultSurface({ title="Evidence & sources" summary={evidenceDrawerSummary({ answer, bestSource, sourceSummary, gaps })} mobileSummary="Evidence" + mobileInline > <div className="space-y-3"> + <AnswerInsightBar + answer={answer} + bestSource={bestSource} + relevance={currentRelevance} + queryMode={queryMode} + sourceGovernanceWarnings={sourceGovernanceWarnings} + /> + <EvidenceVerificationStrip + answer={answer} + bestSource={bestSource} + sourceSummary={sourceSummary} + weakEvidence={weakEvidence} + governanceWarningCount={groupedGovernanceWarningCount} + /> <EvidenceSummaryCard answer={answer} bestSource={bestSource} @@ -5186,6 +3689,7 @@ function SetupChecklist({ checks }: { checks: SetupCheck[] }) { type LibraryHealthTarget = "documents" | "setup" | "indexing" | "failures"; type DocumentDrawerStatusFilter = "all" | "indexed" | "indexing" | "failed"; type IndexingMonitorFilter = "all" | "active" | "failed"; +type UploadIndexingTab = "setup" | "upload" | "jobs"; function documentStatusMatchesFilter(document: ClinicalDocument, filter: DocumentDrawerStatusFilter) { if (filter === "all") return true; @@ -5276,7 +3780,7 @@ function LibraryHealthStrip({ type="button" onClick={() => onSelectTarget?.(item.target)} className={cn( - "rounded-md border px-2.5 py-2 text-left transition hover:-translate-y-px hover:shadow-[var(--shadow-soft)] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--ring)] active:translate-y-0", + "rounded-md border px-2.5 py-2 text-left transition hover:-translate-y-px hover:shadow-[var(--shadow-soft)] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)] active:translate-y-0", item.tone, )} aria-label={item.actionLabel} @@ -5301,6 +3805,7 @@ function UtilityDrawer({ open: controlledOpen, onOpenChange, className, + mobileInline = false, }: { id?: string; title: string; @@ -5312,6 +3817,7 @@ function UtilityDrawer({ open?: boolean; onOpenChange?: (open: boolean) => void; className?: string; + mobileInline?: boolean; }) { const [uncontrolledOpen, setUncontrolledOpen] = useState(defaultOpen); const [usesSheet, setUsesSheet] = useState(false); @@ -5342,6 +3848,7 @@ function UtilityDrawer({ className={cn( "group flex min-h-[56px] w-full cursor-pointer list-none items-center justify-between gap-3 rounded-lg px-4 py-3 text-left transition motion-safe:duration-150 hover:bg-[color:var(--surface-subtle)] sm:hidden", panelSubtle, + mobileInline && "hidden", className, )} > @@ -5361,12 +3868,12 @@ function UtilityDrawer({ <details id={id} - open={open && !usesSheet} + open={open && (!usesSheet || mobileInline)} onToggle={(event) => { const nextOpen = event.currentTarget.open; if (nextOpen !== open) setOpen(nextOpen); }} - className={cn("group hidden sm:block", panelSubtle, className)} + className={cn("group", mobileInline ? "block" : "hidden sm:block", panelSubtle, className)} > <summary className="flex min-h-[56px] cursor-pointer list-none items-center justify-between gap-3 px-4 py-3 transition motion-safe:duration-150 hover:bg-[color:var(--surface-subtle)]"> <span className="flex min-w-0 items-center gap-3"> @@ -5380,11 +3887,11 @@ function UtilityDrawer({ </span> <ChevronDown className="h-4 w-4 shrink-0 text-[color:var(--text-muted)] transition motion-safe:duration-150 group-open:rotate-180" /> </summary> - {open && !usesSheet && <div className={cn(clinicalDivider, "p-4")}>{children}</div>} + {open && (!usesSheet || mobileInline) && <div className={cn(clinicalDivider, "p-4")}>{children}</div>} </details> <Sheet - open={usesSheet && open} + open={usesSheet && open && !mobileInline} onClose={() => setOpen(false)} title={title} description={mobileSummary ?? summary} @@ -5749,7 +4256,7 @@ function MobileSectionFab({ const guideSections = [ { title: "Ask and verify", - body: "Ask a focused guideline question, then verify the answer against linked citations and source passages before clinical use.", + body: "Ask a focused guideline question, then verify linked citations and source passages before use.", }, { title: "Top source and citations", @@ -5761,7 +4268,7 @@ const guideSections = [ }, { title: "Quotes, images, sources", - body: "The bottom nav jumps to exact quotes, extracted diagrams, and source passages. Empty sections simply mean none were cited.", + body: "Bottom nav jumps to quotes, diagrams, and source passages. Empty sections had no citations.", }, { title: "Upload and indexing", @@ -5941,6 +4448,7 @@ export function ClinicalDashboard() { const [guideOpen, setGuideOpen] = useState(false); const [documentsDrawerOpen, setDocumentsDrawerOpen] = useState(false); const [uploadDrawerOpen, setUploadDrawerOpen] = useState(false); + const [uploadMobileTab, setUploadMobileTab] = useState<UploadIndexingTab>("upload"); const [documentDrawerStatusFilter, setDocumentDrawerStatusFilter] = useState<DocumentDrawerStatusFilter>("indexed"); const [indexingMonitorFilter, setIndexingMonitorFilter] = useState<IndexingMonitorFilter>("all"); const [scopeMenuOpen, setScopeMenuOpen] = useState(false); @@ -5974,12 +4482,15 @@ export function ClinicalDashboard() { setDocumentDrawerStatusFilter("indexed"); setDocumentsDrawerOpen(true); } else if (target === "indexing") { + setUploadMobileTab("jobs"); setIndexingMonitorFilter("active"); setUploadDrawerOpen(true); } else if (target === "failures") { + setUploadMobileTab("jobs"); setIndexingMonitorFilter("failed"); setUploadDrawerOpen(true); } else { + setUploadMobileTab("setup"); setIndexingMonitorFilter("all"); setUploadDrawerOpen(true); } @@ -6592,7 +5103,7 @@ export function ClinicalDashboard() { const trimmedQuery = query.trim(); if (!trimmedQuery) return; if (!canRunSearch) { - setError("Search setup is not ready."); + setError("Search setup not ready."); return; } @@ -6689,7 +5200,7 @@ export function ClinicalDashboard() { const trimmedSearchText = searchText.trim(); if (!trimmedSearchText) return; if (!canRunSearch) { - setError("Search setup is not ready."); + setError("Search setup not ready."); return; } @@ -7088,6 +5599,50 @@ export function ClinicalDashboard() { </p> </UtilityDrawer> ); + const setupReadyCount = setupChecks.filter((check) => check.status === "ready").length; + const setupCheckCount = setupChecks.length || fallbackSetupChecks.length; + const activeUploadWork = + jobs.filter((job) => job.status === "pending" || job.status === "processing").length + + batches.filter((batch) => batch.status === "queued" || batch.status === "processing").length; + const failedUploadWork = + jobs.filter((job) => job.status === "failed").length + batches.filter((batch) => batch.status === "failed").length; + const uploadTabs: Array<{ + id: UploadIndexingTab; + label: string; + summary: string; + panelId: string; + icon: typeof UploadCloud; + }> = [ + { + id: "setup", + label: "Setup", + summary: `${setupReadyCount}/${setupCheckCount} ready`, + panelId: "dashboard-setup-section", + icon: ListChecks, + }, + { + id: "upload", + label: "Upload", + summary: uploadReadOnlyMode || !canUsePrivateApis ? "Locked" : "Ready", + panelId: "dashboard-upload-section", + icon: UploadCloud, + }, + { + id: "jobs", + label: "Jobs", + summary: activeUploadWork + ? `${activeUploadWork} active` + : failedUploadWork + ? `${failedUploadWork} failed` + : "Idle", + panelId: "dashboard-indexing-section", + icon: RefreshCw, + }, + ]; + const handleUploadQueued = () => { + setUploadMobileTab("jobs"); + void refresh({ includeSetup: false, includeDashboardData: true, includeDocumentMeta: false }); + }; return ( <div @@ -7119,6 +5674,7 @@ export function ClinicalDashboard() { onScopeOpenChange={setScopeMenuOpen} onOpenGuide={openGuide} onToggleTheme={toggleTheme} + queryModeOptions={clinicalQueryModeOptions} /> <main @@ -7159,7 +5715,7 @@ export function ClinicalDashboard() { title={searchMode === "answer" ? "Answer" : "Document matches"} description={ searchMode === "answer" - ? "Sourced synthesis with quotes, PDFs, and indexed diagrams." + ? "Sourced synthesis with quotes, PDFs, and diagrams." : "Natural-language document search across indexed guideline titles, labels, summaries, and passages." } testId="answer-section-heading" @@ -7355,24 +5911,80 @@ export function ClinicalDashboard() { loading={dashboardDataLoading} onSelectTarget={openLibraryHealthTarget} /> + <div + role="tablist" + aria-label="Upload and indexing sections" + className="grid grid-cols-3 gap-2 lg:hidden" + > + {uploadTabs.map((tab) => { + const active = uploadMobileTab === tab.id; + const Icon = tab.icon; + return ( + <button + key={tab.id} + type="button" + role="tab" + aria-selected={active} + aria-controls={tab.panelId} + onClick={() => setUploadMobileTab(tab.id)} + className={cn( + "min-h-[56px] rounded-lg border px-2.5 py-2 text-left transition focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)] active:translate-y-px", + active + ? "border-[color:var(--primary)] bg-[color:var(--primary-soft)] text-[color:var(--primary)] shadow-[var(--glow-soft)]" + : "border-[color:var(--border)] bg-[color:var(--surface)] text-[color:var(--text-muted)] hover:bg-[color:var(--surface-subtle)]", + )} + > + <span className="flex items-center gap-1.5 text-xs font-bold"> + <Icon className="h-3.5 w-3.5" /> + {tab.label} + </span> + <span className="mt-1 block truncate text-[11px] font-semibold opacity-80">{tab.summary}</span> + </button> + ); + })} + </div> <div className="grid gap-4 lg:grid-cols-2"> - <div id="dashboard-setup-section" className="space-y-3 scroll-mt-4"> + <div + id="dashboard-setup-section" + role="tabpanel" + aria-label="Setup" + className={cn( + "space-y-3 scroll-mt-4 lg:col-start-1 lg:row-start-1", + uploadMobileTab !== "setup" && "hidden lg:block", + )} + > <p className={cn("text-xs font-bold uppercase tracking-[0.08em]", textMuted)}> Developer setup status </p> <SetupChecklist checks={setupChecks} /> {showAuthPanel && <AuthPanel />} - <p className={cn("pt-1 text-xs font-bold uppercase tracking-[0.08em]", textMuted)}>Clinical upload</p> + </div> + <div + id="dashboard-upload-section" + role="tabpanel" + aria-label="Upload" + className={cn( + "space-y-3 scroll-mt-4 lg:col-start-1 lg:row-start-2", + uploadMobileTab !== "upload" && "hidden lg:block", + )} + > + <p className={cn("text-xs font-bold uppercase tracking-[0.08em]", textMuted)}>Clinical upload</p> <UploadPanel - onUploaded={() => { - void refresh({ includeSetup: false, includeDashboardData: true, includeDocumentMeta: false }); - }} + onUploaded={handleUploadQueued} demoMode={uploadReadOnlyMode} canUpload={canUsePrivateApis} authorizationHeader={authorizationHeader} /> </div> - <div id="dashboard-indexing-section" className="space-y-3 scroll-mt-4"> + <div + id="dashboard-indexing-section" + role="tabpanel" + aria-label="Jobs" + className={cn( + "space-y-3 scroll-mt-4 lg:col-start-2 lg:row-span-2 lg:row-start-1", + uploadMobileTab !== "jobs" && "hidden lg:block", + )} + > <p className={cn("text-xs font-bold uppercase tracking-[0.08em]", textMuted)}>Indexing progress</p> <IndexingMonitor jobs={jobs} diff --git a/src/components/DashboardFloatingFab.tsx b/src/components/DashboardFloatingFab.tsx index f73afcd8f..c2d119ed8 100644 --- a/src/components/DashboardFloatingFab.tsx +++ b/src/components/DashboardFloatingFab.tsx @@ -2,14 +2,7 @@ import Link from "next/link"; import { useCallback, useRef, useState } from "react"; -import { - ArrowUp, - ClipboardCopy, - ExternalLink, - Plus, - Search, - X, -} from "lucide-react"; +import { ArrowUp, ClipboardCopy, ExternalLink, Plus, Search, X } from "lucide-react"; import { cn, floatingControl } from "@/components/ui-primitives"; export function DashboardFloatingFab() { @@ -45,7 +38,7 @@ export function DashboardFloatingFab() { const handleFocusSearch = useCallback(() => { const candidates = document.querySelectorAll<HTMLInputElement>( - "input[aria-label*=\"search\" i], input[aria-label*=\"Search\" i], input[placeholder*=\"search\" i], input[type='search']", + 'input[aria-label*="search" i], input[aria-label*="Search" i], input[placeholder*="search" i], input[type=\'search\']', ); const target = candidates[0]; if (target) { @@ -122,7 +115,11 @@ export function DashboardFloatingFab() { <span className="sr-only">{open ? "Close quick actions" : "Open quick actions"}</span> </button> </div> - {copyNotice && <p className="sr-only" aria-live="polite">{copyNotice}</p>} + {copyNotice && ( + <p className="sr-only" aria-live="polite"> + {copyNotice} + </p> + )} </div> ); } diff --git a/src/components/DocumentViewer.tsx b/src/components/DocumentViewer.tsx index 89091265c..54d1bf43f 100644 --- a/src/components/DocumentViewer.tsx +++ b/src/components/DocumentViewer.tsx @@ -18,6 +18,7 @@ import { Loader2, Maximize2, Minus, + MoreHorizontal, Plus, Quote, RefreshCw, @@ -60,6 +61,7 @@ import { isLocalNoAuthMode } from "@/lib/env"; import { useAuthSession } from "@/lib/supabase/client"; import { SafeBoldText } from "@/components/SafeBoldText"; import { DocumentManagementActions } from "@/components/DocumentManagementActions"; +import { Sheet } from "@/components/ui/sheet"; import type { ClinicalDocument, ClinicalDocumentSummaryProfile, @@ -317,6 +319,8 @@ function DocumentImage({ image }: { image: ImageRow }) { ? image.tableTextSnippet : null; const hasStructuredTable = Boolean(tableMarkdown || image.tableRows?.length || image.tableColumns?.length); + const tableCaption = tableHeading || image.caption || "Document table"; + const showImageCaptionLine = image.caption && image.caption !== tableCaption; const displayLabels = smartEvidenceTags( image.labels, [tableHeading, image.caption, image.tableTextSnippet].filter(Boolean).join(" "), @@ -368,14 +372,16 @@ function DocumentImage({ image }: { image: ImageRow }) { )} </div> <figcaption className="mt-3 space-y-2 text-[15px] leading-6 text-[color:var(--text)]"> - {tableHeading && <p className="font-semibold">{tableHeading}</p>} - <p>{image.caption}</p> + {tableHeading ? <p className="font-semibold">{tableHeading}</p> : null} + {showImageCaptionLine ? <p className={textMuted}>{image.caption}</p> : null} <AccessibleTable - caption={tableHeading || image.caption} + caption={tableCaption} markdown={tableMarkdown} rows={image.tableRows} columns={image.tableColumns} - compact + compact={false} + expandOnMobile + dialogTitle={tableCaption} /> {!hasStructuredTable && image.tableTextSnippet ? ( <p className={cn("text-sm leading-6", textMuted)}>{image.tableTextSnippet}</p> @@ -522,6 +528,31 @@ function PinnedSourceEvidence({ ? [`Page ${chunk.page_number ?? "n/a"}`, `chunk ${chunk.chunk_index}`].filter(Boolean).join(" ¡ ") : ""; + if (!loading && !chunk) { + return ( + <section + id={sectionId} + data-testid="pinned-source-evidence" + className={cn( + sourceCard, + "scroll-mt-24 border-[color:var(--primary)]/20 bg-[color:var(--surface-raised)] p-3 shadow-[var(--shadow-tight)]", + )} + > + <div className="flex items-start gap-3"> + <span className="grid h-9 w-9 shrink-0 place-items-center rounded-lg border border-[color:var(--primary)]/20 bg-[color:var(--primary-soft)] text-[color:var(--primary)]"> + <Quote className="h-4 w-4" /> + </span> + <div className="min-w-0"> + <p className="text-sm font-semibold text-[color:var(--text)]">Source evidence</p> + <p className={cn("mt-1 text-xs leading-5", textMuted)}> + Open a cited answer passage to pin the exact indexed excerpt here. + </p> + </div> + </div> + </section> + ); + } + return ( <section id={sectionId} @@ -564,7 +595,7 @@ function PinnedSourceEvidence({ <div className="flex flex-wrap gap-2"> <a href="#pdf-preview-section" className={cn(primaryButton, "min-h-9 px-3 text-xs")}> <ExternalLink className="h-4 w-4" /> - Open page + Open source </a> {compact && isLong ? ( <button @@ -647,7 +678,16 @@ function IndexedSourceText({ } if (block.type === "table") { - return <AccessibleTable key={block.id} caption={block.caption} rows={block.rows} compact={compact} />; + return ( + <AccessibleTable + key={block.id} + caption={block.caption} + rows={block.rows} + compact={false} + expandOnMobile + dialogTitle={block.caption ?? "Document table"} + /> + ); } return ( @@ -925,7 +965,7 @@ function IndexedTextPanel({ <div className="mt-3 flex flex-wrap gap-2 border-t border-[color:var(--border)] pt-3"> <a href="#pdf-preview-section" className={cn(secondaryButton, "min-h-9 px-3 text-xs")}> <ExternalLink className="h-4 w-4" /> - Open page + Open source </a> </div> </div> @@ -1247,7 +1287,7 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri className="inline-flex min-h-[44px] items-center gap-1.5 rounded-md border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-3 text-[color:var(--primary)]" > <ExternalLink className="h-3.5 w-3.5" /> - Open source PDF + Source PDF </a> </div> )} @@ -1267,7 +1307,7 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri </button> <a href={url} target="_blank" rel="noreferrer" className={secondaryButton}> <ExternalLink className="h-4 w-4" /> - Open source PDF + Source PDF </a> </div> </div> @@ -1559,6 +1599,7 @@ export function DocumentViewer({ const [isOnline, setIsOnline] = useState(true); const [authLoadingTimedOut, setAuthLoadingTimedOut] = useState(false); const [localProjectReady, setLocalProjectReady] = useState(true); + const [mobileActionsOpen, setMobileActionsOpen] = useState(false); const generatedSummaryRef = useRef<HTMLElement | null>(null); const { status: authStatus, isConfigured, authorizationHeader, markSessionExpired } = useAuthSession(); const [serverDemoMode, setServerDemoMode] = useState(process.env.NEXT_PUBLIC_DEMO_MODE === "true" || !isConfigured); @@ -1993,19 +2034,76 @@ export function DocumentViewer({ onDeleted={handleDocumentDeleted} /> </div> - <DocumentManagementActions - document={readyDocument} - disabled={!canUsePrivateApis} - className="gap-1 sm:hidden" - onRenamed={handleDocumentRenamed} - onDeleted={handleDocumentDeleted} - /> + <button + type="button" + onClick={() => setMobileActionsOpen(true)} + className={cn( + iconButton, + "border-white/15 bg-white/7 text-slate-100 shadow-[var(--shadow-tight)] hover:border-white/25 hover:bg-white/12 sm:hidden", + )} + aria-label="Open document actions" + > + <MoreHorizontal className="h-4 w-4" /> + </button> </> )} </div> </div> </header> + {readyDocument ? ( + <Sheet + open={mobileActionsOpen} + onClose={() => setMobileActionsOpen(false)} + title="Document actions" + description="Source-backed summary, provenance, and admin controls." + closeLabel="Close document actions" + > + <div className="space-y-3"> + <section className={cn(sourceCard, "p-3")}> + <p className="line-clamp-2 text-sm font-semibold text-[color:var(--text)]">{readyDocument.title}</p> + <p className={cn("mt-1 truncate text-xs", textMuted)}>{readyDocument.file_name}</p> + <div className="mt-3 flex flex-wrap items-center gap-2"> + <SourceStatusBadge metadata={readyDocument.metadata} showTitle={false} /> + <span className={cn("text-xs font-semibold", textMuted)}> + Review {formatClinicalDate(normalizeSourceMetadata(readyDocument.metadata).review_date)} + </span> + {!isOnline ? <span className={cn("text-xs font-semibold", textMuted)}>Offline</span> : null} + </div> + </section> + <button + type="button" + onClick={() => { + setMobileActionsOpen(false); + void summarize(); + }} + disabled={!canSummarizeDocument} + title={summarizeTitle} + className={cn(primaryButton, "w-full justify-center")} + > + {loadingSummary ? <Loader2 className="h-4 w-4 animate-spin" /> : <Sparkles className="h-4 w-4" />} + Summarise document + </button> + <section className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-3"> + <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> + Admin controls + </p> + <DocumentManagementActions + document={readyDocument} + disabled={!canUsePrivateApis} + className="mt-3 justify-start gap-2" + onRenamed={handleDocumentRenamed} + onDeleted={handleDocumentDeleted} + /> + <p className={cn("mt-3 text-xs leading-5", textMuted)}> + Rename changes the display title only. Delete permanently removes this source and its extracted + evidence. + </p> + </section> + </div> + </Sheet> + ) : null} + <section className="mx-auto grid max-w-7xl gap-4 px-3 py-4 pb-24 sm:gap-5 sm:px-4 sm:py-5 lg:grid-cols-[minmax(0,1fr)_420px] lg:px-8"> {(summary || summaryError) && ( <div className="min-w-0 space-y-3 lg:col-span-2"> @@ -2018,7 +2116,7 @@ export function DocumentViewer({ <PanelHeading icon={Sparkles} title="Clinical summary" - description="Generated from indexed source passages and cleaned for practical review." + description="From indexed passages, cleaned for practical use." /> <p className="mt-3 whitespace-pre-wrap text-[15px] leading-6 text-[color:var(--text-muted)]"> <SafeBoldText text={generatedSummaryText} /> @@ -2093,7 +2191,7 @@ export function DocumentViewer({ {signedUrl && ( <a href={signedUrl} target="_blank" rel="noreferrer" className={cn(secondaryButton, "mt-3")}> <ExternalLink className="h-4 w-4" /> - Open source PDF + Source PDF </a> )} </div> @@ -2111,7 +2209,7 @@ export function DocumentViewer({ {signedUrl && ( <a href={signedUrl} target="_blank" rel="noreferrer" className={secondaryButton}> <ExternalLink className="h-4 w-4" /> - Open source PDF + Source PDF </a> )} </div> @@ -2166,7 +2264,7 @@ export function DocumentViewer({ <PanelHeading icon={FileText} title="Evidence status" - description="Source provenance, review date, and indexing health." + description="Source provenance and indexing health." /> <SourceMetadataSummary metadata={document?.metadata} /> {indexHealth ? ( @@ -2258,7 +2356,7 @@ export function DocumentViewer({ <PanelHeading icon={FileImage} title="Tables and diagrams" - description="Indexed clinical tables, diagrams, and image captions extracted from the source document." + description="Indexed tables, diagrams, and image captions." /> <div className="mt-3 space-y-3"> <TableReviewPanel @@ -2270,9 +2368,7 @@ export function DocumentViewer({ {effectiveLoadingDocument ? ( <LoadingPanel label="Loading extracted tables" /> ) : clinicalImages.length === 0 ? ( - <p className={cn("text-[15px]", textMuted)}> - No clinically useful tables or diagrams have been indexed for this document. - </p> + <p className={cn("text-[15px]", textMuted)}>No indexed clinically useful tables or diagrams.</p> ) : ( clinicalImages.map((image) => <DocumentImage key={image.id} image={image} />) )} diff --git a/src/components/clinical-dashboard/ClinicalDashboard.tsx b/src/components/clinical-dashboard/ClinicalDashboard.tsx new file mode 100644 index 000000000..4973c2930 --- /dev/null +++ b/src/components/clinical-dashboard/ClinicalDashboard.tsx @@ -0,0 +1 @@ +export { ClinicalDashboard } from "@/components/ClinicalDashboard"; diff --git a/src/components/clinical-dashboard/answer-status.tsx b/src/components/clinical-dashboard/answer-status.tsx new file mode 100644 index 000000000..80bf4d8d9 --- /dev/null +++ b/src/components/clinical-dashboard/answer-status.tsx @@ -0,0 +1,162 @@ +"use client"; + +import { Clipboard, ClipboardCheck, Search, Sparkles } from "lucide-react"; + +import { cn, EmptyState, floatingControl, LoadingPanel, sourceCard, textMuted } from "@/components/ui-primitives"; + +const sampleQueries = [ + { + label: "Monitoring overview", + query: "What monitoring and escalation issues should I consider across these documents?", + }, + { + label: "Lithium safety-net", + query: "What toxicity safety-net symptoms should be reviewed for lithium?", + }, + { + label: "Clozapine table", + query: "What clozapine monitoring items are shown in the table image?", + }, + { + label: "Risk escalation", + query: "When should acute risk be escalated for senior review?", + }, +] as const; + +export function CopyButton({ + label, + shortLabel, + ariaLabel, + copied, + onClick, +}: { + label: string; + shortLabel?: string; + ariaLabel?: string; + copied: boolean; + onClick: () => void; +}) { + return ( + <button + type="button" + onClick={onClick} + aria-label={ariaLabel ?? label} + className={cn(floatingControl, "px-3 text-xs")} + > + {copied ? <ClipboardCheck className="h-4 w-4" /> : <Clipboard className="h-4 w-4" />} + <span className="sm:hidden">{copied ? "Copied" : (shortLabel ?? label)}</span> + <span className="hidden sm:inline">{copied ? "Copied" : label}</span> + </button> + ); +} + +export function AnswerEmptyState({ + onPickSample, + recentQueries = [], + documentsLoading = false, +}: { + onPickSample: (sample: string) => void; + recentQueries?: string[]; + documentsLoading?: boolean; +}) { + return ( + <div className="space-y-3"> + <EmptyState + icon={Search} + title="Ask indexed guidelines" + body="Results, source quotes, and diagrams will appear here." + /> + {documentsLoading ? ( + <LoadingPanel label="Checking indexed library before showing document status" variant="skeleton" lines={2} /> + ) : null} + {recentQueries.length > 0 ? ( + <section + aria-label="Recent questions" + className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)] p-3 shadow-[var(--shadow-inset)]" + > + <div className="mb-2 flex min-h-7 items-center justify-between gap-2"> + <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> + Recent questions + </p> + <span className={cn("text-[11px] font-semibold", textMuted)}>Resume</span> + </div> + <div className="grid gap-2"> + {recentQueries.map((recent) => ( + <button + key={recent} + type="button" + onClick={() => onPickSample(recent)} + title={recent} + className={cn( + floatingControl, + "min-h-10 justify-start px-3 text-left text-xs font-semibold sm:text-sm", + )} + > + <Search className="h-3.5 w-3.5 shrink-0" /> + <span className="min-w-0 truncate">{recent}</span> + </button> + ))} + </div> + </section> + ) : null} + <section + aria-label="Example questions" + className={cn( + "rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-3 shadow-[var(--shadow-inset)]", + )} + > + <div className="mb-2 flex min-h-7 items-center justify-between gap-2"> + <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> + Starter actions + </p> + <span className={cn("text-[11px] font-semibold", textMuted)}>Set question</span> + </div> + <div className="grid gap-2 sm:grid-cols-2"> + {sampleQueries.map((sample) => ( + <button + key={sample.query} + type="button" + onClick={() => onPickSample(sample.query)} + title={sample.query} + aria-label={`Use sample question: ${sample.query}`} + className={cn( + floatingControl, + "min-h-10 justify-start px-3 text-left text-xs motion-safe:transition-colors motion-safe:duration-150", + )} + > + <Sparkles className="h-3.5 w-3.5" /> + <span className="min-w-0 truncate">{sample.label}</span> + </button> + ))} + </div> + </section> + </div> + ); +} + +export function AnswerSkeleton() { + return ( + <div className="space-y-4" aria-label="Loading answer"> + <div className="space-y-3 rounded-lg border border-[color:var(--primary)]/20 bg-[color:var(--primary-soft)]/45 p-4"> + <div className="h-4 w-10/12 animate-pulse rounded bg-[color:var(--surface-subtle)]" /> + <div className="h-4 w-full animate-pulse rounded bg-[color:var(--surface-subtle)]" /> + <div className="h-4 w-8/12 animate-pulse rounded bg-[color:var(--surface-subtle)]" /> + <div className={cn(sourceCard, "mt-4 flex min-h-[60px] items-center justify-between gap-3 p-3")}> + <div className="min-w-0 flex-1 space-y-2"> + <div className="h-3 w-24 animate-pulse rounded bg-[color:var(--surface-subtle)]" /> + <div className="h-4 w-48 max-w-full animate-pulse rounded bg-[color:var(--surface-subtle)]" /> + </div> + <div className="h-[44px] w-20 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> + </div> + </div> + <div className="flex flex-wrap gap-2"> + <div className="h-[44px] w-48 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> + <div className="h-[44px] w-40 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> + </div> + <div className="grid gap-3 sm:grid-cols-2"> + <div className="h-28 animate-pulse rounded-lg bg-[color:var(--surface-subtle)]" /> + <div className="hidden h-28 animate-pulse rounded-lg bg-[color:var(--surface-subtle)] sm:block" /> + </div> + </div> + ); +} diff --git a/src/components/clinical-dashboard/badges.tsx b/src/components/clinical-dashboard/badges.tsx new file mode 100644 index 000000000..bf4d13feb --- /dev/null +++ b/src/components/clinical-dashboard/badges.tsx @@ -0,0 +1,59 @@ +import { AlertCircle, CheckCircle2, FileText, Loader2 } from "lucide-react"; + +import { cn, toneDanger, toneInfo, toneNeutral, toneSuccess, toneWarning } from "@/components/ui-primitives"; + +function statusTone(status: string) { + if (status === "indexed" || status === "completed") { + return { + icon: CheckCircle2, + className: toneSuccess, + }; + } + if (status === "failed") { + return { + icon: AlertCircle, + className: toneDanger, + }; + } + if (status === "processing") { + return { + icon: Loader2, + className: toneInfo, + }; + } + return { + icon: FileText, + className: toneNeutral, + }; +} + +export function StatusBadge({ status }: { status: string }) { + const tone = statusTone(status); + const Icon = tone.icon; + + return ( + <span + className={cn( + "inline-flex min-h-7 items-center gap-1.5 rounded-md border px-2.5 text-xs font-semibold", + tone.className, + )} + > + <Icon className={cn("h-3.5 w-3.5", status === "processing" && "animate-spin")} /> + {status} + </span> + ); +} + +export function StrengthBadge({ strength }: { strength?: string }) { + const label = strength ?? "source"; + const className = strength === "strong" ? toneSuccess : strength === "limited" ? toneWarning : toneInfo; + + return ( + <span + className={cn("inline-flex min-h-7 items-center gap-1.5 rounded-md border px-2 text-xs font-semibold", className)} + > + <CheckCircle2 className="h-3.5 w-3.5" /> + {label} + </span> + ); +} diff --git a/src/components/clinical-dashboard/document-search-results.tsx b/src/components/clinical-dashboard/document-search-results.tsx new file mode 100644 index 000000000..4fd958c58 --- /dev/null +++ b/src/components/clinical-dashboard/document-search-results.tsx @@ -0,0 +1,403 @@ +"use client"; + +import Link from "next/link"; +import { useMemo, useState } from "react"; +import { + AlertCircle, + ChevronDown, + ExternalLink, + FileText, + Filter, + ListChecks, + ShieldAlert, + Sparkles, + Tag, + Target, + X, + type LucideIcon, +} from "lucide-react"; + +import { DocumentTagCloud } from "@/components/DocumentTagCloud"; +import { SafeBoldText } from "@/components/SafeBoldText"; +import { QueryCoverageChips, RelevanceBadge } from "@/components/clinical-dashboard/relevance"; +import { + cn, + EmptyState, + floatingControl, + LoadingPanel, + metadataPill, + panelSubtle, + primaryControl, + sourceCard, + textMuted, +} from "@/components/ui-primitives"; +import { + buildSmartDocumentTagFacets, + filterDocumentsBySmartTagFacets, + smartDocumentFacetGroups, + type SmartDocumentTag, + type SmartDocumentTagFacet, + type SmartDocumentTagGroup, +} from "@/lib/document-tags"; +import type { DocumentMatch, EvidenceRelevance, SearchResult } from "@/lib/types"; + +type SearchFacet = { value: string; count: number }; +export type SearchFacets = { + status?: SearchFacet[]; + validation?: SearchFacet[]; + extractionQuality?: SearchFacet[]; + sections?: SearchFacet[]; + labels?: SearchFacet[]; + documentTypes?: SearchFacet[]; + evidence?: SearchFacet[]; +}; + +function SearchFacetDisclosure({ facets }: { facets?: SearchFacets | null }) { + const [expanded, setExpanded] = useState(false); + if (!facets) return null; + const chips = [ + ...(facets.status ?? []).map((facet) => ({ ...facet, prefix: "status" })), + ...(facets.documentTypes ?? []).map((facet) => ({ ...facet, prefix: "type" })), + ...(facets.sections ?? []).map((facet) => ({ ...facet, prefix: "section" })), + ...(facets.evidence ?? []).map((facet) => ({ ...facet, prefix: "evidence" })), + ].slice(0, 14); + if (chips.length === 0) return null; + return ( + <div className="w-fit max-w-full"> + <button + type="button" + onClick={() => setExpanded((current) => !current)} + aria-expanded={expanded} + className={cn( + metadataPill, + "min-h-8 cursor-pointer list-none gap-1.5 px-2.5 text-[11px] transition hover:border-[color:var(--border-strong)] hover:text-[color:var(--text)]", + )} + > + <Filter className="h-3.5 w-3.5" /> + Result filters + <span className="text-[color:var(--text-soft)]">({chips.length})</span> + <ChevronDown className={cn("h-3.5 w-3.5 transition", expanded && "rotate-180")} /> + </button> + {expanded ? ( + <div className="mt-2 flex max-w-3xl flex-wrap gap-1.5 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-2"> + {chips.map((facet) => ( + <span key={`${facet.prefix}:${facet.value}`} className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> + {facet.prefix}: {facet.value} ({facet.count}) + </span> + ))} + </div> + ) : null} + </div> + ); +} + +const documentFacetIcons: Record<SmartDocumentTagGroup, LucideIcon> = { + Medication: Target, + Risk: ShieldAlert, + Workflow: ListChecks, + Topic: Tag, + Population: FileText, + Setting: FileText, + Service: Sparkles, + "Document type": FileText, + Manual: Sparkles, +}; + +function DocumentTagFacetRail({ + groups, + activeKeys, + onToggle, + onClear, +}: { + groups: Array<{ group: SmartDocumentTagGroup; facets: SmartDocumentTagFacet[] }>; + activeKeys: string[]; + onToggle: (facet: SmartDocumentTagFacet) => void; + onClear: () => void; +}) { + if (groups.length === 0) return null; + const active = new Set(activeKeys); + + return ( + <aside + aria-label="Document tag filters" + className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-3" + > + <div className="flex flex-wrap items-center justify-between gap-2"> + <p className="text-xs font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]">Tag facets</p> + {activeKeys.length > 0 ? ( + <button type="button" onClick={onClear} className={cn(floatingControl, "min-h-8 px-2 text-[11px]")}> + <X className="h-3.5 w-3.5" /> + Clear + </button> + ) : null} + </div> + <div className="mt-3 grid gap-3 lg:grid-cols-2 xl:grid-cols-3"> + {smartDocumentFacetGroups + .map((group) => groups.find((item) => item.group === group)) + .filter((group): group is { group: SmartDocumentTagGroup; facets: SmartDocumentTagFacet[] } => Boolean(group)) + .map(({ group, facets }) => { + const Icon = documentFacetIcons[group]; + return ( + <section key={group} className="min-w-0"> + <h3 className="flex items-center gap-1.5 text-[11px] font-bold uppercase tracking-[0.08em] text-[color:var(--text-muted)]"> + <Icon className="h-3.5 w-3.5 text-[color:var(--primary)]" /> + {group} + </h3> + <div className="mt-2 flex flex-wrap gap-1.5"> + {facets.map((facet) => { + const selected = active.has(facet.key); + return ( + <button + key={facet.key} + type="button" + onClick={() => onToggle(facet)} + aria-pressed={selected} + title={`Filter to ${facet.label}`} + className={cn( + "inline-flex min-h-7 max-w-full items-center gap-1 rounded-md border px-2 text-[11px] font-semibold shadow-[var(--shadow-inset)] transition", + selected + ? "border-[color:var(--primary)]/35 bg-[color:var(--primary-soft)] text-[color:var(--primary)]" + : "border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] text-[color:var(--text-muted)] hover:border-[color:var(--border-strong)] hover:text-[color:var(--text)]", + )} + > + <span className="truncate">{facet.label}</span> + <span className="rounded bg-[color:var(--surface)] px-1 text-[10px] text-[color:var(--text-soft)]"> + {facet.count} + </span> + </button> + ); + })} + </div> + </section> + ); + })} + </div> + </aside> + ); +} + +export function MatchExplanationChips({ source }: { source: SearchResult }) { + const explanation = source.match_explanation; + const reasons = explanation?.reasons?.length + ? explanation.reasons + : [ + source.score_explanation?.titleBoost ? "title" : "", + source.score_explanation?.textRank ? "text" : "", + source.score_explanation?.vectorScore ? "vector" : "", + source.source_metadata?.document_status ? `status:${source.source_metadata.document_status}` : "", + ].filter(Boolean); + const score = source.score_explanation?.finalScore ?? source.hybrid_score ?? source.similarity; + const chips = [ + ...reasons.slice(0, 5), + Number.isFinite(score) ? `score:${Number(score).toFixed(2)}` : "", + explanation?.indexQualityScore !== undefined && explanation.indexQualityScore !== null + ? `index:${Number(explanation.indexQualityScore).toFixed(2)}` + : "", + explanation?.indexQualityIssues?.length ? "index warning" : "", + explanation?.tableHit ? "table fact" : "", + explanation?.indexUnitType ? `unit:${explanation.indexUnitType.replaceAll("_", " ")}` : "", + ].filter(Boolean); + if (chips.length === 0) return null; + return ( + <div className="mt-2 flex flex-wrap gap-1.5"> + {chips.slice(0, 7).map((chip) => ( + <span key={chip} className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> + {chip} + </span> + ))} + </div> + ); +} + +export function DocumentSearchResultsPanel({ + matches, + query, + loading, + documentCount, + realDataReady, + authUnavailable, + apiUnavailable, + setupWarning, + relevance, + facets, + onScopeDocument, + onAnswerFromDocument, + onTagSearch, +}: { + matches: DocumentMatch[]; + query: string; + loading: boolean; + documentCount: number; + realDataReady: boolean; + authUnavailable: boolean; + apiUnavailable: boolean; + setupWarning: string | null; + relevance?: EvidenceRelevance | null; + facets?: SearchFacets | null; + onScopeDocument: (documentId: string) => void; + onAnswerFromDocument: (documentId: string) => void; + onTagSearch: (tag: SmartDocumentTag | SmartDocumentTagFacet) => void; +}) { + const trimmedQuery = query.trim(); + const [activeFacetState, setActiveFacetState] = useState<{ query: string; keys: string[] }>({ query: "", keys: [] }); + const activeFacetKeys = useMemo( + () => (activeFacetState.query === query ? activeFacetState.keys : []), + [activeFacetState, query], + ); + const tagFacetGroups = useMemo(() => buildSmartDocumentTagFacets(matches, { query }), [matches, query]); + const visibleMatches = useMemo( + () => filterDocumentsBySmartTagFacets(matches, activeFacetKeys), + [matches, activeFacetKeys], + ); + + function toggleTagFacet(facet: SmartDocumentTagFacet) { + setActiveFacetState((current) => { + const keys = current.query === query ? current.keys : []; + return { + query, + keys: keys.includes(facet.key) ? keys.filter((key) => key !== facet.key) : [...keys, facet.key], + }; + }); + } + + if (loading) return <LoadingPanel label="Finding matching documents" />; + + if (apiUnavailable || !realDataReady || authUnavailable) { + return ( + <EmptyState + icon={AlertCircle} + title="Document search unavailable" + body={ + apiUnavailable + ? "The local API is unavailable. Check the app server before searching documents." + : authUnavailable + ? "Sign in or enable local no-auth mode before listing private indexed documents." + : setupWarning || "Complete the search setup before using Documents mode." + } + /> + ); + } + + if (matches.length === 0) { + if (documentCount === 0) { + return ( + <EmptyState + icon={FileText} + title="No indexed documents" + body="Upload and index source documents before using Documents mode." + /> + ); + } + + if (!trimmedQuery) { + return ( + <EmptyState + icon={FileText} + title="Search documents" + body="Enter a clinical topic, medication, workflow, or policy name to list matching source documents." + /> + ); + } + + return ( + <EmptyState + icon={FileText} + title="No matching documents" + body={`No indexed documents matched "${trimmedQuery}". Try a medication, acronym, policy name, or workflow term.`} + /> + ); + } + + return ( + <div className="space-y-3"> + <div className="flex flex-wrap items-center gap-2"> + <div className={cn(metadataPill, "nums inline-flex min-h-8 w-fit max-w-full flex-wrap gap-x-1.5 leading-5")}> + {matches.length} document match{matches.length === 1 ? "" : "es"} for "{query.trim()}" + </div> + {relevance ? <RelevanceBadge relevance={relevance} /> : null} + </div> + <SearchFacetDisclosure facets={facets} /> + <DocumentTagFacetRail + groups={tagFacetGroups} + activeKeys={activeFacetKeys} + onToggle={toggleTagFacet} + onClear={() => setActiveFacetState({ query, keys: [] })} + /> + {activeFacetKeys.length > 0 ? ( + <div className={cn(metadataPill, "min-h-8 w-fit max-w-full text-[11px]")}> + {visibleMatches.length} result{visibleMatches.length === 1 ? "" : "s"} after tag filters + </div> + ) : null} + <div className="grid gap-3"> + {visibleMatches.length === 0 ? ( + <div className={cn(panelSubtle, "p-4 text-sm font-semibold text-[color:var(--text-muted)]")}> + No document matches include all selected tag facets. + </div> + ) : null} + {visibleMatches.map((document) => ( + <article key={document.document_id} className={cn(sourceCard, "p-3 sm:p-4")}> + <div className="grid gap-3 lg:grid-cols-[minmax(0,1fr)_auto] lg:items-start"> + <div className="min-w-0"> + <Link + href={`/documents/${document.document_id}?page=${document.bestPages[0] ?? 1}&chunk=${document.bestChunkIds[0] ?? ""}`} + className="inline-flex min-h-[44px] items-center text-base font-semibold text-[color:var(--text-heading)] transition hover:text-[color:var(--primary)]" + > + <span className="line-clamp-2">{document.title}</span> + </Link> + <p className={cn("text-xs leading-5", textMuted)}> + {document.file_name} ¡ pages {document.bestPages.join(", ") || "n/a"} ¡ {document.tableCount} tables ¡{" "} + {document.imageCount} images + </p> + <p className={cn("mt-1 text-xs leading-5", textMuted)}>{document.matchReason}</p> + <div className="mt-2"> + <QueryCoverageChips relevance={document.relevance} /> + </div> + </div> + <div className="flex flex-wrap items-center gap-2"> + <RelevanceBadge relevance={document.relevance} /> + <Link + href={`/documents/${document.document_id}?page=${document.bestPages[0] ?? 1}&chunk=${document.bestChunkIds[0] ?? ""}`} + className={cn(floatingControl, "min-h-[44px] px-3 text-xs")} + aria-label={`Open ${document.title}`} + > + <ExternalLink className="h-4 w-4" /> + Open + </Link> + <button + type="button" + onClick={() => onScopeDocument(document.document_id)} + className={cn(floatingControl, "min-h-[44px] px-3 text-xs")} + aria-label={`Scope search to ${document.title}`} + > + <Filter className="h-4 w-4" /> + Scope + </button> + <button + type="button" + onClick={() => onAnswerFromDocument(document.document_id)} + className={cn(primaryControl, "min-h-[44px] rounded-lg px-3 text-xs")} + aria-label={`Answer from ${document.title}`} + > + <Sparkles className="h-4 w-4" /> + Answer + </button> + </div> + </div> + {document.summarySnippet && ( + <p className={cn("mt-2 line-clamp-3 text-[15px] leading-6", textMuted)}> + <SafeBoldText text={document.summarySnippet} /> + </p> + )} + <DocumentTagCloud + labels={document.labels} + query={query} + limit={4} + className="mt-3" + onTagClick={onTagSearch} + /> + </article> + ))} + </div> + </div> + ); +} diff --git a/src/components/clinical-dashboard/index.ts b/src/components/clinical-dashboard/index.ts new file mode 100644 index 000000000..2a0f18b17 --- /dev/null +++ b/src/components/clinical-dashboard/index.ts @@ -0,0 +1,14 @@ +export { ClinicalDashboard } from "./ClinicalDashboard"; +export { AnswerEmptyState, AnswerSkeleton, CopyButton } from "./answer-status"; +export { useTheme } from "./use-theme"; +export { StatusBadge, StrengthBadge } from "./badges"; +export { MasterSearchHeader } from "./master-search-header"; +export { DocumentSearchResultsPanel, MatchExplanationChips } from "./document-search-results"; +export type { SearchFacets } from "./document-search-results"; +export { + hasStrongRelevanceIcon, + isWeakRelevance, + QueryCoverageChips, + relevanceChipLabel, + RelevanceBadge, +} from "./relevance"; diff --git a/src/components/clinical-dashboard/master-search-header.tsx b/src/components/clinical-dashboard/master-search-header.tsx new file mode 100644 index 000000000..8502570a0 --- /dev/null +++ b/src/components/clinical-dashboard/master-search-header.tsx @@ -0,0 +1,748 @@ +"use client"; + +import Link from "next/link"; +import { FormEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import { + BookOpen, + ChevronDown, + CheckCircle2, + FileText, + Filter, + ListChecks, + Loader2, + Moon, + Search, + SlidersHorizontal, + Sparkles, + Sun, + X, +} from "lucide-react"; + +import { DocumentTagCloud } from "@/components/DocumentTagCloud"; +import { + cn, + commandInput, + floatingControl, + premiumHeaderSurface, + primaryControl, + shellChip, + eyebrowText, +} from "@/components/ui-primitives"; +import { Sheet } from "@/components/ui/sheet"; +import { type ResolvedTheme } from "@/lib/theme"; +import type { ClinicalDocument, ClinicalQueryMode } from "@/lib/types"; +import type { SearchScopeFilters } from "@/lib/search-scope"; +import { tagSearchText } from "@/lib/document-tags"; + +const mobileSheetMediaQuery = "(max-width: 639px)"; + +function splitFilterText(value: string) { + return value + .split(",") + .map((item) => item.trim()) + .filter(Boolean); +} + +function filterText(values?: string[]) { + return (values ?? []).join(", "); +} + +function documentScopeTitle(document: ClinicalDocument) { + return document.title.replace(/^Synthetic /, "").replace(/\.pdf$/i, ""); +} + +function documentScopeMeta(document: ClinicalDocument) { + const title = documentScopeTitle(document).toLowerCase(); + const fileName = document.file_name; + const fileBase = fileName.replace(/\.pdf$/i, "").toLowerCase(); + if (fileBase === title || fileBase.startsWith(title)) return `${document.page_count ?? "?"} pages`; + return `${fileName} ¡ ${document.page_count ?? "?"} pages`; +} + +export function MasterSearchHeader({ + documents, + query, + searchMode, + loading, + selectedDocumentIds, + queryMode, + scopeFilters, + hasAnswer, + demoMode, + realDataReady, + theme, + onQueryChange, + onSearchModeChange, + onAsk, + onClearQuery, + onClearScope, + onQueryModeChange, + onScopeFiltersChange, + onToggleScope, + onScopeOpenChange, + onOpenGuide, + onToggleTheme, + queryModeOptions, +}: { + documents: ClinicalDocument[]; + query: string; + searchMode: "answer" | "documents"; + loading: boolean; + selectedDocumentIds: string[]; + queryMode: ClinicalQueryMode; + scopeFilters: SearchScopeFilters; + hasAnswer: boolean; + demoMode: boolean; + realDataReady: boolean; + theme: ResolvedTheme; + onQueryChange: (query: string) => void; + onSearchModeChange: (mode: "answer" | "documents") => void; + onAsk: () => void; + onClearQuery: () => void; + onClearScope: () => void; + onQueryModeChange: (mode: ClinicalQueryMode) => void; + onScopeFiltersChange: (filters: SearchScopeFilters) => void; + onToggleScope: (documentId: string) => void; + onScopeOpenChange?: (open: boolean) => void; + onOpenGuide: () => void; + onToggleTheme: () => void; + queryModeOptions: Array<{ value: ClinicalQueryMode; label: string }>; +}) { + const trimmedQuery = query.trim(); + const canAsk = trimmedQuery.length >= 1 && !loading && realDataReady; + const compactMobile = hasAnswer; + const [scopeFilter, setScopeFilter] = useState(""); + const [scopeOpen, setScopeOpen] = useState(false); + const [scopeSheetOpen, setScopeSheetOpen] = useState(false); + const [usesScopeSheet, setUsesScopeSheet] = useState(false); + const scopeDetailsRef = useRef<HTMLDetailsElement | null>(null); + const scopeSummaryRef = useRef<HTMLElement | null>(null); + const scopeFilterInputRef = useRef<HTMLInputElement | null>(null); + const selectedDocuments = selectedDocumentIds + .map((id) => documents.find((document) => document.id === id)) + .filter((document): document is ClinicalDocument => Boolean(document)); + const scopeSummary = selectedDocumentIds.length === 0 ? "All documents" : `${selectedDocumentIds.length} scoped`; + const scopePreview = selectedDocuments + .slice(0, 2) + .map((document) => document?.title.replace(/^Synthetic /, "")) + .filter(Boolean) + .join(", "); + const normalizedScopeFilter = scopeFilter.trim().toLowerCase(); + const recentlyUpdatedDocuments = [...documents].sort((a, b) => { + const bTime = Date.parse(b.updated_at || b.created_at || ""); + const aTime = Date.parse(a.updated_at || a.created_at || ""); + return (Number.isNaN(bTime) ? 0 : bTime) - (Number.isNaN(aTime) ? 0 : aTime); + }); + const matchingDocuments = normalizedScopeFilter + ? recentlyUpdatedDocuments.filter((document) => + [document.title, document.file_name, document.description, tagSearchText(document)] + .filter(Boolean) + .some((value) => value?.toLowerCase().includes(normalizedScopeFilter)), + ) + : recentlyUpdatedDocuments; + const largeScopeSet = documents.length > 12; + const requireScopeFilter = largeScopeSet && !normalizedScopeFilter; + const visibleScopeDocuments = [ + ...selectedDocuments, + ...(requireScopeFilter ? [] : matchingDocuments.filter((document) => !selectedDocumentIds.includes(document.id))), + ].slice(0, 12); + const hiddenScopeMatchCount = requireScopeFilter + ? Math.max(0, selectedDocuments.length ? documents.length - selectedDocumentIds.length : documents.length) + : Math.max(0, matchingDocuments.length - visibleScopeDocuments.length); + const submitLabel = searchMode === "answer" ? (trimmedQuery ? "Answer" : "Ask") : "Docs"; + const collectionOptions = useMemo(() => { + const values = new Set<string>(); + for (const document of documents) { + const metadata = + document.metadata && typeof document.metadata === "object" + ? (document.metadata as Record<string, unknown>) + : {}; + const collection = metadata.collection; + if (typeof collection === "string" && collection.trim()) values.add(collection.trim()); + } + return Array.from(values).sort((a, b) => a.localeCompare(b)); + }, [documents]); + + const closeScope = useCallback((restoreFocus = false) => { + const details = scopeDetailsRef.current; + if (!details?.open) return; + details.open = false; + setScopeOpen(false); + if (restoreFocus) scopeSummaryRef.current?.focus(); + }, []); + + useEffect(() => { + const mediaQuery = window.matchMedia(mobileSheetMediaQuery); + const sync = () => setUsesScopeSheet(mediaQuery.matches); + sync(); + mediaQuery.addEventListener("change", sync); + return () => mediaQuery.removeEventListener("change", sync); + }, []); + + useEffect(() => { + onScopeOpenChange?.(scopeOpen || scopeSheetOpen); + }, [onScopeOpenChange, scopeOpen, scopeSheetOpen]); + + useEffect(() => { + const details = scopeDetailsRef.current; + if (!scopeOpen || !details?.open) return undefined; + + function handlePointerDown(event: PointerEvent) { + const target = event.target; + if (!(target instanceof Node)) return; + if (!scopeDetailsRef.current?.contains(target)) closeScope(false); + } + + function handleKeyDown(event: KeyboardEvent) { + if (event.key === "Escape") { + event.preventDefault(); + closeScope(true); + } + } + + document.addEventListener("pointerdown", handlePointerDown); + document.addEventListener("keydown", handleKeyDown); + return () => { + document.removeEventListener("pointerdown", handlePointerDown); + document.removeEventListener("keydown", handleKeyDown); + }; + }, [closeScope, scopeOpen]); + + function submit(event: FormEvent<HTMLFormElement>) { + event.preventDefault(); + onAsk(); + } + + function renderScopeRows() { + return ( + <div className="grid gap-3"> + <section className="min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)] p-2.5 sm:hidden"> + <div className="mb-2 flex min-h-7 items-center justify-between gap-2 px-0.5"> + <p className={eyebrowText}>Refine search</p> + <span className="text-[11px] font-semibold text-[color:var(--text-soft)]">Mode, status, topics</span> + </div> + <div className="grid gap-2"> + <select + value={queryMode} + onChange={(event) => onQueryModeChange(event.target.value as ClinicalQueryMode)} + aria-label="Clinical query mode" + className="h-10 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + > + {queryModeOptions.map((option) => ( + <option key={option.value} value={option.value}> + {option.label} + </option> + ))} + </select> + <div className="grid grid-cols-2 gap-2"> + <input + value={filterText(scopeFilters.medications)} + onChange={(event) => + onScopeFiltersChange({ ...scopeFilters, medications: splitFilterText(event.target.value) }) + } + placeholder="Medication" + className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + /> + <input + value={filterText(scopeFilters.topics)} + onChange={(event) => + onScopeFiltersChange({ ...scopeFilters, topics: splitFilterText(event.target.value) }) + } + placeholder="Topic" + className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + /> + <select + value={scopeFilters.sourceStatuses?.[0] ?? ""} + onChange={(event) => + onScopeFiltersChange({ + ...scopeFilters, + sourceStatuses: event.target.value + ? [event.target.value as NonNullable<SearchScopeFilters["sourceStatuses"]>[number]] + : [], + }) + } + className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + > + <option value="">Any status</option> + <option value="current">Current</option> + <option value="review_due">Review due</option> + <option value="outdated">Outdated</option> + <option value="unknown">Unknown</option> + </select> + <select + value={scopeFilters.locality ?? ""} + onChange={(event) => + onScopeFiltersChange({ + ...scopeFilters, + locality: event.target.value ? (event.target.value as SearchScopeFilters["locality"]) : undefined, + }) + } + className="h-10 min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + > + <option value="">Any locality</option> + <option value="local">Local only</option> + <option value="non_local">Non-local only</option> + </select> + </div> + <input + value={filterText(scopeFilters.collections)} + onChange={(event) => + onScopeFiltersChange({ ...scopeFilters, collections: splitFilterText(event.target.value) }) + } + placeholder={collectionOptions.length ? `Collection: ${collectionOptions[0]}` : "Collection"} + className="h-10 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2 text-xs font-semibold text-[color:var(--text)] outline-none placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + /> + <button + type="button" + onClick={() => onScopeFiltersChange({})} + className={cn(floatingControl, "min-h-9 px-3 text-xs")} + > + Clear refine filters + </button> + </div> + </section> + <section className="min-w-0 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-subtle)] p-2.5"> + <div className="mb-2 flex min-h-7 items-center justify-between gap-2 px-0.5"> + <p className={eyebrowText}>Document scope</p> + <span className="nums shrink-0 text-[11px] font-semibold text-[color:var(--text-soft)]"> + {selectedDocumentIds.length ? `${selectedDocumentIds.length} selected` : `${documents.length} available`} + </span> + </div> + <div className="space-y-2"> + <label className="relative block"> + <Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-[color:var(--text-soft)]" /> + <input + ref={scopeFilterInputRef} + value={scopeFilter} + onChange={(event) => setScopeFilter(event.target.value)} + data-testid="document-scope-filter" + aria-label="Filter document scope" + placeholder="Filter documents by title or file" + className="h-10 w-full rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] pl-9 pr-3 text-sm font-semibold text-[color:var(--text)] shadow-[var(--shadow-inset)] outline-none transition placeholder:text-[color:var(--text-soft)] focus:border-[color:var(--focus)] focus:ring-4 focus:ring-[color:var(--focus)]/25" + /> + </label> + <div className="flex flex-wrap items-center gap-2"> + <button + type="button" + onClick={onClearScope} + className={cn( + shellChip, + selectedDocumentIds.length === 0 + ? "border-[color:var(--primary)]/40 bg-[color:var(--primary-soft)] text-[color:var(--primary-strong)]" + : "border-[color:var(--border)] bg-[color:var(--surface-raised)] text-[color:var(--text-muted)] hover:bg-[color:var(--surface-subtle)]", + )} + > + All documents + </button> + {scopeFilter ? ( + <span className="nums rounded-md bg-[color:var(--surface-raised)] px-2 py-1 text-[11px] font-semibold text-[color:var(--text-muted)]"> + {matchingDocuments.length} match{matchingDocuments.length === 1 ? "" : "es"} + </span> + ) : ( + <span className="rounded-md bg-[color:var(--surface-raised)] px-2 py-1 text-[11px] font-semibold text-[color:var(--text-muted)]"> + Recently updated first + </span> + )} + </div> + <div className="max-h-72 overflow-y-auto pr-1 polished-scroll"> + <div className="grid gap-1.5"> + {requireScopeFilter && visibleScopeDocuments.length === 0 ? ( + <p className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-3 py-2 text-sm font-medium text-[color:var(--text-muted)]"> + Type to filter {documents.length} documents. Selected documents stay pinned here. + </p> + ) : null} + {visibleScopeDocuments.map((document) => { + const selected = selectedDocumentIds.includes(document.id); + return ( + <button + key={document.id} + type="button" + onClick={() => onToggleScope(document.id)} + title={document.title} + className={cn( + "grid min-h-[44px] w-full grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2 rounded-lg border px-2.5 py-2 text-left transition motion-safe:duration-150", + selected + ? "border-[color:var(--primary)]/40 bg-[color:var(--primary-soft)] text-[color:var(--primary-strong)]" + : "border-[color:var(--border)] bg-[color:var(--surface-raised)] text-[color:var(--text)] hover:border-[color:var(--border-strong)] hover:bg-[color:var(--surface-subtle)]", + )} + > + <span + className={cn( + "grid h-5 w-5 place-items-center rounded-md border", + selected + ? "border-[color:var(--primary)]/50 bg-[color:var(--primary-soft)] text-[color:var(--primary)]" + : "border-[color:var(--border-strong)] bg-[color:var(--surface-subtle)]", + )} + aria-hidden + > + {selected ? <CheckCircle2 className="h-3.5 w-3.5" /> : null} + </span> + <span className="min-w-0"> + <span className="block truncate text-sm font-semibold">{documentScopeTitle(document)}</span> + <span className="nums block truncate text-[11px] font-medium text-[color:var(--text-soft)]"> + {documentScopeMeta(document)} + </span> + <DocumentTagCloud + labels={document.labels} + query={scopeFilter} + limit={2} + compact + expandable={false} + className="mt-1" + /> + </span> + {selected ? ( + <span className="rounded-md bg-[color:var(--primary-soft)] px-2 py-1 text-[11px] font-bold text-[color:var(--primary-strong)]"> + In scope + </span> + ) : null} + </button> + ); + })} + {!requireScopeFilter && visibleScopeDocuments.length === 0 ? ( + <p className="rounded-lg border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-3 py-2 text-sm font-medium text-[color:var(--text-muted)]"> + No documents match that filter. Clear the filter or search by file name. + </p> + ) : null} + </div> + </div> + {hiddenScopeMatchCount > 0 ? ( + <p className="nums px-1 text-xs font-medium text-[color:var(--text-soft)]"> + {requireScopeFilter + ? `${documents.length} documents available. Type a title or file name to narrow the list.` + : `Showing ${visibleScopeDocuments.length} of ${matchingDocuments.length}. Keep typing to narrow the list.`} + </p> + ) : null} + </div> + </section> + </div> + ); + } + + return ( + <header + id="search" + className={cn( + "sticky top-0 z-30 px-3 pb-2 pt-[max(0.5rem,env(safe-area-inset-top))] sm:px-4 lg:px-8", + premiumHeaderSurface, + "sm:py-2.5", + )} + style={{ backgroundColor: "var(--app-shell)" }} + > + <div className="mx-auto max-w-7xl space-y-2"> + <div className="flex items-center justify-between gap-3"> + <div className="flex min-w-0 items-center gap-3"> + <div + className={cn( + "grid shrink-0 place-items-center rounded-lg border border-white/20 bg-[linear-gradient(135deg,var(--primary),var(--primary-strong))] text-[color:var(--primary-contrast)] shadow-[var(--glow-soft)]", + compactMobile ? "h-9 w-9 sm:h-[44px] sm:w-[44px]" : "h-[44px] w-[44px]", + )} + > + <ListChecks className="h-5 w-5" /> + </div> + <div className="min-w-0"> + <div className="flex min-w-0 items-center gap-2"> + <h1 className="truncate text-base font-semibold">Clinical Guide</h1> + {demoMode && ( + <span className="hidden shrink-0 rounded-md border border-amber-300/25 bg-amber-300/12 px-2 py-0.5 text-[11px] font-bold uppercase tracking-[0.08em] text-amber-100 sm:inline-flex"> + Demo data + </span> + )} + </div> + <p className={cn("truncate text-xs font-medium text-slate-300", compactMobile && "hidden sm:block")}> + {demoMode ? "Synthetic data only" : "Ask indexed guidelines"} + </p> + </div> + </div> + <div className="flex shrink-0 items-center gap-2"> + {demoMode && ( + <span className="inline-flex rounded-md border border-amber-300/25 bg-amber-300/12 px-2 py-1 text-[11px] font-bold uppercase tracking-[0.08em] text-amber-100 sm:hidden"> + Demo + </span> + )} + <Link + href="/tools" + aria-label="Open clinical tools" + title="Open clinical tools" + className="group relative grid h-[44px] w-[44px] shrink-0 place-items-center overflow-hidden rounded-lg border border-cyan-100/20 bg-cyan-100/[0.08] text-cyan-50 shadow-[inset_0_1px_0_rgb(255_255_255_/_12%),var(--shadow-tight)] transition hover:-translate-y-0.5 hover:border-cyan-100/35 hover:bg-cyan-100/[0.13] hover:text-white" + > + <span className="pointer-events-none absolute inset-x-2 top-0 h-px bg-gradient-to-r from-transparent via-cyan-100/70 to-transparent" /> + <Sparkles className="relative h-4.5 w-4.5" /> + <span className="sr-only">Clinical tools</span> + </Link> + <button + type="button" + onClick={onOpenGuide} + className="hidden h-[44px] shrink-0 items-center gap-2 rounded-lg border border-white/15 bg-white/7 px-3 text-xs font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition hover:border-white/25 hover:bg-white/12 sm:inline-flex" + aria-label="Open user guide" + > + <BookOpen className="h-4 w-4" /> + <span>Guide</span> + </button> + <button + type="button" + onClick={onToggleTheme} + className="grid h-[44px] w-[44px] shrink-0 place-items-center rounded-lg border border-white/15 bg-white/7 text-slate-100 shadow-[var(--shadow-tight)] transition hover:border-white/25 hover:bg-white/12" + aria-label={`Switch to ${theme === "dark" ? "light" : "dark"} mode`} + > + {theme === "dark" ? <Sun className="h-4.5 w-4.5" /> : <Moon className="h-4.5 w-4.5" />} + </button> + </div> + </div> + + <div className="grid gap-2 rounded-[var(--radius-lg)] border border-white/10 bg-white/6 p-1 shadow-[var(--shadow-inset)] sm:flex sm:flex-wrap sm:items-center sm:justify-between"> + <div role="group" aria-label="Search mode" className="grid grid-cols-2 gap-1 sm:min-w-[14rem]"> + {[ + { mode: "answer" as const, label: "Answer", icon: Sparkles }, + { mode: "documents" as const, label: "Documents", icon: FileText }, + ].map((item) => { + const active = searchMode === item.mode; + const Icon = item.icon; + return ( + <button + key={item.mode} + type="button" + onClick={() => onSearchModeChange(item.mode)} + className={cn( + "inline-flex min-h-[44px] items-center justify-center gap-2 rounded-[var(--radius-md)] px-3 text-sm font-semibold transition", + active + ? "bg-white text-slate-950 shadow-[var(--shadow-tight)]" + : "text-slate-200 hover:bg-white/10 hover:text-white", + )} + aria-pressed={active} + aria-label={item.mode === "answer" ? "Switch to answer mode" : "Switch to document search mode"} + > + <Icon className="h-4 w-4" /> + {item.label} + </button> + ); + })} + </div> + <span className="hidden px-2 text-xs font-medium text-slate-300 lg:inline"> + {searchMode === "answer" ? "Synthesize cited clinical guidance" : "List matching source documents"} + </span> + <div className="ml-auto hidden min-w-0 items-center gap-2 sm:flex"> + <details className="group relative"> + <summary className="flex h-9 cursor-pointer list-none items-center justify-between gap-2 rounded-md border border-white/15 bg-white/7 px-3 text-xs font-semibold text-slate-100"> + <SlidersHorizontal className="h-4 w-4 shrink-0" /> + Refine + <ChevronDown className="h-4 w-4 shrink-0 transition group-open:rotate-180" /> + </summary> + <div className="absolute right-0 top-[calc(100%+0.5rem)] z-40 grid w-[min(42rem,calc(100vw-2rem))] gap-2 rounded-lg border border-white/15 bg-[color:var(--surface-glass)] p-3 shadow-[var(--shadow-elevated)] backdrop-blur-xl sm:grid-cols-2 lg:grid-cols-3"> + <select + value={queryMode} + onChange={(event) => onQueryModeChange(event.target.value as ClinicalQueryMode)} + aria-label="Clinical query mode" + className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" + > + {queryModeOptions.map((option) => ( + <option key={option.value} value={option.value}> + {option.label} + </option> + ))} + </select> + <input + value={filterText(scopeFilters.medications)} + onChange={(event) => + onScopeFiltersChange({ ...scopeFilters, medications: splitFilterText(event.target.value) }) + } + placeholder="Medication labels" + className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" + /> + <input + value={filterText(scopeFilters.topics)} + onChange={(event) => + onScopeFiltersChange({ ...scopeFilters, topics: splitFilterText(event.target.value) }) + } + placeholder="Topic labels" + className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" + /> + <input + value={filterText(scopeFilters.collections)} + onChange={(event) => + onScopeFiltersChange({ ...scopeFilters, collections: splitFilterText(event.target.value) }) + } + placeholder={collectionOptions.length ? `Collection: ${collectionOptions[0]}` : "Collection"} + className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" + /> + <select + value={scopeFilters.sourceStatuses?.[0] ?? ""} + onChange={(event) => + onScopeFiltersChange({ + ...scopeFilters, + sourceStatuses: event.target.value + ? [event.target.value as NonNullable<SearchScopeFilters["sourceStatuses"]>[number]] + : [], + }) + } + className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" + > + <option value="">Any status</option> + <option value="current">Current</option> + <option value="review_due">Review due</option> + <option value="outdated">Outdated</option> + <option value="unknown">Unknown</option> + </select> + <select + value={scopeFilters.locality ?? ""} + onChange={(event) => + onScopeFiltersChange({ + ...scopeFilters, + locality: event.target.value ? (event.target.value as SearchScopeFilters["locality"]) : undefined, + }) + } + className="h-9 rounded-md border border-white/15 bg-white/95 px-2 text-xs font-semibold text-slate-950 outline-none dark:bg-slate-950/90 dark:text-slate-50" + > + <option value="">Any locality</option> + <option value="local">Local only</option> + <option value="non_local">Non-local only</option> + </select> + <button + type="button" + onClick={() => onScopeFiltersChange({})} + className="h-9 rounded-md border border-white/15 bg-white/7 px-2 text-xs font-semibold text-slate-100 hover:bg-white/12" + > + Clear filters + </button> + </div> + </details> + </div> + </div> + + <form + onSubmit={submit} + className="grid grid-cols-[minmax(0,1fr)_52px_52px] gap-2 sm:grid-cols-[minmax(0,1fr)_136px_108px] lg:grid-cols-[minmax(0,1fr)_148px_116px]" + > + <label className="relative min-w-0"> + <Search className="pointer-events-none absolute left-4 top-1/2 h-5 w-5 -translate-y-1/2 text-slate-400" /> + <input + value={query} + onChange={(event) => onQueryChange(event.target.value)} + onKeyDown={(event) => { + if ((event.metaKey || event.ctrlKey) && event.key === "Enter") onAsk(); + }} + aria-label="Search indexed guidelines by question or keyword" + placeholder="Ask a question" + className={commandInput} + /> + {query && ( + <button + type="button" + onClick={onClearQuery} + className="absolute right-2 top-1/2 grid h-[44px] w-[44px] -translate-y-1/2 place-items-center rounded-lg text-slate-500 transition hover:bg-slate-100 hover:text-slate-950 dark:hover:bg-slate-800 dark:hover:text-slate-50" + aria-label="Clear search question" + > + <X className="h-4 w-4" /> + </button> + )} + </label> + <button + type="submit" + disabled={!canAsk} + title={ + !realDataReady + ? "Search setup not ready" + : trimmedQuery.length < 1 + ? searchMode === "answer" + ? "Enter a clinical question" + : "Enter a document search term" + : searchMode === "answer" + ? "Generate a source-backed answer" + : "Find matching documents" + } + className={cn( + primaryControl, + "min-h-[44px] whitespace-nowrap rounded-[var(--radius-lg)] px-0 text-sm sm:px-5", + )} + aria-label={searchMode === "answer" ? "Generate source-backed answer" : "Find matching documents"} + > + {loading ? <Loader2 className="h-4 w-4 animate-spin" /> : <Search className="h-4 w-4" />} + <span className="sr-only sm:not-sr-only">{submitLabel}</span> + </button> + <> + <button + type="button" + data-testid="scope-trigger" + onClick={() => setScopeSheetOpen(true)} + className="flex min-h-[44px] cursor-pointer list-none items-center justify-center gap-1.5 whitespace-nowrap rounded-[var(--radius-lg)] border border-white/15 bg-white/7 px-0 text-sm font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition motion-safe:duration-150 hover:border-white/25 hover:bg-white/12 sm:hidden" + aria-label={usesScopeSheet ? "Open document scope" : "Open mobile document scope"} + aria-expanded={scopeSheetOpen} + > + <Filter className="h-4 w-4" /> + <span className="sr-only">Scope</span> + {selectedDocumentIds.length ? ( + <span className="rounded-md bg-teal-200/15 px-1.5 py-0.5 text-[10px] font-bold text-teal-50"> + {selectedDocumentIds.length} + </span> + ) : null} + </button> + + <details + ref={scopeDetailsRef} + onToggle={(event) => { + const open = event.currentTarget.open; + setScopeOpen(open); + if (open) window.setTimeout(() => scopeFilterInputRef.current?.focus(), 0); + }} + className="group relative hidden sm:block" + > + <summary + ref={scopeSummaryRef} + data-testid="scope-trigger" + className="flex min-h-[44px] cursor-pointer list-none items-center justify-center gap-1.5 whitespace-nowrap rounded-[var(--radius-lg)] border border-white/15 bg-white/7 px-0 text-sm font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition motion-safe:duration-150 hover:border-white/25 hover:bg-white/12 sm:gap-2 sm:px-3 sm:text-xs" + aria-label={usesScopeSheet ? "Open desktop document scope" : "Open document scope"} + aria-expanded={scopeOpen} + > + <Filter className="h-4 w-4" /> + <span className="sr-only sm:not-sr-only">Scope</span> + {selectedDocumentIds.length ? ( + <span className="rounded-md bg-teal-200/15 px-1.5 py-0.5 text-[10px] font-bold text-teal-50"> + {selectedDocumentIds.length} + </span> + ) : null} + </summary> + <div + data-testid={usesScopeSheet ? undefined : "scope-command-popover"} + className="polished-scroll absolute right-0 top-[calc(100%+0.5rem)] z-40 max-h-[min(70dvh,28rem)] w-[28rem] overflow-y-auto overscroll-contain rounded-xl border border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] p-2.5 pb-2.5 text-[color:var(--text)] shadow-[var(--shadow-elevated)] backdrop-blur-xl motion-safe:animate-pop-in" + > + <div className="mb-2 flex min-h-8 items-center justify-between px-1 text-xs font-semibold text-[color:var(--text-muted)]"> + <span>Document scope</span> + <span className="nums">{scopeSummary}</span> + </div> + {scopePreview ? ( + <p className="mb-2 truncate px-1 text-xs text-[color:var(--text-soft)]">{scopePreview}</p> + ) : null} + {renderScopeRows()} + </div> + </details> + + <Sheet + open={scopeSheetOpen} + onClose={() => setScopeSheetOpen(false)} + title="Document scope" + description="Choose documents and filters for the next search." + closeLabel="Close document scope" + initialFocusRef={scopeFilterInputRef} + contentClassName="sm:hidden" + > + <div + data-testid={usesScopeSheet ? "scope-command-popover" : undefined} + className="polished-scroll max-h-[min(70dvh,28rem)] overflow-y-auto overscroll-contain pr-1" + > + <div className="mb-2 flex min-h-8 items-center justify-between px-1 text-xs font-semibold text-[color:var(--text-muted)]"> + <span>Document scope</span> + <span className="nums">{scopeSummary}</span> + </div> + {scopePreview ? ( + <p className="mb-2 truncate px-1 text-xs text-[color:var(--text-soft)]">{scopePreview}</p> + ) : null} + {renderScopeRows()} + </div> + </Sheet> + </> + </form> + </div> + </header> + ); +} diff --git a/src/components/clinical-dashboard/relevance.tsx b/src/components/clinical-dashboard/relevance.tsx new file mode 100644 index 000000000..86bce8547 --- /dev/null +++ b/src/components/clinical-dashboard/relevance.tsx @@ -0,0 +1,95 @@ +import { AlertCircle, CheckCircle2 } from "lucide-react"; + +import { cn, metadataPill } from "@/components/ui-primitives"; +import type { EvidenceRelevance, SourceEvidenceRelevance } from "@/lib/types"; + +export function relevanceChipLabel( + relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, + grounded = false, +) { + if (!relevance) return grounded ? "Source-backed" : "No direct support"; + if (relevance.verdict === "direct") return "Source-backed"; + if (relevance.verdict === "partial") return "Partial support"; + if (relevance.verdict === "nearby") return "Nearby only"; + return "No direct support"; +} + +function relevanceChipClasses( + relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, + grounded = false, +) { + const verdict = relevance?.verdict ?? (grounded ? "direct" : "none"); + if (verdict === "direct") { + return "border-[color:var(--success)]/20 bg-[color:var(--success-soft)]/45 text-[color:var(--success)]"; + } + if (verdict === "partial") { + return "border-[color:var(--primary)]/25 bg-[color:var(--primary-soft)]/45 text-[color:var(--primary)]"; + } + return "border-[color:var(--warning)]/25 bg-[color:var(--warning-soft)]/45 text-[color:var(--warning)]"; +} + +export function hasStrongRelevanceIcon( + relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, + grounded = false, +) { + const verdict = relevance?.verdict ?? (grounded ? "direct" : "none"); + return verdict === "direct" || verdict === "partial"; +} + +export function isWeakRelevance(relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined) { + return !relevance?.isSourceBacked || relevance.verdict === "nearby" || relevance.verdict === "none"; +} + +export function RelevanceBadge({ + relevance, + grounded = false, + testId, +}: { + relevance?: EvidenceRelevance | SourceEvidenceRelevance | null; + grounded?: boolean; + testId?: string; +}) { + const showStrongIcon = hasStrongRelevanceIcon(relevance, grounded); + const label = relevanceChipLabel(relevance, grounded); + return ( + <span + data-testid={testId} + className={cn( + "inline-flex min-h-7 items-center gap-1 rounded-md border px-2 text-[11px] font-semibold leading-none sm:min-h-8 sm:gap-1.5 sm:px-2.5 sm:text-xs", + relevanceChipClasses(relevance, grounded), + )} + aria-label={label} + title={relevance?.supportReason ?? label} + > + {showStrongIcon ? <CheckCircle2 className="h-3.5 w-3.5" /> : <AlertCircle className="h-3.5 w-3.5" />} + <span>{label}</span> + </span> + ); +} + +export function QueryCoverageChips({ + relevance, + limit = 4, +}: { + relevance?: SourceEvidenceRelevance | EvidenceRelevance | null; + limit?: number; +}) { + if (!relevance) return null; + const chips = + "chips" in relevance && relevance.chips.length + ? relevance.chips + : [ + relevance.matchedTerms.length ? `matched: ${relevance.matchedTerms.slice(0, 3).join(", ")}` : "", + relevance.missingTerms.length ? `missing: ${relevance.missingTerms.slice(0, 3).join(", ")}` : "", + relevanceChipLabel(relevance), + ].filter(Boolean); + return ( + <div className="flex flex-wrap gap-1.5"> + {chips.slice(0, limit).map((chip) => ( + <span key={chip} className={cn(metadataPill, "min-h-7 px-2 text-[11px]")}> + {chip} + </span> + ))} + </div> + ); +} diff --git a/src/components/clinical-dashboard/source-actions.tsx b/src/components/clinical-dashboard/source-actions.tsx new file mode 100644 index 000000000..4e646f51e --- /dev/null +++ b/src/components/clinical-dashboard/source-actions.tsx @@ -0,0 +1,114 @@ +"use client"; + +import Link from "next/link"; +import { ExternalLink, FileText, Filter, Search } from "lucide-react"; +import { cn, floatingControl, metadataPill, primaryControl } from "@/components/ui-primitives"; +import type { SearchResult } from "@/lib/types"; + +export function SourceActionRow({ + viewerHref, + sourceTitle, + documentId, + onScopeDocument, + onFollowUp, + imageCount = 0, + divider = true, +}: { + viewerHref: string; + sourceTitle: string; + documentId: string; + onScopeDocument: (documentId: string) => void; + onFollowUp?: () => void; + imageCount?: number; + divider?: boolean; +}) { + return ( + <div className={cn("flex flex-wrap gap-2", divider && "border-t border-[color:var(--border)] pt-3")}> + <Link href={viewerHref} className={cn(primaryControl, "min-h-[44px] px-4 text-xs")}> + <FileText className="h-4 w-4" /> + Open source + </Link> + {onFollowUp && ( + <button + type="button" + onClick={onFollowUp} + className={cn(floatingControl, "px-3 text-xs")} + aria-label={`Ask a follow-up from ${sourceTitle}`} + > + <Search className="h-4 w-4" /> + <span className="sm:hidden">Follow-up</span> + <span className="hidden sm:inline">Ask follow-up</span> + </button> + )} + <button + type="button" + onClick={() => onScopeDocument(documentId)} + className={cn(floatingControl, "px-3 text-xs")} + aria-label={`Search only ${sourceTitle}`} + > + <Filter className="h-4 w-4" /> + <span className="sm:hidden">Scope</span> + <span className="hidden sm:inline">Add scope</span> + </button> + {imageCount > 0 && ( + <span className={cn(metadataPill, "min-h-[44px] rounded-lg px-3")}> + {imageCount} indexed image{imageCount === 1 ? "" : "s"} + </span> + )} + </div> + ); +} + +export function sourceResultHref(source: SearchResult) { + return `/documents/${source.document_id}?page=${source.page_number ?? 1}&chunk=${source.id}`; +} + +export function logSourceOpen(query: string, source: SearchResult) { + if (!query.trim()) return; + void fetch("/api/search/interaction", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query, + documentId: source.document_id, + chunkId: source.id, + fileName: source.file_name, + title: source.title, + }), + keepalive: true, + }).catch(() => undefined); +} + +export function SourcePassageLinks({ + heading, + sources, + compact = false, +}: { + heading: string; + sources: SearchResult[]; + compact?: boolean; +}) { + if (sources.length === 0) return null; + + return ( + <div className="flex flex-wrap items-center gap-1.5"> + {sources.slice(0, compact ? 2 : 3).map((source, index) => ( + <Link + key={`${heading}:${source.id}:${index}`} + href={sourceResultHref(source)} + className={cn( + compact ? metadataPill : floatingControl, + "min-h-8 gap-1.5 px-2.5 text-[11px] sm:min-h-9 sm:px-3", + )} + title={`${source.title} ¡ page ${source.page_number ?? "n/a"} ¡ chunk ${source.chunk_index}`} + aria-label={`Open source passage #${index + 1}`} + > + <ExternalLink className="h-3.5 w-3.5" /> + <span>p.{source.page_number ?? "n/a"}</span> + <span className="hidden sm:inline">chunk {source.chunk_index}</span> + {source.source_strength ? <span className="hidden sm:inline">¡ {source.source_strength}</span> : null} + </Link> + ))} + </div> + ); +} diff --git a/src/components/clinical-dashboard/use-theme.ts b/src/components/clinical-dashboard/use-theme.ts new file mode 100644 index 000000000..0a1ef739e --- /dev/null +++ b/src/components/clinical-dashboard/use-theme.ts @@ -0,0 +1,52 @@ +"use client"; + +import { useEffect, useSyncExternalStore } from "react"; + +import { nextTheme, resolveThemePreference, type ResolvedTheme } from "@/lib/theme"; + +const themeStorageKey = "clinical-kb-theme"; +const themeChangeEvent = "clinical-kb-theme-change"; + +function getThemeSnapshot(): ResolvedTheme { + if (typeof window === "undefined") return "light"; + const storedTheme = window.localStorage.getItem(themeStorageKey); + const prefersDark = window.matchMedia("(prefers-color-scheme: dark)").matches; + return resolveThemePreference(storedTheme, prefersDark); +} + +function getServerThemeSnapshot(): ResolvedTheme { + return "light"; +} + +function subscribeTheme(onStoreChange: () => void) { + if (typeof window === "undefined") return () => undefined; + const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); + const notify = () => onStoreChange(); + + window.addEventListener("storage", notify); + window.addEventListener(themeChangeEvent, notify); + mediaQuery.addEventListener("change", notify); + + return () => { + window.removeEventListener("storage", notify); + window.removeEventListener(themeChangeEvent, notify); + mediaQuery.removeEventListener("change", notify); + }; +} + +export function useTheme() { + const theme = useSyncExternalStore(subscribeTheme, getThemeSnapshot, getServerThemeSnapshot); + + useEffect(() => { + document.documentElement.classList.toggle("dark", theme === "dark"); + }, [theme]); + + function toggleTheme() { + const resolved = nextTheme(theme); + window.localStorage.setItem(themeStorageKey, resolved); + document.documentElement.classList.toggle("dark", resolved === "dark"); + window.dispatchEvent(new Event(themeChangeEvent)); + } + + return { theme, toggleTheme }; +} diff --git a/src/lib/answer-formatting.ts b/src/lib/answer-formatting.ts index 823588102..ecccc5b1f 100644 --- a/src/lib/answer-formatting.ts +++ b/src/lib/answer-formatting.ts @@ -112,6 +112,7 @@ const groupTones: Record<AnswerDisplayGroup, AnswerDisplayTone> = { gap: "gap", summary: "summary", }; +const answerFormatFallbackText = "No usable answer text."; const highSignalGroups = new Set<AnswerDisplayGroup>([ "bottom_line", @@ -183,6 +184,24 @@ function stripBulletPrefix(value: string) { return value.replace(/^(?:[-*â€ĸ]|\d+[.)])\s+/, "").trim(); } +function splitBySemicolonList(value: string) { + const semicolonCount = (value.match(/;/g) ?? []).length; + if (semicolonCount < 2 || semicolonCount > 5) return []; + if (value.length > 340) return []; + + const parts = value + .split(/\s*;\s*/) + .map((part) => normalizeInline(part)) + .filter((part) => part.length >= 24); + + if (parts.length < 3 || parts.length > 6) return []; + + const avgWordsPerPart = parts.reduce((sum, part) => sum + part.split(/\s+/).filter(Boolean).length, 0) / parts.length; + if (avgWordsPerPart < 4) return []; + + return parts; +} + function splitInlineBullets(value: string) { const trimmed = value.trim(); if (!trimmed) return []; @@ -420,7 +439,7 @@ export function parseAnswerDisplayContent( id: "empty", label: null, displayLabel: "Source gap", - text: "No usable answer text for this result.", + text: answerFormatFallbackText, group: "gap", explicitLabel: false, isLead: true, @@ -442,10 +461,7 @@ export function parseAnswerDisplayContent( return { ...parsed, mode: coerceAnswerDisplayMode(preferredMode, parsed.mode) }; } - const semicolonParts = cleaned - .split(/\s*;\s*/) - .map((part) => normalizeInline(part)) - .filter((part) => part.length > 18); + const semicolonParts = splitBySemicolonList(cleaned); if (semicolonParts.length >= 3) { const lines = semicolonParts.map((part, index) => buildAnswerLine(part, index, "semicolon")); const parsed = parsedAnswer("bullets", lines); diff --git a/src/lib/answer-ranking.ts b/src/lib/answer-ranking.ts index 73506719a..422c48618 100644 --- a/src/lib/answer-ranking.ts +++ b/src/lib/answer-ranking.ts @@ -10,7 +10,7 @@ import type { RagAnswer, RagQueryClass, SearchResult } from "@/lib/types"; const answerRankStrategy = "query_focused_answer_evidence_v2"; -// Hardened boilerplate patterns to suppress generic medical disclaimers and +// Hardened boilerplate patterns to suppress generic medical disclaimers and // document control noise that can bury unique clinical instructions. const answerBoilerplatePattern = /\b(?:uncontrolled when printed|document control|review date|version\s+\d|page\s+\d+\s+of\s+\d+|copyright|confidential|all rights reserved|refer to the electronic version|consult your doctor|seek medical advice|this is not medical advice|intended for healthcare professionals|disclaimer)\b/i; @@ -183,12 +183,12 @@ function answerEvidenceScore(query: string, result: SearchResult, queryClass: Ra contentCoverage * 0.34 + titleCoverage * 0.12 + sectionCoverage * 0.1 + metadataCoverage * 0.08; const weakOverlapPenalty = combinedCoverage < 0.2 ? -0.18 : combinedCoverage < 0.34 ? -0.07 : 0; const adjacentOnlyPenalty = contentCoverage < 0.16 && adjacentCoverage > contentCoverage ? -0.08 : 0; - - // Dynamic boilerplate suppression: heavier penalty for common disclaimers + + // Dynamic boilerplate suppression: heavier penalty for common disclaimers // that don't match the specific clinical query. const boilerplatePenalty = answerBoilerplatePattern.test(result.content) && contentCoverage < 0.35 ? -0.15 : 0; const lowYieldPenalty = lowYieldSourceNoiseScore(result.content) >= 0.35 && contentCoverage < 0.45 ? -0.12 : 0; - + const coreConceptTokens = tokens.filter( (token) => ![ diff --git a/src/lib/bulk-import.ts b/src/lib/bulk-import.ts index de8bf1af5..066bd6ba1 100644 --- a/src/lib/bulk-import.ts +++ b/src/lib/bulk-import.ts @@ -51,8 +51,7 @@ export function parseImportCliArgs(argv: string[]): ImportCliArgs { if (!token.startsWith("--")) continue; const key = token.slice(2); if (key === "dry-run" || key === "force" || key === "force-large-import") { - const normalizedKey = - key === "dry-run" ? "dryRun" : key === "force-large-import" ? "forceLargeImport" : "force"; + const normalizedKey = key === "dry-run" ? "dryRun" : key === "force-large-import" ? "forceLargeImport" : "force"; args[normalizedKey] = true; continue; } diff --git a/src/lib/clinical-safety.ts b/src/lib/clinical-safety.ts index 9c85d7470..39ad2b975 100644 --- a/src/lib/clinical-safety.ts +++ b/src/lib/clinical-safety.ts @@ -1,6 +1,10 @@ import { documentCitationHref, formatCitationLabel } from "@/lib/citations"; import { queryCoreTerms } from "@/lib/evidence-relevance"; -import { sourceTextForDisplay } from "@/lib/source-text-sanitizer"; +import { + clinicalProseUsefulness, + sourceTextForCompactDisplay, + sourceTextForDisplay, +} from "@/lib/source-text-sanitizer"; import type { Citation, RagAnswer, SearchResult } from "@/lib/types"; export type SafetyFindingKind = @@ -64,7 +68,14 @@ function normalizeText(text: string) { } function conciseSourceText(text: string) { - const normalized = normalizeText(sourceTextForDisplay(text)); + const useful = clinicalProseUsefulness(text); + const normalized = normalizeText( + (sourceTextForCompactDisplay(useful.text || text) || sourceTextForDisplay(text)) + .replace(/\bsource mentions\s*:?\s*/gi, "") + .replace(/\b(?:procedure|policy|protocol)\s+[A-Z]{2,}(?:-[A-Z0-9]+)+(?:\/\d+)?\b[\s.:-]*/gi, "") + .replace(/\bpage\s+\d+\s+of\s+\d+\b[\s.:-]*/gi, "") + .replace(/\bchunk\s*(?:id|index)?\s*[:#=-]?\s*[a-z0-9_-]+\b[\s.:-]*/gi, ""), + ); if (normalized.length <= 260) return normalized; return `${normalized.slice(0, 257).trim()}...`; } @@ -141,7 +152,7 @@ export function extractSafetyFindings(answer: RagAnswer | null | undefined, limi id: `${match.kind}:${candidate.id}`, kind: match.kind, label: match.label, - text: `Source mentions: ${text}`, + text, citation: candidate.citation, href: documentCitationHref(candidate.citation), }); diff --git a/src/lib/clinical-vocabulary.ts b/src/lib/clinical-vocabulary.ts index 24ae07ab5..f4a5e5d32 100644 --- a/src/lib/clinical-vocabulary.ts +++ b/src/lib/clinical-vocabulary.ts @@ -92,16 +92,58 @@ const entries: ClinicalVocabularyEntry[] = [ { canonical: "metabolic monitoring", aliases: ["metbolic monitoring", "metabolic screening"], type: "workflow" }, { canonical: "patient safety plan", aliases: ["safety plan", "pt safety plan"], type: "form", weight: 1.1 }, // Systematic Medical Term Expansion (RAG-E1) - { canonical: "extrapyramidal side effects", aliases: ["epse", "eps", "dystonia", "akathisia", "parkinsonism", "tardive dyskinesia"], type: "risk", weight: 1.1 }, - { canonical: "neuroleptic malignant syndrome", aliases: ["nms", "hyperthermia", "muscle rigidity", "autonomic instability"], type: "risk", weight: 1.2 }, - { canonical: "serotonin syndrome", aliases: ["serotonin toxicity", "shivering", "diarrhea", "muscle rigidity", "fever", "seizures"], type: "risk", weight: 1.2 }, - { canonical: "anticholinergic side effects", aliases: ["anticholinergic toxicity", "dry mouth", "blurred vision", "constipation", "urinary retention", "tachycardia"], type: "risk", weight: 1.1 }, - { canonical: "metabolic syndrome", aliases: ["weight gain", "dyslipidemia", "hyperglycemia", "hypertension", "waist circumference"], type: "risk", weight: 1.1 }, - { canonical: "renal function", aliases: ["egfr", "creatinine", "kidney function", "urea", "electrolytes", "u&e"], type: "lab", weight: 1.1 }, + { + canonical: "extrapyramidal side effects", + aliases: ["epse", "eps", "dystonia", "akathisia", "parkinsonism", "tardive dyskinesia"], + type: "risk", + weight: 1.1, + }, + { + canonical: "neuroleptic malignant syndrome", + aliases: ["nms", "hyperthermia", "muscle rigidity", "autonomic instability"], + type: "risk", + weight: 1.2, + }, + { + canonical: "serotonin syndrome", + aliases: ["serotonin toxicity", "shivering", "diarrhea", "muscle rigidity", "fever", "seizures"], + type: "risk", + weight: 1.2, + }, + { + canonical: "anticholinergic side effects", + aliases: [ + "anticholinergic toxicity", + "dry mouth", + "blurred vision", + "constipation", + "urinary retention", + "tachycardia", + ], + type: "risk", + weight: 1.1, + }, + { + canonical: "metabolic syndrome", + aliases: ["weight gain", "dyslipidemia", "hyperglycemia", "hypertension", "waist circumference"], + type: "risk", + weight: 1.1, + }, + { + canonical: "renal function", + aliases: ["egfr", "creatinine", "kidney function", "urea", "electrolytes", "u&e"], + type: "lab", + weight: 1.1, + }, { canonical: "thyroid function", aliases: ["tft", "tsh", "free t4", "t3"], type: "lab", weight: 1.1 }, { canonical: "prolactin level", aliases: ["hyperprolactinemia", "prolactin"], type: "lab", weight: 1.1 }, { canonical: "body mass index", aliases: ["bmi", "weight", "height"], type: "lab", weight: 1.05 }, - { canonical: "blood pressure", aliases: ["bp", "hypertension", "hypotension", "orthostatic hypotension"], type: "lab", weight: 1.1 }, + { + canonical: "blood pressure", + aliases: ["bp", "hypertension", "hypotension", "orthostatic hypotension"], + type: "lab", + weight: 1.1, + }, ]; function normalize(value: string) { diff --git a/src/lib/deep-memory.ts b/src/lib/deep-memory.ts index f6f2de0b8..3bce46d18 100644 --- a/src/lib/deep-memory.ts +++ b/src/lib/deep-memory.ts @@ -521,7 +521,7 @@ async function stampDeepMemoryVersion(args: { supabase: SupabaseClient; document }) .eq("id", chunk.id); if (error) throw new Error(error.message); - }) + }), ); } } @@ -572,8 +572,8 @@ export async function upsertDocumentDeepMemory(args: { const embeddings = await embedTexts(cards.map(embeddingText)); if (embeddings.length !== cards.length) throw new Error("OpenAI returned an unexpected memory-card embedding count."); - for (let start = 0; start < cards.length; start += 10) { - const batch = cards.slice(start, start + 10).map((card, index) => { + for (let start = 0; start < cards.length; start += 50) { + const batch = cards.slice(start, start + 50).map((card, index) => { const { section_index: sectionIndex, ...row } = card; return { ...row, @@ -594,8 +594,8 @@ export async function upsertDocumentDeepMemory(args: { }); if (indexUnits.length > 0) { const indexUnitEmbeddings = await embedTexts(indexUnits.map(embeddingTextForDocumentIndexUnit)); - for (let start = 0; start < indexUnits.length; start += 10) { - const batch = indexUnits.slice(start, start + 10).map((unit, index) => ({ + for (let start = 0; start < indexUnits.length; start += 50) { + const batch = indexUnits.slice(start, start + 50).map((unit, index) => ({ ...unit, embedding: indexUnitEmbeddings[start + index], })); diff --git a/src/lib/env.ts b/src/lib/env.ts index 2b99ec3e7..18e47b14c 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -61,11 +61,11 @@ const envSchema = z.object({ MAX_IMPORT_BYTES_PER_RUN: z.coerce.number().int().positive().default(157286400), CHUNK_SIZE: z.coerce.number().int().positive().default(2000), CHUNK_OVERLAP: z.coerce.number().int().nonnegative().default(200), - WORKER_POLL_MS: z.coerce.number().int().positive().default(30000), - WORKER_BATCH_SIZE: z.coerce.number().int().positive().default(3), - WORKER_CONCURRENCY: z.coerce.number().int().positive().default(1), - WORKER_MAX_ATTEMPTS: z.coerce.number().int().positive().default(3), - WORKER_STALE_AFTER_MINUTES: z.coerce.number().int().positive().default(45), + WORKER_POLL_MS: z.coerce.number().int().positive().default(1500), + WORKER_BATCH_SIZE: z.coerce.number().int().positive().default(25), + WORKER_CONCURRENCY: z.coerce.number().int().positive().default(8), + WORKER_MAX_ATTEMPTS: z.coerce.number().int().positive().default(5), + WORKER_STALE_AFTER_MINUTES: z.coerce.number().int().positive().default(5), WORKER_HEALTH_BACKOFF_MS: z.coerce.number().int().positive().default(120000), WORKER_MAX_CLAIM_FAILURES: z.coerce.number().int().positive().default(3), WORKER_PROGRESS_UPDATE_MIN_INTERVAL_MS: z.coerce.number().int().positive().default(60000), diff --git a/src/lib/extractors/document.ts b/src/lib/extractors/document.ts index 1228a1708..d0e9da987 100644 --- a/src/lib/extractors/document.ts +++ b/src/lib/extractors/document.ts @@ -119,9 +119,7 @@ async function extractPdf(buffer: Buffer) { const warnings = ["Used JavaScript PDF fallback; install Python PDF/OCR prerequisites for scanned PDFs."]; const ocrNeededPages = pages.filter((page) => page.needsOcr).length; if (ocrNeededPages > 0) { - warnings.push( - `needs_ocr: ${ocrNeededPages} page(s) appear image-only and were not OCR'd by the JS fallback.`, - ); + warnings.push(`needs_ocr: ${ocrNeededPages} page(s) appear image-only and were not OCR'd by the JS fallback.`); } return { pages, images, warnings }; diff --git a/src/lib/image-filtering.ts b/src/lib/image-filtering.ts index 6e71cf52b..dda0f90e3 100644 --- a/src/lib/image-filtering.ts +++ b/src/lib/image-filtering.ts @@ -152,7 +152,11 @@ export function lowSignalImageTextSkipReason(input: { if (sourceKind === "table_crop" && tokens.length >= 12 && uniqueRatio < 0.34 && clinicalHits < 2) { return "repetitive noisy table OCR"; } - if (sourceKind === "table_crop" && noisyTablePatterns.some((pattern) => pattern.test(text)) && adminHits > clinicalHits) { + if ( + sourceKind === "table_crop" && + noisyTablePatterns.some((pattern) => pattern.test(text)) && + adminHits > clinicalHits + ) { return "document-control table OCR"; } if (sourceKind !== "table_crop" && shortestSide !== null && shortestSide < 96 && clinicalHits === 0) { diff --git a/src/lib/ingestion-recovery.ts b/src/lib/ingestion-recovery.ts index ee8f25110..989bc9d40 100644 --- a/src/lib/ingestion-recovery.ts +++ b/src/lib/ingestion-recovery.ts @@ -35,9 +35,7 @@ export function buildIngestionRecoveryPlan(args: { const chunkCount = Number(job.documents?.chunk_count ?? 0); const isIndexedDocument = documentStatus === "indexed" && chunkCount > 0; const isRecoverableStatus = - job.status === "failed" || - job.status === "pending" || - isStaleProcessingJob(job, now, args.staleAfterMinutes); + job.status === "failed" || job.status === "pending" || isStaleProcessingJob(job, now, args.staleAfterMinutes); if (isIndexedDocument && job.status !== "completed") { actions.push({ action: "supersede", jobId: job.id, documentId: job.document_id }); diff --git a/src/lib/ingestion.ts b/src/lib/ingestion.ts index 082975f1f..9a0f7507d 100644 --- a/src/lib/ingestion.ts +++ b/src/lib/ingestion.ts @@ -1,8 +1,11 @@ export function isRetryableIngestionError(error: unknown) { const message = error instanceof Error ? error.message : String(error); if (isPartialIndexWriteConflict(error)) return false; - return /\b(429|rate limit|timeout|temporar|network|fetch failed|ECONNRESET|ETIMEDOUT|5\d\d|502|503|504|bad gateway|gateway timeout|service unavailable)\b/i.test(message) || - /document_pages_document_id_page_number_key/i.test(message); + return ( + /\b(429|rate limit|timeout|temporar|network|fetch failed|ECONNRESET|ETIMEDOUT|5\d\d|502|503|504|bad gateway|gateway timeout|service unavailable)\b/i.test( + message, + ) || /document_pages_document_id_page_number_key/i.test(message) + ); } export function isPartialIndexWriteConflict(error: unknown) { diff --git a/src/lib/openai.ts b/src/lib/openai.ts index 72b7e0f85..d2c63c086 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -414,11 +414,9 @@ export async function embedTexts(texts: string[]) { const byIndex = new Array<number[]>(uniqueTexts.length); for (const item of response.data) { if (item.index < 0 || item.index >= uniqueTexts.length) { - throw new PublicApiError( - `OpenAI returned an out-of-range embedding index ${item.index}.`, - 502, - { code: "openai_embedding_index_range" }, - ); + throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { + code: "openai_embedding_index_range", + }); } // IDX-C2: guard against a model whose dimension does not match the schema. if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { diff --git a/src/lib/rag.ts b/src/lib/rag.ts index d9172a409..5a5da812d 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -370,6 +370,9 @@ export type SearchTelemetry = { search_cache_hit: boolean; shared_cache_hit?: boolean; query_class?: RagQueryClass; + vector_candidate_count?: number; + text_candidate_count?: number; + embedding_field_count?: number; retrieval_query_variant_count?: number; rag_alias_count?: number; rag_alias_expansion_count?: number; @@ -602,7 +605,10 @@ type SanitizedCitations = { modelCited: boolean; }; -function sanitizeCitations(proposed: Array<{ chunk_id: string }> | undefined, results: SearchResult[]): SanitizedCitations { +function sanitizeCitations( + proposed: Array<{ chunk_id: string }> | undefined, + results: SearchResult[], +): SanitizedCitations { const chunks = allowedChunkMap(results); const citations: Citation[] = []; const seen = new Set<string>(); @@ -1221,6 +1227,9 @@ async function getSharedCachedSearch(args: SearchChunksArgs) { search_cache_hit: true, shared_cache_hit: true, query_class: payload.telemetry?.query_class, + vector_candidate_count: payload.telemetry?.vector_candidate_count, + text_candidate_count: payload.telemetry?.text_candidate_count, + embedding_field_count: payload.telemetry?.embedding_field_count, retrieval_query_variant_count: payload.telemetry?.retrieval_query_variant_count ?? 0, text_fast_path_latency_ms: 0, embedding_skipped: true, @@ -3082,6 +3091,9 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { const telemetry: SearchTelemetry = { search_cache_hit: false, query_class: queryClassification.queryClass, + vector_candidate_count: 0, + text_candidate_count: 0, + embedding_field_count: 0, retrieval_query_variant_count: 0, rag_alias_count: 0, rag_alias_expansion_count: 0, @@ -3138,6 +3150,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentIds: documentFilterList, matchCount: candidateCount, }); + telemetry.text_candidate_count = textData.length; telemetry.text_fast_path_latency_ms = Date.now() - textRpcStartedAt; telemetry.supabase_rpc_latency_ms += telemetry.text_fast_path_latency_ms; @@ -3293,6 +3306,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentIds: documentFilterList, matchCount: Math.min(candidateCount, 48), }); + telemetry.embedding_field_count = embeddingFieldCandidates.length; telemetry.supabase_rpc_latency_ms += Date.now() - embeddingFieldStartedAt; if (embeddingFieldCandidates.length > 0) { textFastResults = mergeSearchResults(embeddingFieldCandidates, textFastResults); @@ -3326,6 +3340,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { owner_filter: args.ownerId ?? null, }); telemetry.supabase_rpc_latency_ms += Date.now() - hybridRpcStartedAt; + telemetry.vector_candidate_count = hybridData?.length ?? 0; if (!hybridError) { const rerankStartedAt = Date.now(); @@ -3382,6 +3397,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { throw error; }); telemetry.supabase_rpc_latency_ms += Date.now() - fallbackRpcStartedAt; + telemetry.vector_candidate_count = resultSets.reduce((count, resultSet) => count + resultSet.length, 0); const rerankStartedAt = Date.now(); const mergedWithMetadata = await attachDocumentRankingMetadata( @@ -3502,11 +3518,11 @@ function neutralizeInstructions(text: string): string { let cleaned = text; cleaned = cleaned.replace( /\bignore\s+(?:all\s+)?(?:previous\s+)?instructions(?:\s+and\s+\w+(?:\s+\d+\s+\w+)?)?/gi, - "[neutralized-instruction: ignore instructions]" + "[neutralized-instruction: ignore instructions]", ); cleaned = cleaned.replace( /\byou\s+are\s+now\s+an?\s+(?:unrestricted|jailbroken|assistant)(?:\s+\w+){0,3}/gi, - "[neutralized-instruction: unrestricted assistant]" + "[neutralized-instruction: unrestricted assistant]", ); return cleaned; } @@ -3593,20 +3609,20 @@ export function parseAnswerJson(raw: string, results: SearchResult[], query?: st machineReadableFallbackAnswer; const answerSections = sanitizeAnswerSections(parsed.answerSections, results, query); const grounded = modelCited && citations.length > 0 && confidence !== "unsupported"; - const answer: RagAnswer = { - answer: boldHighYieldClinicalText(sanitizedAnswer, query), - grounded, - confidence, - citations, - sources: results, + const answer: RagAnswer = { + answer: boldHighYieldClinicalText(sanitizedAnswer, query), + grounded, + confidence, + citations, + sources: results, answerSections, - conflictsOrGaps: sanitizeConflictsOrGaps(parsed.conflictsOrGaps, results), - quoteCards: sanitizeQuoteCards(parsed.quoteCards, results), - visualEvidence: [], - bestSource: null, - documentBreakdown: [], - routingReason: undefined, - }; + conflictsOrGaps: sanitizeConflictsOrGaps(parsed.conflictsOrGaps, results), + quoteCards: sanitizeQuoteCards(parsed.quoteCards, results), + visualEvidence: [], + bestSource: null, + documentBreakdown: [], + routingReason: undefined, + }; if (!modelCited) { answer.routingReason = "ungrounded_no_model_citation"; } diff --git a/src/lib/supabase/health.ts b/src/lib/supabase/health.ts index a3b4521a9..61da9133a 100644 --- a/src/lib/supabase/health.ts +++ b/src/lib/supabase/health.ts @@ -1,7 +1,12 @@ type SupabaseProbeClient = { from(table: string): { - select(columns: string, options?: Record<string, unknown>): { - limit(count: number): PromiseLike<{ error: { message?: string; code?: string; details?: string; hint?: string } | null }>; + select( + columns: string, + options?: Record<string, unknown>, + ): { + limit( + count: number, + ): PromiseLike<{ error: { message?: string; code?: string; details?: string; hint?: string } | null }>; }; }; }; @@ -28,14 +33,19 @@ export function isSupabaseUnavailableError(error: unknown) { export function formatSupabaseUnavailableError(error: unknown) { const message = errorMessage(error); - const title = message.match(/<title>\s*([^<]+?)\s*<\/title>/i)?.[1]?.replace(/\s+/g, " ").trim(); + const title = message + .match(/<title>\s*([^<]+?)\s*<\/title>/i)?.[1] + ?.replace(/\s+/g, " ") + .trim(); if (title) return `Supabase is temporarily unavailable (${title}).`; if (/connection terminated/i.test(message)) return "Supabase SQL connection was terminated due to a timeout."; if (/statement timeout/i.test(message)) return "Supabase cancelled the query due to statement timeout."; if (/544/.test(message)) return "Supabase Storage is timing out with a 544 response."; if (/522/.test(message)) return "Supabase API is timing out with a 522 response."; if (/504/.test(message)) return "Supabase API is timing out with a 504 response."; - return message ? `Supabase is temporarily unavailable: ${message.slice(0, 240)}` : "Supabase is temporarily unavailable."; + return message + ? `Supabase is temporarily unavailable: ${message.slice(0, 240)}` + : "Supabase is temporarily unavailable."; } export async function probeSupabaseHealth(supabase: SupabaseProbeClient): Promise<SupabaseHealthResult> { diff --git a/src/lib/ward-output.ts b/src/lib/ward-output.ts index ecd35860b..ffd00b496 100644 --- a/src/lib/ward-output.ts +++ b/src/lib/ward-output.ts @@ -295,15 +295,15 @@ function buildVerifySourceItems(answer: RagAnswer) { return uniqueShortItems( [ citationCount - ? `${citationCount} linked citation${citationCount === 1 ? "" : "s"} available for source verification.` - : "No linked citations were returned for this answer.", + ? `${citationCount} linked citation${citationCount === 1 ? "" : "s"} for verification.` + : "No linked citations.", quoteCount - ? `${quoteCount} exact source quote${quoteCount === 1 ? "" : "s"} available before clinical use.` - : "No exact source quote was returned; verify the answer against the source document.", + ? `${quoteCount} exact source quote${quoteCount === 1 ? "" : "s"} available.` + : "No exact source quote returned.", sourceStrength && sourceStrength !== "none" ? `Strongest retrieved source support: ${sourceStrength}.` : "", answer.confidence === "unsupported" - ? "Treat as unsupported: do not use clinically without opening and checking source text." - : "Open the cited source passage before copying into the medical record.", + ? "Unsupported: verify against source text before use." + : "Verify source passage before copying into the medical record.", ], 4, ); @@ -336,6 +336,19 @@ const sectionKindLabels: Record<AnswerSectionKind, string> = { verification: "Verify source", }; +const genericBottomLinePlaceholders = [ + "no usable answer text.", + "no usable answer text for this result", + "no usable section text available", + "no usable section text for this result", +]; + +function isGenericBottomLine(value: string) { + const normalized = normalizeText(value).toLowerCase(); + if (!normalized) return true; + return genericBottomLinePlaceholders.some((placeholder) => normalized.includes(placeholder)); +} + function sectionKindToClinicalSection(kind?: AnswerSectionKind): { id: ClinicalOutputSectionId; title: string } | null { if (kind === "bottom_line") return { id: "bottom-line", title: "Bottom line" }; if (kind === "required_actions") return { id: "action", title: "Action" }; @@ -367,7 +380,7 @@ function sectionDisplayLines(answer: RagAnswer) { .filter((section) => isPromotableAnswerSection(section, answer)) .flatMap((section) => { const label = section.kind ? sectionKindLabels[section.kind] : section.heading; - return parseAnswerDisplayContent(`${label}: ${section.body}`).lines; + return parseAnswerDisplayContent(`${label}: ${section.body}`, answer.responseMode).lines; }); } @@ -377,6 +390,12 @@ function compactTableCell(value: string, limit = 180) { return `${normalized.slice(0, limit - 3).trim()}...`; } +function compactEvidencePassage(value: string, limit = 140) { + const normalized = normalizeClinicalItem(value); + if (normalized.length <= limit) return normalized; + return `${normalized.slice(0, limit - 3).trim()}...`; +} + function clinicalTableCaption(value: string) { return normalizeText(value) .replace(/\btable\s+\d+\s*[:.-]?\s*/i, "") @@ -526,7 +545,7 @@ export function buildAnswerEvidenceMap(answer: RagAnswer | null | undefined): An citationCount: citationIds.length, sourceStatus: sourceStatusSummary(metadata), bestSourceLabel, - bestLinkedPassage, + bestLinkedPassage: compactEvidencePassage(bestLinkedPassage, 130), href, }); } @@ -536,12 +555,12 @@ export function buildAnswerEvidenceMap(answer: RagAnswer | null | undefined): An return answer.citations.slice(0, 6).map((citation, index) => ({ id: `citation:${citation.chunk_id}:${index}`, section: "Citation", - detail: "Linked source passage available for review.", + detail: "Source passage available.", supportLevel: supportLevelLabel(answer.evidenceSummary?.source_strength), citationCount: 1, sourceStatus: sourceStatusSummary(citation.source_metadata), bestSourceLabel: formatCitationLabel(citation), - bestLinkedPassage: "Open the linked passage to verify the source text.", + bestLinkedPassage: "Open source passage.", href: evidenceHref(citation.document_id, citation.page_number, citation.chunk_id), })); } @@ -586,9 +605,14 @@ function clinicalTableToTextRows(table: ClinicalThresholdTable) { export function buildClinicalOutputSections(answer: RagAnswer | null | undefined) { if (!answer) return []; - const answerContent = parseAnswerDisplayContent(answer.answer); - const answerLead = answerContent.lead ?? answerContent.lines[0]; - const answerDetailLines = answerContent.lines.filter((line) => line.id !== answerLead?.id); + const answerContent = parseAnswerDisplayContent(answer.answer, answer.responseMode); + const answerLead = + (answerContent.lead && !isGenericBottomLine(answerContent.lead.text) ? answerContent.lead : undefined) ?? + answerContent.lines.find((line) => !isGenericBottomLine(line.text) && line.group === "bottom_line") ?? + answerContent.lines.find((line) => !isGenericBottomLine(line.text)); + const answerDetailLines = answerLead + ? answerContent.lines.filter((line) => line.id !== answerLead.id) + : answerContent.lines; const parsedLines = [...answerDetailLines, ...sectionDisplayLines(answer)]; const quoteTexts = answer.quoteCards?.map((quote) => quote.quote) ?? []; const allTexts = [...parsedLines.map((line) => line.text), ...quoteTexts]; @@ -636,7 +660,12 @@ export function buildClinicalOutputSections(answer: RagAnswer | null | undefined group === "bottom_line" ? 1 : 4, ); if (items.length === 0) continue; - sections.push({ ...section, items }); + const existing = sections.find((candidate) => candidate.id === section.id); + if (existing) { + existing.items = uniqueShortItems([...existing.items, ...items], group === "bottom_line" ? 1 : 4); + } else { + sections.push({ ...section, items }); + } } for (const section of answer.answerSections ?? []) { @@ -685,24 +714,108 @@ export function buildClinicalOutputSections(answer: RagAnswer | null | undefined return sections; } -export function formatAnswerForClipboard(answer: RagAnswer) { - const citations = answer.citations.map((citation, index) => `${index + 1}. ${formatCitationLabel(citation)}`); - const provenance = answer.citations.map( +function normalizedOutputText(text: string) { + return normalizeClinicalToken(normalizeText(text)); +} + +function isRepeatedBottomLine(item: string, bottomLine: string) { + const normalizedItem = normalizedOutputText(item); + const normalizedBottomLine = normalizedOutputText(bottomLine); + if (!normalizedItem || !normalizedBottomLine) return false; + if (normalizedItem === normalizedBottomLine) return true; + if (item.length >= 80 && normalizedBottomLine.includes(normalizedItem)) return true; + if (bottomLine.length >= 80 && normalizedItem.includes(normalizedBottomLine)) return true; + return false; +} + +function bottomLineForOutput(answer: RagAnswer, sections: ClinicalOutputSection[]) { + const bottomLineSection = sections.find((section) => section.id === "bottom-line"); + const parsedLead = parseAnswerDisplayContent(answer.answer, answer.responseMode).lead; + const fallbackFromSections = (answer.answerSections ?? []) + .map((section) => normalizeClinicalItem(section.body)) + .find((item) => Boolean(item) && !isGenericBottomLine(item)); + const fallbackFromParsed = parseAnswerDisplayContent(answer.answer, answer.responseMode) + .lines.map((line) => normalizeClinicalItem(line.text)) + .find((item) => Boolean(item) && !isGenericBottomLine(item)); + + const candidateBottomLine = + bottomLineSection?.items[0] ?? + fallbackFromParsed ?? + fallbackFromSections ?? + parsedLead?.text ?? + normalizeClinicalItem(answer.answer) ?? + normalizeText(answer.answer); + return isGenericBottomLine(candidateBottomLine) + ? "No stable source-backed lead. Review cited passages before reuse." + : candidateBottomLine; +} + +function highYieldSectionsForOutput(answer: RagAnswer, bottomLine: string) { + return buildHighYieldClinicalOutputSections(answer) + .filter((section) => section.id !== "bottom-line" && section.id !== "verify-source") + .map((section) => ({ + ...section, + items: uniqueShortItems( + section.items.filter((item) => !isRepeatedBottomLine(item, bottomLine)), + 4, + ), + })) + .filter((section) => section.items.length > 0 || Boolean(section.tables?.length)); +} + +function sectionOutputLines(section: ClinicalOutputSection) { + return [ + section.title, + ...section.items.map((item) => `- ${item}`), + ...(section.tables ?? []).flatMap(clinicalTableToTextRows), + "", + ]; +} + +function citationOutputLines(answer: RagAnswer) { + if (answer.citations.length === 0) return ["No linked citations."]; + return answer.citations.map((citation, index) => `${index + 1}. ${formatCitationLabel(citation)}`); +} + +function sourceStatusOutputLines(answer: RagAnswer) { + if (answer.citations.length === 0) return ["No source provenance."]; + return answer.citations.map( (citation, index) => `${index + 1}. ${formatCitationLabel(citation)} | ${clipboardProvenanceLine(citation.source_metadata)}`, ); - return [ +} + +const clinicalReviewRequirement = + "Draft for clinician review only. Verify source text, local policy, patient context, and medication details before use."; + +function compactOutputDocument(lines: string[]) { + return lines + .filter((line, index, allLines) => line || (allLines[index - 1] && allLines[index + 1])) + .join("\n") + .trim(); +} + +export function formatAnswerForClipboard(answer: RagAnswer) { + const sections = buildClinicalOutputSections(answer); + const bottomLine = bottomLineForOutput(answer, sections); + const highYieldSections = highYieldSectionsForOutput(answer, bottomLine); + return compactOutputDocument([ "Source-backed answer draft", - "Clinician must verify against linked source documents before clinical use.", + "Verify against linked source documents before clinical use.", "", - normalizeText(answer.answer), - citations.length ? "\nCitations" : "", - ...citations, - provenance.length ? "\nSource status" : "", - ...provenance, - ] - .filter(Boolean) - .join("\n"); + "Bottom line", + `- ${bottomLine}`, + "", + ...highYieldSections.flatMap(sectionOutputLines), + "Citations", + ...citationOutputLines(answer), + "", + "Source status", + ...sourceStatusOutputLines(answer), + "", + "Review requirement", + clinicalReviewRequirement, + ]); } export function formatQuotesForClipboard(quotes: QuoteCard[] = []) { @@ -713,6 +826,8 @@ export function formatQuotesForClipboard(quotes: QuoteCard[] = []) { export function formatWardNote(answer: RagAnswer, demoMode = false) { const clinicalSections = buildClinicalOutputSections(answer); + const bottomLine = bottomLineForOutput(answer, clinicalSections); + const highYieldSections = highYieldSectionsForOutput(answer, bottomLine); const generatedAt = new Intl.DateTimeFormat("en-AU", { day: "2-digit", month: "2-digit", @@ -723,38 +838,25 @@ export function formatWardNote(answer: RagAnswer, demoMode = false) { timeZone: "Australia/Perth", timeZoneName: "short", }).format(new Date()); - const sourceStatus = answer.citations.map( - (citation, index) => - `${index + 1}. ${formatCitationLabel(citation)} | ${clipboardProvenanceLine(citation.source_metadata)}`, - ); - const body = [ + return compactOutputDocument([ "Source-backed clinical draft", - "Clinician must verify against linked source documents before clinical use.", + "Verify against linked source documents before clinical use.", demoMode ? "Synthetic demo only: not clinical guidance." : "Generated only from indexed source documents.", `Generated: ${generatedAt}`, "", - normalizeText(answer.answer), + "Bottom line", + `- ${bottomLine}`, "", - ...clinicalSections.flatMap((section) => [ - section.title, - ...section.items.map((item) => `- ${item}`), - ...(section.tables ?? []).flatMap(clinicalTableToTextRows), - "", - ]), + ...highYieldSections.flatMap(sectionOutputLines), "Citations", - ...answer.citations.map((citation, index) => `${index + 1}. ${formatCitationLabel(citation)}`), + ...citationOutputLines(answer), "", "Source status", - ...sourceStatus, + ...sourceStatusOutputLines(answer), "", "Review requirement", - "This is a draft for clinician review only. Verify source text, source status, local policy, patient context, and medication details before use.", - ]; - - return body - .filter((line, index, lines) => line || lines[index - 1]) - .join("\n") - .trim(); + clinicalReviewRequirement, + ]); } export function createQuoteFollowUp(quote: QuoteCard) { diff --git a/supabase/migrations/20260620000000_retrieval_accuracy_speed.sql b/supabase/migrations/20260620000000_retrieval_accuracy_speed.sql new file mode 100644 index 000000000..081fd42dd --- /dev/null +++ b/supabase/migrations/20260620000000_retrieval_accuracy_speed.sql @@ -0,0 +1,634 @@ +-- Migration: Maximize RAG retrieval accuracy + speed +-- 1. Tune HNSW indexes (m=24, ef_construction=128) +-- 2. Add rag_retrieval_logs table for diagnostics +-- 3. Add content_hash dedup on document_embedding_fields +-- 4. Drop duplicate/redundant indexes +-- 5. Enhance hybrid scoring with quality + recency weighting +-- 6. Add analyze_rag_tables() utility function + +set search_path = public, extensions; + +-- ============================================================ +-- 1. HNSW INDEX TUNING +-- Rebuild with m=24 (graph connectivity) and ef_construction=128 +-- (build-time recall). Default was m=16, ef_construction=64. +-- ============================================================ + +drop index if exists document_chunks_embedding_hnsw_idx; +create index document_chunks_embedding_hnsw_idx + on public.document_chunks using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +drop index if exists document_memory_cards_embedding_hnsw_idx; +create index document_memory_cards_embedding_hnsw_idx + on public.document_memory_cards using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +drop index if exists document_index_units_embedding_hnsw_idx; +create index document_index_units_embedding_hnsw_idx + on public.document_index_units using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +drop index if exists document_embedding_fields_embedding_hnsw_idx; +create index document_embedding_fields_embedding_hnsw_idx + on public.document_embedding_fields using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +-- ============================================================ +-- 2. RAG RETRIEVAL LOGS TABLE +-- Per-query diagnostics: candidate scores, latency breakdown, +-- miss detection. +-- ============================================================ + +create table if not exists public.rag_retrieval_logs ( + id uuid primary key default gen_random_uuid(), + owner_id uuid references auth.users(id) on delete set null, + query text not null, + normalized_query text, + query_class text, + retrieval_strategy text, + + -- Candidate scores + candidate_count integer not null default 0, + top_similarity double precision, + top_text_rank double precision, + top_hybrid_score double precision, + top_rrf_score double precision, + mean_hybrid_score double precision, + + -- Selected citations + selected_chunk_ids uuid[] not null default '{}', + selected_document_ids uuid[] not null default '{}', + selected_count integer not null default 0, + + -- Latency breakdown + embedding_latency_ms integer, + rpc_latency_ms integer, + rerank_latency_ms integer, + total_latency_ms integer, + + -- Source counts + vector_candidate_count integer, + text_candidate_count integer, + memory_card_count integer, + index_unit_count integer, + embedding_field_count integer, + + -- Miss detection + is_miss boolean not null default false, + miss_reason text, + + -- Metadata + embedding_cache_hit boolean, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); + +create index if not exists rag_retrieval_logs_owner_created_idx + on public.rag_retrieval_logs(owner_id, created_at desc); +create index if not exists rag_retrieval_logs_miss_idx + on public.rag_retrieval_logs(is_miss, created_at desc) + where is_miss = true; +create index if not exists rag_retrieval_logs_strategy_idx + on public.rag_retrieval_logs(retrieval_strategy, created_at desc); + +alter table public.rag_retrieval_logs enable row level security; +grant select, insert, update, delete on table public.rag_retrieval_logs to service_role; +grant select on table public.rag_retrieval_logs to authenticated; + +create policy "rag retrieval logs owner read" on public.rag_retrieval_logs + for select to authenticated using (owner_id = (select auth.uid())); + +-- ============================================================ +-- 3. CONTENT_HASH DEDUP ON DOCUMENT_EMBEDDING_FIELDS +-- Replace the problematic unique(document_id, source_chunk_id, +-- field_type, content) constraint that indexes full text. +-- ============================================================ + +alter table public.document_embedding_fields + add column if not exists content_hash text; + +-- Populate for existing rows (md5 is fast and sufficient for dedup) +update public.document_embedding_fields + set content_hash = md5(content) + where content_hash is null; + +-- Drop the old constraint. The constraint name is auto-generated by Postgres +-- from the unique(...) declaration. Try both possible names. +alter table public.document_embedding_fields + drop constraint if exists document_embedding_fields_document_id_source_chunk_id_field_key; +alter table public.document_embedding_fields + drop constraint if exists document_embedding_fields_document_id_source_chunk_id_fiel_key; + +-- Create hash-based unique index +create unique index if not exists document_embedding_fields_dedup_idx + on public.document_embedding_fields(document_id, source_chunk_id, field_type, content_hash); + +-- ============================================================ +-- 4. DROP DUPLICATE/REDUNDANT INDEXES +-- These single-column indexes are covered by composite indexes +-- with the same leading column. +-- ============================================================ + +-- document_chunks_document_id_idx(document_id) +-- superseded by document_chunks_document_idx(document_id, chunk_index) +drop index if exists document_chunks_document_id_idx; + +-- document_sections_document_id_idx(document_id) +-- superseded by document_sections_document_idx(document_id, section_index) +drop index if exists document_sections_document_id_idx; + +-- document_memory_cards_document_id_idx(document_id) +-- superseded by document_memory_cards_document_idx(document_id, card_type, confidence) +drop index if exists document_memory_cards_document_id_idx; + +-- document_images_document_id_idx(document_id) +-- superseded by document_images_document_idx(document_id, page_number) +drop index if exists document_images_document_id_idx; + +-- document_labels_document_id_idx(document_id) +-- exact duplicate of document_labels_document_idx(document_id) +drop index if exists document_labels_document_id_idx; + +-- document_embedding_fields_document_id_idx(document_id) +-- superseded by document_embedding_fields_document_idx(document_id, field_type) +drop index if exists document_embedding_fields_document_id_idx; + +-- document_table_facts_document_id_idx(document_id) +-- superseded by document_table_facts_document_idx(document_id, page_number) +drop index if exists document_table_facts_document_id_idx; + +-- document_index_quality_document_id_idx(document_id) +-- PK on document_index_quality(document_id) already serves as the index +drop index if exists document_index_quality_document_id_idx; + +-- ============================================================ +-- 5. ENHANCED HYBRID SCORING +-- Add quality_score, confidence, and recency factors to the +-- ranking formulas. +-- ============================================================ + +-- 5a. match_document_chunks_hybrid: add quality and recency weighting +create or replace function public.match_document_chunks_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 12, + min_similarity double precision default 0.12, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + vector_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + row_number() over (order by c.embedding <=> query_embedding) as vector_rank, + null::bigint as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' + and 1 - (c.embedding <=> query_embedding) >= min_similarity + order by c.embedding <=> query_embedding + limit greatest(match_count * 6, 48) + ), + text_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + null::bigint as vector_rank, + row_number() over ( + order by + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc, + c.embedding <=> query_embedding + ) as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' + and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit greatest(match_count * 6, 48) + ), + combined as ( + select * from vector_ranked + union all + select * from text_ranked + ), + scored as ( + select + id, + document_id, + page_number, + chunk_index, + section_heading, + content, + retrieval_synopsis, + image_ids, + max(similarity)::double precision as similarity, + max(text_rank)::double precision as text_rank, + min(vector_rank) as vector_rank, + min(text_match_rank) as text_match_rank, + max(quality_score)::double precision as quality_score, + bool_or(has_deep_index) as has_deep_index, + max(doc_updated_at) as doc_updated_at + from combined + group by id, document_id, page_number, chunk_index, section_heading, content, retrieval_synopsis, image_ids + ), + scored_metrics as ( + select + scored.*, + ( + (scored.similarity * 0.62) + + (least(scored.text_rank, 1) * 0.22) + + (scored.quality_score * 0.10) + + (case when scored.doc_updated_at > now() - interval '90 days' then 0.06 else 0 end) + )::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + scored.vector_rank), 0) + + coalesce(1.0 / (60 + scored.text_match_rank), 0) + )::double precision as rrf_score + from scored + ), + hybrid_candidates as ( + select id + from scored_metrics + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count + ), + vector_candidates as ( + select id + from scored_metrics + order by similarity desc, hybrid_score desc + limit match_count + ), + text_candidates as ( + select id + from scored_metrics + order by text_rank desc, hybrid_score desc + limit match_count + ), + rrf_candidates as ( + select id + from scored_metrics + order by rrf_score desc, hybrid_score desc + limit match_count + ), + candidate_ids as ( + select id from hybrid_candidates + union + select id from vector_candidates + union + select id from text_candidates + union + select id from rrf_candidates + ) + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + c.similarity, + c.text_rank, + c.hybrid_score, + c.rrf_score, + public.chunk_image_metadata(c.image_ids) as images + from scored_metrics c + join candidate_ids candidates on candidates.id = c.id + join public.documents d on d.id = c.document_id + order by c.hybrid_score desc, c.rrf_score desc, c.similarity desc, c.text_rank desc + limit match_count; +$$; + +-- 5b. match_document_index_units_hybrid: add extraction_mode boost +create or replace function public.match_document_index_units_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + ranked as ( + select u.id, u.document_id, u.source_chunk_id, u.source_image_id, u.unit_type, u.title, u.content, u.page_start, + u.page_end, u.heading_path, u.normalized_terms, u.source_span, u.quality_score, u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + (ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact') then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 end + )::double precision as text_rank, + u.metadata + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and u.source_chunk_id is not null + and (u.search_tsv @@ query.tsq or u.normalized_terms && query.terms) + order by text_rank desc, similarity desc + limit greatest(match_count * 3, 48) + ) + select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, heading_path, + normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case when unit_type = 'askable_question' then 0.04 else 0 end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count; +$$; + +-- ============================================================ +-- 6. ANALYZE UTILITY FUNCTION +-- Call after large ingestion batches to update query planner +-- statistics on key tables. +-- ============================================================ + +create or replace function public.analyze_rag_tables() +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + analyze public.document_chunks; + analyze public.document_memory_cards; + analyze public.document_index_units; + analyze public.document_embedding_fields; + analyze public.document_table_facts; + analyze public.documents; +end; +$$; + +revoke execute on function public.analyze_rag_tables() from public, anon, authenticated; +grant execute on function public.analyze_rag_tables() to service_role; + +-- ============================================================ +-- 7. UPDATE reset_document_index TO HANDLE NEW TABLE +-- ============================================================ + +create or replace function public.reset_document_index(p_document_id uuid) +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + perform set_config('statement_timeout', '180000', true); + delete from public.document_index_units where document_id = p_document_id; + delete from public.document_memory_cards where document_id = p_document_id; + delete from public.document_sections where document_id = p_document_id; + delete from public.document_table_facts where document_id = p_document_id; + delete from public.document_embedding_fields where document_id = p_document_id; + delete from public.document_index_quality where document_id = p_document_id; + delete from public.document_chunks where document_id = p_document_id; + delete from public.document_images where document_id = p_document_id; + delete from public.document_pages where document_id = p_document_id; +end; +$$; + +-- ============================================================ +-- 8. UPDATE search_schema_health TO CHECK NEW TABLE +-- ============================================================ + +create or replace function public.search_schema_health() +returns jsonb +language plpgsql +stable +set search_path = public, extensions, pg_catalog, pg_temp +as $$ +declare + missing text[] := array[]::text[]; + vector_type_oid oid; + vector_schema text; + index_name text; +begin + select t.oid, n.nspname + into vector_type_oid, vector_schema + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where t.typname = 'vector' + and n.nspname = 'extensions' + limit 1; + + if vector_type_oid is null then + missing := array_append(missing, 'extensions.vector_type'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_chunks_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_memory_cards_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); + end if; + + if vector_type_oid is not null and not exists ( + select 1 + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.proname = 'match_document_index_units_hybrid' + and p.proargtypes[0] = vector_type_oid + ) then + missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); + end if; + + if to_regclass('public.document_index_units') is null then + missing := array_append(missing, 'document_index_units.table'); + end if; + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); + end if; + if not exists (select 1 from pg_class where relname = 'documents_title_trgm_idx') then + missing := array_append(missing, 'documents_title_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_chunks_content_trgm_idx') then + missing := array_append(missing, 'document_chunks_content_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_labels_label_trgm_idx') then + missing := array_append(missing, 'document_labels_label_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_summaries_summary_trgm_idx') then + missing := array_append(missing, 'document_summaries_summary_trgm_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_index_units_embedding_hnsw_idx') then + missing := array_append(missing, 'document_index_units_embedding_hnsw_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_embedding_fields_owner_idx', 'document_embedding_fields_owner_id_idx')) then + missing := array_append(missing, 'document_embedding_fields_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname in ('document_table_facts_owner_idx', 'document_table_facts_owner_id_idx')) then + missing := array_append(missing, 'document_table_facts_owner_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_table_facts_source_image_idx') then + missing := array_append(missing, 'document_table_facts_source_image_idx'); + end if; + if not exists (select 1 from pg_class where relname = 'document_embedding_fields_dedup_idx') then + missing := array_append(missing, 'document_embedding_fields_dedup_idx'); + end if; + foreach index_name in array array[ + 'document_pages_document_idx', + 'document_images_document_idx', + 'document_sections_document_idx', + 'document_memory_cards_document_idx', + 'document_chunks_document_idx', + 'document_table_facts_document_idx', + 'document_embedding_fields_document_idx', + 'document_index_units_document_idx', + 'ingestion_jobs_status_next_run_idx', + 'ingestion_jobs_document_status_idx', + 'documents_owner_status_idx', + 'import_batches_status_created_idx', + 'storage_cleanup_jobs_status_created_idx' + ] loop + if not exists (select 1 from pg_class where relname = index_name) then + missing := array_append(missing, index_name); + end if; + end loop; + + return jsonb_build_object( + 'ok', cardinality(missing) = 0, + 'missing', missing, + 'vector_extension_schema', vector_schema, + 'checked_at', now() + ); +end; +$$; + +revoke execute on function public.search_schema_health() from public, anon, authenticated; +grant execute on function public.search_schema_health() to service_role; diff --git a/supabase/migrations/20260622000000_reconcile_schema_health_vector_privileges.sql b/supabase/migrations/20260622000000_reconcile_schema_health_vector_privileges.sql new file mode 100644 index 000000000..01b78bb83 --- /dev/null +++ b/supabase/migrations/20260622000000_reconcile_schema_health_vector_privileges.sql @@ -0,0 +1,355 @@ +-- Reconcile live schema-health drift, vector index strategy, and RPC surface posture. +-- This migration is additive and safe to apply after existing history. + +set search_path = public, extensions, pg_temp; + +-- ============================================================ +-- 1) Required indexes checked by health validation +-- ============================================================ + +create index if not exists documents_title_trgm_idx + on public.documents using gin (lower(coalesce(title, '') || ' ' || coalesce(file_name, '')) gin_trgm_ops); + +create index if not exists document_chunks_content_trgm_idx + on public.document_chunks using gin (lower(coalesce(section_heading, '') || ' ' || coalesce(content, '')) gin_trgm_ops); + +create index if not exists document_labels_label_trgm_idx + on public.document_labels using gin (lower(label) gin_trgm_ops); + +create index if not exists document_summaries_summary_trgm_idx + on public.document_summaries using gin (left(summary, 2500) gin_trgm_ops); + +create index if not exists rag_retrieval_logs_owner_created_idx + on public.rag_retrieval_logs(owner_id, created_at desc); + +create index if not exists rag_retrieval_logs_miss_idx + on public.rag_retrieval_logs(is_miss, created_at desc) + where is_miss = true; + +create index if not exists rag_retrieval_logs_strategy_idx + on public.rag_retrieval_logs(retrieval_strategy, created_at desc); + +create index if not exists document_index_units_embedding_hnsw_idx + on public.document_index_units using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +create index if not exists document_table_facts_source_image_idx + on public.document_table_facts(source_image_id) + where source_image_id is not null; + +create index if not exists document_pages_document_idx + on public.document_pages(document_id, page_number); + +create index if not exists document_sections_document_idx + on public.document_sections(document_id, section_index); + +create index if not exists document_chunks_document_idx + on public.document_chunks(document_id, chunk_index); + +-- ============================================================ +-- 2) HNSW index reconciliation +-- Create missing HNSW index variants first. Legacy ivfflat cleanup is not automatic. +-- ============================================================ + +create index if not exists document_chunks_embedding_hnsw_idx + on public.document_chunks using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +-- document_embedding_fields is covered by the existing ivfflat index in this rollout. +-- Its HNSW replacement is intentionally deferred to a direct-connection maintenance window: +-- Supabase API/connector execution timed out before the long HNSW build could commit. + +create or replace function public.detect_legacy_ivfflat_indexes() +returns text[] +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select coalesce(array_agg(idx.relname order by idx.relname), array[]::text[]) + from pg_class idx + join pg_index ix on ix.indexrelid = idx.oid + join pg_class tab on tab.oid = ix.indrelid + join pg_namespace tab_ns on tab_ns.oid = tab.relnamespace + join pg_am am on idx.relam = am.oid + where idx.relkind = 'i' + and am.amname = 'ivfflat' + and tab_ns.nspname = 'public' + and tab.relname = any ( + array[ + 'document_chunks', + 'document_embedding_fields', + 'document_index_units', + 'document_memory_cards', + 'document_table_facts' + ] + ); +$$; + +do $$ +declare + legacy_ivfflat_indexes text[]; +begin + select public.detect_legacy_ivfflat_indexes() into legacy_ivfflat_indexes; + if array_length(legacy_ivfflat_indexes, 1) > 0 then + raise notice + 'Legacy ivfflat vector indexes still present and should be reviewed for controlled deprecation: %', + legacy_ivfflat_indexes; + end if; +end $$; + +-- ============================================================ +-- 3) Add hardening for document_embedding_fields.content_hash +-- ============================================================ + +create or replace function public.set_document_embedding_field_content_hash() +returns trigger +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + if tg_op = 'INSERT' then + NEW.content_hash := md5(coalesce(NEW.content, '')); + elsif tg_op = 'UPDATE' then + if NEW.content_hash is null or NEW.content is distinct from OLD.content then + NEW.content_hash := md5(coalesce(NEW.content, '')); + end if; + end if; + return NEW; +end; +$$; + +drop trigger if exists set_document_embedding_field_content_hash on public.document_embedding_fields; +create trigger set_document_embedding_field_content_hash +before insert or update +on public.document_embedding_fields +for each row +execute function public.set_document_embedding_field_content_hash(); + +update public.document_embedding_fields +set content_hash = md5(coalesce(content, '')) +where content_hash is null + or content_hash = ''; + +create unique index if not exists document_embedding_fields_dedup_idx + on public.document_embedding_fields(document_id, source_chunk_id, field_type, content_hash); + +do $$ +begin + if exists (select 1 from public.document_embedding_fields where content_hash is null or content_hash = '') then + raise notice 'document_embedding_fields content_hash backfill is incomplete; NOT NULL constraint deferred'; + else + alter table public.document_embedding_fields + alter column content_hash set not null; + end if; +end $$; + +-- ============================================================ +-- 4) Runtime schema health check alignment +-- ============================================================ + +create or replace function public.search_schema_health() +returns jsonb +language plpgsql +stable +set search_path = public, extensions, pg_temp +as $$ +declare + missing text[] := array[]::text[]; + vector_type_oid oid; + vector_schema text; + index_name text; + legacy_ivfflat_indexes text[]; + required_indexes constant text[] := array[ + 'documents_title_trgm_idx', + 'document_chunks_content_trgm_idx', + 'document_labels_label_trgm_idx', + 'document_summaries_summary_trgm_idx', + 'document_index_units_embedding_hnsw_idx', + 'document_chunks_embedding_hnsw_idx', + 'document_table_facts_source_image_idx', + 'document_pages_document_idx', + 'document_sections_document_idx', + 'document_chunks_document_idx', + 'document_memory_cards_document_idx', + 'document_embedding_fields_document_idx', + 'document_table_facts_document_idx', + 'document_index_units_document_idx', + 'rag_retrieval_logs_owner_created_idx', + 'rag_retrieval_logs_miss_idx', + 'rag_retrieval_logs_strategy_idx' + ]; +begin + select t.oid, n.nspname + into vector_type_oid, vector_schema + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where t.typname = 'vector' + and n.nspname = 'extensions' + limit 1; + + if vector_type_oid is null then + missing := array_append(missing, 'extensions.vector_type'); + end if; + + if to_regprocedure('public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid)') is null then + missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_chunks_text(text, integer, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_chunks_text.signature'); + end if; + if to_regprocedure('public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_embedding_fields_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_documents_for_query(text, integer, uuid)') is null then + missing := array_append(missing, 'match_documents_for_query.signature'); + end if; + if to_regprocedure('public.match_document_table_facts_text(text, integer, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_table_facts_text.signature'); + end if; + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); + end if; + + foreach index_name in array required_indexes loop + if not exists ( + select 1 + from pg_class c + join pg_namespace ns on ns.oid = c.relnamespace + where ns.nspname = 'public' + and c.relname = index_name + and c.relkind = 'i' + ) then + missing := array_append(missing, index_name); + end if; + end loop; + + select public.detect_legacy_ivfflat_indexes() into legacy_ivfflat_indexes; + + return jsonb_build_object( + 'ok', cardinality(missing) = 0, + 'missing', missing, + 'vector_extension_schema', vector_schema, + 'legacy_ivfflat_indexes', coalesce(legacy_ivfflat_indexes, array[]::text[]), + 'deferred_hnsw_indexes', array['document_embedding_fields_embedding_hnsw_idx'], + 'checked_at', now() + ); +end; +$$; + +revoke execute on function public.search_schema_health() from public, anon, authenticated; +grant execute on function public.search_schema_health() to service_role; + +-- ============================================================ +-- 5) Restrict function execution to service-side pathways +-- ============================================================ + +do $$ +begin + if to_regprocedure('public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid)') is not null then + revoke execute on function public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid) from public, anon, authenticated; + grant execute on function public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid) to service_role; + end if; + if to_regprocedure('public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is not null then + revoke execute on function public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) from public, anon, authenticated; + grant execute on function public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) to service_role; + end if; + if to_regprocedure('public.match_document_chunks_text(text, integer, uuid[], uuid)') is not null then + revoke execute on function public.match_document_chunks_text(text, integer, uuid[], uuid) from public, anon, authenticated; + grant execute on function public.match_document_chunks_text(text, integer, uuid[], uuid) to service_role; + end if; + if to_regprocedure('public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is not null then + revoke execute on function public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) from public, anon, authenticated; + grant execute on function public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) to service_role; + end if; + if to_regprocedure('public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is not null then + revoke execute on function public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) from public, anon, authenticated; + grant execute on function public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) to service_role; + end if; + if to_regprocedure('public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is not null then + revoke execute on function public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) from public, anon, authenticated; + grant execute on function public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) to service_role; + end if; + if to_regprocedure('public.match_documents_for_query(text, integer, uuid)') is not null then + revoke execute on function public.match_documents_for_query(text, integer, uuid) from public, anon, authenticated; + grant execute on function public.match_documents_for_query(text, integer, uuid) to service_role; + end if; + if to_regprocedure('public.match_document_table_facts_text(text, integer, uuid[], uuid)') is not null then + revoke execute on function public.match_document_table_facts_text(text, integer, uuid[], uuid) from public, anon, authenticated; + grant execute on function public.match_document_table_facts_text(text, integer, uuid[], uuid) to service_role; + end if; + + if to_regprocedure('public.analyze_rag_tables()') is not null then + revoke execute on function public.analyze_rag_tables() from public, anon, authenticated; + grant execute on function public.analyze_rag_tables() to service_role; + end if; + if to_regprocedure('public.claim_ingestion_jobs(text, integer, integer)') is not null then + revoke execute on function public.claim_ingestion_jobs(text, integer, integer) from public, anon, authenticated; + grant execute on function public.claim_ingestion_jobs(text, integer, integer) to service_role; + end if; + if to_regprocedure('public.refresh_import_batch_status(uuid)') is not null then + revoke execute on function public.refresh_import_batch_status(uuid) from public, anon, authenticated; + grant execute on function public.refresh_import_batch_status(uuid) to service_role; + end if; + if to_regprocedure('public.complete_ingestion_job(uuid, uuid, uuid, text)') is not null then + revoke execute on function public.complete_ingestion_job(uuid, uuid, uuid, text) from public, anon, authenticated; + grant execute on function public.complete_ingestion_job(uuid, uuid, uuid, text) to service_role; + end if; + if to_regprocedure('public.fail_or_retry_ingestion_job(uuid, uuid, uuid, boolean, text, text, text, timestamptz)') is not null then + revoke execute on function public.fail_or_retry_ingestion_job(uuid, uuid, uuid, boolean, text, text, text, timestamptz) from public, anon, authenticated; + grant execute on function public.fail_or_retry_ingestion_job(uuid, uuid, uuid, boolean, text, text, text, timestamptz) to service_role; + end if; + if to_regprocedure('public.reset_document_index(uuid)') is not null then + revoke execute on function public.reset_document_index(uuid) from public, anon, authenticated; + grant execute on function public.reset_document_index(uuid) to service_role; + end if; + if to_regprocedure('public.stamp_document_deep_memory_version(uuid, text)') is not null then + revoke execute on function public.stamp_document_deep_memory_version(uuid, text) from public, anon, authenticated; + grant execute on function public.stamp_document_deep_memory_version(uuid, text) to service_role; + end if; +end $$; + +-- ============================================================ +-- 6) Expand stats freshness scope for hot tables +-- ============================================================ + +create or replace function public.analyze_rag_tables() +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +declare + table_names constant text[] := array[ + 'documents', + 'document_chunks', + 'document_pages', + 'document_images', + 'document_sections', + 'document_memory_cards', + 'document_table_facts', + 'document_embedding_fields', + 'document_index_quality', + 'document_index_units', + 'rag_queries', + 'rag_query_misses', + 'rag_retrieval_logs', + 'rag_aliases', + 'ingestion_jobs' + ]; + table_name text; +begin + foreach table_name in array table_names loop + if to_regclass(format('public.%I', table_name)) is not null then + execute format('analyze public.%I', table_name); + end if; + end loop; +end; +$$; diff --git a/supabase/migrations/20260623014639_finalize_embedding_fields_hnsw_health.sql b/supabase/migrations/20260623014639_finalize_embedding_fields_hnsw_health.sql new file mode 100644 index 000000000..0581d62fe --- /dev/null +++ b/supabase/migrations/20260623014639_finalize_embedding_fields_hnsw_health.sql @@ -0,0 +1,112 @@ +-- Finalize deferred HNSW coverage for document_embedding_fields. +-- Production rollout note: +-- Build document_embedding_fields_embedding_hnsw_idx with CREATE INDEX CONCURRENTLY +-- over a direct database connection before pushing this migration to live. + +set search_path = public, extensions, pg_temp; + +create index if not exists document_embedding_fields_embedding_hnsw_idx + on public.document_embedding_fields using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); + +create or replace function public.search_schema_health() +returns jsonb +language plpgsql +stable +set search_path = public, extensions, pg_temp +as $$ +declare + missing text[] := array[]::text[]; + vector_type_oid oid; + vector_schema text; + index_name text; + legacy_ivfflat_indexes text[]; + required_indexes constant text[] := array[ + 'documents_title_trgm_idx', + 'document_chunks_content_trgm_idx', + 'document_labels_label_trgm_idx', + 'document_summaries_summary_trgm_idx', + 'document_index_units_embedding_hnsw_idx', + 'document_chunks_embedding_hnsw_idx', + 'document_embedding_fields_embedding_hnsw_idx', + 'document_table_facts_source_image_idx', + 'document_pages_document_idx', + 'document_sections_document_idx', + 'document_chunks_document_idx', + 'document_memory_cards_document_idx', + 'document_embedding_fields_document_idx', + 'document_table_facts_document_idx', + 'document_index_units_document_idx', + 'rag_retrieval_logs_owner_created_idx', + 'rag_retrieval_logs_miss_idx', + 'rag_retrieval_logs_strategy_idx' + ]; +begin + select t.oid, n.nspname + into vector_type_oid, vector_schema + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where t.typname = 'vector' + and n.nspname = 'extensions' + limit 1; + + if vector_type_oid is null then + missing := array_append(missing, 'extensions.vector_type'); + end if; + + if to_regprocedure('public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid)') is null then + missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_chunks_text(text, integer, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_chunks_text.signature'); + end if; + if to_regprocedure('public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_embedding_fields_hybrid.extensions_vector_signature'); + end if; + if to_regprocedure('public.match_documents_for_query(text, integer, uuid)') is null then + missing := array_append(missing, 'match_documents_for_query.signature'); + end if; + if to_regprocedure('public.match_document_table_facts_text(text, integer, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_table_facts_text.signature'); + end if; + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); + end if; + + foreach index_name in array required_indexes loop + if not exists ( + select 1 + from pg_class c + join pg_namespace ns on ns.oid = c.relnamespace + where ns.nspname = 'public' + and c.relname = index_name + and c.relkind = 'i' + ) then + missing := array_append(missing, index_name); + end if; + end loop; + + select public.detect_legacy_ivfflat_indexes() into legacy_ivfflat_indexes; + + return jsonb_build_object( + 'ok', cardinality(missing) = 0, + 'missing', missing, + 'vector_extension_schema', vector_schema, + 'legacy_ivfflat_indexes', coalesce(legacy_ivfflat_indexes, array[]::text[]), + 'deferred_hnsw_indexes', array[]::text[], + 'checked_at', now() + ); +end; +$$; + +revoke execute on function public.search_schema_health() from public, anon, authenticated; +grant execute on function public.search_schema_health() to service_role; diff --git a/supabase/schema.sql b/supabase/schema.sql index 8708681d4..b426b3ea3 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -263,7 +263,7 @@ create table if not exists public.document_chunks ( anchor_id text, content text not null, retrieval_synopsis text, - content_hash text, + content_hash text not null, index_generation_id uuid, token_estimate integer not null default 0, image_ids uuid[] not null default '{}', @@ -322,11 +322,11 @@ create table if not exists public.document_embedding_fields ( ) ), content text not null, + content_hash text, embedding extensions.vector(1536) not null, metadata jsonb not null default '{}'::jsonb, search_tsv tsvector generated always as (to_tsvector('english', content)) stored, - created_at timestamptz not null default now(), - unique (document_id, source_chunk_id, field_type, content) + created_at timestamptz not null default now() ); create table if not exists public.document_index_quality ( @@ -431,6 +431,38 @@ create table if not exists public.rag_response_cache ( updated_at timestamptz not null default now() ); +create table if not exists public.rag_retrieval_logs ( + id uuid primary key default gen_random_uuid(), + owner_id uuid references auth.users(id) on delete set null, + query text not null, + normalized_query text, + query_class text, + retrieval_strategy text, + candidate_count integer not null default 0, + top_similarity double precision, + top_text_rank double precision, + top_hybrid_score double precision, + top_rrf_score double precision, + mean_hybrid_score double precision, + selected_chunk_ids uuid[] not null default '{}', + selected_document_ids uuid[] not null default '{}', + selected_count integer not null default 0, + embedding_latency_ms integer, + rpc_latency_ms integer, + rerank_latency_ms integer, + total_latency_ms integer, + vector_candidate_count integer, + text_candidate_count integer, + memory_card_count integer, + index_unit_count integer, + embedding_field_count integer, + is_miss boolean not null default false, + miss_reason text, + embedding_cache_hit boolean, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); + create table if not exists public.storage_cleanup_jobs ( id uuid primary key default gen_random_uuid(), owner_id uuid references auth.users(id) on delete set null, @@ -462,7 +494,7 @@ create index if not exists documents_owner_hash_idx on public.documents(owner_id create index if not exists documents_search_idx on public.documents using gin(search_tsv); create index if not exists documents_title_search_idx on public.documents using gin(title_search_tsv); create index if not exists documents_title_trgm_idx - on public.documents using gin ((lower(coalesce(title, '') || ' ' || coalesce(file_name, ''))) gin_trgm_ops); + on public.documents using gin (lower(coalesce(title, '') || ' ' || coalesce(file_name, '')) gin_trgm_ops); create index if not exists document_pages_document_idx on public.document_pages(document_id, page_number); create index if not exists document_images_document_idx on public.document_images(document_id, page_number); create index if not exists document_images_searchable_idx @@ -476,10 +508,10 @@ create index if not exists document_labels_owner_label_idx on public.document_labels(owner_id, label_type, label); create index if not exists document_labels_document_idx on public.document_labels(document_id); create index if not exists document_labels_label_trgm_idx - on public.document_labels using gin ((lower(label)) gin_trgm_ops); + on public.document_labels using gin (lower(label) gin_trgm_ops); create index if not exists document_summaries_owner_idx on public.document_summaries(owner_id, generated_at desc); create index if not exists document_summaries_summary_trgm_idx - on public.document_summaries using gin ((lower(summary)) gin_trgm_ops); + on public.document_summaries using gin (lower(summary) gin_trgm_ops); create index if not exists document_sections_document_idx on public.document_sections(document_id, section_index); create index if not exists document_sections_owner_idx @@ -493,7 +525,8 @@ create index if not exists document_memory_cards_section_idx create index if not exists document_memory_cards_search_idx on public.document_memory_cards using gin(search_tsv); create index if not exists document_memory_cards_embedding_hnsw_idx - on public.document_memory_cards using hnsw (embedding vector_cosine_ops); + on public.document_memory_cards using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); create index if not exists document_chunks_document_idx on public.document_chunks(document_id, chunk_index); create index if not exists document_chunks_generation_idx on public.document_chunks(document_id, index_generation_id); create index if not exists document_chunks_content_hash_idx on public.document_chunks(document_id, content_hash); @@ -504,9 +537,10 @@ create index if not exists document_chunks_anchor_idx where anchor_id is not null; create index if not exists document_chunks_search_idx on public.document_chunks using gin(search_tsv); create index if not exists document_chunks_content_trgm_idx - on public.document_chunks using gin ((lower(coalesce(section_heading, '') || ' ' || content)) gin_trgm_ops); + on public.document_chunks using gin (lower(coalesce(section_heading, '') || ' ' || coalesce(content, '')) gin_trgm_ops); create index if not exists document_chunks_embedding_hnsw_idx - on public.document_chunks using hnsw (embedding vector_cosine_ops); + on public.document_chunks using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); create index if not exists document_table_facts_document_idx on public.document_table_facts(document_id, page_number); create index if not exists document_table_facts_chunk_idx @@ -530,7 +564,35 @@ create index if not exists document_embedding_fields_chunk_idx create index if not exists document_embedding_fields_search_idx on public.document_embedding_fields using gin(search_tsv); create index if not exists document_embedding_fields_embedding_hnsw_idx - on public.document_embedding_fields using hnsw (embedding vector_cosine_ops); + on public.document_embedding_fields using hnsw (embedding vector_cosine_ops) + with (m = 24, ef_construction = 128); +create unique index if not exists document_embedding_fields_dedup_idx + on public.document_embedding_fields(document_id, source_chunk_id, field_type, content_hash); + +create or replace function public.set_document_embedding_field_content_hash() +returns trigger +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + if tg_op = 'INSERT' then + NEW.content_hash := md5(coalesce(NEW.content, '')); + elsif tg_op = 'UPDATE' then + if NEW.content_hash is null or NEW.content is distinct from OLD.content then + NEW.content_hash := md5(coalesce(NEW.content, '')); + end if; + end if; + return NEW; +end; +$$; + +drop trigger if exists set_document_embedding_field_content_hash on public.document_embedding_fields; +create trigger set_document_embedding_field_content_hash +before insert or update +on public.document_embedding_fields +for each row +execute function public.set_document_embedding_field_content_hash(); + create index if not exists document_index_quality_owner_score_idx on public.document_index_quality(owner_id, quality_score, updated_at desc); create index if not exists ingestion_jobs_document_idx on public.ingestion_jobs(document_id); @@ -565,7 +627,7 @@ create index if not exists rag_aliases_owner_enabled_idx create index if not exists rag_aliases_type_enabled_idx on public.rag_aliases(alias_type, enabled); create index if not exists rag_aliases_alias_trgm_idx - on public.rag_aliases using gin ((lower(alias)) gin_trgm_ops); + on public.rag_aliases using gin (lower(alias) gin_trgm_ops); create index if not exists rag_response_cache_expiry_idx on public.rag_response_cache(expires_at); create index if not exists rag_response_cache_owner_kind_idx @@ -584,14 +646,16 @@ create index if not exists storage_cleanup_jobs_owner_status_idx create index if not exists storage_cleanup_jobs_document_idx on public.storage_cleanup_jobs(document_id); -create index if not exists document_chunks_document_id_idx on public.document_chunks(document_id); -create index if not exists document_sections_document_id_idx on public.document_sections(document_id); -create index if not exists document_memory_cards_document_id_idx on public.document_memory_cards(document_id); -create index if not exists document_images_document_id_idx on public.document_images(document_id); -create index if not exists document_labels_document_id_idx on public.document_labels(document_id); -create index if not exists document_embedding_fields_document_id_idx on public.document_embedding_fields(document_id); -create index if not exists document_table_facts_document_id_idx on public.document_table_facts(document_id); -create index if not exists document_index_quality_document_id_idx on public.document_index_quality(document_id); +create index if not exists rag_retrieval_logs_owner_created_idx + on public.rag_retrieval_logs(owner_id, created_at desc); +create index if not exists rag_retrieval_logs_miss_idx + on public.rag_retrieval_logs(is_miss, created_at desc) + where is_miss = true; +create index if not exists rag_retrieval_logs_strategy_idx + on public.rag_retrieval_logs(retrieval_strategy, created_at desc); + +-- Redundant single-column FK indexes removed (covered by composite indexes +-- with the same leading column, e.g. document_chunks_document_idx). create or replace function public.set_updated_at() returns trigger @@ -1103,7 +1167,13 @@ as $$ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) )::double precision as text_rank, row_number() over (order by c.embedding <=> query_embedding) as vector_rank, - null::bigint as text_match_rank + null::bigint as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score from public.document_chunks c join public.documents d on d.id = c.document_id cross join query @@ -1137,7 +1207,13 @@ as $$ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) ) desc, c.embedding <=> query_embedding - ) as text_match_rank + ) as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score from public.document_chunks c join public.documents d on d.id = c.document_id cross join query @@ -1169,14 +1245,22 @@ as $$ max(similarity)::double precision as similarity, max(text_rank)::double precision as text_rank, min(vector_rank) as vector_rank, - min(text_match_rank) as text_match_rank + min(text_match_rank) as text_match_rank, + max(quality_score)::double precision as quality_score, + bool_or(has_deep_index) as has_deep_index, + max(doc_updated_at) as doc_updated_at from combined group by id, document_id, page_number, chunk_index, section_heading, content, retrieval_synopsis, image_ids ), scored_metrics as ( select scored.*, - ((scored.similarity * 0.72) + (least(scored.text_rank, 1) * 0.28))::double precision as hybrid_score, + ( + (scored.similarity * 0.62) + + (least(scored.text_rank, 1) * 0.22) + + (scored.quality_score * 0.10) + + (case when scored.doc_updated_at > now() - interval '90 days' then 0.06 else 0 end) + )::double precision as hybrid_score, ( coalesce(1.0 / (60 + scored.vector_rank), 0) + coalesce(1.0 / (60 + scored.text_match_rank), 0) @@ -1376,17 +1460,64 @@ as $$ limit match_count; $$; +create or replace function public.detect_legacy_ivfflat_indexes() +returns text[] +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select coalesce(array_agg(idx.relname order by idx.relname), array[]::text[]) + from pg_class idx + join pg_index ix on ix.indexrelid = idx.oid + join pg_class tab on tab.oid = ix.indrelid + join pg_namespace tab_ns on tab_ns.oid = tab.relnamespace + join pg_am am on idx.relam = am.oid + where idx.relkind = 'i' + and am.amname = 'ivfflat' + and tab_ns.nspname = 'public' + and tab.relname = any ( + array[ + 'document_chunks', + 'document_embedding_fields', + 'document_index_units', + 'document_memory_cards', + 'document_table_facts' + ] + ); +$$; + create or replace function public.search_schema_health() returns jsonb language plpgsql stable -set search_path = public, extensions, pg_catalog, pg_temp +set search_path = public, extensions, pg_temp as $$ declare missing text[] := array[]::text[]; vector_type_oid oid; vector_schema text; index_name text; + legacy_ivfflat_indexes text[]; + required_indexes constant text[] := array[ + 'documents_title_trgm_idx', + 'document_chunks_content_trgm_idx', + 'document_labels_label_trgm_idx', + 'document_summaries_summary_trgm_idx', + 'document_index_units_embedding_hnsw_idx', + 'document_chunks_embedding_hnsw_idx', + 'document_embedding_fields_embedding_hnsw_idx', + 'document_table_facts_source_image_idx', + 'document_pages_document_idx', + 'document_sections_document_idx', + 'document_chunks_document_idx', + 'document_memory_cards_document_idx', + 'document_embedding_fields_document_idx', + 'document_table_facts_document_idx', + 'document_index_units_document_idx', + 'rag_retrieval_logs_owner_created_idx', + 'rag_retrieval_logs_miss_idx', + 'rag_retrieval_logs_strategy_idx' + ]; begin select t.oid, n.nspname into vector_type_oid, vector_schema @@ -1400,101 +1531,55 @@ begin missing := array_append(missing, 'extensions.vector_type'); end if; - if vector_type_oid is not null and not exists ( - select 1 - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = 'public' - and p.proname = 'match_document_chunks' - and p.proargtypes[0] = vector_type_oid - ) then + if to_regprocedure('public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid)') is null then missing := array_append(missing, 'match_document_chunks.extensions_vector_signature'); end if; - - if vector_type_oid is not null and not exists ( - select 1 - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = 'public' - and p.proname = 'match_document_chunks_hybrid' - and p.proargtypes[0] = vector_type_oid - ) then + if to_regprocedure('public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then missing := array_append(missing, 'match_document_chunks_hybrid.extensions_vector_signature'); end if; - - if vector_type_oid is not null and not exists ( - select 1 - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = 'public' - and p.proname = 'match_document_memory_cards_hybrid' - and p.proargtypes[0] = vector_type_oid - ) then + if to_regprocedure('public.match_document_chunks_text(text, integer, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_chunks_text.signature'); + end if; + if to_regprocedure('public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then missing := array_append(missing, 'match_document_memory_cards_hybrid.extensions_vector_signature'); end if; - - if vector_type_oid is not null and not exists ( - select 1 - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = 'public' - and p.proname = 'match_document_index_units_hybrid' - and p.proargtypes[0] = vector_type_oid - ) then + if to_regprocedure('public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then missing := array_append(missing, 'match_document_index_units_hybrid.extensions_vector_signature'); end if; - - if to_regclass('public.document_index_units') is null then - missing := array_append(missing, 'document_index_units.table'); - end if; - if not exists (select 1 from pg_class where relname = 'documents_title_trgm_idx') then - missing := array_append(missing, 'documents_title_trgm_idx'); - end if; - if not exists (select 1 from pg_class where relname = 'document_chunks_content_trgm_idx') then - missing := array_append(missing, 'document_chunks_content_trgm_idx'); - end if; - if not exists (select 1 from pg_class where relname = 'document_labels_label_trgm_idx') then - missing := array_append(missing, 'document_labels_label_trgm_idx'); - end if; - if not exists (select 1 from pg_class where relname = 'document_summaries_summary_trgm_idx') then - missing := array_append(missing, 'document_summaries_summary_trgm_idx'); + if to_regprocedure('public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_embedding_fields_hybrid.extensions_vector_signature'); end if; - if not exists (select 1 from pg_class where relname = 'document_index_units_embedding_hnsw_idx') then - missing := array_append(missing, 'document_index_units_embedding_hnsw_idx'); + if to_regprocedure('public.match_documents_for_query(text, integer, uuid)') is null then + missing := array_append(missing, 'match_documents_for_query.signature'); end if; - if not exists (select 1 from pg_class where relname = 'document_embedding_fields_owner_idx') then - missing := array_append(missing, 'document_embedding_fields_owner_idx'); + if to_regprocedure('public.match_document_table_facts_text(text, integer, uuid[], uuid)') is null then + missing := array_append(missing, 'match_document_table_facts_text.signature'); end if; - if not exists (select 1 from pg_class where relname = 'document_table_facts_owner_idx') then - missing := array_append(missing, 'document_table_facts_owner_idx'); + if to_regclass('public.rag_retrieval_logs') is null then + missing := array_append(missing, 'rag_retrieval_logs.table'); end if; - if not exists (select 1 from pg_class where relname = 'document_table_facts_source_image_idx') then - missing := array_append(missing, 'document_table_facts_source_image_idx'); - end if; - foreach index_name in array array[ - 'document_pages_document_idx', - 'document_images_document_idx', - 'document_sections_document_idx', - 'document_memory_cards_document_idx', - 'document_chunks_document_idx', - 'document_table_facts_document_idx', - 'document_embedding_fields_document_idx', - 'document_index_units_document_idx', - 'ingestion_jobs_status_next_run_idx', - 'ingestion_jobs_document_status_idx', - 'documents_owner_status_idx', - 'import_batches_status_created_idx', - 'storage_cleanup_jobs_status_created_idx' - ] loop - if not exists (select 1 from pg_class where relname = index_name) then + + foreach index_name in array required_indexes loop + if not exists ( + select 1 + from pg_class c + join pg_namespace ns on ns.oid = c.relnamespace + where ns.nspname = 'public' + and c.relname = index_name + and c.relkind = 'i' + ) then missing := array_append(missing, index_name); end if; end loop; + select public.detect_legacy_ivfflat_indexes() into legacy_ivfflat_indexes; + return jsonb_build_object( 'ok', cardinality(missing) = 0, 'missing', missing, 'vector_extension_schema', vector_schema, + 'legacy_ivfflat_indexes', coalesce(legacy_ivfflat_indexes, array[]::text[]), + 'deferred_hnsw_indexes', array[]::text[], 'checked_at', now() ); end; @@ -1933,7 +2018,8 @@ grant select, insert, update, delete on table public.rag_query_misses, public.rag_aliases, public.rag_response_cache, - public.storage_cleanup_jobs + public.storage_cleanup_jobs, + public.rag_retrieval_logs to service_role; grant usage, select on all sequences in schema public to service_role; @@ -1954,7 +2040,8 @@ grant select on table public.rag_queries, public.rag_query_misses, public.rag_aliases, - public.storage_cleanup_jobs + public.storage_cleanup_jobs, + public.rag_retrieval_logs to authenticated; grant insert, update, delete on table public.document_labels to authenticated; @@ -1978,6 +2065,7 @@ alter table public.rag_query_misses enable row level security; alter table public.rag_aliases enable row level security; alter table public.rag_response_cache enable row level security; alter table public.storage_cleanup_jobs enable row level security; +alter table public.rag_retrieval_logs enable row level security; create policy "import batches owner read" on public.import_batches for select to authenticated using (owner_id = (select auth.uid())); @@ -2053,6 +2141,9 @@ create policy "rag owner read" on public.rag_queries create policy "rag misses owner read" on public.rag_query_misses for select to authenticated using (owner_id = (select auth.uid())); +create policy "rag retrieval logs owner read" on public.rag_retrieval_logs + for select to authenticated using (owner_id = (select auth.uid())); + create policy "rag aliases owner read" on public.rag_aliases for select to authenticated using (owner_id is null or owner_id = (select auth.uid())); @@ -2102,7 +2193,7 @@ create index if not exists document_index_units_image_idx on public.document_ind create index if not exists document_index_units_terms_idx on public.document_index_units using gin(normalized_terms); create index if not exists document_index_units_heading_path_idx on public.document_index_units using gin(heading_path); create index if not exists document_index_units_search_idx on public.document_index_units using gin(search_tsv); -create index if not exists document_index_units_embedding_hnsw_idx on public.document_index_units using hnsw (embedding vector_cosine_ops); +create index if not exists document_index_units_embedding_hnsw_idx on public.document_index_units using hnsw (embedding vector_cosine_ops) with (m = 24, ef_construction = 128); drop trigger if exists document_index_units_updated_at on public.document_index_units; create trigger document_index_units_updated_at @@ -2149,7 +2240,12 @@ as $$ select u.id, u.document_id, u.source_chunk_id, u.source_image_id, u.unit_type, u.title, u.content, u.page_start, u.page_end, u.heading_path, u.normalized_terms, u.source_span, u.quality_score, u.extraction_mode, (1 - (u.embedding <=> query_embedding))::double precision as similarity, - (ts_rank_cd(u.search_tsv, query.tsq) + case when u.normalized_terms && query.terms then 0.25 else 0 end + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact') then 0.06 when u.unit_type = 'section_summary' then 0.03 else 0 end)::double precision as text_rank, + (ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact') then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 end + )::double precision as text_rank, u.metadata from public.document_index_units u join public.documents d on d.id = u.document_id @@ -2164,7 +2260,13 @@ as $$ ) select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, heading_path, normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, - ((similarity * 0.58) + (least(text_rank, 1) * 0.32) + (quality_score * 0.1))::double precision as hybrid_score, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case when unit_type = 'askable_question' then 0.04 else 0 end) + )::double precision as hybrid_score, metadata from ranked order by hybrid_score desc, similarity desc, text_rank desc @@ -2190,6 +2292,24 @@ begin end; $$; +create or replace function public.analyze_rag_tables() +returns void +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + analyze public.document_chunks; + analyze public.document_memory_cards; + analyze public.document_index_units; + analyze public.document_embedding_fields; + analyze public.document_table_facts; + analyze public.documents; +end; +$$; + +revoke execute on function public.analyze_rag_tables() from public, anon, authenticated; +grant execute on function public.analyze_rag_tables() to service_role; + alter table public.document_index_units enable row level security; grant select, insert, update, delete on table public.document_index_units to service_role; grant select on table public.document_index_units to authenticated; diff --git a/tests/accessible-table-normalization.test.ts b/tests/accessible-table-normalization.test.ts index f3d602f42..422a7bd75 100644 --- a/tests/accessible-table-normalization.test.ts +++ b/tests/accessible-table-normalization.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { normalizeAccessibleTable } from "../src/lib/accessible-table-normalization"; @@ -105,4 +106,13 @@ describe("normalizeAccessibleTable", () => { "Facilitating appointment with alternative provider", ]); }); + + it("keeps the mobile table presentation on one visible semantic table", () => { + const source = readFileSync(new URL("../src/components/AccessibleTable.tsx", import.meta.url), "utf8"); + + expect(source).not.toContain("<dl"); + expect(source).not.toContain("hidden md:block"); + expect(source).toContain("md:table-row"); + expect(source).toContain("rowActions"); + }); }); diff --git a/tests/answer-verification.test.ts b/tests/answer-verification.test.ts index f0a798360..2b537995b 100644 --- a/tests/answer-verification.test.ts +++ b/tests/answer-verification.test.ts @@ -53,11 +53,7 @@ describe("answer-verification (GEN-C2 / GEN-H2)", () => { it("only credits chunks the answer actually cites", () => { const cited = source({ id: "chunk-1", content: "Monitor weekly." }); const uncited = source({ id: "chunk-2", content: "Dose is 12.5 mg." }); - const verification = verifyAnswerNumbers( - "Give 12.5 mg.", - [{ chunk_id: "chunk-1" }], - [cited, uncited], - ); + const verification = verifyAnswerNumbers("Give 12.5 mg.", [{ chunk_id: "chunk-1" }], [cited, uncited]); expect(verification.hasUnverifiedNumbers).toBe(true); expect(verification.unverifiedTokens).toContain("12.5mg"); }); diff --git a/tests/bulk-import.test.ts b/tests/bulk-import.test.ts index f0d5f5814..abeabfed5 100644 --- a/tests/bulk-import.test.ts +++ b/tests/bulk-import.test.ts @@ -124,9 +124,7 @@ describe("bulk import helpers", () => { it("chunks imports into configured queue batches", () => { expect(chunkImportFiles([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); expect(chunkImportFiles(Array.from({ length: 41 }, (_, index) => index)).map((batch) => batch.length)).toEqual([ - 20, - 20, - 1, + 20, 20, 1, ]); }); diff --git a/tests/clinical-safety.test.ts b/tests/clinical-safety.test.ts index 7fd58f33d..dac34e12c 100644 --- a/tests/clinical-safety.test.ts +++ b/tests/clinical-safety.test.ts @@ -53,7 +53,9 @@ describe("clinical safety findings", () => { const findings = extractSafetyFindings(answer); expect(findings).toHaveLength(1); - expect(findings[0].text).toContain("Source mentions:"); + expect(findings[0].label).toBe("Red flag"); + expect(findings[0].text).toContain("Escalate for urgent review"); + expect(findings[0].text).not.toContain("Source mentions:"); expect(findings[0].href).toBe("/documents/doc-1?page=1&chunk=chunk-1"); }); @@ -100,4 +102,28 @@ describe("clinical safety findings", () => { expect(findings[0].text).not.toContain("Image ID:"); expect(findings[0].text).not.toContain("Table text:"); }); + + it("removes provenance boilerplate from extracted finding text", () => { + const findings = extractSafetyFindings({ + ...answer, + quoteCards: [ + { + chunk_id: "chunk-1", + document_id: "doc-1", + title: "Risk source", + file_name: "risk.pdf", + page_number: 1, + chunk_index: 0, + section_heading: null, + quote: + "Source mentions: Procedure PAE-PRO-0338/16 Page 5 of 5. Chunk index: 12. Monitor FBC weekly and escalate urgent toxicity symptoms.", + }, + ], + sources: [], + }); + + expect(findings).toHaveLength(1); + expect(findings[0].text).toContain("Monitor FBC weekly"); + expect(findings[0].text).not.toMatch(/Source mentions|PAE-PRO-0338|Page 5 of 5|Chunk index/i); + }); }); diff --git a/tests/deep-memory.test.ts b/tests/deep-memory.test.ts index 55dcb4685..9325560b7 100644 --- a/tests/deep-memory.test.ts +++ b/tests/deep-memory.test.ts @@ -303,19 +303,19 @@ describe("deep RAG memory indexing", () => { expect(insertedIndexUnits.some((row) => row.unit_type === "document_profile")).toBe(true); expect(insertedMemoryRows.every((row) => !("section_index" in row))).toBe(true); expect(insertedMemoryRows.every((row) => typeof row.section_id === "string" || row.section_id === null)).toBe(true); - + // Verify JS version stamping updates expect(updatedRows.get("documents")?.[0]?.metadata).toEqual( expect.objectContaining({ rag_indexing_version: ragDeepMemoryVersion, rag_memory_version: ragDeepMemoryVersion, - }) + }), ); expect(updatedRows.get("document_chunks")?.[0]?.metadata).toEqual( expect.objectContaining({ rag_indexing_version: ragDeepMemoryVersion, rag_memory_version: ragDeepMemoryVersion, - }) + }), ); }); }); diff --git a/tests/embed-texts-integrity.test.ts b/tests/embed-texts-integrity.test.ts index 8767a9913..0e37e6220 100644 --- a/tests/embed-texts-integrity.test.ts +++ b/tests/embed-texts-integrity.test.ts @@ -22,9 +22,7 @@ describe("embedTexts integrity (IDX-C1, IDX-C2)", () => { default: class MockOpenAI { embeddings = { create: vi.fn(async ({ input }: { input: string[] }) => ({ - data: input - .map((_text, index) => ({ index, embedding: [index, index] as number[] })) - .reverse(), + data: input.map((_text, index) => ({ index, embedding: [index, index] as number[] })).reverse(), })), }; diff --git a/tests/evidence.test.ts b/tests/evidence.test.ts index 7d621a6a2..62195e659 100644 --- a/tests/evidence.test.ts +++ b/tests/evidence.test.ts @@ -134,11 +134,7 @@ The haematologist can assist with altering WCC and ANC thresholds for specific c result({ id: "m-chunk", document_id: "doc-m", hybrid_score: 0.7, similarity: 0.7 }), ]; - expect(diversifySearchResults(sources, 3).map((source) => source.id)).toEqual([ - "a-chunk", - "m-chunk", - "z-chunk", - ]); + expect(diversifySearchResults(sources, 3).map((source) => source.id)).toEqual(["a-chunk", "m-chunk", "z-chunk"]); }); it("encodes document ids in citation links", () => { diff --git a/tests/ingestion.test.ts b/tests/ingestion.test.ts index f8818602d..d02f0e257 100644 --- a/tests/ingestion.test.ts +++ b/tests/ingestion.test.ts @@ -18,7 +18,9 @@ describe("ingestion retry helpers", () => { }); it("classifies duplicate unique-key failures as partial-write conflicts", () => { - const error = new Error('duplicate key value violates unique constraint "document_chunks_document_id_chunk_index_key"'); + const error = new Error( + 'duplicate key value violates unique constraint "document_chunks_document_id_chunk_index_key"', + ); expect(isPartialIndexWriteConflict(error)).toBe(true); expect(isRetryableIngestionError(error)).toBe(false); }); diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index e6e0e53e0..7e014c588 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -719,10 +719,9 @@ describe("private document API access", () => { mockRuntime(client); const { POST } = await import("../src/app/api/ingestion/jobs/[id]/retry/route"); - const response = await POST( - authenticatedRequest(`/api/ingestion/jobs/job-1/retry`, { method: "POST" }), - { params: Promise.resolve({ id: "job-1" }) }, - ); + const response = await POST(authenticatedRequest(`/api/ingestion/jobs/job-1/retry`, { method: "POST" }), { + params: Promise.resolve({ id: "job-1" }), + }); expect(response.status).toBe(409); expect(String((await payload(response)).error)).toContain("still being processed"); @@ -749,10 +748,9 @@ describe("private document API access", () => { mockRuntime(client); const { POST } = await import("../src/app/api/ingestion/jobs/[id]/retry/route"); - const response = await POST( - authenticatedRequest(`/api/ingestion/jobs/job-1/retry`, { method: "POST" }), - { params: Promise.resolve({ id: "job-1" }) }, - ); + const response = await POST(authenticatedRequest(`/api/ingestion/jobs/job-1/retry`, { method: "POST" }), { + params: Promise.resolve({ id: "job-1" }), + }); const documentUpdate = client.calls.find((call) => call.table === "documents" && call.operation === "update"); expect(response.status).toBe(200); diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index 322146f09..d53490886 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -38,7 +38,7 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("public.document_summaries,"); expect(schema).toContain("public.storage_cleanup_jobs"); expect(schema).toMatch( - /grant select, insert, update, delete on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.image_caption_cache, .*public\.document_labels, .*public\.document_summaries, .*public\.document_sections, .*public\.document_memory_cards, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs to service_role;/, + /grant select, insert, update, delete on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.image_caption_cache, .*public\.document_labels, .*public\.document_summaries, .*public\.document_sections, .*public\.document_memory_cards, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs.* to service_role;/, ); expect(schema).toContain("grant execute on all functions in schema public to service_role;"); }); @@ -47,7 +47,7 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("revoke all privileges on all tables in schema public from anon, authenticated;"); expect(schema).toContain("revoke execute on all functions in schema public from public, anon, authenticated;"); expect(schema).toMatch( - /grant select on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs to authenticated;/, + /grant select on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs.* to authenticated;/, ); expect(schema).not.toContain("grant select, insert, update, delete on table public.documents to authenticated;"); expect(schema).not.toContain("grant select, insert on table public.rag_queries to authenticated;"); diff --git a/tests/ui-accessibility.spec.ts b/tests/ui-accessibility.spec.ts new file mode 100644 index 000000000..2019ec1a4 --- /dev/null +++ b/tests/ui-accessibility.spec.ts @@ -0,0 +1,82 @@ +import { expect, test, type Page } from "playwright/test"; + +const readySetupChecks = [ + { id: "env", label: ".env.local configured", status: "ready", detail: "Test environment ready." }, + { id: "project", label: "Clinical KB Database target", status: "ready", detail: "Test Supabase project ready." }, + { id: "schema", label: "supabase/schema.sql applied", status: "ready", detail: "Test schema ready." }, + { id: "search", label: "Search RPC and vector indexes", status: "ready", detail: "Test search schema ready." }, + { id: "openai", label: "OpenAI API key available", status: "ready", detail: "Test OpenAI ready." }, + { id: "worker", label: "npm run worker running", status: "unknown", detail: "Worker not required for UI smoke." }, +]; + +async function mockMinimalDashboardApi(page: Page) { + await page.route(/\/api\/setup-status$/, async (route) => { + await route.fulfill({ + json: { demoMode: false, checks: readySetupChecks }, + }); + }); + await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => { + await route.fulfill({ + json: { + documents: [], + pagination: { limit: 150, offset: 0, total: 0, nextOffset: 0, hasMore: false }, + }, + }); + }); + await page.route(/\/api\/ingestion\/jobs(?:\?.*)?$/, async (route) => { + await route.fulfill({ json: { jobs: [] } }); + }); + await page.route(/\/api\/ingestion\/batches(?:\?.*)?$/, async (route) => { + await route.fulfill({ json: { batches: [] } }); + }); +} + +async function gotoApp(page: Page) { + await page.goto("/", { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); +} + +async function expectNoPageHorizontalOverflow(page: Page) { + const overflow = await page.evaluate(() => { + const documentWidth = Math.max(document.documentElement.scrollWidth, document.body?.scrollWidth ?? 0); + return documentWidth - document.documentElement.clientWidth; + }); + + expect(overflow).toBeLessThanOrEqual(2); +} + +async function expectDashboardUsable(page: Page) { + await expect(page.getByRole("heading", { level: 1, name: "Clinical Guide" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Answer" })).toBeVisible(); + await expect(page.getByLabel("Search indexed guidelines by question or keyword")).toBeVisible(); + await expect(page.locator('[data-testid="scope-trigger"]:visible')).toBeVisible(); + await expectNoPageHorizontalOverflow(page); +} + +test.describe("Clinical KB accessibility media smoke", () => { + test.describe.configure({ timeout: 60_000 }); + + test("dashboard remains usable with reduced motion", async ({ page }) => { + await page.emulateMedia({ reducedMotion: "reduce" }); + await page.setViewportSize({ width: 390, height: 820 }); + await mockMinimalDashboardApi(page); + await gotoApp(page); + + await expectDashboardUsable(page); + await page.locator('[data-testid="scope-trigger"]:visible').click(); + await expect(page.locator('[data-testid="scope-command-popover"]:visible')).toBeVisible(); + await expectNoPageHorizontalOverflow(page); + }); + + test("dashboard remains usable with forced colors", async ({ page }) => { + await page.emulateMedia({ forcedColors: "active" }); + await page.setViewportSize({ width: 390, height: 820 }); + await mockMinimalDashboardApi(page); + await gotoApp(page); + + await expectDashboardUsable(page); + await page.locator('[data-testid="scope-trigger"]:visible').click(); + await expect(page.locator('[data-testid="scope-command-popover"]:visible')).toBeVisible(); + await expectNoPageHorizontalOverflow(page); + }); +}); diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index fa7c9d221..08b26a058 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -701,7 +701,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByText("Retrieval details")).toHaveCount(0); await scopeTrigger(page).click(); - const scopePopover = page.getByTestId("scope-command-popover"); + const scopePopover = page.locator('[data-testid="scope-command-popover"]:visible'); await expect(scopePopover).toBeVisible(); const scopeFilter = scopePopover.locator('[data-testid="document-scope-filter"]'); await expect(scopeFilter).toBeVisible(); @@ -1016,11 +1016,13 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByLabel("Search indexed guidelines by question or keyword")).toBeVisible(); await scrollDashboardToBottom(page); - const uploadSummary = page.locator("summary").filter({ hasText: "Upload and indexing" }).first(); - await uploadSummary.scrollIntoViewIfNeeded(); - await uploadSummary.click({ force: true }); - const uploadDrawer = page.locator("details").filter({ hasText: "Upload and indexing" }).first(); + const uploadTrigger = page.locator("#dashboard-upload-drawer-mobile-trigger"); + await uploadTrigger.scrollIntoViewIfNeeded(); + await uploadTrigger.click(); + const uploadDrawer = page.getByRole("dialog", { name: "Upload and indexing" }); + await expect(uploadDrawer).toBeVisible(); + await uploadDrawer.getByRole("tab", { name: /Setup/ }).click(); await expect(uploadDrawer.getByText("First-run setup checklist")).toBeVisible(); await expect(uploadDrawer.getByText(".env.local configured")).toBeVisible(); await expect(uploadDrawer.getByText("Clinical KB Database target")).toBeVisible(); @@ -1028,6 +1030,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(uploadDrawer.getByText("Search RPC and vector indexes")).toBeVisible(); await expect(uploadDrawer.getByText("OpenAI API key available")).toBeVisible(); await expect(uploadDrawer.getByText("npm run worker running")).toBeVisible(); + await uploadDrawer.getByRole("tab", { name: /Upload/ }).click(); await expect(uploadDrawer.getByText("Document title optional")).toBeVisible(); await expect(uploadDrawer.getByText("Guideline files required")).toBeVisible(); await expectNoPageHorizontalOverflow(page); @@ -1040,13 +1043,16 @@ test.describe("Clinical KB UI smoke coverage", () => { await gotoApp(page, "/"); await scrollDashboardToBottom(page); - const uploadSummary = page.locator("summary").filter({ hasText: "Upload and indexing" }).first(); - await uploadSummary.scrollIntoViewIfNeeded(); - await uploadSummary.click({ force: true }); - const uploadDrawer = page.locator("details").filter({ hasText: "Upload and indexing" }).first(); + const uploadTrigger = page.locator("#dashboard-upload-drawer-mobile-trigger"); + await uploadTrigger.scrollIntoViewIfNeeded(); + await uploadTrigger.click(); + const uploadDrawer = page.getByRole("dialog", { name: "Upload and indexing" }); + await expect(uploadDrawer).toBeVisible(); - await expect(uploadDrawer.getByRole("button", { name: "Queue document" })).toBeEnabled({ timeout: 30000 }); + await uploadDrawer.getByRole("tab", { name: /Jobs/ }).click(); await expect(uploadDrawer.getByText("2 exact copies skipped")).toBeVisible(); + await uploadDrawer.getByRole("tab", { name: /Upload/ }).click(); + await expect(uploadDrawer.getByRole("button", { name: "Queue document" })).toBeEnabled({ timeout: 30000 }); await uploadDrawer.locator('input[name="file"]').setInputFiles({ name: "guideline.pdf", mimeType: "application/pdf", diff --git a/tests/ui-tools.spec.ts b/tests/ui-tools.spec.ts new file mode 100644 index 000000000..02948f256 --- /dev/null +++ b/tests/ui-tools.spec.ts @@ -0,0 +1,38 @@ +import { expect, test, type Page } from "playwright/test"; + +async function gotoTools(page: Page) { + await page.goto("/tools", { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); +} + +async function expectNoPageHorizontalOverflow(page: Page) { + const overflow = await page.evaluate(() => { + const documentWidth = Math.max(document.documentElement.scrollWidth, document.body?.scrollWidth ?? 0); + return documentWidth - document.documentElement.clientWidth; + }); + + expect(overflow).toBeLessThanOrEqual(2); +} + +test.describe("Clinical KB tools launcher", () => { + test.describe.configure({ timeout: 60_000 }); + + for (const viewport of [ + { name: "mobile", width: 390, height: 820 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + test(`tools launcher is usable at ${viewport.name}`, async ({ page }) => { + await page.setViewportSize({ width: viewport.width, height: viewport.height }); + await gotoTools(page); + + await expect(page.getByRole("heading", { level: 1, name: "Open the right clinical tool." })).toBeVisible(); + await expect(page.getByRole("heading", { name: "All clinical tools" })).toBeVisible(); + await expect(page.getByLabel("Return to Clinical KB dashboard")).toBeVisible(); + await expect(page.getByLabel("Launch priority tool Formulation")).toBeVisible(); + await expect(page.getByLabel("Launch Formulation")).toBeVisible(); + await expect(page.getByLabel("Launch Psychiatry Notes")).toBeVisible(); + await expect(page.getByRole("link", { name: "Launchers" })).toHaveAttribute("href", "#launchers"); + await expectNoPageHorizontalOverflow(page); + }); + } +}); diff --git a/tests/ward-output.test.ts b/tests/ward-output.test.ts index 58f041659..c83166c3f 100644 --- a/tests/ward-output.test.ts +++ b/tests/ward-output.test.ts @@ -383,7 +383,9 @@ describe("ward output helpers", () => { quoteCards: [], }); - expect(sections.find((section) => section.id === "source-gap")?.items[0]).toContain("do not provide"); + const sourceGapSections = sections.filter((section) => section.id === "source-gap"); + expect(sourceGapSections).toHaveLength(1); + expect(sourceGapSections[0]?.items.join(" ")).toContain("do not provide"); expect(sections.find((section) => section.id === "thresholds")).toBeUndefined(); }); @@ -552,7 +554,17 @@ describe("ward output helpers", () => { }); it("formats copyable answer and quote text with citations", () => { - expect(formatAnswerForClipboard(answer)).toContain("Lithium source, p. 1"); + const copy = formatAnswerForClipboard(answer); + + expect(copy).toContain("Bottom line"); + expect(copy).toContain("- Monitor renal function"); + expect(copy).toContain("Monitoring"); + expect(copy).toContain("Thresholds"); + expect(copy).toContain("Citations"); + expect(copy).toContain("Lithium source, p. 1"); + expect(copy).toContain("Source status"); + expect(copy).toContain("Review requirement"); + expect(copy.match(/Monitor renal function and escalate review/g)).toHaveLength(1); expect(formatQuotesForClipboard(answer.quoteCards)).toContain('"Escalate review'); }); @@ -560,7 +572,10 @@ describe("ward output helpers", () => { const note = formatWardNote(answer, true); expect(note).toContain("Synthetic demo only"); + expect(note).toContain("Bottom line"); + expect(note).toContain("Review requirement"); expect(note).toContain("Citations"); + expect(note.match(/Monitor renal function and escalate review/g)).toHaveLength(1); }); it("creates a focused follow-up question from a quote", () => { diff --git a/worker/embedding-fields.ts b/worker/embedding-fields.ts index 3a6adf999..0306327c4 100644 --- a/worker/embedding-fields.ts +++ b/worker/embedding-fields.ts @@ -163,13 +163,18 @@ export function buildAdditionalEmbeddingFieldInputs(args: { const text = `${chunk.section_heading ?? ""} ${chunk.content ?? ""}`; if (!highYieldPattern.test(text)) continue; - addField(chunk, "chunk_high_yield", `High-yield clinical context: ${chunkContext(args.job, chunk, chunkContextLimit)}`, { - source: "chunk_high_yield", - chunk_index: chunk.chunk_index ?? null, - page_number: chunk.page_number, - duplicate_chunk_ratio: Number.isFinite(duplicateRatio) ? duplicateRatio : null, - heading_density: Number.isFinite(headingDensity) ? headingDensity : null, - }); + addField( + chunk, + "chunk_high_yield", + `High-yield clinical context: ${chunkContext(args.job, chunk, chunkContextLimit)}`, + { + source: "chunk_high_yield", + chunk_index: chunk.chunk_index ?? null, + page_number: chunk.page_number, + duplicate_chunk_ratio: Number.isFinite(duplicateRatio) ? duplicateRatio : null, + heading_density: Number.isFinite(headingDensity) ? headingDensity : null, + }, + ); const actionText = extractClinicalSentences(chunk.content, actionPattern); if (actionText) { diff --git a/worker/main.ts b/worker/main.ts index db254ad45..e59d349c9 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -59,8 +59,13 @@ const progressUpdateState = new Map<string, { updatedAt: number; progress: numbe const progressUpdateMinIntervalMs = env.WORKER_PROGRESS_UPDATE_MIN_INTERVAL_MS; const progressUpdateMinDelta = 4; const maxSupabaseBackoffMs = env.WORKER_HEALTH_BACKOFF_MS; +const analyzeRagTablesThrottleMs = 45_000; +let lastAnalyzeRagTablesAt = 0; -function supabaseStageError(stage: string, error: { message?: string; code?: string; details?: string; hint?: string }) { +function supabaseStageError( + stage: string, + error: { message?: string; code?: string; details?: string; hint?: string }, +) { const wrapped = new Error(`${stage}: ${error.message ?? "Supabase request failed"}`); Object.assign(wrapped, { code: error.code, @@ -93,7 +98,10 @@ async function updateJobProgress(jobId: string, patch: { stage: string; progress const { error } = await supabase.from("ingestion_jobs").update(patch).eq("id", jobId); if (error) { - console.warn("Ingestion progress update failed", safeErrorLogDetails(supabaseStageError("update ingestion progress", error))); + console.warn( + "Ingestion progress update failed", + safeErrorLogDetails(supabaseStageError("update ingestion progress", error)), + ); return; } progressUpdateState.set(jobId, { updatedAt: now, progress: patch.progress, stage: patch.stage }); @@ -224,6 +232,17 @@ function optionalIndexWriteWarning(stage: string, error: unknown) { console.warn(`Optional ${stage} write failed`, safeErrorLogDetails(error)); } +async function refreshRagTableStats() { + const now = Date.now(); + if (now - lastAnalyzeRagTablesAt < analyzeRagTablesThrottleMs) return; + + const { error } = await supabase.rpc("analyze_rag_tables"); + lastAnalyzeRagTablesAt = now; + if (!error) return; + + optionalIndexWriteWarning("rag table statistics refresh", supabaseStageError("analyze_rag_tables", error)); +} + function noteSkippedImage(skipReasons: Map<string, number>, reason: string) { skipReasons.set(reason, (skipReasons.get(reason) ?? 0) + 1); } @@ -252,7 +271,10 @@ async function downloadDocument(storagePath: string) { function cleanString(val: string): string { if (typeof val !== "string") return val; - return val.replace(/\u0000/g, "").replace(/\\u0000/g, "").toWellFormed(); + return val + .replace(/\u0000/g, "") + .replace(/\\u0000/g, "") + .toWellFormed(); } type JsonbValue = string | number | boolean | null | { [key: string]: JsonbValue } | JsonbValue[]; @@ -306,6 +328,10 @@ function hashText(text: string) { return createHash("sha256").update(text.replace(/\s+/g, " ").trim()).digest("hex"); } +function hashEmbeddingFieldContent(content: string) { + return createHash("md5").update(content).digest("hex"); +} + function compactSearchText(value: unknown, limit = 900) { const compact = String(value ?? "") .replace(/\s+/g, " ") @@ -837,6 +863,7 @@ async function insertDocumentLevelEmbeddingFields(args: { source_chunk_id: sourceChunkId, field_type: field.field_type, content: field.content, + content_hash: hashEmbeddingFieldContent(field.content), embedding: embeddings[index], metadata: { source: "document_level", @@ -916,11 +943,12 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { const fieldRows = fieldInputs.map((field, index) => ({ ...field, content: cleanString(field.content), + content_hash: hashEmbeddingFieldContent(cleanString(field.content)), embedding: fieldEmbeddings[index], metadata: sanitizeJsonbRecord(field.metadata), })); - for (let start = 0; start < fieldRows.length; start += 10) { - const batch = fieldRows.slice(start, start + 10); + for (let start = 0; start < fieldRows.length; start += 50) { + const batch = fieldRows.slice(start, start + 50); const { error: fieldsError } = await supabase.from("document_embedding_fields").insert(batch); if (fieldsError) throw supabaseStageError("insert section-context embedding fields", fieldsError); } @@ -954,16 +982,21 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { if (additionalFieldInputs.length > 0) { try { const additionalEmbeddings = await embedTexts(additionalFieldInputs.map((field) => field.content)); - const additionalRows = additionalFieldInputs.map((field, index) => ({ - ...field, - content: cleanString(field.content), - embedding: additionalEmbeddings[index], - metadata: sanitizeJsonbRecord(field.metadata), - })); - for (let start = 0; start < additionalRows.length; start += 10) { - const batch = additionalRows.slice(start, start + 10); + const additionalRows = additionalFieldInputs.map((field, index) => { + const content = cleanString(field.content); + return { + ...field, + content, + content_hash: hashEmbeddingFieldContent(content), + embedding: additionalEmbeddings[index], + metadata: sanitizeJsonbRecord(field.metadata), + }; + }); + for (let start = 0; start < additionalRows.length; start += 50) { + const batch = additionalRows.slice(start, start + 50); const { error: additionalFieldsError } = await supabase.from("document_embedding_fields").insert(batch); - if (additionalFieldsError) throw supabaseStageError("insert supplemental embedding fields", additionalFieldsError); + if (additionalFieldsError) + throw supabaseStageError("insert supplemental embedding fields", additionalFieldsError); } } catch (error) { optionalIndexWriteWarning("supplemental embedding field", error); @@ -1083,8 +1116,8 @@ async function processJob(job: JobRow) { const { error: initialQualityError } = await supabase .from("document_index_quality") .upsert(sanitizeJsonbRecord(initialQuality), { - onConflict: "document_id", - }); + onConflict: "document_id", + }); if (initialQualityError) throw new Error(initialQualityError.message); const indexedAt = new Date().toISOString(); @@ -1197,6 +1230,7 @@ async function processJob(job: JobRow) { }); await completeJob(job, enrichmentStatus === "completed" ? "indexed" : "indexed; enrichment deferred"); + await refreshRagTableStats(); } catch (error) { console.error(`Ingestion job ${job.id} failed:`, error); const message = error instanceof Error ? error.message : String(error);