diff --git a/.gitignore b/.gitignore index 06c8cac..38f0905 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ lib/* node_modules/ tests/output/*.png tests/output/*.jpg +tests/output/*.pdf tests/output/*.cpuprofile *.DS_Store* .aider* diff --git a/README.md b/README.md index 5e6a921..42d8c4f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,23 @@ Ratta Supernote has often commented that the file-format is yet unstable and sho For some quick snippets, take a look at the [smoke tests](./tests/main.test.ts). +### Generating searchable PDFs + +`toPdf` renders each page's raster image into a PDF page and overlays the recognized handwriting (RTR) text invisibly at the location it was written, so the PDF is searchable and words can be selected/copied from the image. See [tests/pdf.test.ts](./tests/pdf.test.ts) for a full example. + +```ts +import { SupernoteX, toPdf } from 'supernote-typescript'; + +const note = new SupernoteX(buffer); +const pdfBytes = await toPdf(note); +``` + +The default font (Helvetica) only supports Latin text. Pass `fontBytes` with a Unicode TTF/OTF to support other scripts: + +```ts +const pdfBytes = await toPdf(note, { fontBytes: await fs.readFile('NotoSans-Regular.ttf') }); +``` + ## Developer Notes ### Test Individual Suite diff --git a/package-lock.json b/package-lock.json index 72ed482..6e705cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,11 @@ "version": "0.4.2", "license": "GPL-3.0-or-later", "dependencies": { + "@pdf-lib/fontkit": "^1.1.1", "color": "^5.0.3", "fs-extra": "^11.4.0", - "image-js": "^1.7.0" + "image-js": "^1.7.0", + "pdf-lib": "^1.17.1" }, "devDependencies": { "@eslint/js": "10.0.1", @@ -22,6 +24,7 @@ "@vitest/coverage-v8": "4.1.10", "eslint": "10.8.0", "globals": "17.7.0", + "pdf-parse": "^2.4.5", "prettier": "3.9.6", "typescript": "6.0.3", "vitest": "4.1.10" @@ -330,6 +333,201 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", + "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==", + "dev": true, + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.80", + "@napi-rs/canvas-darwin-arm64": "0.1.80", + "@napi-rs/canvas-darwin-x64": "0.1.80", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.80", + "@napi-rs/canvas-linux-arm64-musl": "0.1.80", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-musl": "0.1.80", + "@napi-rs/canvas-win32-x64-msvc": "0.1.80" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz", + "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz", + "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz", + "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz", + "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz", + "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz", + "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz", + "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz", + "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz", + "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz", + "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", @@ -359,6 +557,33 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@pdf-lib/fontkit": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@pdf-lib/fontkit/-/fontkit-1.1.1.tgz", + "integrity": "sha512-KjMd7grNapIWS/Dm0gvfHEilSyAmeLvrEGVcqLGi0VYebuqqzTbgF29efCx7tvx+IEbG3zQciRSWl3GkUSvjZg==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.6" + } + }, + "node_modules/@pdf-lib/standard-fonts": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz", + "integrity": "sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.6" + } + }, + "node_modules/@pdf-lib/upng": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@pdf-lib/upng/-/upng-1.0.1.tgz", + "integrity": "sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.10" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", @@ -2631,6 +2856,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parenthesis": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/parenthesis/-/parenthesis-3.1.8.tgz", @@ -2665,6 +2896,58 @@ "dev": true, "license": "MIT" }, + "node_modules/pdf-lib": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/pdf-lib/-/pdf-lib-1.17.1.tgz", + "integrity": "sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==", + "license": "MIT", + "dependencies": { + "@pdf-lib/standard-fonts": "^1.0.0", + "@pdf-lib/upng": "^1.0.1", + "pako": "^1.0.11", + "tslib": "^1.11.1" + } + }, + "node_modules/pdf-lib/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "license": "0BSD" + }, + "node_modules/pdf-parse": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz", + "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@napi-rs/canvas": "0.1.80", + "pdfjs-dist": "5.4.296" + }, + "bin": { + "pdf-parse": "bin/cli.mjs" + }, + "engines": { + "node": ">=20.16.0 <21 || >=22.3.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/package.json b/package.json index 7a01e54..f033c56 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ ], "scripts": { "build": "tsc", - "clean": "rm -f tests/output/*.png tests/output/*.jpg tests/output/*.cpuprofile", + "clean": "rm -f tests/output/*.png tests/output/*.jpg tests/output/*.pdf tests/output/*.cpuprofile", "lint": "eslint .", "test": "vitest run", "test-watch": "vitest", @@ -43,13 +43,16 @@ "@vitest/coverage-v8": "4.1.10", "eslint": "10.8.0", "globals": "17.7.0", + "pdf-parse": "^2.4.5", "prettier": "3.9.6", "typescript": "6.0.3", "vitest": "4.1.10" }, "dependencies": { + "@pdf-lib/fontkit": "^1.1.1", "color": "^5.0.3", "fs-extra": "^11.4.0", - "image-js": "^1.7.0" + "image-js": "^1.7.0", + "pdf-lib": "^1.17.1" } } diff --git a/plans/rtr-searchable-pdf-workers.md b/plans/rtr-searchable-pdf-workers.md new file mode 100644 index 0000000..8e7ab87 --- /dev/null +++ b/plans/rtr-searchable-pdf-workers.md @@ -0,0 +1,46 @@ +# Plan: Worker-parallel page rendering for `toPdf()` + +## Context + +`toPdf()` (`src/pdf.ts:43`) currently does everything on one thread, in a loop over `pages`: call `toImage(note, pageNumbers)` (`src/conversion.ts:71`) once up front for all pages, then per page, embed the PNG and push invisible-text operators. The CPU-heavy part is inside `toImage` — `RattaRLEDecoder.decode()` and `compositeImages()` — which is pure, page-independent, synchronous pixel work. The PDF-assembly part (`embedPng`, `drawImage`, `pushOperators` for text) is comparatively cheap. `toImage` already accepts a `pageNumbers` array (`src/conversion.ts:73-75`), so per-page rendering is already a well-defined unit of work — the missing piece is a way to hand that unit to a worker and feed the result back into PDF assembly, since `toPdf` currently couples render and assemble into one inseparable loop. + +## Goal + +Let an application render pages in parallel across Web Workers (browser) or `worker_threads` (Node), without duplicating this library's PDF-assembly logic in application code, by splitting `toPdf` into a render step (worker-safe, already mostly exists) and an assemble step (main-thread only, since `pdf-lib` document objects aren't transferable). + +## Step-by-step plan + +1. **Spike: profile the split before committing to it.** Measure wall-clock time spent in `toImage` vs. the rest of `toPdf`'s per-page loop (PNG embed + text operators) on a multi-page fixture. This determines whether parallelizing render-only (Option A below) captures most of the win, or whether PDF assembly is expensive enough to also need offloading (Option B). Don't guess — the earlier coordinate-scale spike (see `plans/rtr-searchable-pdf.md`) showed assumptions here are easy to get wrong. + +2. **Refactor `toPdf` into three pieces**, keeping its existing public signature and behavior unchanged (backward compatible): + - `createPdfContext(options?: ToPdfOptions): Promise<{ pdfDoc: PDFDocument, font: PDFFont }>` — the current setup code (`src/pdf.ts:49-57`): creates the `PDFDocument`, embeds the font (default Helvetica or custom `fontBytes`). Runs once, main thread only. + - `addPdfPage(ctx: { pdfDoc, font }, page: IPage, image: Image | Uint8Array, options?: { dpi?: number }): Promise` — the current per-page body (`src/pdf.ts:59-109`): sizes the page from the image, embeds it (accept either an `Image` or already-encoded PNG bytes, so a worker can hand back raw PNG bytes without the main thread needing an `Image` reconstruction step), and draws the invisible text layer from `page.recognitionElements`. Main thread only, since `pdfDoc`/`font` are `pdf-lib` objects. + - `toPdf(note, options)` stays as the convenience wrapper: `createPdfContext` → loop calling `toImage` + `addPdfPage` → `pdfDoc.save()`. Single-threaded, unchanged from the caller's perspective. + +3. **Confirm (don't assume) that per-page render input is structured-clone-safe.** `toImage(note, [n])` only touches `note.pageWidth`, `note.pageHeight`, and `page.LAYERSEQ`-referenced layers' `bitmapBuffer` (`src/conversion.ts:80-95`). A worker doesn't need the whole `SupernoteX` instance (which also carries methods that won't clone, and every other page's buffers, wasteful for large notebooks) — it needs a minimal plain-object slice per page. Add a small extraction helper, e.g. `extractPageRenderData(note: ISupernote, pageNumber: number): IPageRenderData` returning just `{ pageWidth, pageHeight, layers }`, so applications don't need to reach into internals to build a transferable payload. Write a real `node:worker_threads` round-trip test that posts this slice to a worker, renders it, and posts PNG bytes back — this is the actual risk area (something not surviving structured clone), not something to assume works. + +4. **Document the application-side pattern** (README + example), since the worker orchestration itself is the app's responsibility, not this library's: + ```ts + const note = new SupernoteX(buffer); + const ctx = await createPdfContext(); + const pngBuffers = await Promise.all( + note.pages.map((_, i) => renderInWorker(extractPageRenderData(note, i + 1))) + ); // each worker: toImage(sliceAsNote, [1]) + encodePng, both already exported + for (let i = 0; i < note.pages.length; i++) { + await addPdfPage(ctx, note.pages[i], pngBuffers[i]); + } + const pdfBytes = await ctx.pdfDoc.save(); + ``` + Note `toImage` and `encodePng` (from `image-js`, already a dependency) are sufficient for the worker side today — no new export needed there, only documentation that they're safe to call off-main-thread. + +5. **Tests**: unit-test that `addPdfPage` given a pre-rendered `Image` vs. given pre-encoded PNG bytes produces equivalent output; unit-test that `toPdf` built from the three pieces still produces byte-identical (or search-equivalent, via the existing `pdf-parse` check) output to the current implementation, so the refactor is provably behavior-preserving; add the `worker_threads` round-trip test from step 3. + +## Non-goals + +- No bundled Worker/browser glue code or a full example app — orchestration is application-specific. +- Not parallelizing the invisible-text-drawing loop itself (cheap relative to RLE decode; not worth the complexity per the spike in step 1, pending its result). +- Not pursuing per-page PDF merge (Option B: build single-page PDFs in workers, merge via `PDFDocument.copyPages`) unless step 1's profiling shows assembly is a meaningful fraction of total time — it adds real complexity (duplicated embedded fonts inflate file size unless deduped). + +## Open question for whoever implements this + +Step 1's profiling result decides whether this plan is even the right shape — if PDF assembly turns out to be non-trivial too, Option B needs its own design pass before implementation starts. diff --git a/plans/rtr-searchable-pdf.md b/plans/rtr-searchable-pdf.md new file mode 100644 index 0000000..3542e3b --- /dev/null +++ b/plans/rtr-searchable-pdf.md @@ -0,0 +1,53 @@ +# Plan: Add a `toPdf()` API that renders searchable PDFs with an invisible RTR text layer + +## Context for the implementing agent + +This is `supernote-typescript`, a parser/renderer for Ratta Supernote `.note` files. Relevant facts already established: + +- **Exports** (`src/index.ts`): `SupernoteX` (parser), `toImage` (rasterizes pages to `image-js` `Image` objects, see `src/conversion.ts`), `fetchMirrorFrame`. +- **No PDF code exists anywhere in the repo.** `package.json` dependencies are just `color`, `fs-extra`, `image-js` — no PDF library. +- **RTR** = "Real Time Recognition" (Supernote's on-device handwriting recognition). Confirmed by test fixture `tests/input/rtr.note` and its expected text `'Real time recognition paragraph test'`. +- **Data model** (`src/format.ts:132`): `IRecognitionElement { label, type, words: [{ label, "bounding-box"?: { x, y, width, height } }] }`. Populated per-page at `page.recognitionElements` (`src/format.ts:181`) by `SupernoteX._parseRecognition` (`src/parsing.ts:357`). +- Existing helpers `_extractText` (`src/parsing.ts:377`) and `_extractParagraphs` (`src/parsing.ts:385`) already show the pattern for walking `type === 'Text'` elements and reading `e.words[0]['bounding-box']`, but they discard position info once flattened to a string — the new code needs to keep per-word boxes. +- Page pixel size is `note.pageWidth` / `note.pageHeight` (`src/format.ts:3-6`), the same dimensions `toImage()` rasterizes to (`src/conversion.ts:94-97`). +- Useful test fixtures already in the repo: `tests/input/rtr.note` (has paragraph/text expectations in `tests/main.test.ts:149-174`) and `tests/input/nomad-3.15.27-blank-shapes-and-RTR.note`. + +## Goal + +Add a public API, e.g. `toPdf(note: SupernoteX, options?): Promise`, that produces a multi-page PDF per Supernote note where: +1. Each PDF page shows the rasterized page image (from `toImage`) as the visible content. +2. Each recognized word is drawn as **invisible** text (PDF text-rendering mode 3) positioned directly over the handwriting that produced it, so PDF viewers/search/copy-paste find the right word in the right place — the standard "OCR text layer" technique used by scanned-PDF tools. + +## Step-by-step plan + +1. **Spike: verify the coordinate space before writing real code.** This is the biggest risk in the task. Write a throwaway script that loads `tests/input/rtr.note`, calls `toImage`, and dumps `page.recognitionElements[*].words[*]['bounding-box']` alongside `note.pageWidth`/`pageHeight` and the actual rendered image dimensions. Confirm bounding boxes are already in page-pixel space (same origin/scale as the raster image) with no separate DPI/scale factor. If they're not 1:1, work out the correct scale/offset here before proceeding — don't guess in the main implementation. + +2. **Add dependencies.** Add `pdf-lib` for PDF construction, and `@pdf-lib/fontkit` for embedding a Unicode-capable TTF (the standard 14 PDF fonts don't cover most recognized text, and Supernote recognition may include non-Latin scripts). Pick and vendor/bundle a permissively-licensed Unicode font (e.g. Noto Sans) or accept a font path via options. + +3. **New module `src/pdf.ts`** exporting `toPdf(note: SupernoteX, options?: { fontBytes?: Uint8Array }): Promise`: + - For each page: call `toImage`-equivalent per-page rendering (check whether `toImage` needs to run once for the whole note or supports per-page — read `src/conversion.ts` signature), embed the resulting PNG/bitmap into a new `PDFPage` sized to `pageWidth`/`pageHeight` (convert pixels → PDF points using a fixed assumed DPI, e.g. 96 or whatever matches Supernote's real DPI — confirm this too, since it affects physical print size, not searchability). + - For each `page.recognitionElements` entry with `type === 'Text'`, for each `word` with a `bounding-box`: compute PDF coordinates. PDF origin is bottom-left, so `pdfY = pageHeightPts - (box.y + box.height) * scale`. Set font size from `box.height * scale`. + - Draw invisible text. `pdf-lib`'s high-level `drawText` doesn't expose text-rendering mode, so use `page.pushOperators` with raw operators (`BT`, `Tr 3` to set invisible mode, `Tf`, `Td`, `Tj`, `ET`) — or check if a newer `pdf-lib` version added a `renderMode` option to `drawText` and prefer that if available. + - Decode word labels the same way `_extractText` does (`decodeURIComponent(escape(e.label))`) before drawing. + - Handle pages with `RECOGNSTATUS === RecognitionStatuses.NONE` (or missing `recognitionElements`) by emitting the image with no text layer — not an error case. + +4. **Export from `src/index.ts`**: `export { toPdf } from './pdf';` + +5. **Tests** (`tests/main.test.ts` or a new `tests/pdf.test.ts`): + - Generate a PDF from `tests/input/rtr.note`, write it to `tests/output/` like the existing image tests do. + - Use a PDF text-extraction library (e.g. `pdf-parse` or `pdfjs-dist`, dev dependency only) to assert the extracted text matches (or is a superset of) the known-good text from the existing `rtr.note` test (`tests/main.test.ts:152-174`). + - Optionally assert extracted text *item positions* (if the chosen extraction library exposes them) fall within the page bounds and in roughly the expected reading order, as a sanity check on the coordinate mapping from step 1. + - Also run against `nomad-3.15.27-blank-shapes-and-RTR.note` and a note with no recognition data, to confirm the no-text-layer path doesn't throw. + +6. **Docs**: add a short usage example to `README.md` alongside the existing `toImage` example (check `README.md` for the existing example format and match its style). + +## Non-goals + +- No reflowed/native PDF text (this is an image-plus-invisible-text overlay, not a real text document). +- No attempt to lay out shapes/drawings/other layer types as vector PDF content — only the rasterized page image plus the text layer. + +## Open questions the implementing agent should resolve early, not assume + +- Exact page-pixel → PDF-point DPI conversion (affects only physical page size, not searchability, but should be correct). +- Whether `toImage` can be called per-page or only for the whole note (affects the render loop structure). +- Confirm license terms before bundling any font file. diff --git a/src/conversion.ts b/src/conversion.ts index f1ffae7..999410c 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -87,7 +87,13 @@ export function toImage(note: ISupernote, pageNumbers?: number[]) { layer.LAYERNAME == 'BGLAYER' && page.PAGESTYLE.startsWith('user_') ) { - return decodePng(layer.bitmapBuffer as Uint8Array); + // User-uploaded background templates are arbitrary PNGs and may + // decode to any bit depth/color model (e.g. 8-bit RGB with no + // alpha channel). compositeImages() requires 8-bit RGBA on both + // sides, so normalize regardless of the source PNG's format. + return decodePng(layer.bitmapBuffer as Uint8Array) + .convertBitDepth(8) + .convertColor(ImageColorModel.RGBA); } const buffer = decoder.decode( layer.bitmapBuffer as Uint8Array, diff --git a/src/index.ts b/src/index.ts index b2f31ab..07c46f6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ export { SupernoteX } from './parsing'; export { toImage } from './conversion'; export { fetchMirrorFrame } from './mirror'; +export { toPdf } from './pdf'; +export type { ToPdfOptions } from './pdf'; diff --git a/src/pdf.ts b/src/pdf.ts new file mode 100644 index 0000000..30cf4fb --- /dev/null +++ b/src/pdf.ts @@ -0,0 +1,128 @@ +import { + PDFDocument, + PDFFont, + StandardFonts, + TextRenderingMode, + beginText, + endText, + moveText, + setCharacterSqueeze, + setFontAndSize, + setTextRenderingMode, + showText, +} from 'pdf-lib'; +import fontkit from '@pdf-lib/fontkit'; +import { encodePng } from 'image-js'; +import { toImage } from './conversion'; +import { ISupernote } from './format'; + +// Empirically-verified constant used by Supernote's own recognition format: +// recognized word bounding boxes are stored in raster-pixel units divided by +// this factor. See plans/rtr-searchable-pdf.md for how this was confirmed. +const RECOGNITION_COORDINATE_SCALE = 11.9; + +export interface ToPdfOptions { + /** Page numbers to export (1-indexed). Defaults to all pages. */ + pageNumbers?: number[]; + /** Bytes of a Unicode-capable TTF/OTF font to embed for the invisible text + * layer. Defaults to the standard Helvetica font, which only supports + * Latin (WinAnsi) text and will throw for other scripts. */ + fontBytes?: Uint8Array; + /** Assumed pixel density of the source page raster, used to size PDF + * pages in points. Supernote doesn't record this in the file; 300 matches + * known device screen densities. Only affects the physical/print page + * size, not searchability. */ + dpi?: number; +} + +/** + * Render a Supernote note to a PDF where each page shows the rasterized + * page image with the recognized handwriting (RTR) text drawn invisibly on + * top of it, at the position it was written, so PDF viewers can search for + * and select the handwritten words. + */ +export async function toPdf(note: ISupernote, options: ToPdfOptions = {}): Promise { + const { pageNumbers, fontBytes, dpi = 300 } = options; + const pages = pageNumbers ? pageNumbers.map((n) => note.pages[n - 1]) : note.pages; + const images = await toImage(note, pageNumbers); + const pointsPerPixel = 72 / dpi; + + const pdfDoc = await PDFDocument.create(); + + let font: PDFFont; + if (fontBytes) { + pdfDoc.registerFontkit(fontkit); + font = await pdfDoc.embedFont(fontBytes, { subset: true }); + } else { + font = await pdfDoc.embedFont(StandardFonts.Helvetica); + } + + for (let i = 0; i < pages.length; i++) { + const page = pages[i]; + const image = images[i]; + + const widthPts = image.width * pointsPerPixel; + const heightPts = image.height * pointsPerPixel; + + const pdfPage = pdfDoc.addPage([widthPts, heightPts]); + const fontKey = pdfPage.node.newFontDictionary(font.name, font.ref); + + const pngImage = await pdfDoc.embedPng(encodePng(image)); + pdfPage.drawImage(pngImage, { x: 0, y: 0, width: widthPts, height: heightPts }); + + for (const element of page.recognitionElements) { + if (element.type !== 'Text') continue; + + for (const word of element.words) { + const box = word['bounding-box']; + if (!box) continue; + + const label = decodeURIComponent(escape(word.label)); + if (!label) continue; + + const xPx = box.x * RECOGNITION_COORDINATE_SCALE; + const yPx = box.y * RECOGNITION_COORDINATE_SCALE; + const widthPx = box.width * RECOGNITION_COORDINATE_SCALE; + const heightPx = box.height * RECOGNITION_COORDINATE_SCALE; + + const boxWidthPts = widthPx * pointsPerPixel; + const boxHeightPts = heightPx * pointsPerPixel; + const x = xPx * pointsPerPixel; + // PDF's y-axis runs bottom-up; recognition boxes are top-down. + const y = heightPts - (yPx * pointsPerPixel + boxHeightPts); + + // Size the font to the box height, then use horizontal scaling + // (the PDF `Tz` operator) to stretch or squeeze the text to + // exactly match the box width in both directions — handwriting + // is rarely the same width as print at a given height (cursive + // runs narrower, print can run wider) — so that PDF viewers' + // search-hit highlight rectangle lines up with the ink instead + // of just not overflowing it. + try { + const fontSize = boxHeightPts; + const naturalWidth = font.widthOfTextAtSize(label, fontSize); + const horizontalScale = naturalWidth > 0 ? (boxWidthPts / naturalWidth) * 100 : 100; + + pdfPage.pushOperators( + beginText(), + setTextRenderingMode(TextRenderingMode.Invisible), + setFontAndSize(fontKey, fontSize), + setCharacterSqueeze(horizontalScale), + moveText(x, y), + showText(font.encodeText(label)), + endText(), + ); + } catch { + // The active font (Helvetica by default) can't encode every + // character recognition may produce (e.g. superscripts, smart + // punctuation). Skip this word rather than losing the whole + // PDF over one unsearchable word; pass a Unicode `fontBytes` + // font via ToPdfOptions to cover more characters. + continue; + } + } + } + } + + return pdfDoc.save(); +} diff --git a/tests/input/moonchild-user-bg-and-bad-glyph.note b/tests/input/moonchild-user-bg-and-bad-glyph.note new file mode 100644 index 0000000..e0e4a86 Binary files /dev/null and b/tests/input/moonchild-user-bg-and-bad-glyph.note differ diff --git a/tests/pdf.test.ts b/tests/pdf.test.ts new file mode 100644 index 0000000..422b834 --- /dev/null +++ b/tests/pdf.test.ts @@ -0,0 +1,70 @@ +import * as fs from "fs-extra" +import { toPdf } from "../src/pdf" +import { SupernoteX } from "../src/parsing" +import { PDFParse } from "pdf-parse" +import { describe, test, expect } from 'vitest' + +function readFileToUint8Array(filePath: string): Promise { + return new Promise((resolve, reject) => { + fs.readFile(`tests/input/${filePath}`, (err, data) => { + if (err) { + reject(err); + } else { + resolve(new Uint8Array(data.buffer)); + } + }); + }); +} + +describe("pdf", () => { + test("generates a searchable PDF with an RTR text layer", { timeout: 30000 }, async () => { + const sn = new SupernoteX(await readFileToUint8Array("rtr.note")) + const pdfBytes = await toPdf(sn) + await fs.writeFile("tests/output/rtr.note.pdf", pdfBytes) + + const parser = new PDFParse({ data: pdfBytes }) + const result = await parser.getText() + await parser.destroy() + + for (const word of ["Real", "time", "recognition", "paragraph", "reflow", "together"]) { + expect(result.text).toContain(word) + } + }) + + test("handles a note with recognition data from a nomad device", { timeout: 30000 }, async () => { + const sn = new SupernoteX(await readFileToUint8Array("nomad-3.15.27-blank-shapes-and-RTR.note")) + const pdfBytes = await toPdf(sn) + expect(pdfBytes.byteLength).toBeGreaterThan(0) + + const parser = new PDFParse({ data: pdfBytes }) + const result = await parser.getText() + await parser.destroy() + expect(result.text).not.toBeUndefined() + }) + + test("handles a note with no recognition data without throwing", { timeout: 30000 }, async () => { + const sn = new SupernoteX(await readFileToUint8Array("test.note")) + const pdfBytes = await toPdf(sn) + expect(pdfBytes.byteLength).toBeGreaterThan(0) + }) + + test("handles a user-uploaded background template and unencodable recognition glyphs", { timeout: 30000 }, async () => { + // Regression fixture trimmed from a real note that hit two bugs together: + // (1) a user-uploaded PNG background template that decodes to 8-bit RGB + // with no alpha channel, which compositeImages() previously rejected + // (it requires 8-bit RGBA); and + // (2) recognized handwriting containing characters (e.g. "→") the default + // Helvetica font can't encode, which previously aborted the whole PDF. + const sn = new SupernoteX(await readFileToUint8Array("moonchild-user-bg-and-bad-glyph.note")) + const pdfBytes = await toPdf(sn) + expect(pdfBytes.byteLength).toBeGreaterThan(0) + + const parser = new PDFParse({ data: pdfBytes }) + const result = await parser.getText() + await parser.destroy() + + for (const word of ["Saturn", "Mercury", "Moon", "MAGUS"]) { + expect(result.text).toContain(word) + } + }) +})