From a934acf91e5e3a1a0e6a0f4aca3310da7f7f8b79 Mon Sep 17 00:00:00 2001 From: Brandon Philips's Clanker Date: Fri, 31 Jul 2026 10:02:25 -0700 Subject: [PATCH] Support decoding at a reduced resolution, for cheap thumbnails Closes #40. toImage() always rasterized every layer at full pageWidth x pageHeight, even for a small thumbnail -- the motivating case (a ~140px sidebar preview in the Obsidian plugin) paid the same decode/memory cost as actually viewing the page, contributing to memory pressure on iOS. Goes with the "real fix" approach flagged in the issue (decode-time downsampling) rather than decode-full-then-resize, since only the former avoids ever allocating the full-resolution buffer -- resizing afterward still hits that peak allocation, which is what actually matters for the motivating crash. - RattaRLEDecoder.decodeAtScale(buffer, width, height, factor, ...): like decode(), but samples directly at 1/factor resolution (nearest-neighbor) by walking the same RLE runs and only writing output pixels that land on a sample point, skipping whole sample rows/columns arithmetically rather than iterating every full-resolution pixel a run covers. Never allocates a width x height buffer. On a 1404x1872 page, factor 10 produces a ~104 KB buffer instead of the ~10 MB decode() needs. - decode()'s run-parsing loop (the two-byte length-extension "holder" logic) is now shared with decodeAtScale via a private _walkRuns helper, parameterized by how a run gets written; decode()'s own behavior is unchanged (only how it's factored), confirmed by the full existing test suite plus a new byte-for-byte cross-check against decodeAtScale(factor: 1). - toImage(note, pageNumbers?, { scale? }): scale (default 1) is a downsample factor. Output pages are ceil(pageWidth/scale) x ceil(pageHeight/scale). The BGLAYER PNG path (user-uploaded background templates) is resized to match via image-js's resize() so compositeImages()' equal-dimensions requirement still holds across layers. Tests: a hand-crafted decodeAtScale case, decode()-vs-decodeAtScale cross-checks (factor 1 byte-for-byte; factor 5, which doesn't evenly divide the page, pixel-by-pixel against a naive downsample of the full decode) against a real fixture, invalid-factor/-scale rejection, and a toImage({ scale }) integration test with visual inspection of the output. --- README.md | 12 +++ src/conversion.ts | 187 ++++++++++++++++++++++++++++++++++++--- src/index.ts | 2 +- tests/conversion.test.ts | 138 +++++++++++++++++++++++++++++ 4 files changed, 326 insertions(+), 13 deletions(-) create mode 100644 tests/conversion.test.ts diff --git a/README.md b/README.md index c94a1df..0173821 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,18 @@ const pdfBytes = await ctx.pdfDoc.save(); Note that only page rendering (`toImage`/`encodePng`) is parallelizable this way — PDF assembly, including the final `pdfDoc.save()`, is a single main-thread operation regardless of how many Workers rendered pages. +### Cheap thumbnails: rendering at a reduced resolution + +`toImage` always rasterizes at the note's native `pageWidth`×`pageHeight` — fine for a main view or export, but wasteful for something like a small thumbnail sidebar, where decoding and holding a full-resolution page in memory per thumbnail adds up fast on memory-constrained devices (this is what motivated [#40](https://github.com/philips/supernote-typescript/issues/40)). Pass `{ scale }` to render directly at a reduced resolution instead: + +```ts +const thumbnails = await toImage(note, undefined, { scale: 10 }); +``` + +`scale` is an integer downsample factor; output pages are `ceil(pageWidth / scale)` × `ceil(pageHeight / scale)`. This isn't full-resolution decoding followed by a resize — each layer is decoded directly at the reduced resolution (`RattaRLEDecoder.decodeAtScale`, nearest-neighbor sampled), so the full-resolution buffer is never allocated at all. On a 1404×1872 page, `scale: 10` produces a ~104 KB output buffer instead of the ~10 MB a full decode would need. + +Omitting `scale` (or passing `{ scale: 1 }`) renders at full resolution exactly as before. + ### Reading Atelier `.spd` files `.spd` files, produced by the Supernote Atelier app, are a different format from `.note` files: a SQLite database of image tiles rather than the custom binary layout `SupernoteX` parses. `SupernoteAtelier.open` reads it (via [sql.js](https://github.com/sql-js/sql.js)) and exposes the tiles per surface (layer — surface names vary per file, e.g. `surface_1` or a `surface_9999` "Reference Layer"), plus best-effort decoded metadata (viewport, canvas size, layer names). Its `.spd` schema and `ls` layer encoding aren't officially documented; the reverse-engineered details are noted in [src/atelier.ts](./src/atelier.ts). diff --git a/src/conversion.ts b/src/conversion.ts index 36790ba..63b2fae 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -93,13 +93,33 @@ export function compositeImages(sourceImage: Image, destinationImage: Image) { } } +export interface ToImageOptions { + /** Downsample factor (positive integer, default 1 = full resolution). + * Output pages are `ceil(pageWidth / scale)` x `ceil(pageHeight / scale)`. + * At `scale > 1`, each layer is decoded directly at the reduced + * resolution via `RattaRLEDecoder.decodeAtScale` rather than decoded at + * full resolution and downscaled afterward, so a full `pageWidth` x + * `pageHeight` buffer is never allocated -- meant for cheap thumbnails + * on memory-constrained devices, see + * https://github.com/philips/supernote-typescript/issues/40. */ + scale?: number; +} + /** * Convert a Supernote file to one or more image-js image objects. * @param note Parsed Supernote, or the minimal `IRenderableNote` slice * produced by `extractPageRenderData` (e.g. when rendering off-main-thread). * @param pageNumbers Optional page numbers to export (defaults to all). Indexing starts at 1. + * @param options See `ToImageOptions`. */ -export function toImage(note: IRenderableNote, pageNumbers?: number[]) { +export function toImage(note: IRenderableNote, pageNumbers?: number[], options?: ToImageOptions) { + const scale = options?.scale ?? 1; + if (!Number.isInteger(scale) || scale < 1) { + throw new RangeError(`scale must be a positive integer, received ${scale}`); + } + const outWidth = Math.ceil(note.pageWidth / scale); + const outHeight = Math.ceil(note.pageHeight / scale); + const pages = pageNumbers ? pageNumbers.map((n) => note.pages[n - 1]) : note.pages; @@ -119,16 +139,31 @@ export function toImage(note: IRenderableNote, pageNumbers?: number[]) { // decode to any bit depth/color model (e.g. 8-bit RGB with no // alpha channel). compositeImages() requires 8-bit RGBA on both // sides, so normalize regardless of the source PNG's format. - return decodePng(layer.bitmapBuffer as Uint8Array) + let image = decodePng(layer.bitmapBuffer as Uint8Array) .convertBitDepth(8) .convertColor(ImageColorModel.RGBA); + if (scale > 1) { + // Matches the RLE layers' output size exactly (see below) so + // compositeImages()' equal-dimensions requirement still holds. + image = image.resize({ width: outWidth, height: outHeight, preserveAspectRatio: false }); + } + return image; } - const buffer = decoder.decode( + if (scale === 1) { + const buffer = decoder.decode( + layer.bitmapBuffer as Uint8Array, + note.pageWidth, + note.pageHeight, + ); + return new Image(note.pageWidth, note.pageHeight, { colorModel: ImageColorModel.RGBA, data: buffer }); + } + const decoded = decoder.decodeAtScale( layer.bitmapBuffer as Uint8Array, note.pageWidth, note.pageHeight, + scale, ); - return new Image(note.pageWidth, note.pageHeight, { colorModel: ImageColorModel.RGBA, data: buffer }); + return new Image(decoded.width, decoded.height, { colorModel: ImageColorModel.RGBA, data: decoded.data }); }); let images = await Promise.all(promises); @@ -268,6 +303,87 @@ export class RattaRLEDecoder { const result = new Uint8Array(expectedLength); const pixels = new Uint32Array(result.buffer); + const cursor = this._walkRuns(buffer, totalPixels, allBlank, (cursor, color, length) => + this.fillRun(pixels, cursor, color, length, translation), + ); + + if (cursor !== totalPixels) + throw new Error( + `Uint8Array length ${cursor * 4} doesn't match expected length ${expectedLength}.`, + ); + return result; + } + + /** + * Like `decode`, but samples directly at a reduced resolution instead of + * decoding a full `width` x `height` buffer and downscaling afterward, so + * the full-resolution buffer is never allocated -- for cheap thumbnails + * on memory-constrained devices, see + * https://github.com/philips/supernote-typescript/issues/40. Uses + * nearest-neighbor sampling: output pixel `(x, y)` takes the color of + * full-resolution pixel `(x * factor, y * factor)`. + * @param buffer Input buffer following Ratta RLE protocol. + * @param width Full-resolution page width. + * @param height Full-resolution page height. + * @param factor Downsample factor (positive integer). `1` samples every + * pixel -- equivalent to, but slower than, `decode` itself, so prefer + * `decode` directly at factor `1`. + * @param palette Optionally custom palette. + * @param allBlank Blank toggle. + * @returns Decoded buffer sized `ceil(width / factor)` x + * `ceil(height / factor)` (not necessarily an exact divisor of the input + * dimensions), plus those output dimensions. + */ + decodeAtScale( + buffer: Uint8Array, + width: number, + height: number, + factor: number, + palette?: IColorPalette, + allBlank = false, + ): { data: Uint8Array; width: number; height: number } { + if (!Number.isInteger(factor) || factor < 1) { + throw new RangeError(`factor must be a positive integer, received ${factor}`); + } + const outWidth = Math.ceil(width / factor); + const outHeight = Math.ceil(height / factor); + + const pal = palette ?? defaultPalette; + const translation = this.buildPackedTranslation(pal); + + const totalPixels = width * height; + const outResult = new Uint8Array(outWidth * outHeight * 4); + const outPixels = new Uint32Array(outResult.buffer); + + const cursor = this._walkRuns(buffer, totalPixels, allBlank, (cursor, color, length) => + this.fillRunAtScale(outPixels, outWidth, width, totalPixels, factor, cursor, length, color, translation), + ); + + if (cursor !== totalPixels) + throw new Error( + `Uint8Array length ${cursor * 4} doesn't match expected length ${totalPixels * 4}.`, + ); + return { data: outResult, width: outWidth, height: outHeight }; + } + + /** Walks `buffer`'s encoded (color, length) runs following Ratta RLE's + * two-byte length-extension scheme, invoking `fill(cursor, color, + * length)` for each run in turn and threading its return value through + * as the next cursor. Shared by `decode` and `decodeAtScale` so both + * stay in sync with this parsing logic; only how a run gets written + * (`fillRun` vs. `fillRunAtScale`) differs between them. + * @param totalPixels Full-resolution pixel count (`width * height`), + * needed to size the trailing run correctly regardless of any + * downsampling `fill` itself does. + * @returns The final cursor, which callers compare against + * `totalPixels` to confirm the buffer decoded to the expected size. + */ + private _walkRuns( + buffer: Uint8Array, + totalPixels: number, + allBlank: boolean, + fill: (cursor: number, color: number, length: number) => number, + ): number { let cursor = 0; let waiting: [number, number][] = []; let holder: [number, number] | null = null; @@ -305,24 +421,20 @@ export class RattaRLEDecoder { } } for (const [runColor, runLength] of waiting.values()) { - cursor = this.fillRun(pixels, cursor, runColor, runLength, translation); + cursor = fill(cursor, runColor, runLength); } waiting = []; } if (holder !== null) { [color, length] = holder as [number, number]; - length = this.adjustTailLength(length, cursor * 4, expectedLength); + length = this.adjustTailLength(length, cursor * 4, totalPixels * 4); if (length > 0) { - cursor = this.fillRun(pixels, cursor, color, length, translation); + cursor = fill(cursor, color, length); } } - if (cursor !== totalPixels) - throw new Error( - `Uint8Array length ${cursor * 4} doesn't match expected length ${expectedLength}.`, - ); - return result; + return cursor; } /** Fills `length` pixels starting at `cursor` with the color for @@ -343,6 +455,57 @@ export class RattaRLEDecoder { return cursor + length; } + /** Like `fillRun`, but writes into a `factor`-downsampled output buffer + * instead of a full `fullWidth` x (`totalPixels`/`fullWidth`) one, only + * computing/writing the sample pixels (nearest-neighbor, the top-left + * corner of each `factor` x `factor` block) that fall within this run -- + * skipping whole sample rows/columns between them arithmetically rather + * than iterating every full-resolution pixel the run covers, so the cost + * of a run scales with the *output* pixels it contains, not the + * full-resolution ones. */ + fillRunAtScale( + outPixels: Uint32Array, + outWidth: number, + fullWidth: number, + totalPixels: number, + factor: number, + cursor: number, + length: number, + encodedColor: number, + translation: Record, + ): number { + const packed = translation[encodedColor] ?? this.unknownColorPacked; + const start = cursor; + // Mirrors fillRun's clamp against the full-resolution pixel count + // (there, `pixels.length`), even though nothing here is actually + // sized to totalPixels -- it's just the bound for what full-res rows + // this run could possibly touch. + const end = Math.min(cursor + length, totalPixels); + + if (start < end) { + const rowStart = Math.floor(start / fullWidth); + const rowEndInclusive = Math.floor((end - 1) / fullWidth); + const firstSampleRow = Math.ceil(rowStart / factor) * factor; + for (let row = firstSampleRow; row <= rowEndInclusive; row += factor) { + const rowLinearStart = row * fullWidth; + const rowLinearEnd = rowLinearStart + fullWidth; + const segStart = Math.max(start, rowLinearStart); + const segEnd = Math.min(end, rowLinearEnd); + if (segStart >= segEnd) continue; + + const colStart = segStart - rowLinearStart; + const colEndInclusive = segEnd - 1 - rowLinearStart; + const firstSampleCol = Math.ceil(colStart / factor) * factor; + const outRowBase = (row / factor) * outWidth; + for (let col = firstSampleCol; col <= colEndInclusive; col += factor) { + outPixels[outRowBase + col / factor] = packed; + } + } + } + + return cursor + length; + } + /** RGBA used for encoded colors outside the known palette: fully * transparent, matching the previous per-pixel fallback. */ private readonly unknownColorPacked = packRGBA(255, 255, 255, 0); diff --git a/src/index.ts b/src/index.ts index c9ad574..dc4fb4f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ export { SupernoteX, extractText, extractParagraphs } from './parsing.js'; export { toImage, extractPageRenderData } from './conversion.js'; -export type { IRenderableNote, IRenderablePage, IRenderableLayer } from './conversion.js'; +export type { IRenderableNote, IRenderablePage, IRenderableLayer, ToImageOptions } from './conversion.js'; export type { ILink, IPage } from './format.js'; export { RecognitionStatuses } from './format.js'; export { fetchMirrorFrame } from './mirror.js'; diff --git a/tests/conversion.test.ts b/tests/conversion.test.ts new file mode 100644 index 0000000..ca3cfab --- /dev/null +++ b/tests/conversion.test.ts @@ -0,0 +1,138 @@ +import * as fs from "fs-extra" +import * as imagejs from "image-js" +import { describe, test, expect } from 'vitest' +import { toImage, RattaRLEDecoder } from "../src/conversion" +import { SupernoteX } from "../src/parsing" + +function readFileToUint8Array(filePath: string): Promise { + return new Promise((resolve, reject) => { + fs.readFile(`tests/input/${filePath}`, (err, data) => { + if (err) { + reject(err); + } else { + resolve(new Uint8Array(data.buffer)); + } + }); + }); +} + +describe("RattaRLEDecoder.decodeAtScale", () => { + test("nearest-neighbor samples a hand-crafted 4x4 buffer down to 2x2", () => { + const decoder = new RattaRLEDecoder(); + // width=4, height=4. Rows alternate which columns are black/white (row0, + // row2) vs. a filler color (row1, row3, not sampled at factor 2) so the + // expected 2x2 output is a deterministic checkerboard: + // row0: black black | white white (cols 0-1 black, 2-3 white) + // row1: gray gray | gray gray (uniform filler, unsampled) + // row2: white white | black black (cols 0-1 white, 2-3 black) + // row3: darkGray x4 (uniform filler, unsampled) + // Byte pairs are (color, lengthByte), where lengthByte = runLength - 1 + // for a simple (non-extended) run. + const black = 0x61, white = 0x65, gray = 0x64, darkGray = 0x63; + const buffer = new Uint8Array([ + black, 1, white, 1, // row0: 2 black, 2 white + gray, 3, // row1: 4 gray + white, 1, black, 1, // row2: 2 white, 2 black + darkGray, 3, // row3: 4 darkGray + ]); + + const { data, width, height } = decoder.decodeAtScale(buffer, 4, 4, 2); + expect(width).toBe(2); + expect(height).toBe(2); + + const image = new imagejs.Image(width, height, { colorModel: imagejs.ImageColorModel.RGBA, data }); + expect(image.getPixel(0, 0)).toEqual([0, 0, 0, 255]); // sampled from row0 col0: black + expect(image.getPixel(1, 0)).toEqual([255, 255, 255, 255]); // row0 col2: white + expect(image.getPixel(0, 1)).toEqual([255, 255, 255, 255]); // row2 col0: white + expect(image.getPixel(1, 1)).toEqual([0, 0, 0, 255]); // row2 col2: black + }) + + test("factor 1 matches decode() exactly, byte-for-byte, on a real layer's buffer", async () => { + const sn = new SupernoteX(await readFileToUint8Array("test.note")); + const layer = sn.pages[0].MAINLAYER; + expect(layer.bitmapBuffer).not.toBeNull(); + + const decoder = new RattaRLEDecoder(); + const full = decoder.decode(layer.bitmapBuffer!, sn.pageWidth, sn.pageHeight); + const atScale = decoder.decodeAtScale(layer.bitmapBuffer!, sn.pageWidth, sn.pageHeight, 1); + + expect(atScale.width).toBe(sn.pageWidth); + expect(atScale.height).toBe(sn.pageHeight); + // These buffers are several MB (a full page at 8 bits/channel RGBA) -- + // vitest's toEqual() does a generic, per-element deep-equal on typed + // arrays that's wildly expensive (and can OOM) at this size. Buffer.compare + // is a fast native byte comparison instead. + const same = Buffer.compare(Buffer.from(full.buffer, full.byteOffset, full.length), Buffer.from(atScale.data.buffer, atScale.data.byteOffset, atScale.data.length)) === 0; + expect(same).toBe(true); + }) + + test("factor N matches a naive nearest-neighbor downsample of decode()'s full output, on a real layer's buffer", async () => { + const sn = new SupernoteX(await readFileToUint8Array("test.note")); + const layer = sn.pages[0].MAINLAYER; + expect(layer.bitmapBuffer).not.toBeNull(); + + const factor = 5; // deliberately doesn't evenly divide pageWidth/pageHeight + const decoder = new RattaRLEDecoder(); + const full = decoder.decode(layer.bitmapBuffer!, sn.pageWidth, sn.pageHeight); + const fullPixels = new Uint32Array(full.buffer, full.byteOffset, full.length / 4); + + const atScale = decoder.decodeAtScale(layer.bitmapBuffer!, sn.pageWidth, sn.pageHeight, factor); + const outPixels = new Uint32Array(atScale.data.buffer, atScale.data.byteOffset, atScale.data.length / 4); + + expect(atScale.width).toBe(Math.ceil(sn.pageWidth / factor)); + expect(atScale.height).toBe(Math.ceil(sn.pageHeight / factor)); + + // Collect mismatches instead of asserting per-pixel (tens of thousands + // of individual expect() calls in a hot loop adds up) -- a single + // assertion at the end with a useful failure message if anything's off. + const mismatches: string[] = []; + for (let oy = 0; oy < atScale.height && mismatches.length < 5; oy++) { + for (let ox = 0; ox < atScale.width && mismatches.length < 5; ox++) { + const sx = ox * factor, sy = oy * factor; + const expected = fullPixels[sy * sn.pageWidth + sx]; + const actual = outPixels[oy * atScale.width + ox]; + if (actual !== expected) { + mismatches.push(`(${ox}, ${oy}): expected ${expected.toString(16)}, got ${actual.toString(16)}`); + } + } + } + expect(mismatches).toEqual([]); + }) + + test("rejects a non-positive-integer factor", () => { + const decoder = new RattaRLEDecoder(); + const buffer = new Uint8Array([0x61, 0]); + expect(() => decoder.decodeAtScale(buffer, 1, 1, 0)).toThrow(RangeError); + expect(() => decoder.decodeAtScale(buffer, 1, 1, 1.5)).toThrow(RangeError); + expect(() => decoder.decodeAtScale(buffer, 1, 1, -1)).toThrow(RangeError); + }) +}) + +describe("toImage scale option", () => { + test("renders a downscaled page directly, without decoding at full resolution first", { timeout: 30000 }, async () => { + const sn = new SupernoteX(await readFileToUint8Array("nomad-3.15.27-blank-shapes-and-RTR.note")); + const scale = 10; + const images = await toImage(sn, [1], { scale }); + expect(images.length).toBe(1); + expect(images[0].width).toBe(Math.ceil(sn.pageWidth / scale)); + expect(images[0].height).toBe(Math.ceil(sn.pageHeight / scale)); + await imagejs.writeSync(`tests/output/scaled-thumbnail.png`, images[0]); + }) + + test("scale: 1 (default) still matches the previous full-resolution output size", { timeout: 30000 }, async () => { + const sn = new SupernoteX(await readFileToUint8Array("test.note")); + const [withoutOptions] = await toImage(sn, [1]); + const [withScale1] = await toImage(sn, [1], { scale: 1 }); + expect(withoutOptions.width).toBe(sn.pageWidth); + expect(withoutOptions.height).toBe(sn.pageHeight); + expect(withScale1.width).toBe(sn.pageWidth); + expect(withScale1.height).toBe(sn.pageHeight); + }) + + test("rejects a non-positive-integer scale", async () => { + const sn = new SupernoteX(await readFileToUint8Array("test.note")); + // toImage validates scale synchronously (before ever returning a + // promise), so this throws immediately rather than rejecting. + expect(() => toImage(sn, [1], { scale: 0 })).toThrow(RangeError); + }) +})