diff --git a/CHANGELOG.md b/CHANGELOG.md
index b64223e4..82f041fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,50 @@ in the README).
same validated style properties, so an explicit `style=` still wins and specs
that don't use them are byte-identical.
+### Changed
+- Peak memory cut on four paths, with byte-identical output everywhere (89
+ payload/export/view fingerprints pinned before and after):
+ - The indexed-palette PNG encoder stages one scanline buffer and hands it to
+ zlib directly instead of building a per-row `bytes` list, joining it, and
+ narrowing an `intp`-per-pixel `np.unique` inverse. A 1800×840 export peaks
+ at 6 MB instead of 50 MB (33 MB instead of 274 MB at 4K) and encodes ~1.8x
+ faster; the truecolor branch drops ~17%.
+ - Full-column color quantization (density mean-color planes, `direct_rgba`
+ channels, u8 live-wire channels) runs chunk-bounded and in place. Its chunk
+ was 4M rows, so every real column still paid the one-shot peak: a 2.1M-row
+ continuous color channel resolved in 44 MB and now resolves in 7 MB, taking
+ a colored 2.1M-point first paint from 181 MB to 147 MB of RSS.
+ - Direct-tier scatter/line/area payloads skip the all-visible row mask. Zone
+ maps already count NaN *and* ±inf as null, so on linear axes with no nulls
+ the mask is provably all-true; it was three O(N) passes and two N-byte
+ temporaries per build. Emit is 16–35% faster, and area no longer allocates
+ an identity index vector (nor gathers animation keys through it).
+ - SVG documents assemble in one flat join with block-buffered markers, and the
+ native rasterizer borrows the display list instead of freezing a `bytes`
+ copy. A 100k-point SVG export peaks at 27 MB instead of 39 MB.
+ - The JPEG encoder streams instead of exploding: the entropy packer works in
+ bounded bit passes (it cost 17 bytes per output *bit*, so a 2.8 MB stream
+ peaked over 1.5 GB), the YCbCr planes are released as they are consumed, the
+ quantize chain rounds in place via `trunc(x + copysign(0.5, x))`, and the
+ per-component token fields are freed as they are gathered. A 1800x840 export
+ peaks at 48 MB instead of 108 (photographic: **400 MB instead of 1516 MB**)
+ and is 5-19% faster.
+ - Standalone HTML export joins the document once from parts, with every large
+ string — the client bundle, the spec, each base64 chunk — as its own part, so
+ the join copies it exactly once. Previously the chunks were folded through a
+ `"\n".join(...)` and then into an f-string, duplicating 4/3 of the payload. A
+ 1M-point export peaks at 33 MB instead of 41; a small export (where the
+ ~330 KB client bundle is the document) is ~2% faster.
+ - The native rasterizer's serial point and segment passes no longer
+ materialize an `0..n` index vector to read back in order (4 bytes per mark),
+ and the per-point density quantizer keeps one temporary instead of three.
+- `memory_report()` reports `capacity_bytes` per column and
+ `canonical_capacity_bytes` per store, and builds `resident_array_bytes` from
+ the capacity total. A streamed column's `values` is a prefix view of its
+ capacity-doubling buffer, so up to half of what it holds was invisible to the
+ report (§27: if a number isn't in the report, it isn't real). Figures that
+ never appended report exactly what they did before.
+
### Fixed
- The colorbar stringified its colormap, so a custom ramp reached it as an
unparseable name and silently painted viridis while the marks beside it
diff --git a/python/xy/_figure.py b/python/xy/_figure.py
index 19985bca..fbbddcf8 100644
--- a/python/xy/_figure.py
+++ b/python/xy/_figure.py
@@ -1871,8 +1871,10 @@ def memory_report(self) -> dict[str, Any]:
report["transport_bytes_per_point"] = len(blob) / n_total
report["pyramid_bytes"] = interaction.pyramid_report_bytes(self)
report["bin_color_bytes"] = interaction.bin_color_cache_bytes(self)
+ # Capacity, not live length: a streamed column's growth-buffer slack is
+ # resident RAM (§27), and equals `canonical_bytes` when nothing appended.
report["resident_array_bytes"] = (
- report["canonical_bytes"]
+ report["canonical_capacity_bytes"]
+ report["channel_bytes"]
+ report["pyramid_bytes"]
+ report["bin_color_bytes"]
diff --git a/python/xy/_jpeg.py b/python/xy/_jpeg.py
index 2a08b526..222eaf0c 100644
--- a/python/xy/_jpeg.py
+++ b/python/xy/_jpeg.py
@@ -20,7 +20,9 @@
from __future__ import annotations
+import itertools
import struct
+from typing import Optional
import numpy as np
@@ -168,6 +170,9 @@ def _dct_matrix() -> np.ndarray:
# searchsorted keeps this exact where a float log2 could round at the edges.
_POW2 = np.int64(1) << np.arange(12, dtype=np.int64)
+#: Placeholder that releases a token field's storage while its list slot lives on.
+_EMPTY_I64 = np.empty(0, dtype=np.int64)
+
def _bit_size(magnitude: np.ndarray) -> np.ndarray:
return np.searchsorted(_POW2, magnitude, side="right").astype(np.int64)
@@ -255,22 +260,61 @@ def _component_tokens(
return merged[0], merged[1], merged[2], merged[3], merged[4], merged[5]
+#: Bits packed per `_pack_entropy` pass. The exploded form costs 17 bytes per
+#: output *bit* (two int64 index vectors plus the bit itself), so packing a whole
+#: image at once peaked at ~130 MB for a 1 MB entropy stream. Bit packing is not
+#: chunk-independent — group boundaries land mid-byte — so each pass carries its
+#: leftover bits into the next one, which keeps the byte stream identical.
+_ENTROPY_BIT_CHUNK = 1 << 18
+
+
def _pack_entropy(chunk: np.ndarray, nbits: np.ndarray) -> bytes:
"""MSB-first bit packing of (value, bit-count) chunks, with 1-padding to a
- byte boundary and 0x00 stuffing after every 0xFF (both per T.81)."""
+ byte boundary and 0x00 stuffing after every 0xFF (both per T.81).
+
+ Packed in bounded passes: token groups of at most `_ENTROPY_BIT_CHUNK` bits,
+ with sub-byte remainders carried across group boundaries so the emitted bytes
+ are exactly those of a single whole-image pass. Byte stuffing is per-byte
+ local, so it applies per group.
+ """
total = int(nbits.sum())
- start = np.cumsum(nbits) - nbits
- idx = np.repeat(np.arange(chunk.size, dtype=np.int64), nbits)
- offset = np.arange(total, dtype=np.int64) - np.repeat(start, nbits)
- bits = ((chunk[idx] >> (nbits[idx] - 1 - offset)) & 1).astype(np.uint8)
- pad = (-total) % 8
- if pad:
- bits = np.concatenate((bits, np.ones(pad, dtype=np.uint8)))
- stream = np.packbits(bits)
- ff = np.flatnonzero(stream == 0xFF)
- if ff.size:
- stream = np.insert(stream, ff + 1, np.uint8(0))
- return stream.tobytes()
+ cum = np.cumsum(nbits)
+ # Token index at each bit-budget boundary: a group is [t0, t1) tokens, and a
+ # single token never spans groups (its bits stay contiguous, as before).
+ edges = np.searchsorted(
+ cum, np.arange(_ENTROPY_BIT_CHUNK, total, _ENTROPY_BIT_CHUNK), side="left"
+ )
+ bounds = [0, *(int(e) + 1 for e in edges), chunk.size]
+ parts: list[bytes] = []
+ carry = np.empty(0, dtype=np.uint8)
+ for t0, t1 in itertools.pairwise(bounds):
+ if t1 <= t0:
+ continue
+ values = chunk[t0:t1]
+ widths = nbits[t0:t1]
+ group_total = int(widths.sum())
+ start = np.cumsum(widths) - widths
+ idx = np.repeat(np.arange(values.size, dtype=np.int64), widths)
+ offset = np.arange(group_total, dtype=np.int64) - np.repeat(start, widths)
+ bits = ((values[idx] >> (widths[idx] - 1 - offset)) & 1).astype(np.uint8)
+ if carry.size:
+ bits = np.concatenate((carry, bits))
+ whole = (bits.size // 8) * 8
+ if t1 == chunk.size:
+ # Final group: 1-pad the tail to a byte boundary (T.81).
+ pad = (-bits.size) % 8
+ if pad:
+ bits = np.concatenate((bits, np.ones(pad, dtype=np.uint8)))
+ whole = bits.size
+ carry = np.empty(0, dtype=np.uint8)
+ else:
+ carry = bits[whole:].copy()
+ stream = np.packbits(bits[:whole])
+ ff = np.flatnonzero(stream == 0xFF)
+ if ff.size:
+ stream = np.insert(stream, ff + 1, np.uint8(0))
+ parts.append(stream.tobytes())
+ return b"".join(parts)
def _headers(h: int, w: int, qy_zz: np.ndarray, qc_zz: np.ndarray) -> bytes:
@@ -310,6 +354,26 @@ def _headers(h: int, w: int, qy_zz: np.ndarray, qc_zz: np.ndarray) -> bytes:
return b"\xff\xd8" + app0 + dqt + sof0 + dht + sos
+def _gathered(fields: list[list[np.ndarray]], index: int, order: np.ndarray) -> np.ndarray:
+ """One token field, concatenated across components into MCU order.
+
+ The per-component pieces are released as they are joined, so a field costs
+ one joined copy plus the gathered result rather than both plus the parts —
+ each is an int64 per token, and a photographic image has millions. `parts`
+ is dropped explicitly rather than left to the frame: a local stays alive
+ until the function returns, so `return np.concatenate(parts)[order]` would
+ hold the pieces, the join, *and* the gather at once.
+ """
+ parts = [f[index] for f in fields]
+ for f in fields:
+ f[index] = _EMPTY_I64
+ joined = np.concatenate(parts)
+ del parts
+ gathered = joined[order]
+ del joined
+ return gathered
+
+
def encode(rgba: np.ndarray, *, quality: int = 90) -> bytes:
"""Encode an `(h, w, 4)` RGBA (alpha ignored) or `(h, w, 3)` RGB uint8
image as a baseline JFIF JPEG. Deterministic for identical input."""
@@ -338,6 +402,10 @@ def encode(rgba: np.ndarray, *, quality: int = 90) -> bytes:
y = 0.299 * r + 0.587 * g + 0.114 * b - 128.0
cb = -0.168736 * r - 0.331264 * g + 0.5 * b
cr = 0.5 * r - 0.418688 * g - 0.081312 * b
+ # The interleaved f32 source is dead once the planes exist, and it is three
+ # floats per pixel — hand it back before the per-component pipeline runs.
+ # (`r`/`g`/`b` are views into it, so all four references have to go.)
+ del rgb, r, g, b
qy = _scaled_quant(_QUANT_LUMA, quality)
qc = _scaled_quant(_QUANT_CHROMA, quality)
@@ -346,37 +414,77 @@ def encode(rgba: np.ndarray, *, quality: int = 90) -> bytes:
pad_h, pad_w = (-h) % 8, (-w) % 8
h8, w8 = h + pad_h, w + pad_w
- fields: list[tuple[np.ndarray, ...]] = []
+ fields: list[list[np.ndarray]] = []
keys: list[np.ndarray] = []
- for comp, (plane, q_zz) in enumerate(zip((y, cb, cr), (qy_zz, qc_zz, qc_zz), strict=True)):
+ # Planes are released as they are consumed (one float per pixel each), so the
+ # pipeline holds one component's working set instead of all three.
+ planes: list[Optional[np.ndarray]] = [y, cb, cr]
+ del y, cb, cr
+ for comp, q_zz in enumerate((qy_zz, qc_zz, qc_zz)):
+ plane = planes[comp]
+ planes[comp] = None
+ assert plane is not None
# Edge replication avoids the ringing a zero/black pad would inject
# into every border block.
padded = np.pad(plane, ((0, pad_h), (0, pad_w)), mode="edge")
+ del plane
blocks = padded.reshape(h8 // 8, 8, w8 // 8, 8).swapaxes(1, 2).reshape(-1, 8, 8)
+ del padded
coef = _DCT @ blocks @ _DCT.T
- scaled = coef.reshape(-1, 64)[:, _ZIGZAG] / q_zz.astype(np.float32)
+ del blocks
+ # The zigzag gather is a fresh array, so the divide, the round, and the
+ # sign restore all run in place on it: same f32 operations in the same
+ # order as the one-shot chain, five fewer full-size temporaries.
+ scaled = coef.reshape(-1, 64)[:, _ZIGZAG]
+ del coef
+ scaled /= q_zz.astype(np.float32)
# Round half away from zero: any deterministic tie rule is valid JPEG;
- # this one matches the common integer implementations.
- quant = (np.sign(scaled) * np.floor(np.abs(scaled) + 0.5)).astype(np.int64)
+ # this one matches the common integer implementations. `trunc(x + ±0.5)`
+ # is that rule in three in-place passes and one temporary, where
+ # `sign(x) * floor(|x| + 0.5)` took four passes and four full-size
+ # temporaries. It is exact, not approximate: IEEE addition is
+ # sign-symmetric, so for x < 0 the sum is the exact negation of
+ # `|x| + 0.5`, and truncation toward zero then matches the floor of that
+ # magnitude. The ±0 cases land on ±0.0 either way and integer-cast to 0.
+ bias = np.copysign(np.float32(0.5), scaled)
+ scaled += bias
+ del bias
+ np.trunc(scaled, out=scaled)
+ quant = scaled.astype(np.int64)
+ del scaled
# Exact-math AC magnitudes cap at 1020 (category 10); clamp is one-LSB
# insurance against float rounding ever minting category 11, which the
# baseline AC tables cannot code.
quant[:, 1:] = np.clip(quant[:, 1:], -1023, 1023)
dc_tbl, ac_tbl = (0, 1) if comp == 0 else (2, 3)
toks = _component_tokens(quant, dc_tbl, ac_tbl)
+ del quant
# 4:4:4 → one block per component per MCU, so the MCU-interleaved
# order Y, Cb, Cr is a stable sort on (block, component, in-block seq).
keys.append(((toks[0] * 3 + comp) << 8) | toks[1])
- fields.append(toks)
+ # Only the four emitted fields outlive the key: the block and in-block
+ # sequence numbers are folded into it, and on a photographic image each
+ # dropped field is an int64 per token.
+ fields.append(list(toks[2:]))
+ del toks
order = np.argsort(np.concatenate(keys), kind="stable")
- tbl = np.concatenate([f[2] for f in fields])[order]
- sym = np.concatenate([f[3] for f in fields])[order]
- ampl = np.concatenate([f[4] for f in fields])[order]
- abits = np.concatenate([f[5] for f in fields])[order]
+ keys.clear()
+ tbl = _gathered(fields, 0, order)
+ sym = _gathered(fields, 1, order)
+ ampl = _gathered(fields, 2, order)
+ abits = _gathered(fields, 3, order)
+ del fields
code = _HUFF_CODES[tbl, sym]
clen = _HUFF_LENS[tbl, sym]
- # Huffman code then amplitude bits, as one ≤27-bit chunk per token.
- entropy = _pack_entropy((code << abits) | ampl, clen + abits)
+ del tbl, sym
+ # Huffman code then amplitude bits, as one ≤27-bit chunk per token. Both
+ # gathers are fresh arrays, so the combine runs in place.
+ code <<= abits
+ code |= ampl
+ clen += abits
+ del ampl, abits
+ entropy = _pack_entropy(code, clen)
+ del code, clen
return _headers(h, w, qy_zz, qc_zz) + entropy + b"\xff\xd9"
diff --git a/python/xy/_native.py b/python/xy/_native.py
index b9c07c41..79d5b16b 100644
--- a/python/xy/_native.py
+++ b/python/xy/_native.py
@@ -17,7 +17,7 @@
import os
import sys
from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
import numpy as np
import numpy.typing as npt
@@ -3093,8 +3093,11 @@ def _byte_span_arrays(spans): # noqa: ANN001, ANN202 - private ctypes adapter
return arenas, pointers, lengths
-def rasterize_spans(cmds: bytes, spans, w: int, h: int) -> npt.NDArray[np.uint8]: # noqa: ANN001
- """Paint a display list borrowing multiple call-scoped byte arenas."""
+def rasterize_spans(cmds: Any, spans, w: int, h: int) -> npt.NDArray[np.uint8]: # noqa: ANN001
+ """Paint a display list borrowing multiple call-scoped byte arenas.
+
+ `cmds` is any read-only-safe buffer (`bytes`, `bytearray`, `memoryview`):
+ it is borrowed through `np.frombuffer`, never copied."""
w = _positive_int(w, "raster width")
h = _positive_int(h, "raster height")
buf = np.frombuffer(cmds, dtype=np.uint8)
@@ -3115,8 +3118,9 @@ def rasterize_spans(cmds: bytes, spans, w: int, h: int) -> npt.NDArray[np.uint8]
return out
-def rasterize_png_spans(cmds: bytes, spans, w: int, h: int) -> bytes: # noqa: ANN001
- """Paint and encode a display list borrowing multiple byte arenas."""
+def rasterize_png_spans(cmds: Any, spans, w: int, h: int) -> bytes: # noqa: ANN001
+ """Paint and encode a display list borrowing multiple byte arenas
+ (`cmds` is borrowed, as in `rasterize_spans`)."""
w = _positive_int(w, "raster width")
h = _positive_int(h, "raster height")
buf = np.frombuffer(cmds, dtype=np.uint8)
diff --git a/python/xy/_payload.py b/python/xy/_payload.py
index 20ca1c53..98d3dea0 100644
--- a/python/xy/_payload.py
+++ b/python/xy/_payload.py
@@ -403,6 +403,29 @@ def _finite_sel(t: Trace, xv: np.ndarray, yv: np.ndarray) -> np.ndarray | None:
return None
return np.flatnonzero(np.isfinite(xv) & np.isfinite(yv))
+ def _visible_mask_needed(
+ self,
+ t: Trace,
+ *,
+ prefiltered: bool,
+ base_column: Optional[Column] = None,
+ ) -> bool:
+ """Whether `_log_visible_mask` can drop any row for this trace.
+
+ The mask is three O(N) passes plus two N-byte temporaries, and on the
+ common shape (linear axes, no nulls) it is provably all-True: zone maps
+ count NaN *and* ±inf as null (§22/§19), so a null-free column has no
+ row for `isfinite` to reject, and `prefiltered` rows already went
+ through `_finite_sel`. Only a log axis (which additionally rejects
+ non-positive values) or a baseline column outside the x/y zone maps can
+ actually remove something.
+ """
+ if self._axis_scale(t.x_axis) == "log" or self._axis_scale(t.y_axis) == "log":
+ return True
+ if base_column is not None and base_column.zone.null_count:
+ return True
+ return not prefiltered and bool(t.x.zone.null_count or t.y.zone.null_count)
+
def _log_visible_mask(
self,
t: Trace,
@@ -410,6 +433,14 @@ def _log_visible_mask(
yv: np.ndarray,
base: Optional[np.ndarray] = None,
) -> np.ndarray:
+ """Rows this trace may actually ship.
+
+ Paired with `_visible_mask_needed`, which decides whether calling this
+ can drop anything at all: **a new rejection rule here needs a matching
+ condition there**, or the emitters will skip the mask on data it would
+ now reject. (`tests/test_figure.py` pins one case per rule; the
+ predicate going stale is otherwise silent.)
+ """
mask = np.isfinite(xv) & np.isfinite(yv)
if self._axis_scale(t.x_axis) == "log":
mask &= xv > 0
@@ -483,7 +514,7 @@ def _emit_line(
sel = self._finite_sel(t, xv, yv)
if sel is not None:
xv, yv = xv[sel], yv[sel]
- if len(xv):
+ if len(xv) and self._visible_mask_needed(t, prefiltered=sel is not None):
finite = self._log_visible_mask(t, xv, yv)
if not bool(np.all(finite)):
sel = np.flatnonzero(finite) if sel is None else sel[finite]
@@ -507,9 +538,11 @@ def _emit_area(
tier, (xv, yv, bv) = self._m4_decimate(
t, xr, px_width, t.x.values, t.y.values, t.base.values
)
- sel = np.flatnonzero(self._log_visible_mask(t, xv, yv, bv))
- if len(sel) != len(xv):
- xv, yv, bv = xv[sel], yv[sel], bv[sel]
+ sel = None
+ if self._visible_mask_needed(t, prefiltered=False, base_column=t.base):
+ sel = np.flatnonzero(self._log_visible_mask(t, xv, yv, bv))
+ if len(sel) != len(xv):
+ xv, yv, bv = xv[sel], yv[sel], bv[sel]
entry = self._base_entry(t, pw, xv, yv, tier, self._default_styled(t))
if tier == "decimated":
entry["decimation_px"] = int(px_width)
@@ -535,7 +568,7 @@ def _emit_scatter(
sel = self._finite_sel(t, xv, yv)
if sel is not None:
xv, yv = xv[sel], yv[sel]
- if len(xv):
+ if len(xv) and self._visible_mask_needed(t, prefiltered=sel is not None):
visible = self._log_visible_mask(t, xv, yv)
if not bool(np.all(visible)):
sel = np.flatnonzero(visible) if sel is None else sel[visible]
diff --git a/python/xy/_png.py b/python/xy/_png.py
index 6f5d02c8..4ed8f48b 100644
--- a/python/xy/_png.py
+++ b/python/xy/_png.py
@@ -9,6 +9,13 @@
one byte per pixel instead of four shrinks native-PNG exports several-fold.
Falls back to truecolor otherwise.
+Both stage the filtered scanlines in a single NumPy buffer and hand that buffer
+straight to zlib. The image is large (4 bytes per pixel, ×4 again at scale 2),
+so every avoided intermediate is a full frame of peak RSS: building the rows as
+one array instead of a list of per-row `bytes` drops two full copies (the row
+objects and their join), and compressing the buffer directly drops a third
+(`tobytes`).
+
This balanced/indexed path stays pure Python/stdlib. The separate latency-first
`xy.pyplot` path fuses rasterization with the Rust PNG encoder.
"""
@@ -28,36 +35,59 @@ def _chunk(tag: bytes, data: bytes) -> bytes:
_SIG = b"\x89PNG\r\n\x1a\n"
_COMPRESSION_LEVEL = 6
+# Row-block length for the palette-index lookup. `np.unique(return_inverse=True)`
+# would hand back one intp per pixel (8 bytes/px — twice the image itself) only
+# to be narrowed to u8; searching the ≤256-entry palette per block writes the
+# final bytes in place with a bounded temporary instead.
+_INDEX_ROW_BLOCK = 64
+
+
+def _filtered_rows(h: int, stride: int) -> np.ndarray:
+ """An `(h, stride + 1)` scanline buffer with the per-row filter byte set to
+ 0 (PNG filter type "None"), ready for the pixel columns to be filled."""
+ rows = np.empty((h, stride + 1), dtype=np.uint8)
+ rows[:, 0] = 0
+ return rows
def png_truecolor(
- w: int, h: int, rgba: bytes, *, compression_level: int = _COMPRESSION_LEVEL
+ w: int,
+ h: int,
+ rgba: bytes | bytearray | memoryview | np.ndarray,
+ *,
+ compression_level: int = _COMPRESSION_LEVEL,
) -> bytes:
- """RGBA8 PNG (color type 6). `rgba` is row-major `w*h*4` bytes, top row first."""
+ """RGBA8 PNG (color type 6). `rgba` is row-major `w*h*4` bytes, top row
+ first.
+
+ Any buffer works and none is copied: `bytes`/`bytearray`/`memoryview`, or a
+ **C-contiguous** uint8 array (pass `np.ascontiguousarray` if that is not
+ guaranteed — a strided view would be read in memory order, not row order).
+ Only the first `w * h * 4` bytes are read.
+ """
ihdr = struct.pack(">IIBBBBB", w, h, 8, 6, 0, 0, 0)
stride = w * 4
- raw = b"".join(b"\x00" + rgba[y * stride : (y + 1) * stride] for y in range(h))
+ rows = _filtered_rows(h, stride)
+ rows[:, 1:] = np.frombuffer(rgba, dtype=np.uint8, count=h * stride).reshape(h, stride)
return (
_SIG
+ _chunk(b"IHDR", ihdr)
- + _chunk(b"IDAT", zlib.compress(raw, compression_level))
+ + _chunk(b"IDAT", zlib.compress(rows, compression_level))
+ _chunk(b"IEND", b"")
)
-def _png_indexed(w: int, h: int, idx: np.ndarray, palette: np.ndarray) -> bytes:
- """Indexed PNG (color type 3). `idx` is `(h, w)` uint8 palette indices;
- `palette` is `(n, 4)` uint8 RGBA. `tRNS` carries per-entry alpha."""
+def _png_indexed(w: int, h: int, rows: np.ndarray, palette: np.ndarray) -> bytes:
+ """Indexed PNG (color type 3). `rows` is the `(h, w + 1)` filtered scanline
+ buffer holding palette indices; `palette` is `(n, 4)` uint8 RGBA. `tRNS`
+ carries per-entry alpha."""
ihdr = struct.pack(">IIBBBBB", w, h, 8, 3, 0, 0, 0)
plte = palette[:, :3].astype(np.uint8).tobytes()
trns = palette[:, 3].astype(np.uint8).tobytes()
- rows = np.concatenate(
- [np.zeros((h, 1), dtype=np.uint8), idx.astype(np.uint8)], axis=1
- ) # a 0 filter byte per scanline
out = _SIG + _chunk(b"IHDR", ihdr) + _chunk(b"PLTE", plte)
# tRNS may omit trailing opaque (255) entries; keep it simple and always emit.
out += _chunk(b"tRNS", trns)
- out += _chunk(b"IDAT", zlib.compress(rows.tobytes(), _COMPRESSION_LEVEL)) + _chunk(b"IEND", b"")
+ out += _chunk(b"IDAT", zlib.compress(rows, _COMPRESSION_LEVEL)) + _chunk(b"IEND", b"")
return out
@@ -77,10 +107,17 @@ def encode(img: np.ndarray) -> bytes:
# image is too, so the truecolor result is identical either way.
probe = keys[:: keys.size // 65_536]
if np.unique(probe).size > 256:
- return png_truecolor(w, h, np.ascontiguousarray(img).tobytes())
- palette_keys, inverse = np.unique(keys, return_inverse=True)
+ return png_truecolor(w, h, flat)
+ palette_keys = np.unique(keys)
if palette_keys.size <= 256:
palette = palette_keys.view(np.uint8).reshape(-1, 4)
- idx = inverse.astype(np.uint8).reshape(h, w)
- return _png_indexed(w, h, idx, palette)
- return png_truecolor(w, h, np.ascontiguousarray(img).tobytes())
+ # `searchsorted` over the sorted palette is exactly `np.unique`'s
+ # inverse for values that are present (all of them, by construction),
+ # written straight into the scanline buffer a row-block at a time.
+ rows = _filtered_rows(h, w)
+ keyed = keys.reshape(h, w)
+ for start in range(0, h, _INDEX_ROW_BLOCK):
+ end = start + _INDEX_ROW_BLOCK
+ rows[start:end, 1:] = np.searchsorted(palette_keys, keyed[start:end]).astype(np.uint8)
+ return _png_indexed(w, h, rows, palette)
+ return png_truecolor(w, h, flat)
diff --git a/python/xy/_raster.py b/python/xy/_raster.py
index 5733b54a..9928b3ec 100644
--- a/python/xy/_raster.py
+++ b/python/xy/_raster.py
@@ -1032,9 +1032,13 @@ def emit_axis_title(axis: dict[str, Any], *, is_x: bool) -> None:
from . import _native
spans = (blob, *borrowed)
+ # The command buffer ships as a borrowed buffer, not a `bytes` copy: the
+ # ctypes seam wraps it with `np.frombuffer` and the native rasterizer only
+ # reads it, so freezing it would duplicate a display list that is O(marks)
+ # (megabytes on a direct-tier scatter) for nothing.
if fast_png:
- return _native.rasterize_png_spans(bytes(cmd.buf), spans, w_px, h_px)
- return _native.rasterize_spans(bytes(cmd.buf), spans, w_px, h_px)
+ return _native.rasterize_png_spans(cmd.buf, spans, w_px, h_px)
+ return _native.rasterize_spans(cmd.buf, spans, w_px, h_px)
def _emit_line(
diff --git a/python/xy/_svg.py b/python/xy/_svg.py
index 1227afa2..c39b11af 100644
--- a/python/xy/_svg.py
+++ b/python/xy/_svg.py
@@ -1647,7 +1647,7 @@ def line_attrs(style: dict[str, Any], color: str) -> str:
)
elif kind == "scatter":
- marks.append(_scatter_marks(t, blob, cols, trace_sx, trace_sy, style, color))
+ marks.extend(_scatter_marks(t, blob, cols, trace_sx, trace_sy, style, color))
elif kind == "hexbin":
marks.append(_hexbin_marks(t, blob, cols, trace_sx, trace_sy, style, color))
@@ -1872,17 +1872,29 @@ def tick_span(style: dict[str, Any]) -> tuple[float, float, float]:
f''
)
- return (
- f'"
+ # One flat join over the pieces rather than nested `join`s inside an
+ # f-string: the mark list is the whole document for a per-point chart (tens
+ # of MB at 100k markers), and joining it separately would materialize a
+ # second full copy of it before the result string is built.
+ return "".join(
+ [
+ f'",
+ ]
)
@@ -2080,9 +2092,18 @@ def read(index: int) -> np.ndarray:
)
+#: Markers per emitted string block. One SVG element per point means the mark
+#: list is the document, and a list of N short strings costs ~50 bytes of object
+#: header each on top of the markup — 40% overhead at 100k points, live at the
+#: same time as the joined result. Collapsing every block keeps the per-object
+#: overhead bounded while staying a single linear pass (byte-identical output:
+#: concatenation is associative).
+_SVG_MARK_BLOCK = 4096
+
+
def _scatter_marks(
t: dict, blob: bytes, cols: list, sx: _Scale, sy: _Scale, style: dict, fallback: str
-) -> str:
+) -> list[str]:
xv = _column(blob, cols[t["x"]])
yv = _column(blob, cols[t["y"]])
px, py = sx(xv), sy(yv)
@@ -2150,9 +2171,10 @@ def read(index: int) -> np.ndarray:
if grouped_alpha:
fill_group = float(scalar_artist) * _fill_opacity(style, 1.0)
stroke_group = float(scalar_artist) * _stroke_opacity(style, 1.0)
- out = [f'']
+ blocks = [f'']
else:
- out = [""]
+ blocks = [""]
+ out: list[str] = []
for i in range(n):
fill = face_rgba[i]
fill_value = (
@@ -2197,8 +2219,13 @@ def read(index: int) -> np.ndarray:
out.append(
builder(float(px[i]), float(py[i]), marker_radius) + f"{fill_attr}{stroke_attr}/>"
)
- out.append("")
- return "".join(out)
+ if len(out) >= _SVG_MARK_BLOCK:
+ blocks.append("".join(out))
+ out.clear()
+ if out:
+ blocks.append("".join(out))
+ blocks.append("")
+ return blocks
_SYMBOL_NAMES = (
diff --git a/python/xy/channels.py b/python/xy/channels.py
index e24ac1f4..3c459a54 100644
--- a/python/xy/channels.py
+++ b/python/xy/channels.py
@@ -626,9 +626,21 @@ def quantize_unit_u8(values: npt.NDArray[np.float64], domain: tuple[float, float
The lossy sibling of :func:`normalize_to_unit`, for wire paths where the
value is only ever a GPU LUT/ramp coordinate (a colormap texture has 256
texels; a size ramp spans ~16 px) and is never read back into a displayed
- number — 75% less traffic than f32, same rendered output (§29)."""
- unit = normalize_to_unit(values, domain)
- return np.rint(np.clip(unit, 0.0, 1.0) * 255.0).astype(np.uint8)
+ number — 75% less traffic than f32, same rendered output (§29).
+
+ Chunk-bounded like the other quantizers (`_QUANTIZE_CHUNK`): the arithmetic
+ is element-wise and stays in f32 exactly as the one-shot chain did, so the
+ bytes are identical while the transient stays independent of N."""
+ out = np.empty(len(values), dtype=np.uint8)
+ for start in range(0, len(values), _QUANTIZE_CHUNK):
+ end = start + _QUANTIZE_CHUNK
+ # Fresh f32 kernel output: clip/scale/round can all run in place on it.
+ unit = normalize_to_unit(values[start:end], domain)
+ np.clip(unit, 0.0, 1.0, out=unit)
+ unit *= 255.0
+ np.rint(unit, out=unit)
+ out[start:end] = unit.astype(np.uint8)
+ return out
def colormap_lut_rgba8(colormap: Colormap) -> npt.NDArray[np.uint8]:
@@ -723,7 +735,15 @@ def bins_mean_color(cc: Optional[ColorChannel]) -> bool:
# (~20 GB at 1e9 rows — the difference between a colored billion-point build
# fitting in RAM or not), while chunked passes keep every temporary at chunk
# size and the only N-sized allocation is the u8 result.
-_QUANTIZE_CHUNK = 1 << 22
+#
+# The chunk is sized so a whole pass (one f32 normalize output + one f64 stage
+# array + the u8 slice) stays inside a core's private cache rather than
+# streaming through DRAM: at 2^18 rows that is ~3 MB of live temporary, versus
+# ~50 MB at 2^22, where "chunked" still meant a 4M-row f64 pipeline for every
+# real-world column (a 2.1M-row colored trace fit in a single chunk and paid
+# the full one-shot peak). Bigger chunks buy nothing — the per-chunk Python
+# overhead is already amortized thousands of elements ago.
+_QUANTIZE_CHUNK = 1 << 18
def _quantized_lut_idx(values: npt.NDArray[np.float64], domain: tuple[float, float]) -> np.ndarray:
@@ -732,12 +752,17 @@ def _quantized_lut_idx(values: npt.NDArray[np.float64], domain: tuple[float, flo
Per-element math is exactly the historical one-shot chain —
`normalize_to_unit` (f32), widen to f64, ×255, `rint`, cast u8 — applied
per chunk, so results are bitwise identical while peak memory stays
- O(chunk) + the N-byte output."""
+ O(chunk) + the N-byte output. The ×255/`rint` stages run in place on the
+ widened copy, so one f64 chunk buffer serves the whole pipeline."""
out = np.empty(len(values), dtype=np.uint8)
for start in range(0, len(values), _QUANTIZE_CHUNK):
end = start + _QUANTIZE_CHUNK
- unit = normalize_to_unit(values[start:end], domain)
- out[start:end] = np.rint(np.asarray(unit, dtype=np.float64) * 255.0).astype(np.uint8)
+ # `normalize_to_unit` hands back a fresh f32 kernel output, so the
+ # widened copy below is ours to mutate.
+ scaled = np.asarray(normalize_to_unit(values[start:end], domain), dtype=np.float64)
+ scaled *= 255.0
+ np.rint(scaled, out=scaled)
+ out[start:end] = scaled.astype(np.uint8)
return out
@@ -746,8 +771,11 @@ def _quantized_rgba8(values: npt.NDArray[np.float64]) -> np.ndarray:
out = np.empty(values.shape, dtype=np.uint8)
for start in range(0, len(values), _QUANTIZE_CHUNK):
end = start + _QUANTIZE_CHUNK
- seg = values[start:end]
- out[start:end] = np.rint(np.clip(seg, 0.0, 1.0) * 255.0).astype(np.uint8)
+ # `clip` copies the input rows; the rest of the chain reuses that copy.
+ seg = np.clip(values[start:end], 0.0, 1.0)
+ seg *= 255.0
+ np.rint(seg, out=seg)
+ out[start:end] = seg.astype(np.uint8)
return out
@@ -826,9 +854,9 @@ def ship_channels(
`quantize_continuous` ships continuous color/size as u8 LUT coordinates
(`dtype: "u8"` marker) instead of unit f32. Live-interaction paths opt in:
their hover/pick answers come from the server's canonical columns, so the
- quantization is invisible. The build path must NOT opt in — it retains the
- shipped columns CPU-side (`_cpu.color`/`_cpu.size`) and denormalizes them
- for tooltip readouts, where 8-bit steps would show as wrong digits.
+ quantization is invisible. The build path must NOT opt in — the client keeps
+ the shipped columns CPU-side and denormalizes them for tooltip readouts,
+ where 8-bit steps would show as wrong digits.
Returns (color_spec, size_spec)."""
cc = trace.color_ch or ColorChannel(mode="constant", constant=None)
color_spec = ship_color_channel(
@@ -865,7 +893,10 @@ def ship_color_channel(
if rgba is None:
raise ValueError("direct RGBA color channel missing values")
values = rgba if sel is None else rgba[sel]
- packed = np.rint(np.clip(values, 0.0, 1.0) * 255.0).astype(np.uint8)
+ # Same per-element chain as the historical one-shot expression, but
+ # chunk-bounded: a full-column direct-RGBA trace otherwise holds three
+ # 32-bytes-per-point f64 temporaries at once (§27).
+ packed = _quantized_rgba8(values)
color_spec["buf"] = ship_u8(packed.reshape(-1))
color_spec["n"] = int(len(values))
elif cc.mode == "match_fill":
diff --git a/python/xy/columns.py b/python/xy/columns.py
index 83d94d45..7d37a745 100644
--- a/python/xy/columns.py
+++ b/python/xy/columns.py
@@ -209,6 +209,17 @@ class Column:
def __len__(self) -> int:
return len(self.values)
+ @property
+ def capacity_bytes(self) -> int:
+ """Bytes this column actually holds, slack included.
+
+ `append` keeps values as a prefix view of a capacity-doubling buffer, so
+ after a stream of appends the allocation can be up to twice
+ `values.nbytes`. The memory report needs the real number (§27).
+ """
+ grow = getattr(self, "_grow", None)
+ return int(grow.nbytes if grow is not None else self.values.nbytes)
+
@property
def zone(self) -> ZoneMaps:
"""Materialize deferred statistics at most once."""
@@ -456,23 +467,36 @@ def memory_report(self) -> dict[str, Any]:
OS as a reclaimable cache, so they do not sit in the process's resident
set the way an in-RAM column does. ``canonical_bytes`` therefore stays
the honest RAM-resident canonical figure (unchanged for all-RAM
- figures, where ``canonical_mapped_bytes`` is 0)."""
+ figures, where ``canonical_mapped_bytes`` is 0).
+
+ A streamed column's values are a prefix *view* of its capacity-doubling
+ growth buffer (`Column.append`), so up to half of what it holds is slack
+ that `values.nbytes` cannot see. That slack is resident RAM like any
+ other allocation, so each column also reports ``capacity_bytes`` and the
+ report totals them as ``canonical_capacity_bytes`` — equal to
+ ``canonical_bytes`` for every never-appended figure, and the number the
+ resident total is built from (channels already report their own growth
+ buffers this way)."""
resident = 0
+ resident_capacity = 0
mapped = 0
columns = []
for c in self._columns:
nbytes = int(c.values.nbytes)
+ capacity = int(c.capacity_bytes)
memmapped = _ooc.is_memmapped(c.values)
if memmapped:
mapped += nbytes
else:
resident += nbytes
+ resident_capacity += capacity
columns.append(
{
"id": c.id,
"kind": c.kind,
"len": len(c),
"bytes": nbytes,
+ "capacity_bytes": capacity,
"backing": "memmap" if memmapped else "ram",
"ingest_copies": c.ingest_copies,
"null_count": c.zone.null_count,
@@ -480,6 +504,7 @@ def memory_report(self) -> dict[str, Any]:
)
return {
"canonical_bytes": resident,
+ "canonical_capacity_bytes": resident_capacity,
"canonical_mapped_bytes": mapped,
"columns": columns,
}
diff --git a/python/xy/export.py b/python/xy/export.py
index 86abafbc..f44ef604 100644
--- a/python/xy/export.py
+++ b/python/xy/export.py
@@ -21,6 +21,8 @@
from typing import TYPE_CHECKING, Any, Optional, SupportsFloat, SupportsIndex, cast
if TYPE_CHECKING:
+ from collections.abc import Iterator
+
from ._figure import Figure
@@ -168,6 +170,21 @@ def _javascript_for_inline_script(source: str) -> str:
return source.replace(" "Iterator[str]":
+ """`_base64_chunks` as a generator, for callers that consume chunks once.
+
+ The encoded text is 4/3 of the payload, so materializing the list *and* the
+ string built from it doubles that; yielding lets each chunk be wrapped and
+ released as it is produced.
+ """
+ if not blob:
+ return
+ view = memoryview(blob)
+ step = _B64_CHUNK_BYTES
+ for i in range(0, len(view), step):
+ yield base64.b64encode(view[i : i + step]).decode("ascii")
+
+
def _base64_chunks(blob: bytes) -> list[str]:
"""Base64 the payload as 3-byte-aligned chunks (see `_B64_CHUNK_BYTES`).
@@ -175,11 +192,7 @@ def _base64_chunks(blob: bytes) -> list[str]:
interior `=` padding and decodes to an exact byte length — letting the client
reassemble one contiguous buffer without tracking base64 boundaries. A
memoryview avoids copying the (potentially huge) blob per slice."""
- if not blob:
- return []
- view = memoryview(blob)
- step = _B64_CHUNK_BYTES
- return [base64.b64encode(view[i : i + step]).decode("ascii") for i in range(0, len(view), step)]
+ return list(_iter_base64_chunks(blob))
# Inline decoder for the chunked base64 payload. Prefers the native Uint8Array
@@ -310,10 +323,16 @@ def to_html(
# valid JS string literal verbatim and can never close the ' for c in _base64_chunks(blob)
- )
- doc = f"""
+ # Assembled as parts joined once. Every large string — the client bundle, the
+ # spec, each base64 chunk — is its own part, so the join copies it exactly
+ # once and nothing is formatted into an intermediate first. Interpolating one
+ # of them costs a second full copy of it: folding the chunks through
+ # `"\n".join(...)` duplicated 4/3 of the payload, and (measured on
+ # `test_html_export_line`, where the ~330 KB client bundle *is* the document)
+ # interpolating `client_js` into a header part cost ~10% of a small export.
+ blob_len = len(blob)
+ parts = [
+ f"""
@@ -325,18 +344,32 @@ def to_html(
{_custom_css_block(custom_css)}
-
+
-{chunk_scripts}
-')
+ del blob
+ parts.append("\n
-"""
+