diff --git a/CHANGELOG.md b/CHANGELOG.md index da223dcb..9f2a5204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,16 @@ in the README). contract without importing the widget stack. ### Changed +- **Transport allocation cleanup.** Scalar area baselines now avoid an N-row + canonical and wire column; constant baseline arrays and error-band lower + bounds retain their canonical data but ship one `base_const` number; + native-endian fixed-unit NumPy datetimes convert + directly from their original (even strided) i64 ticks into one f64-ms output + (native ABI 38); hexbin center payloads reuse their canonical zone-map bounds; + `memory_report()` counts the exact payload shape without offset-encoding + geometry or joining a full payload blob (channel preparation still follows + its normal path); and notebook repr streams standalone parts into escaped + `srcdoc` output rather than retaining a second full standalone document. - **Responsive, author-defeatable browser chrome.** XY's visual defaults now live in a low-priority cascade layer, so Tailwind utilities, ordinary author CSS, and slot styles override them without `!important`. Long legends remain @@ -325,6 +335,16 @@ in the README). - `LICENSE` (Apache-2.0), `CHANGELOG.md`, `SECURITY.md`, root `CONTRIBUTING.md`. ### Changed +- **Transport hot paths:** split-layout `u8` columns no longer copy packed-only + alignment tails; direct RGBA8 packing and rectangle midpoint construction use + bounded scratch; stacked bars reuse shared category geometry; static Reflex + assets hash/write XYBF frame parts incrementally through unique atomic temps; + and equal-sized line/area tier refinements update existing GPU storage with + `bufferSubData` while size changes retain the safe `bufferData` path. Client + instance styles now upload only their dynamic components (4 B/item for one + channel instead of a padded 16 B/item; scalar artist alpha stays uniform), + and native-color density blends preserve their unchanged pick snapshot while + retaining invalidation for the geometry-changing frame that starts them. - **Rendering hardening:** context loss now quiesces draw/animation/re-bin work, invalidates pre-loss replies, retains streamed canonical payloads, reports recovery state, and rebuilds without throwing an unhandled event error. The diff --git a/benchmarks/bench_client_transport_quick_wins.py b/benchmarks/bench_client_transport_quick_wins.py new file mode 100644 index 00000000..5598abd9 --- /dev/null +++ b/benchmarks/bench_client_transport_quick_wins.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Focused real-Chromium measurements for issue #176 client findings. + +Measures dense style packing, append-key alternatives, and GPU pick readback. +The append alternatives are diagnostic: only an exact identity relation is a +candidate for production, regardless of its timing. + +Usage: + PYTHONPATH=python python benchmarks/bench_client_transport_quick_wins.py +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import xy # noqa: E402 +from _xy_browser import chart_payload, page_for_charts, run_json_probe # noqa: E402 + +PROBE = r""" +(async () => { + try { + const payload = XY_CHARTS[0]; + const bytes = xyBytesFromPayload(payload); + const root = document.createElement("div"); + root.style.cssText = "width:900px;height:420px"; + document.getElementById("root").appendChild(root); + const view = xy.renderStandalone(root, payload.spec, bytes); + view._drawNow(); + if (view._raf) cancelAnimationFrame(view._raf); + view._raf = null; + const gl = view.gl; + gl.finish(); + const g = view.gpuTraces[0]; + const opacity = g.trace.channels.opacity; + const source = view._columnView(bytes, payload.spec.columns[opacity.buf]); + const n = g.n; + const reps = window.XY_REPS; + const median = (values) => values.slice().sort((a, b) => a - b)[values.length >> 1]; + + const legacyPack = () => { + const values = new Float32Array(n * 4); + for (let i = 0; i < n; i++) { + values[i * 4] = 1; + values[i * 4 + 1] = -1; + values[i * 4 + 2] = -1; + values[i * 4 + 3] = -1; + } + for (let i = 0; i < n; i++) values[i * 4] = source[i]; + const buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, values, gl.STATIC_DRAW); + gl.finish(); + gl.deleteBuffer(buffer); + return values.byteLength; + }; + const compactPack = () => { + const target = {}; + view._packInstanceStyleChannels( + target, n, + (name) => name === "opacity" ? opacity : null, + NaN, () => source, "stroke_width", + ); + gl.finish(); + const result = target.styleBuf._fcBytes; + gl.deleteBuffer(target.styleBuf); + return result; + }; + legacyPack(); compactPack(); + const legacyPackMs = [], compactPackMs = []; + for (let i = 0; i < reps; i++) { + let start = performance.now(); + const legacyBytes = legacyPack(); + legacyPackMs.push(performance.now() - start); + start = performance.now(); + const compactBytes = compactPack(); + compactPackMs.push(performance.now() - start); + if (legacyBytes !== n * 16 || compactBytes !== n * 4) throw Error("style byte oracle"); + } + + const appendN = Math.min(n, 200000); + // Mirror the real append path: coordinates are f32 residuals decoded with + // a large f64 offset. These epoch-scale values are all distinct under the + // legacy 12-significant-digit relation. A f32 key is not: at 1.7e12 its + // ULP is 131072, so many adjacent timestamps alias one Map entry. + const epochBase = 1700000000000; + const epochStep = 1024; + const oldOffset = epochBase + 65536; + const newOffset = epochBase + 131072; + const oldEncoded = new Float32Array(appendN); + const newEncoded = new Float32Array(appendN); + const oldValues = new Float64Array(appendN); + const newValues = new Float64Array(appendN); + for (let i = 0; i < appendN; i++) { + const value = epochBase + i * epochStep; + oldEncoded[i] = value - oldOffset; + newEncoded[i] = value - newOffset; + oldValues[i] = oldEncoded[i] + oldOffset; + newValues[i] = newEncoded[i] + newOffset; + } + const mathKey = (value) => { + if (value === 0) return 0; + const magnitude = Math.abs(value); + let exponent = Math.floor(Math.log10(magnitude)); + const scale = 10 ** (11 - exponent); + let mantissa = Math.round(magnitude * scale); + if (mantissa >= 1e12) { mantissa = 1e11; exponent += 1; } + return ((exponent + 400) * 2 + (value < 0 ? 1 : 0)) * 1000000000001 + mantissa; + }; + const f32Storage = new ArrayBuffer(4); + const f32Value = new Float32Array(f32Storage); + const f32Bits = new Uint32Array(f32Storage); + const f32BitsKey = (value) => { + // SameValueZero (Map's relation) treats -0 and +0 as one identity; keep + // that legacy behavior while testing the exact proposed f32-bit key. + if (value === 0) return 0; + f32Value[0] = value; + return f32Bits[0]; + }; + const appendPass = (key) => { + const index = new Map(); + for (let i = 0; i < appendN; i++) index.set(key(oldValues[i]), i); + let matches = 0, exactMatches = 0; + for (let i = 0; i < appendN; i++) { + const oldIndex = index.get(key(newValues[i])); + if (oldIndex !== undefined) matches++; + if (oldIndex === i) exactMatches++; + } + return { + matches, + exact_matches: exactMatches, + unique_old_keys: index.size, + old_key_collisions: appendN - index.size, + }; + }; + const keyFns = { + string: (value) => value.toPrecision(12), + parsed: (value) => Number(value.toPrecision(12)), + math: mathKey, + fround: (value) => Math.fround(value), + f32_bits: f32BitsKey, + }; + const append = {}; + for (const [name, key] of Object.entries(keyFns)) { + appendPass(key); + const samples = []; + let pass = null; + for (let i = 0; i < reps; i++) { + const start = performance.now(); + pass = appendPass(key); + samples.push(performance.now() - start); + } + append[name] = { median_ms: median(samples), ...pass }; + } + const boundary = { + shouldMatch: [8.952695812915, 8.95269581290906], + shouldMiss: [1234567890125, 1234567890124.9973], + }; + const relation = (key, pair) => key(pair[0]) === key(pair[1]); + const epochDistinct = [epochBase, epochBase + epochStep]; + for (const [name, key] of Object.entries(keyFns)) { + append[name].boundary_match = relation(key, boundary.shouldMatch); + append[name].boundary_miss = !relation(key, boundary.shouldMiss); + append[name].epoch_adjacent_distinct = !relation(key, epochDistinct); + } + // Semantic oracles are part of the benchmark: timings must never make a + // lossy candidate look like a production win. + if (append.string.unique_old_keys !== appendN || + append.string.exact_matches !== appendN || + !append.string.boundary_match || !append.string.boundary_miss || + !append.string.epoch_adjacent_distinct) { + throw Error("legacy append-key oracle"); + } + for (const name of ["fround", "f32_bits"]) { + if (append[name].unique_old_keys >= appendN || + append[name].exact_matches >= appendN || + append[name].boundary_miss || + append[name].epoch_adjacent_distinct) { + throw Error(`${name} rejection oracle`); + } + } + + // Populate the pick snapshot once, then separate stable readback cost from + // the O(N) redraw that a dirty snapshot forces. + const px = Math.floor(view.canvas.width / 2); + const py = Math.floor(view.canvas.height / 2); + view._pickDirty = true; + view._pickAt(px / view.dpr, view.plot.h - py / view.dpr); + const cleanPickMs = []; + for (let i = 0; i < Math.max(50, reps * 10); i++) { + const start = performance.now(); + view._pickAt(px / view.dpr, view.plot.h - py / view.dpr); + cleanPickMs.push(performance.now() - start); + } + const dirtyPickMs = []; + for (let i = 0; i < Math.max(5, reps); i++) { + view._pickDirty = true; + const start = performance.now(); + view._pickAt(px / view.dpr, view.plot.h - py / view.dpr); + dirtyPickMs.push(performance.now() - start); + } + + const pbo = gl.createBuffer(); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, pbo); + gl.bufferData(gl.PIXEL_PACK_BUFFER, 4, gl.STREAM_READ); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); + const pboSubmitMs = [], pboCompleteMs = []; + let pboForcedFinishes = 0; + const pboOut = new Uint8Array(4); + for (let i = 0; i < Math.max(12, reps * 2); i++) { + gl.bindFramebuffer(gl.FRAMEBUFFER, view.pickFbo); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, pbo); + const start = performance.now(); + gl.readPixels(px, py, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, 0); + const fence = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0); + gl.flush(); + pboSubmitMs.push(performance.now() - start); + await new Promise((resolve) => setTimeout(resolve, 0)); + let status = gl.clientWaitSync(fence, gl.SYNC_FLUSH_COMMANDS_BIT, 0); + if (status === gl.TIMEOUT_EXPIRED) { + // A production async path would defer the tooltip and keep polling. + // Finish here only so this bounded diagnostic can report that first- + // task availability miss instead of hanging --dump-dom. + pboForcedFinishes += 1; + gl.finish(); + status = gl.CONDITION_SATISFIED; + } + if (status === gl.WAIT_FAILED) throw Error("PBO fence failed"); + gl.getBufferSubData(gl.PIXEL_PACK_BUFFER, 0, pboOut); + pboCompleteMs.push(performance.now() - start); + gl.deleteSync(fence); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + } + gl.deleteBuffer(pbo); + + xyReport("XY_CLIENT_QUICK_WINS", { + n, + reps, + style: { + legacy_bytes: n * 16, + compact_bytes: n * 4, + legacy_median_ms: median(legacyPackMs), + compact_median_ms: median(compactPackMs), + }, + append, + append_dataset: { + n: appendN, + epoch_base: epochBase, + epoch_step: epochStep, + old_offset: oldOffset, + new_offset: newOffset, + }, + pick: { + clean_sync_median_ms: median(cleanPickMs), + dirty_sync_median_ms: median(dirtyPickMs), + pbo_submit_median_ms: median(pboSubmitMs), + pbo_complete_median_ms: median(pboCompleteMs), + pbo_forced_finishes: pboForcedFinishes, + pbo_reps: pboCompleteMs.length, + }, + gl_error: gl.getError(), + }); + } catch (err) { + xyFail("XY_CLIENT_QUICK_WINS", err); + } +})(); +""" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--n", type=int, default=250_000) + parser.add_argument("--reps", type=int, default=7) + parser.add_argument("--chromium") + args = parser.parse_args() + + x = np.linspace(0.0, 1.0, args.n, dtype=np.float64) + opacity = np.linspace(0.2, 1.0, args.n, dtype=np.float64) + chart = xy.scatter_chart( + xy.scatter(x=x, y=x, opacity=opacity, density=False, size=3.0), + width=900, + height=420, + ).figure() + spec, blob = chart.build_payload() + probe = f"window.XY_REPS = {args.reps};\n" + PROBE + page = page_for_charts( + [chart_payload("client-quick-wins", spec, blob)], + probe, + title="xy client quick wins", + ) + result = run_json_probe( + page, + marker="XY_CLIENT_QUICK_WINS", + chromium=args.chromium, + virtual_time_ms=30_000, + timeout_s=180, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if result.get("status") == "ok" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/test_codspeed_kernels.py b/benchmarks/test_codspeed_kernels.py index c5f207fc..dadc70a8 100644 --- a/benchmarks/test_codspeed_kernels.py +++ b/benchmarks/test_codspeed_kernels.py @@ -18,6 +18,7 @@ import xy from xy import kernels as k from xy._figure import Figure # harness type annotations only +from xy.columns import ColumnStore # Small/medium/large sizes keep CodSpeed honest across normal dashboard charts, # exact WebGL workloads, and screen-bounded large-data paths without turning it @@ -644,6 +645,12 @@ def _hexbin_payload(x: np.ndarray, y: np.ndarray) -> int: return sum(b.nbytes for b in buffers) +def _datetime_seconds_ingest(values: np.ndarray) -> int: + col = ColumnStore().ingest(values) + assert col.ingest_copies == 1 + return col.values.nbytes + + def _contour_payload(z: np.ndarray) -> int: fig = xy.chart(xy.contour(z=z, levels=12, filled=True)).figure() _spec, buffers = fig.build_payload_split(N_BUCKETS) @@ -747,6 +754,28 @@ def build(): assert sum(b.nbytes for b in buffers) == 4 * len(x) * 4 +def test_first_payload_scatter_direct_rgba(benchmark, medium_data): + """Direct RGBA8 packing without payload-sized chained temporaries.""" + x, y = medium_data + rgba = np.column_stack( + ( + np.linspace(0.0, 1.0, len(x)), + np.linspace(1.0, 0.0, len(x)), + np.full(len(x), 0.5), + np.full(len(x), 0.25), + ) + ) + + def build(): + fig = xy.chart(xy.scatter(x=x, y=y, color=rgba)).figure() + return fig.build_payload_split(N_BUCKETS) + + spec, buffers = benchmark(build) + color = spec["traces"][0]["color"] + assert color["mode"] == "direct_rgba" and color["dtype"] == "u8" + assert sum(b.nbytes for b in buffers) == 12 * len(x) + + def test_first_payload_line_unsorted_x(benchmark, medium_data): """Large line ingestion through the sort-and-reingest branch.""" x, y = medium_data @@ -822,6 +851,30 @@ def test_memory_report_density_medium(benchmark, medium_data): assert report["transport_bytes_per_point"] > 0 +def test_memory_report_counts_without_payload_blob(benchmark, medium_data): + """Exact channel-rich accounting skips geometry encoding and packed join.""" + x, y = medium_data + rgba = np.column_stack( + ( + np.linspace(0.0, 1.0, len(x)), + np.linspace(1.0, 0.0, len(x)), + np.full(len(x), 0.5), + np.full(len(x), 0.25), + ) + ) + size = 4.0 + 3.0 * np.abs(np.cos(x * 0.0003)) + fig = xy.chart(xy.scatter(x=x, y=y, color=rgba, size=size)).figure() + expected = fig.payload_nbytes() + report = benchmark(fig.memory_report) + assert report["transport_bytes_first_paint"] == expected + + +def test_datetime_seconds_fused_one_copy_ingest(benchmark): + """Non-ms datetime ticks convert directly into one canonical f64 output.""" + values = np.datetime64("2026-01-01", "s") + np.arange(MEDIUM_N).astype("timedelta64[s]") + assert benchmark(_datetime_seconds_ingest, values) == values.nbytes + + def test_first_payload_histogram_core_2d(benchmark, core_2d_data): """Core 2D payload prep: histogram binning plus rectangle transport.""" values = core_2d_data["hist_values"] @@ -850,6 +903,24 @@ def test_first_payload_bar_core_2d(benchmark, core_2d_data): assert 0 < payload_bytes < values.nbytes * 2 +def test_first_payload_stacked_bar_reuses_category_geometry(benchmark): + """Stacked series build shared rectangle edges/centers once.""" + n = 100_000 + categories = [f"C{i:06d}" for i in range(n)] + x = np.arange(n, dtype=np.float64) + values = np.vstack([1.0 + np.sin(x * 0.0001 + i) ** 2 for i in range(8)]) + + def build(): + fig = xy.chart(xy.bar(categories, values, mode="stacked")).figure() + spec, buffers = fig.build_payload_split(N_BUCKETS) + return fig, spec, buffers + + fig, spec, buffers = benchmark(build) + assert all(trace.x is fig.traces[0].x for trace in fig.traces[1:]) + assert len(spec["traces"]) == 8 + assert sum(buffer.nbytes for buffer in buffers) > 0 + + def test_first_payload_heatmap_core_2d(benchmark, core_2d_data): """Core 2D payload prep: dense cell grid normalization and binary transport.""" z = core_2d_data["heatmap_z"] @@ -884,6 +955,19 @@ def test_first_payload_hexbin_core_2d(benchmark, medium_data): assert payload_bytes < x.nbytes + y.nbytes +def test_hexbin_payload_reuses_precomputed_center_bounds(benchmark, medium_data): + """Steady payload encode reuses canonical center zone maps (no min/max scan).""" + x, y = medium_data + fig = xy.chart(xy.hexbin(x=x, y=y, gridsize=HEXBIN_GRIDSIZE)).figure() + + def encode_centers(): + return fig.build_payload_split(N_BUCKETS) + + spec, buffers = benchmark(encode_centers) + assert spec["traces"][0]["n_marks"] > 0 + assert buffers + + def test_first_payload_errorbar_large(benchmark, data): """Large error bars ship per-point decimated segment groups, not 3N marks.""" x, y = data @@ -981,6 +1065,14 @@ def test_html_export_line(benchmark, export_data): assert "= 0.0 ? a_style.z : u_width; + vec4 itemStyle = xyStyle(a_style); + float itemWidth = itemStyle.z >= 0.0 ? itemStyle.z : u_width; float half_w = itemWidth * 0.5 + 0.5; vec2 pos = mix(pix0, pix1, c.x) + dir * (c.x * 2.0 - 1.0) * 0.5 + n * c.y * half_w; gl_Position = vec4(pos / u_res * 2.0 - 1.0, 0.0, 1.0); v_off = c.y * half_w; v_cval = u_colorMode == 2 ? (a_cval + 0.5) / 256.0 : a_cval; v_dash = a_dash0 + c.x * len * a_dashDir; - v_rgba = a_rgba; v_style = a_style; + v_rgba = a_rgba; v_style = itemStyle; }`; export const SEGMENT_FS = `#version 300 es @@ -582,6 +611,7 @@ uniform int u_y0mode; uniform int u_y1mode; uniform int u_y2mode; uniform int u_colorMode; out float v_cval; out vec3 v_bary; out vec4 v_rgba; out vec4 v_style; out vec4 v_stroke; ${AXIS_GLSL} +${STYLE_CHANNEL_GLSL} void main() { int vertex = gl_VertexID % 3; float x = vertex == 0 ? ax0 : (vertex == 1 ? ax1 : ax2); @@ -593,7 +623,7 @@ void main() { gl_Position = vec4(xyMap(x, u_xmap, xm, xmode), xyMap(y, u_ymap, ym, ymode), 0.0, 1.0); v_cval = u_colorMode == 2 ? (a_cval + 0.5) / 256.0 : a_cval; v_bary = vertex == 0 ? vec3(1.,0.,0.) : (vertex == 1 ? vec3(0.,1.,0.) : vec3(0.,0.,1.)); - v_rgba = a_rgba; v_style = a_style; v_stroke = a_stroke; + v_rgba = a_rgba; v_style = xyStyle(a_style); v_stroke = a_stroke; }`; export const MESH_FS = `#version 300 es @@ -726,6 +756,7 @@ out vec2 v_local; out vec2 v_half; out float v_t; out vec4 v_rgba; out vec4 v_style; out vec4 v_stroke; out vec2 v_radius; const vec2 corners[4] = vec2[4](vec2(0.,0.), vec2(1.,0.), vec2(0.,1.), vec2(1.,1.)); ${AXIS_GLSL} +${STYLE_CHANNEL_GLSL} void main() { vec2 c = corners[gl_VertexID]; float x0 = xyMap(ax0, u_x0map, u_x0meta, u_xmode) + u_edgePad.x; @@ -740,7 +771,7 @@ void main() { v_half = abs(pB - pA) * 0.5; v_local = mix(pA, pB, c) - (pA + pB) * 0.5; v_t = c.y; - v_rgba = a_rgba; v_style = a_style; v_stroke = a_stroke; v_radius = a_radius; + v_rgba = a_rgba; v_style = xyStyle(a_style); v_stroke = a_stroke; v_radius = a_radius; gl_Position = vec4(mix(x0, x1, c.x), mix(y0, y1, c.y), 0.0, 1.0); }`; @@ -766,6 +797,7 @@ out vec2 v_local; out vec2 v_half; out float v_t; out vec4 v_rgba; out vec4 v_style; out vec4 v_stroke; out vec2 v_radius; const vec2 corners[4] = vec2[4](vec2(0.,0.), vec2(1.,0.), vec2(0.,1.), vec2(1.,1.)); ${AXIS_GLSL} +${STYLE_CHANNEL_GLSL} void main() { vec2 c = corners[gl_VertexID]; float nextP = xyMap(a_pos, u_pmap, u_pmeta, u_pmode); @@ -803,7 +835,7 @@ void main() { vec2 pB = (clipB * 0.5 + 0.5) * u_res; v_half = abs(pB - pA) * 0.5; v_local = vec2(mix(pA.x, pB.x, c.x), mix(pA.y, pB.y, c.y)) - (pA + pB) * 0.5; - v_rgba = a_rgba; v_style = a_style; v_stroke = a_stroke; v_radius = a_radius; + v_rgba = a_rgba; v_style = xyStyle(a_style); v_stroke = a_stroke; v_radius = a_radius; }`; // Shared by the rect and compact-bar programs: flat fill or LUT color, then an diff --git a/js/src/45_lod.ts b/js/src/45_lod.ts index c2cbe561..48b1c1de 100644 --- a/js/src/45_lod.ts +++ b/js/src/45_lod.ts @@ -288,33 +288,16 @@ export function lodApplyDrill(view, g, upd, buffers) { } const styleChannel = (name) => upd.channels && upd.channels[name]; const artistScalar = Number(d.trace.style && d.trace.style.artist_alpha); - if (styleChannel("opacity") || styleChannel("artist_alpha") || - styleChannel("stroke_width") || styleChannel("symbol") || Number.isFinite(artistScalar)) { - const values = new Float32Array(d.n * 4); - for (let i = 0; i < d.n; i++) { - values[i * 4] = 1; - values[i * 4 + 1] = Number.isFinite(artistScalar) ? artistScalar : -1; - values[i * 4 + 2] = -1; - values[i * 4 + 3] = -1; - } - const copy = (name, component, scale = 1) => { - const spec = styleChannel(name); - if (!spec) return; - const source = spec.dtype === "u8" - ? view._asU8(buffers[spec.buf]) - : view._asF32(buffers[spec.buf]); - const components = spec.components || 1; - for (let i = 0; i < d.n; i++) values[i * 4 + component] = source[i * components] * scale; - }; - copy("opacity", 0); - copy("artist_alpha", 1); - copy("stroke_width", 2, view.dpr); - copy("symbol", 3); - if (!d.styleBuf) d.styleBuf = gl.createBuffer(); - d.styleBuf._fcType = gl.FLOAT; - gl.bindBuffer(gl.ARRAY_BUFFER, d.styleBuf); - gl.bufferData(gl.ARRAY_BUFFER, values, gl.STATIC_DRAW); - } + view._packInstanceStyleChannels( + d, + d.n, + styleChannel, + artistScalar, + (spec) => spec.dtype === "u8" + ? view._asU8(buffers[spec.buf]) + : view._asF32(buffers[spec.buf]), + "stroke_width", + ); if (upd.stroke && upd.stroke.mode === "direct_rgba") { const values = view._asU8(buffers[upd.stroke.buf]); if (!d.strokeBuf) d.strokeBuf = gl.createBuffer(); diff --git a/js/src/50_chartview.ts b/js/src/50_chartview.ts index b4697fca..d0a95818 100644 --- a/js/src/50_chartview.ts +++ b/js/src/50_chartview.ts @@ -2099,31 +2099,65 @@ export class ChartView { g.yBuf = this._upload(y); } - _buildInstanceStyleChannels(g, t, buffer, widthName) { - const channel = (name) => t.channels && t.channels[name]; - const artistScalar = Number(t.style && t.style.artist_alpha); - const hasStyle = channel("opacity") || channel("artist_alpha") || - channel(widthName) || channel("symbol") || Number.isFinite(artistScalar); - if (hasStyle) { - const values = new Float32Array(g.n * 4); - for (let i = 0; i < g.n; i++) { - values[i * 4] = 1; - values[i * 4 + 1] = Number.isFinite(artistScalar) ? artistScalar : -1; - values[i * 4 + 2] = -1; - values[i * 4 + 3] = -1; + // Pack only the dynamic style components. The shaders reconstruct the + // semantic [opacity, artist_alpha, width, symbol] vec4 from these slots and + // styleBase. This makes the common one-channel case 4 B/item instead of + // 16 B/item and keeps scalar artist alpha entirely in a uniform. + _packInstanceStyleChannels(g, n, channel, artistScalar, sourceOf, widthName) { + const base = [1, Number.isFinite(artistScalar) ? artistScalar : -1, -1, -1]; + const definitions = [ + { name: "opacity", component: 0, scale: 1 }, + { name: "artist_alpha", component: 1, scale: 1 }, + { name: widthName, component: 2, scale: this.dpr }, + { name: "symbol", component: 3, scale: 1 }, + ]; + const active = definitions + .map(({ name, component, scale }) => ({ spec: channel(name), component, scale })) + .filter(({ spec }) => !!spec); + const slots = [-1, -1, -1, -1]; + for (let packed = 0; packed < active.length; packed++) { + slots[active[packed].component] = packed; + } + g.styleBase = base; + g.styleSlots = slots; + const previousSize = g.styleSize || 0; + g.styleSize = active.length; + if (!active.length) { + if (g.styleBuf) { + this._deleteVaos(g); + this.gl.deleteBuffer(g.styleBuf); + g.styleBuf = null; } - const copy = (name, component, scale = 1) => { - const spec = channel(name); - if (!spec) return; - const source = this._columnView(buffer, this.spec.columns[spec.buf]); - for (let i = 0; i < g.n; i++) values[i * 4 + component] = source[i * (spec.components || 1)] * scale; - }; - copy("opacity", 0); - copy("artist_alpha", 1); - copy(widthName, 2, this.dpr); - copy("symbol", 3); + return; + } + const values = new Float32Array(n * active.length); + for (let packed = 0; packed < active.length; packed++) { + const { spec, scale } = active[packed]; + const source = sourceOf(spec); + const components = spec.components || 1; + for (let i = 0; i < n; i++) { + values[i * active.length + packed] = source[i * components] * scale; + } + } + if (g.styleBuf) { + if (previousSize !== active.length) this._deleteVaos(g); + this._uploadTierBuffer(g.styleBuf, values); + } else { g.styleBuf = this._upload(values); } + } + + _buildInstanceStyleChannels(g, t, buffer, widthName) { + const channel = (name) => t.channels && t.channels[name]; + const artistScalar = Number(t.style && t.style.artist_alpha); + this._packInstanceStyleChannels( + g, + g.n, + channel, + artistScalar, + (spec) => this._columnView(buffer, this.spec.columns[spec.buf]), + widthName, + ); const radius = channel("corner_radius"); if (radius) { const source = this._columnView(buffer, this.spec.columns[radius.buf]); @@ -2304,30 +2338,16 @@ export class ChartView { } const channel = (name) => sample.channels && sample.channels[name]; const artistScalar = Number(trace.style && trace.style.artist_alpha); - if (channel("opacity") || channel("artist_alpha") || channel("stroke_width") || - channel("symbol") || Number.isFinite(artistScalar)) { - const values = new Float32Array(s.n * 4); - for (let i = 0; i < s.n; i++) { - values[i * 4] = 1; - values[i * 4 + 1] = Number.isFinite(artistScalar) ? artistScalar : -1; - values[i * 4 + 2] = -1; - values[i * 4 + 3] = -1; - } - const copy = (name, component, scale = 1) => { - const spec = channel(name); - if (!spec) return; - const source = spec.dtype === "u8" - ? this._asU8(buffers[spec.buf]) - : this._asF32(buffers[spec.buf]); - const components = spec.components || 1; - for (let i = 0; i < s.n; i++) values[i * 4 + component] = source[i * components] * scale; - }; - copy("opacity", 0); - copy("artist_alpha", 1); - copy("stroke_width", 2, this.dpr); - copy("symbol", 3); - s.styleBuf = this._upload(values); - } + this._packInstanceStyleChannels( + s, + s.n, + channel, + artistScalar, + (spec) => spec.dtype === "u8" + ? this._asU8(buffers[spec.buf]) + : this._asF32(buffers[spec.buf]), + "stroke_width", + ); if (sample.stroke && sample.stroke.mode === "direct_rgba") { s.strokeBuf = this._upload(this._asU8(buffers[sample.stroke.buf])); } @@ -2615,10 +2635,16 @@ export class ChartView { _buildAreaMark(g, t, buffer) { const x = this._columnView(buffer, this.spec.columns[t.x]); const y = this._columnView(buffer, this.spec.columns[t.y]); - const base = this._columnView(buffer, this.spec.columns[t.base]); g.xMeta = { ...this.spec.columns[t.x] }; g.yMeta = { ...this.spec.columns[t.y] }; - g.baseMeta = { ...this.spec.columns[t.base] }; + const baseConst = Number(t.base_const); + const scalarBase = t.base === undefined && Number.isFinite(baseConst); + const base = scalarBase + ? new Float32Array(Math.min(x.length, y.length)) + : this._columnView(buffer, this.spec.columns[t.base]); + g.baseMeta = scalarBase + ? { offset: baseConst, scale: 1 } + : { ...this.spec.columns[t.base] }; g.n = Math.min(x.length, y.length, base.length); g._cpu = { x, y, base, xMeta: g.xMeta, yMeta: g.yMeta }; const sm = this._smoothArrays(t, x, y, base, g.n); @@ -2628,6 +2654,7 @@ export class ChartView { if (sm) g.n = sm.n; g._dashX = sm ? sm.x : x; g._dashY = sm ? sm.y : y; + g._dashBase = sm ? sm.extra : base; g.color = parseColor(this.root, t.style && t.style.color, [0.3, 0.47, 0.66, 1]); g.lineColor = parseColor(this.root, t.style && (t.style.line_color || t.style.color), g.color); g.grad = this._resolveMarkFill(t.style, g.color); @@ -2829,11 +2856,23 @@ export class ChartView { // drill swap) gets a new id, so any VAO built over the old one rebuilds. buf._fcId = ++this._bufSeq; buf._fcType = view instanceof Uint8Array ? gl.UNSIGNED_BYTE : gl.FLOAT; + buf._fcBytes = view.byteLength; gl.bindBuffer(gl.ARRAY_BUFFER, buf); gl.bufferData(gl.ARRAY_BUFFER, view, gl.STATIC_DRAW); return buf; } + _uploadTierBuffer(buf, view) { + const gl = this.gl; + gl.bindBuffer(gl.ARRAY_BUFFER, buf); + if (buf._fcBytes === view.byteLength) { + gl.bufferSubData(gl.ARRAY_BUFFER, 0, view); + } else { + gl.bufferData(gl.ARRAY_BUFFER, view, gl.STATIC_DRAW); + buf._fcBytes = view.byteLength; + } + } + // -- vertex-array objects --------------------------------------------------- // // One VAO per (trace × draw-config). Attribute slots are fixed at link time @@ -2948,11 +2987,20 @@ export class ChartView { gl.uniform1i(u(`${prefix}mode`), this._axisMode(axisId)); } - // `keepPick` marks a frame whose ONLY trigger is hover-highlight state: the - // highlight lives in the color pass, so the pick framebuffer's geometry/view - // snapshot stays valid and the frame must not invalidate it. Coalescing is - // conservative: if any caller of a pending frame needs invalidation, the - // frame invalidates (§17 — steady hover must not re-render N-point picks). + _setInstanceStyleUniforms(prog, g) { + const gl = this.gl; + const base = g.styleBase || [1, -1, -1, -1]; + const slots = g.styleSlots || [-1, -1, -1, -1]; + gl.uniform4f(uniformOf(gl, prog, "u_styleBase"), base[0], base[1], base[2], base[3]); + gl.uniform4i(uniformOf(gl, prog, "u_styleSlots"), slots[0], slots[1], slots[2], slots[3]); + } + + // `keepPick` marks a color-only frame with no geometry or view change. The + // hover highlight and native-color density blend live in the color pass, so + // the pick framebuffer's geometry/view snapshot stays valid and the frame + // must not invalidate it. Coalescing is conservative: if any caller of a + // pending frame needs invalidation, the frame invalidates (§17 — steady hover + // must not re-render N-point picks). draw(keepPick = false) { if (this._destroyed || this._glLost || !this.gl) return; this._updateZoomMenuLabel?.(); @@ -2971,6 +3019,12 @@ export class ChartView { _drawNow() { if (this._destroyed || !this.gl || this._glLost) return; + // Consume this frame's coalesced invalidation policy before drawing. A + // color-only animation may schedule its *next* keep-pick frame from inside + // _drawPoints; that next-frame flag must not retroactively make the current + // (possibly geometry-changing) frame preserve a stale pick snapshot. + const keepPick = this._rafKeepPick === true; + this._rafKeepPick = false; this._healStaleTheme(); const gl = this.gl; const { x0, x1, y0, y1 } = this.view; @@ -3002,8 +3056,7 @@ export class ChartView { this._repositionTooltip(); // Hover-only frames leave the pick snapshot valid (see draw()); direct // _drawNow() callers never set the flag, so they invalidate as before. - if (!this._rafKeepPick) this._pickDirty = true; - this._rafKeepPick = false; + if (!keepPick) this._pickDirty = true; this._drawChrome(); this._renderLassoSelection?.(); } @@ -3018,8 +3071,10 @@ export class ChartView { _canDrawSimplePoints(g) { + const style = g.styleBase || [1, -1, -1, -1]; return g.colorMode === 0 && g.sizeMode === 0 && !g.selActive && !g.rgbaBuf && !g.styleBuf && !g.strokeBuf && + style[0] === 1 && style[1] === -1 && style[2] === -1 && style[3] === -1 && (g.symbol || 0) === 0 && (g.pointStrokeWidth || 0) <= 0 && Math.max(g.lodBlendShown ?? 0, g.lodBlend ?? 0) <= 0.001; } @@ -3035,6 +3090,7 @@ export class ChartView { const prog = this.pointProg; gl.useProgram(prog); const u = (n) => uniformOf(gl, prog, n); + this._setInstanceStyleUniforms(prog, g); gl.uniform2f(u("u_xmap"), xm[0], xm[1]); gl.uniform2f(u("u_ymap"), ym[0], ym[1]); this._setAxisUniforms(prog, "u_x", g.xMeta, g.xAxis); @@ -3095,7 +3151,10 @@ export class ChartView { g._blendTick = now; blend += (blendTarget - blend) * (1 - Math.exp(-dt / 90)); g.lodBlendShown = blend; - this.draw(); + // This tween changes only fragment color. Preserve the existing pick + // geometry so pointer movement during the blend cannot trigger an O(N) + // pick redraw on every frame. + this.draw(true); } else { g.lodBlendShown = blend = blendTarget; g._blendTick = 0; @@ -3135,7 +3194,7 @@ export class ChartView { this._vaoAttr(ATTR_SLOTS.a_prevy, g._transitionPrevYBuf, 0, 0); } if (rgbaOn) this._vaoAttr(ATTR_SLOTS.a_rgba, g.rgbaBuf, 0, 0, 4, true); - if (styleOn) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 0, 4); + if (styleOn) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 0, g.styleSize || 4); if (strokeOn) this._vaoAttr(ATTR_SLOTS.a_stroke, g.strokeBuf, 0, 0, 4, true); } ); @@ -3371,6 +3430,7 @@ export class ChartView { const prog = this.segmentProg; gl.useProgram(prog); const u = (n) => uniformOf(gl, prog, n); + this._setInstanceStyleUniforms(prog, g); gl.uniform2f(u("u_xmap"), xm[0], xm[1]); gl.uniform2f(u("u_ymap"), ym[0], ym[1]); this._setAxisUniforms(prog, "u_x0", g.x0Meta, g.xAxis); @@ -3406,7 +3466,7 @@ export class ChartView { this._vaoAttr(ATTR_SLOTS.ay1, g.y1Buf, 0, 1); if (g.colorMode && g.cBuf) this._vaoAttr(ATTR_SLOTS.a_cval, g.cBuf, 0, 1); if (g.rgbaBuf) this._vaoAttr(ATTR_SLOTS.a_rgba, g.rgbaBuf, 0, 1, 4, true); - if (g.styleBuf) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, 4); + if (g.styleBuf) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, g.styleSize || 4); if (dashed) { this._vaoAttr(ATTR_SLOTS.a_dash0, g._segmentDashOffsetBuf, 0, 1); this._vaoAttr(ATTR_SLOTS.a_dashDir, g._segmentDashDirBuf, 0, 1); @@ -3499,6 +3559,7 @@ export class ChartView { const prog = this.meshProg; gl.useProgram(prog); const u = (n) => uniformOf(gl, prog, n); + this._setInstanceStyleUniforms(prog, g); gl.uniform2f(u("u_xmap"), xm[0], xm[1]); gl.uniform2f(u("u_ymap"), ym[0], ym[1]); for (const name of ["x0", "x1", "x2"]) this._setAxisUniforms(prog, "u_" + name, g[name + "Meta"], g.xAxis); @@ -3527,7 +3588,7 @@ export class ChartView { } if (g.cBuf) this._vaoAttr(ATTR_SLOTS.a_cval, g.cBuf, 0, 1); if (g.rgbaBuf) this._vaoAttr(ATTR_SLOTS.a_rgba, g.rgbaBuf, 0, 1, 4, true); - if (g.styleBuf) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, 4); + if (g.styleBuf) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, g.styleSize || 4); if (g.strokeBuf) this._vaoAttr(ATTR_SLOTS.a_stroke, g.strokeBuf, 0, 1, 4, true); }); if (!g.cBuf) gl.vertexAttrib1f(ATTR_SLOTS.a_cval, 0); @@ -3622,6 +3683,7 @@ export class ChartView { const prog = this.rectProg; gl.useProgram(prog); const u = (n) => uniformOf(gl, prog, n); + this._setInstanceStyleUniforms(prog, g); gl.uniform2f(u("u_x0map"), x0[0], x0[1]); gl.uniform2f(u("u_x1map"), x1[0], x1[1]); gl.uniform2f(u("u_y0map"), y0[0], y0[1]); @@ -3662,7 +3724,7 @@ export class ChartView { this._vaoAttr(ATTR_SLOTS.ay1, g.y1Buf, 0, 1); if (colorOn) this._vaoAttr(ATTR_SLOTS.a_cval, g.cBuf, 0, 1); if (rgbaOn) this._vaoAttr(ATTR_SLOTS.a_rgba, g.rgbaBuf, 0, 1, 4, true); - if (styleOn) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, 4); + if (styleOn) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, g.styleSize || 4); if (strokeOn) this._vaoAttr(ATTR_SLOTS.a_stroke, g.strokeBuf, 0, 1, 4, true); if (radiusOn) this._vaoAttr(ATTR_SLOTS.a_radius, g.radiusBuf, 0, 1, 2); } @@ -3681,6 +3743,7 @@ export class ChartView { const prog = this.barProg; gl.useProgram(prog); const u = (n) => uniformOf(gl, prog, n); + this._setInstanceStyleUniforms(prog, g); gl.uniform2f(u("u_pmap"), pmap[0], pmap[1]); gl.uniform2f(u("u_v1map"), v1map[0], v1map[1]); gl.uniform2f(u("u_v0map"), v0map ? v0map[0] : 1, v0map ? v0map[1] : 0); @@ -3747,7 +3810,7 @@ export class ChartView { this._vaoAttr(ATTR_SLOTS.a_prevx1, g._transitionPrevValue0Buf, 0, 1); } if (rgbaOn) this._vaoAttr(ATTR_SLOTS.a_rgba, g.rgbaBuf, 0, 1, 4, true); - if (styleOn) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, 4); + if (styleOn) this._vaoAttr(ATTR_SLOTS.a_style, g.styleBuf, 0, 1, g.styleSize || 4); if (strokeOn) this._vaoAttr(ATTR_SLOTS.a_stroke, g.strokeBuf, 0, 1, 4, true); if (radiusOn) this._vaoAttr(ATTR_SLOTS.a_radius, g.radiusBuf, 0, 1, 2); } @@ -4456,7 +4519,9 @@ export class ChartView { const px = Math.round(cssX * this.dpr); const py = Math.round((this.plot.h - cssY) * this.dpr); // GL origin bottom-left if (px < 0 || py < 0 || px >= this.canvas.width || py >= this.canvas.height) return null; - const buf = new Uint8Array(4); + // A single chart cannot issue overlapping synchronous reads on the main + // thread; reuse this tiny result buffer instead of allocating per move. + const buf = this._pickPixel || (this._pickPixel = new Uint8Array(4)); gl.bindFramebuffer(gl.FRAMEBUFFER, this.pickFbo); gl.readPixels(px, py, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, buf); gl.bindFramebuffer(gl.FRAMEBUFFER, null); diff --git a/js/src/54_kernel.ts b/js/src/54_kernel.ts index d2e3e30a..e628d697 100644 --- a/js/src/54_kernel.ts +++ b/js/src/54_kernel.ts @@ -278,12 +278,15 @@ Object.assign(ChartView.prototype, { for (const upd of msg.traces) { const g = this.gpuTraces.find((t) => t.trace.id === upd.id); if (!g) continue; - const gl = this.gl; const xArr = this._asF32(buffers[upd.x.buf]); const yArr = this._asF32(buffers[upd.y.buf]); - const bArr = upd.base && g.baseBuf ? this._asF32(buffers[upd.base.buf]) : null; let n = Math.min(upd.x.len, upd.y.len); - if (bArr) n = Math.min(n, upd.base.len); + const baseConst = Number(upd.base_const); + const scalarBase = !upd.base && g.baseBuf && Number.isFinite(baseConst); + const bArr = upd.base && g.baseBuf + ? this._asF32(buffers[upd.base.buf]) + : scalarBase ? new Float32Array(n) : null; + if (upd.base && bArr) n = Math.min(n, upd.base.len); // curve:"smooth" traces re-smooth every refined window, so the curve // survives zoom-driven re-decimation instead of snapping to segments. // style.step traces likewise re-expand so the step corners survive @@ -291,18 +294,18 @@ Object.assign(ChartView.prototype, { const sm = this._smoothArrays(g.trace, xArr, yArr, bArr, n); const src = sm || { x: xArr, y: yArr, n }; const st = this._stepArrays(g.trace, src.x, src.y, src.n); - gl.bindBuffer(gl.ARRAY_BUFFER, g.xBuf); - gl.bufferData(gl.ARRAY_BUFFER, st ? st.x : src.x, gl.STATIC_DRAW); - gl.bindBuffer(gl.ARRAY_BUFFER, g.yBuf); - gl.bufferData(gl.ARRAY_BUFFER, st ? st.y : src.y, gl.STATIC_DRAW); + this._uploadTierBuffer(g.xBuf, st ? st.x : src.x); + this._uploadTierBuffer(g.yBuf, st ? st.y : src.y); g.xMeta = { ...g.xMeta, offset: upd.x.offset, scale: upd.x.scale }; g.yMeta = { ...g.yMeta, offset: upd.y.offset, scale: upd.y.scale }; g._dashX = st ? st.x : src.x; g._dashY = st ? st.y : src.y; if (bArr) { - gl.bindBuffer(gl.ARRAY_BUFFER, g.baseBuf); - gl.bufferData(gl.ARRAY_BUFFER, sm ? sm.extra : bArr, gl.STATIC_DRAW); - g.baseMeta = { ...g.baseMeta, offset: upd.base.offset, scale: upd.base.scale }; + this._uploadTierBuffer(g.baseBuf, sm ? sm.extra : bArr); + g.baseMeta = scalarBase + ? { offset: baseConst, scale: 1 } + : { ...g.baseMeta, offset: upd.base.offset, scale: upd.base.scale }; + g._dashBase = sm ? sm.extra : bArr; } g.n = st ? st.n : src.n; } diff --git a/js/src/55_marks.ts b/js/src/55_marks.ts index 43a0d03a..738be795 100644 --- a/js/src/55_marks.ts +++ b/js/src/55_marks.ts @@ -126,7 +126,7 @@ const AREA_MARK = { const yBuf = g.yBuf, yMeta = g.yMeta, dashY = g._dashY; g.yBuf = g.baseBuf; g.yMeta = g.baseMeta; - g._dashY = g._cpu.base; + g._dashY = g._dashBase; view._drawLine(g, xm, ym, g.lineColor, g.trace.style.line_width, g.trace.style.line_opacity ?? 1); g.yBuf = yBuf; g.yMeta = yMeta; diff --git a/python/reflex-xy/reflex_xy/payload_asset.py b/python/reflex-xy/reflex_xy/payload_asset.py index 4854f2a8..4bcf2976 100644 --- a/python/reflex-xy/reflex_xy/payload_asset.py +++ b/python/reflex-xy/reflex_xy/payload_asset.py @@ -34,9 +34,10 @@ import hashlib from pathlib import Path +from secrets import token_hex from typing import Any -from xy.channel import encode_frame +from xy.channel import encode_frame_parts from .registry import _figure_of @@ -65,8 +66,14 @@ def payload_asset(chart_or_figure: Any) -> str: figure = _figure_of(chart_or_figure) spec, blob = figure.build_payload() - frame = encode_frame(spec, [blob]) - digest = hashlib.sha256(frame).hexdigest()[:_DIGEST_CHARS] + # Keep the packed payload owner alive while the scatter/gather views are + # hashed and consumed. This preserves the exact XYBF bytes without a + # second payload-sized `b"".join(...)` allocation. + frame_parts = encode_frame_parts(spec, [blob]) + frame_hash = hashlib.sha256() + for part in frame_parts: + frame_hash.update(part) + digest = frame_hash.hexdigest()[:_DIGEST_CHARS] name = f"{digest}{_SUFFIX}" if _should_write(): @@ -75,10 +82,22 @@ def payload_asset(chart_or_figure: Any) -> str: dest = asset_dir / name if not dest.exists(): # Content-addressed, so concurrent writers (multiple workers - # importing the app module) produce identical bytes; the rename - # keeps a racing reader from ever seeing a partial file. - tmp = asset_dir / f".{name}.tmp" - tmp.write_bytes(frame) - tmp.replace(dest) + # importing the app module) produce identical bytes. A unique + # sibling temp avoids writers clobbering one another, and the + # final rename keeps readers from ever seeing a partial frame. + tmp: Path | None = None + try: + # `xb` gives the temp the same umask-governed permissions as + # the old direct `write_bytes` path while still refusing an + # existing name. The random suffix makes that name private to + # this writer without the 0600 mode imposed by mkstemp. + tmp = asset_dir / f".{name}.{token_hex(16)}.tmp" + with tmp.open("xb") as stream: + for part in frame_parts: + stream.write(part) + tmp.replace(dest) + finally: + if tmp is not None: + tmp.unlink(missing_ok=True) return AssetPathStr(f"/{ASSET_SUBDIR}/{name}") diff --git a/python/xy/_figure.py b/python/xy/_figure.py index 5493d5e8..ab2e560f 100644 --- a/python/xy/_figure.py +++ b/python/xy/_figure.py @@ -472,6 +472,7 @@ def _append_bar_rect( color_ch: Optional[ColorChannel] = None, stroke_ch: Optional[ColorChannel] = None, style_channels: Optional[dict[str, Any]] = None, + x_center: Optional[np.ndarray] = None, ) -> None: if orientation == "vertical": self._append_rect_trace( @@ -489,6 +490,7 @@ def _append_bar_rect( color_ch=color_ch, stroke_ch=stroke_ch, style_channels=style_channels, + x_center=x_center, ) else: self._append_rect_trace( @@ -936,6 +938,7 @@ def _append_rect_trace( style_channels: Optional[dict[str, Any]] = None, count: Optional[int] = None, extra_style: Optional[dict[str, Any]] = None, + x_center: Optional[np.ndarray] = None, ) -> None: name = self._optional_text(name, f"{kind} name") opacity = self._opacity(opacity, f"{kind} opacity") @@ -945,6 +948,8 @@ def _append_rect_trace( self._rect_edge_len(y0, f"{kind} y0"), self._rect_edge_len(y1, f"{kind} y1"), } + if x_center is not None: + lengths.add(self._rect_edge_len(x_center, f"{kind} x center")) if len(lengths) != 1: raise ValueError(f"{kind} rectangle columns must have equal length") checkpoint = self._checkpoint() @@ -953,7 +958,10 @@ def _append_rect_trace( x1c = self.store.ingest(x1) y0c = self.store.ingest(y0) y1c = self.store.ingest(y1) - xc = self.store.ingest(x0c.values + (x1c.values - x0c.values) / 2.0) + center = ( + x_center if x_center is not None else self._rect_midpoint(x0c.values, x1c.values) + ) + xc = self.store.ingest(center) yc = self.store.ingest(y1c.values) style: dict[str, Any] = {"color": color, "opacity": opacity, "role": role} if orientation is not None: @@ -983,6 +991,19 @@ def _append_rect_trace( self._rollback(checkpoint) raise + @staticmethod + def _rect_midpoint(left: np.ndarray, right: np.ndarray) -> np.ndarray: + """Overflow-safe midpoint with one owned temporary. + + Keep the historical operation order (`left + (right-left)/2`) so + encoded bytes do not drift, but reuse the subtraction output for the + divide and add instead of materializing three full-size temporaries. + """ + center = np.subtract(right, left) + np.divide(center, 2.0, out=center) + np.add(left, center, out=center) + return center + @staticmethod def _rect_edge_len(values: Any, label: str) -> int: if hasattr(values, "to_numpy"): @@ -1014,6 +1035,10 @@ def _range(self, axis_id: str, *, use_domain: bool = True) -> tuple[float, float for col in self._range_columns(t, axis_id): lo = min(lo, col.min) hi = max(hi, col.max) + base_const = self._range_base_const(t, axis_id) + if base_const is not None: + lo = min(lo, base_const) + hi = max(hi, base_const) if not np.isfinite(lo) or not np.isfinite(hi): lo, hi = 0.0, 1.0 scale = self._axis_scale(axis_id) @@ -1025,6 +1050,10 @@ def _range(self, axis_id: str, *, use_domain: bool = True) -> tuple[float, float if np.isfinite(col.zone.positive_min): positive_los.append(col.zone.positive_min) positive_his.append(col.zone.positive_max) + base_const = self._range_base_const(t, axis_id) + if base_const is not None and base_const > 0: + positive_los.append(base_const) + positive_his.append(base_const) if not positive_los: raise ValueError(f"{axis_id} log axis requires at least one positive value") lo, hi = min(positive_los), max(positive_his) @@ -1204,6 +1233,19 @@ def _range_columns(self, t: Trace, axis_id: str) -> list[Column]: return [t.x0, t.x1] if axis == "x" else [t.y0, t.y1] return [t.x if axis == "x" else t.y] + def _range_base_const(self, t: Trace, axis_id: str) -> Optional[float]: + """Scalar area baseline contributing to a matching y-axis range.""" + if ( + self._axis_dim(axis_id) == "y" + and t.y_axis == axis_id + and t.kind in {"area", "error_band"} + and t.base is None + and t.base_const is not None + and t.n_points > 0 + ): + return t.base_const + return None + # -- payload -------------------------------------------------------------- def _interaction_spec(self) -> dict[str, Any]: @@ -1642,7 +1684,7 @@ def html( def _repr_html_(self) -> str: """Notebook HTML repr isolated from the host document's styles.""" - return export.notebook_iframe(self.to_html(), width=self.width, height=self.height) + return export.notebook_figure_iframe(self, width=self.width, height=self.height) def to_svg( self, @@ -1768,7 +1810,6 @@ def memory_report(self) -> dict[str, Any]: """Every byte class itemized; if it isn't in the report it isn't real.""" from . import interaction # method-local: no load-time cycle - spec, blob = self.build_payload() report = self.store.memory_report() channel_arrays: list[np.ndarray] = [] store_arrays = [column.values for column in self.store.columns] @@ -1798,9 +1839,10 @@ def memory_report(self) -> dict[str, Any]: seen_channels.add(key) channel_arrays.append(array) report["channel_bytes"] = int(sum(array.nbytes for array in channel_arrays)) - report["transport_bytes_first_paint"] = len(blob) + transport_bytes = self.payload_nbytes() + report["transport_bytes_first_paint"] = transport_bytes n_total = sum(t.n_points for t in self.traces) or 1 - report["transport_bytes_per_point"] = len(blob) / n_total + report["transport_bytes_per_point"] = transport_bytes / n_total report["pyramid_bytes"] = interaction.pyramid_report_bytes(self) report["resident_array_bytes"] = ( report["canonical_bytes"] + report["channel_bytes"] + report["pyramid_bytes"] diff --git a/python/xy/_native.py b/python/xy/_native.py index fb5182c5..4d33467e 100644 --- a/python/xy/_native.py +++ b/python/xy/_native.py @@ -24,7 +24,7 @@ from .config import MAX_CONTOUR_WORK, MAX_SCREEN_DIM -ABI_VERSION = 37 +ABI_VERSION = 38 # Rust reports invalid arguments (and, via the ffi_guard panic shield, any # internal panic) by returning `usize::MAX` from size-returning entry points. @@ -148,6 +148,15 @@ def _load() -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_void_p, ] + lib.xy_datetime64_to_ms.restype = ctypes.c_int32 + lib.xy_datetime64_to_ms.argtypes = [ + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_ssize_t, + ctypes.c_int64, + ctypes.c_int64, + ctypes.c_void_p, + ] lib.xy_encode_f32.restype = ctypes.c_int32 lib.xy_encode_f32.argtypes = [ ctypes.c_void_p, @@ -1103,6 +1112,42 @@ def unpack(records: np.ndarray) -> tuple[np.ndarray, ...]: return unpack(x_records), unpack(y_records) +def datetime64_to_ms( + values: npt.NDArray[np.int64], numerator: int, denominator: int +) -> npt.NDArray[np.float64]: + """Convert datetime ticks to whole-ms f64 with one output allocation. + + ``values`` may be strided (including reversed); the native loop reads the + source view directly. NumPy's NaT sentinel maps to NaN and finer-than-ms + units use exact floor division, including for negative pre-epoch values. + """ + arr = np.asarray(values) + if arr.ndim != 1 or arr.dtype != np.dtype(np.int64): + raise ValueError("datetime64 ticks must be a 1-D int64 array") + try: + numerator = int(numerator) + denominator = int(denominator) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError("datetime64 conversion ratio must be positive int64") from exc + i64_max = np.iinfo(np.int64).max + if not (0 < numerator <= i64_max and 0 < denominator <= i64_max): + raise ValueError("datetime64 conversion ratio must be positive int64") + out = np.empty(len(arr), dtype=np.float64) + status = _lib.xy_datetime64_to_ms( + arr.ctypes.data, + len(arr), + arr.strides[0], + numerator, + denominator, + out.ctypes.data, + ) + if status == -1: + raise OverflowError("Overflow when converting between datetime64 units") + if status != 1: + raise ValueError("invalid datetime64 conversion arguments") + return out + + def encode_f32( data: npt.NDArray[np.float64], offset: float, scale: float = 1.0 ) -> npt.NDArray[np.float32]: diff --git a/python/xy/_payload.py b/python/xy/_payload.py index f6c10eaa..e61bcdeb 100644 --- a/python/xy/_payload.py +++ b/python/xy/_payload.py @@ -73,20 +73,20 @@ def ship_scalar(self, values: np.ndarray) -> int: return self._append(enc, {}) def ship_u8(self, values: np.ndarray) -> int: - """Raw byte column, padded so every later f32 column stays aligned.""" + """Raw byte column; packed mode pads only between shared-blob columns.""" enc = np.ascontiguousarray(values, dtype=np.uint8).reshape(-1) index = len(self.columns) if self._split: - # One buffer per column: fold the alignment padding into the u8 - # buffer itself (spec `len` still counts only real values), so the - # split layout stays a byte-identical repack of the packed blob. - padding = (-len(enc)) % 4 - padded = np.concatenate([enc, np.zeros(padding, np.uint8)]) if padding else enc + # A split column starts at offset zero in its own wire buffer, so + # padding that only aligned the *next* packed-blob column is dead + # transport. Retain the encoded owner and ship its meaningful + # bytes directly; the client already addresses it by (`buf`, 0, + # `len`) and therefore observes the exact same column values. self.columns.append( {"buf": len(self._chunks), "byte_offset": 0, "len": int(len(enc)), "dtype": "u8"} ) - self._chunks.append(padded) - self._pos += padded.nbytes + self._chunks.append(enc) + self._pos += enc.nbytes return index self.columns.append({"byte_offset": self._pos, "len": int(len(enc)), "dtype": "u8"}) self._chunks.append(enc) @@ -160,6 +160,55 @@ def buffers(self) -> list[memoryview]: ] +class _PayloadSizeWriter: + """Count packed first-paint bytes without encoding geometry or joining. + + Emitters still make the same tier/selection decisions, so the report is + exact rather than a source-row estimate. Offset-encoded geometry arrays and + the joined payload blob are skipped. Emitters may still prepare temporary + channel values (for example continuous normalization or RGBA8 packing) + before passing their lengths to this shared writer. + """ + + def __init__(self) -> None: + self.columns: list[dict[str, Any]] = [] + self.nbytes = 0 + self.borrow_heatmaps = False + + def _append_count(self, length: int, itemsize: int, dtype: str | None = None) -> int: + index = len(self.columns) + meta: dict[str, Any] = {"byte_offset": self.nbytes, "len": int(length)} + if dtype is not None: + meta["dtype"] = dtype + self.columns.append(meta) + self.nbytes += int(length) * itemsize + return index + + def ship(self, values: np.ndarray, col: "Column") -> int: + del col + return self._append_count(np.asarray(values).size, 4) + + def ship_scalar(self, values: np.ndarray) -> int: + return self._append_count(np.asarray(values).size, 4) + + def ship_u8(self, values: np.ndarray) -> int: + index = self._append_count(np.asarray(values).size, 1, "u8") + self.nbytes += (-self.nbytes) % 4 + return index + + def ship_u32(self, values: np.ndarray) -> int: + return self._append_count(np.asarray(values).size, 4, "u32") + + def ship_values(self, values: np.ndarray, *, kind: str = "float") -> int: + del kind + return self._append_count(np.asarray(values).size, 4) + + def borrow_f64(self, values: np.ndarray) -> int: + # Not used by ordinary first paint; keep the interface complete for a + # future emitter shared with raster-only payloads. + return self._append_count(np.asarray(values).size, 8, "f64") + + class PayloadMixin(_Host): def build_payload(self, px_width: Optional[int] = None) -> tuple[dict[str, Any], bytes]: """Encode every trace for first paint: (spec, binary buffer blob). @@ -192,6 +241,12 @@ def build_payload_split( spec["buffer_layout"] = "split" return spec, pw.buffers() + def payload_nbytes(self, px_width: Optional[int] = None) -> int: + """Exact packed bytes without geometry encoding or a joined blob.""" + pw = _PayloadSizeWriter() + self._payload_spec(pw, self._resolve_px_width(px_width)) + return pw.nbytes + def _build_raster_payload( self, px_width: Optional[int] = None ) -> tuple[dict[str, Any], bytes, tuple[np.ndarray, ...]]: @@ -432,18 +487,35 @@ def _emit_line( def _emit_area( self, t: Trace, pw: "_PayloadWriter", xr: tuple, yr: tuple, px_width: int ) -> dict[str, Any]: - if t.base is None: + base_const = t.resolved_base_const() + if t.base is None and base_const is None: raise ValueError("area trace missing baseline column") - tier, (xv, yv, bv) = self._m4_decimate( - t, xr, px_width, t.x.values, t.y.values, t.base.values - ) - sel = np.flatnonzero(self._log_visible_mask(t, xv, yv, bv)) - if len(sel) != len(xv): - xv, yv, bv = xv[sel], yv[sel], bv[sel] + if base_const is not None: + tier, (xv, yv) = self._m4_decimate(t, xr, px_width, t.x.values, t.y.values) + visible = self._log_visible_mask(t, xv, yv) + if self._axis_scale(t.y_axis) == "log" and base_const <= 0: + visible &= False + bv = None + else: + assert t.base is not None + tier, (xv, yv, bv) = self._m4_decimate( + t, xr, px_width, t.x.values, t.y.values, t.base.values + ) + visible = self._log_visible_mask(t, xv, yv, bv) + sel = None + if not bool(np.all(visible)): + sel = np.flatnonzero(visible) + xv, yv = xv[sel], yv[sel] + if bv is not None: + bv = bv[sel] entry = self._base_entry(t, pw, xv, yv, tier, self._default_styled(t)) if t.transition_keys is not None: self._transition_entry(entry, t, pw, sel) - entry["base"] = pw.ship(bv, t.base) + if base_const is not None: + entry["base_const"] = float(base_const) + else: + assert bv is not None and t.base is not None + entry["base"] = pw.ship(bv, t.base) return entry def _emit_error_band( @@ -498,8 +570,11 @@ def _emit_hexbin( "n_marks": int(len(xv)), "x_axis": t.x_axis, "y_axis": t.y_axis, - "x": pw.ship_values(xv), - "y": pw.ship_values(yv), + # Hexbin centers already live in canonical Columns with materialized + # zone maps; reuse their offset/bounds instead of scanning min/max + # again in ``ship_values``. + "x": pw.ship(xv, t.x), + "y": pw.ship(yv, t.y), } entry["color"], _size = self._ship_channels(t, sel, pw.ship_scalar, pw.ship_u8) return entry diff --git a/python/xy/_raster.py b/python/xy/_raster.py index 24109955..513b036d 100644 --- a/python/xy/_raster.py +++ b/python/xy/_raster.py @@ -1221,7 +1221,11 @@ def _emit_area( ) -> None: xv = _column(blob, cols[t["x"]]) yv = _column(blob, cols[t["y"]]) - bv = _column(blob, cols[t["base"]]) + bv = ( + _column(blob, cols[t["base"]]) + if "base" in t + else np.full(xv.shape, float(t["base_const"]), dtype=np.float64) + ) smooth = style.get("curve") == "smooth" top = _scene.curve_points(xv, yv, sx, sy, smooth) base = _scene.curve_points(xv[::-1], bv[::-1], sx, sy, smooth) diff --git a/python/xy/_svg.py b/python/xy/_svg.py index 34c20ecb..380291ce 100644 --- a/python/xy/_svg.py +++ b/python/xy/_svg.py @@ -1539,7 +1539,11 @@ def line_attrs(style: dict[str, Any], color: str) -> str: elif kind in ("area", "error_band"): xv = _column(blob, cols[t["x"]]) yv = _column(blob, cols[t["y"]]) - bv = _column(blob, cols[t["base"]]) + bv = ( + _column(blob, cols[t["base"]]) + if "base" in t + else np.full(xv.shape, float(t["base_const"]), dtype=np.float64) + ) smooth = style.get("curve") == "smooth" top_path = _curve_path(xv, yv, trace_sx, trace_sy, smooth) base_path = _curve_path(xv[::-1], bv[::-1], trace_sx, trace_sy, smooth) diff --git a/python/xy/_trace.py b/python/xy/_trace.py index 14e4e4d5..14c9eac3 100644 --- a/python/xy/_trace.py +++ b/python/xy/_trace.py @@ -26,6 +26,11 @@ class Trace: # Area-style marks keep an explicit baseline column; rectangle-like marks # use x0/x1/y0/y1 below. base: Optional[Column] = None + # Scalar area baselines stay scalar end-to-end: no N-length canonical + # column and no N-length wire buffer. A constant error-band lower column + # may retain ``base`` for canonical/reporting purposes while also exposing + # this wire optimization. + base_const: Optional[float] = None # Grid-like marks (heatmap/image) ship one scalar grid plus metadata instead # of four rectangle columns per cell. grid: Optional[Column] = None @@ -83,6 +88,17 @@ def n_points(self) -> int: return self.count return len(self.x) + def resolved_base_const(self) -> Optional[float]: + """Return a safe scalar baseline without rescanning canonical data.""" + if self.base_const is not None: + return self.base_const + if self.base is None or len(self.base) == 0: + return None + zone = self.base.zone + if zone.null_count == 0 and zone.count == len(self.base) and self.base.min == self.base.max: + return self.base.min + return None + def per_item_channel_names(self) -> tuple[str, ...]: """Names of channels whose values vary independently per rendered item.""" names: list[str] = [] diff --git a/python/xy/channels.py b/python/xy/channels.py index e91482fa..75e253cb 100644 --- a/python/xy/channels.py +++ b/python/xy/channels.py @@ -537,7 +537,7 @@ def ship_color_channel( if rgba is None: raise ValueError("direct RGBA color channel missing values") values = rgba if sel is None else rgba[sel] - packed = np.rint(np.clip(values, 0.0, 1.0) * 255.0).astype(np.uint8) + packed = _pack_direct_rgba(values, source=rgba) color_spec["buf"] = ship_u8(packed.reshape(-1)) color_spec["n"] = int(len(values)) elif cc.mode == "match_fill": @@ -569,6 +569,29 @@ def ship_color_channel( return color_spec +def _pack_direct_rgba( + values: npt.NDArray[np.float64], *, source: npt.NDArray[np.float64] +) -> npt.NDArray[np.uint8]: + """Pack canonical straight-alpha floats to RGBA8 with bounded scratch. + + A fancy-indexed row selection is already a detached temporary, so it can + serve as the float scratch. Full columns and slice views still belong to + the canonical channel and must never be mutated. Clipping stays in place + to preserve the defensive behavior if internal channel storage is edited + after validation, and `rint` keeps the existing round-to-even bytes. + """ + if values.flags.writeable and not np.shares_memory(values, source): + scaled = values + np.clip(scaled, 0.0, 1.0, out=scaled) + else: + scaled = np.empty_like(values) + np.clip(values, 0.0, 1.0, out=scaled) + np.multiply(scaled, 255.0, out=scaled) + packed = np.empty(scaled.shape, dtype=np.uint8) + np.rint(scaled, out=packed, casting="unsafe") + return packed + + def resolve_style_channel( value: Any, n: int, diff --git a/python/xy/columns.py b/python/xy/columns.py index 549f5010..719c5a0a 100644 --- a/python/xy/columns.py +++ b/python/xy/columns.py @@ -19,6 +19,7 @@ from __future__ import annotations import datetime as dt +import math from dataclasses import dataclass from functools import cached_property from typing import Any @@ -426,6 +427,22 @@ def _datetime_to_float_ms( arr: npt.NDArray[Any], copies: int ) -> tuple[npt.NDArray[np.float64], int]: """Canonicalize datetime-like columns to f64 ms, preserving nulls as NaN.""" + if np.issubdtype(arr.dtype, np.datetime64) and arr.dtype.isnative: + ratio = _fixed_datetime_ms_ratio(arr.dtype) + if ratio is not None: + numerator, denominator = ratio + try: + # The native loop consumes the datetime array's i64 view at + # its original stride and writes canonical f64 ms directly: + # one full-size output, rather than datetime64[ms] plus f64. + return ( + kernels.datetime64_to_ms(arr.view(np.int64), numerator, denominator), + copies + 1, + ) + except ValueError: + # Exotic dtype multipliers outside the int64 ABI ratio keep + # NumPy's general calendar-aware fallback below. + pass try: dt_ms, copies = _astype_counted(arr, "datetime64[ms]", copies) except (TypeError, ValueError) as e: @@ -437,6 +454,39 @@ def _datetime_to_float_ms( return out, copies +_FIXED_DATETIME_MS_RATIOS: dict[str, tuple[int, int]] = { + "W": (7 * 24 * 60 * 60 * 1000, 1), + "D": (24 * 60 * 60 * 1000, 1), + "h": (60 * 60 * 1000, 1), + "m": (60 * 1000, 1), + "s": (1000, 1), + "ms": (1, 1), + "us": (1, 1000), + "ns": (1, 1_000_000), + "ps": (1, 1_000_000_000), + "fs": (1, 1_000_000_000_000), + "as": (1, 1_000_000_000_000_000), +} + + +def _fixed_datetime_ms_ratio(dtype: np.dtype[Any]) -> tuple[int, int] | None: + """Milliseconds per datetime tick for fixed-duration NumPy units. + + Years and months deliberately return ``None``: their conversion is + calendar-dependent, so NumPy remains the correctness oracle for them. + """ + unit, step = np.datetime_data(dtype) + base = _FIXED_DATETIME_MS_RATIOS.get(unit) + if base is None: + return None + numerator = int(base[0]) * int(step) + denominator = int(base[1]) + # Reduce before crossing the fixed-width C ABI. This also handles dtypes + # such as datetime64[1000us] as the exact 1 ms/tick ratio. + divisor = math.gcd(numerator, denominator) + return numerator // divisor, denominator // divisor + + def _astype_counted(arr: npt.NDArray[Any], dtype: Any, copies: int) -> tuple[npt.NDArray[Any], int]: out = arr.astype(dtype, copy=False) if out is not arr and not np.shares_memory(out, arr): diff --git a/python/xy/export.py b/python/xy/export.py index e4915375..7d679d52 100644 --- a/python/xy/export.py +++ b/python/xy/export.py @@ -14,6 +14,7 @@ import subprocess import tempfile import warnings +from collections.abc import Iterator from contextlib import suppress from enum import StrEnum from os import PathLike @@ -262,22 +263,13 @@ def _custom_css_block(custom_css: Optional[str]) -> str: return f"\n" -def to_html( +def _standalone_parts( fig: "Figure", - path: Optional[str | PathLike[str]] = None, *, custom_css: Optional[str] = None, animation_progress: Optional[float] = None, -) -> str: - """Render `fig` to a standalone interactive HTML string (optionally saved). - - User strings (title, names, labels) ride inside ' for c in _base64_chunks(blob) - ) - doc = f""" + yield f""" @@ -319,10 +308,14 @@ def to_html( {_custom_css_block(custom_css)}
- + -{chunk_scripts} -\n' + yield f""" """ + + +def to_html( + fig: "Figure", + path: Optional[str | PathLike[str]] = None, + *, + custom_css: Optional[str] = None, + animation_progress: Optional[float] = None, +) -> str: + """Render `fig` to a standalone interactive HTML string (optionally saved). + + User strings (title, names, labels) ride inside