From 757318e5b22df3d197f386132299206776066127 Mon Sep 17 00:00:00 2001 From: Alek Date: Thu, 9 Jul 2026 13:27:16 -0700 Subject: [PATCH 1/3] Improve dashboard telemetry and PNG export speed --- README.md | 5 +- benchmarks/README.md | 5 +- benchmarks/bench_dashboard.py | 169 +++++++++++++++++++---- benchmarks/bench_interaction.py | 6 +- benchmarks/categories.py | 4 +- docs/benchmark.md | 7 +- docs/design/benchmark-methodology.md | 6 +- python/fastcharts/_png.py | 8 +- scripts/verify_benchmark_report.py | 188 +++++++++++++++++++++++++- tests/test_benchmark_environment.py | 29 ++++ tests/test_png_export.py | 18 +++ tests/test_verify_benchmark_report.py | 113 ++++++++++++---- 12 files changed, 489 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 460f76e3..9c9b6f5e 100644 --- a/README.md +++ b/README.md @@ -501,8 +501,9 @@ fails if any of those required interaction rows disappear. `bench_workflows.py` covers contiguous/converted/strided/datetime/list/Arrow ingestion, line append, density append followed by a real 2M+ pyramid rebuild, and HTML/SVG/native-PNG/Chromium-PNG export. The dashboard benchmark attempts 10, 20, and 50 charts, -records JS heap and steady-redraw pacing, and reports the largest fully nonblank -count rather than assuming every attempted dashboard succeeded. +records per-chart context loss/restoration, initial and scrolled visibility, JS +heap, and redraw submission pacing, then reports the largest loss-free nonblank +count without discarding partial-dashboard metrics. ### 10M-point native benchmark diff --git a/benchmarks/README.md b/benchmarks/README.md index 3f34d4a1..18e3d9af 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -67,7 +67,8 @@ JSON artifacts, retain failed/over-budget rows, and label the table - Interactive TTFR is build + HTML serialization + chart-ready time. - Interaction browser rows are standalone client input-to-pixel-readback; backend LOD work is in CodSpeed and workflow rows. -- Dashboard rows attempt 10/20/50 charts and publish the largest fully nonblank - count, including context-limit failures. +- Dashboard rows attempt 10/20/50 charts, retain timings for partial dashboards, + record per-chart context loss/restoration plus initial/scrolled nonblank IDs, + and publish the largest stable loss-free count. - Density rows must include a count-conservation oracle and explicit aggregate dimensions. A density result is not an exact-marker result. diff --git a/benchmarks/bench_dashboard.py b/benchmarks/bench_dashboard.py index bcfd7895..61c4b1a7 100644 --- a/benchmarks/bench_dashboard.py +++ b/benchmarks/bench_dashboard.py @@ -116,43 +116,128 @@ def _probe_js() -> str: (async () => { try { const root = document.getElementById("root"); - const views = []; + const slots = []; + const contextEvents = []; + const creationFailureIds = []; + let phase = "create"; const heapBefore = performance.memory ? performance.memory.usedJSHeapSize : null; const t0 = performance.now(); for (const payload of FC_CHARTS) { const cell = document.createElement("div"); cell.className = "chart-cell"; + cell.dataset.chartId = payload.id; root.appendChild(cell); - const view = fastcharts.renderStandalone(cell, payload.spec, fcBytesFromB64(payload.b64)); - view._drawNow(); - views.push(view); + try { + const view = fastcharts.renderStandalone(cell, payload.spec, fcBytesFromB64(payload.b64)); + const state = {lost: false}; + view.canvas.addEventListener("webglcontextlost", () => { + state.lost = true; + contextEvents.push({id: payload.id, type: "lost", phase, at_ms: performance.now() - t0}); + }); + view.canvas.addEventListener("webglcontextrestored", () => { + state.lost = false; + contextEvents.push({id: payload.id, type: "restored", phase, at_ms: performance.now() - t0}); + }); + view._drawNow(); + slots.push({id: payload.id, cell, view, state}); + } catch (err) { + creationFailureIds.push(payload.id); + slots.push({id: payload.id, cell, view: null, state: {lost: true}}); + } } - await fcRaf(); - let nonblank = 0; - for (const view of views) { - if (fcNonblankPixels(view) > 0) nonblank++; + phase = "initial"; + await new Promise((resolve) => setTimeout(resolve, 0)); + + function contextLost(slot) { + if (!slot.view || !slot.view.gl) return true; + return slot.state.lost || slot.view._glLost || slot.view.gl.isContextLost(); + } + function nonblankPixels(slot) { + if (contextLost(slot)) return 0; + try { + return fcNonblankPixels(slot.view); + } catch (_err) { + return 0; + } + } + function sampleNonblank() { + const lit = []; + for (const slot of slots) if (nonblankPixels(slot) > 0) lit.push(slot.id); + return lit; } - if (nonblank !== views.length) { - throw new Error(`blank dashboard chart: ${nonblank}/${views.length} nonblank`); + function complement(ids) { + const present = new Set(ids); + return slots.filter((slot) => !present.has(slot.id)).map((slot) => slot.id); } + + const initialNonblankIds = sampleNonblank(); const renderMs = performance.now() - t0; const navigationReadyMs = performance.now(); + + phase = "scroll"; + const scrollStart = performance.now(); + const scrollNonblankIds = []; + for (const slot of slots) { + slot.cell.scrollIntoView({block: "center", inline: "center"}); + await new Promise((resolve) => setTimeout(resolve, 0)); + if (nonblankPixels(slot) > 0) scrollNonblankIds.push(slot.id); + } + window.scrollTo(0, 0); + await new Promise((resolve) => setTimeout(resolve, 0)); + const scrollPassMs = performance.now() - scrollStart; + + phase = "redraw"; const steadyRedraws = []; for (let i = 0; i < 12; i++) { const redrawStart = performance.now(); - for (const view of views) view._drawNow(); + for (const slot of slots) if (!contextLost(slot)) slot.view._drawNow(); steadyRedraws.push(performance.now() - redrawStart); } + await new Promise((resolve) => setTimeout(resolve, 0)); + phase = "report"; + const heapAfter = performance.memory ? performance.memory.usedJSHeapSize : null; + const lostEvents = contextEvents.filter((event) => event.type === "lost"); + const restoredEvents = contextEvents.filter((event) => event.type === "restored"); + const uniqueIds = (events) => Array.from(new Set(events.map((event) => event.id))); + const currentlyLostIds = slots + .filter((slot) => slot.view && contextLost(slot)) + .map((slot) => slot.id); + const createdCharts = slots.length - creationFailureIds.length; + const fullyNonblank = + createdCharts === slots.length && + initialNonblankIds.length === slots.length && + scrollNonblankIds.length === slots.length && + lostEvents.length === 0 && + currentlyLostIds.length === 0; fcReport("FC_DASHBOARD", { status: "ok", + render_status: fullyNonblank ? "complete" : "partial", + fully_nonblank: fullyNonblank, render_ms: renderMs, navigation_ready_ms: navigationReadyMs, + scroll_pass_ms: scrollPassMs, steady_redraw_p95_ms: fcPercentile(steadyRedraws, 95), + steady_redraw_active_charts: slots.filter((slot) => slot.view && !contextLost(slot)).length, js_heap_before_bytes: heapBefore, js_heap_bytes: heapAfter, js_heap_delta_bytes: heapBefore == null || heapAfter == null ? null : heapAfter - heapBefore, - nonblank_charts: nonblank, + created_charts: createdCharts, + creation_failed_charts: creationFailureIds.length, + creation_failure_ids: creationFailureIds, + nonblank_charts: initialNonblankIds.length, + initial_nonblank_charts: initialNonblankIds.length, + initial_nonblank_chart_ids: initialNonblankIds, + initial_blank_chart_ids: complement(initialNonblankIds), + scroll_nonblank_charts: scrollNonblankIds.length, + scroll_nonblank_chart_ids: scrollNonblankIds, + scroll_blank_chart_ids: complement(scrollNonblankIds), + context_lost_events: lostEvents.length, + context_restored_events: restoredEvents.length, + context_lost_chart_ids: uniqueIds(lostEvents), + context_restored_chart_ids: uniqueIds(restoredEvents), + currently_lost_chart_ids: currentlyLostIds, + context_events: contextEvents, }); } catch (err) { fcFail("FC_DASHBOARD", err); @@ -196,20 +281,44 @@ def run(*, chart_counts: list[int], chromium: str | None = None) -> dict[str, An "status": result.get("status", "failed(no status)"), } if row["status"] == "ok": - row["render_ms"] = result.get("render_ms") + metric_keys = ( + "render_status", + "fully_nonblank", + "render_ms", + "navigation_ready_ms", + "scroll_pass_ms", + "steady_redraw_p95_ms", + "steady_redraw_active_charts", + "js_heap_before_bytes", + "js_heap_bytes", + "js_heap_delta_bytes", + "created_charts", + "creation_failed_charts", + "creation_failure_ids", + "nonblank_charts", + "initial_nonblank_charts", + "initial_nonblank_chart_ids", + "initial_blank_chart_ids", + "scroll_nonblank_charts", + "scroll_nonblank_chart_ids", + "scroll_blank_chart_ids", + "context_lost_events", + "context_restored_events", + "context_lost_chart_ids", + "context_restored_chart_ids", + "currently_lost_chart_ids", + "context_events", + ) + row.update({key: result.get(key) for key in metric_keys}) row["ms_per_chart"] = ( result.get("render_ms") / count if result.get("render_ms") is not None else None ) - row["nonblank_charts"] = result.get("nonblank_charts") - row["navigation_ready_ms"] = result.get("navigation_ready_ms") - row["steady_redraw_p95_ms"] = result.get("steady_redraw_p95_ms") - row["js_heap_before_bytes"] = result.get("js_heap_before_bytes") - row["js_heap_bytes"] = result.get("js_heap_bytes") - row["js_heap_delta_bytes"] = result.get("js_heap_delta_bytes") rows.append(row) successful_counts = [ - row["chart_count"] for row in rows if str(row.get("status", "")).startswith("ok") + row["chart_count"] + for row in rows + if str(row.get("status", "")).startswith("ok") and row.get("fully_nonblank") is True ] return { "schema_version": SCHEMA_VERSION, @@ -249,34 +358,42 @@ def to_markdown(report: dict[str, Any]) -> str: "Tracked in this run: " + ", ".join(f"`{category['id']}`" for category in report["tracked_categories"]), "", - f"Successful nonblank chart-count ceiling: `{report.get('chart_count_ceiling')}`.", + f"Stable loss-free chart-count ceiling: `{report.get('chart_count_ceiling')}`.", "", "## Results", "", - "| scenario | charts | prep | navigation ready | render | redraw p95 | JS heap | payload | html | nonblank charts | status |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|", + "| scenario | charts | prep | navigation ready | render | scroll pass | redraw submit p95 | active | JS heap | payload | html | initial/scroll nonblank | loss/restore | health |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] for row in report["rows"]: lines.append( - "| {scenario} | {count} | {prep} | {navigation} | {render} | {idle} | {heap} | {payload} | {html} | {nonblank} | {status} |".format( + "| {scenario} | {count} | {prep} | {navigation} | {render} | {scroll} | {idle} | {active} | {heap} | {payload} | {html} | {nonblank}/{scroll_nonblank} | {lost}/{restored} | {health} |".format( scenario=row["scenario"], count=row["chart_count"], prep=_fmt_ms(row.get("payload_prep_ms")), navigation=_fmt_ms(row.get("navigation_ready_ms")), render=_fmt_ms(row.get("render_ms")), + scroll=_fmt_ms(row.get("scroll_pass_ms")), idle=_fmt_ms(row.get("steady_redraw_p95_ms")), + active=row.get("steady_redraw_active_charts", "—"), heap=_fmt_bytes(row.get("js_heap_bytes")), payload=_fmt_bytes(row.get("total_payload_bytes")), html=_fmt_bytes(row.get("html_bytes")), nonblank=row.get("nonblank_charts", "—"), - status=row.get("status", "unknown"), + scroll_nonblank=row.get("scroll_nonblank_charts", "—"), + lost=row.get("context_lost_events", "—"), + restored=row.get("context_restored_events", "—"), + health=row.get("render_status", row.get("status", "unknown")), ) ) lines += [ "", "Notes:", "", - "- `render` is measured inside the page from first chart decode to post-RAF WebGL readback.", + "- `render` is measured inside the page from first chart decode through a context-event task yield and WebGL readback.", + "- `redraw submit p95` measures synchronous command submission for every currently live context; `active` states how many charts contributed.", + "- Partial rows retain startup, heap, redraw, context-event, and scrolling metrics; they do not raise the loss-free ceiling.", + "- Context event IDs and phases are retained in JSON so LRU eviction and restoration churn are directly observable.", "- The chart mix cycles through line, scatter, histogram, grouped bar, and heatmap.", ] return "\n".join(lines) diff --git a/benchmarks/bench_interaction.py b/benchmarks/bench_interaction.py index 26386659..b70ba414 100644 --- a/benchmarks/bench_interaction.py +++ b/benchmarks/bench_interaction.py @@ -251,7 +251,7 @@ def _probe_js(reps: int) -> str: const warmY1 = before.y0 + (before.y1 - before.y0) * 0.65; view._selectLocal(warmX0, warmX1, warmY0, warmY1); view._clearSelection(); - view._drawNow(); + settlePixels(); let viewChanged = false; function noteViewChanged() {{ @@ -275,6 +275,10 @@ def _probe_js(reps: int) -> str: view._cancelViewAnimation(); view.view = target; }} + if (view._raf) {{ + cancelAnimationFrame(view._raf); + view._raf = null; + }} const gl = view.gl; view._drawNow(); const pixel = new Uint8Array(4); diff --git a/benchmarks/categories.py b/benchmarks/categories.py index 41386a8f..49db082a 100644 --- a/benchmarks/categories.py +++ b/benchmarks/categories.py @@ -67,10 +67,10 @@ "id": "many_chart_dashboards", "name": "Many-chart dashboards", "why": "Plotly-class apps often fail from total page weight and many live canvases, not one chart.", - "metrics": "payload prep, navigation readiness, JS heap, steady redraw, chart count ceiling", + "metrics": "payload prep, navigation readiness, JS heap, redraw submission, scroll visibility, context loss/restore, stable chart-count ceiling", "harness": "benchmarks/bench_dashboard.py", "status": "tracked", - "goal": "Measure the 10-50 chart scaling curve and expose browser context ceilings.", + "goal": "Measure the 10-50 chart scaling curve and expose LRU context eviction without discarding partial-row metrics.", }, { "id": "interaction_smoothness", diff --git a/docs/benchmark.md b/docs/benchmark.md index a34c1342..f0b71079 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -55,7 +55,7 @@ commit so CI artifacts are quick to inspect from logs. | `huge_scatter_overview` | Huge scatter overview | tracked | Proves screen-bounded rendering for datasets larger than the browser should draw point-for-point. | ingest/bin time, density payload size, peak memory, TTFR | `bench_scatter_native.py`, `bench_vs.py`, `test_first_payload_density_large`, example app assets | Keep resident/render payload flat in N while showing truthful density summaries. | | `adaptive_scatter_drilldown` | Adaptive scatter drilldown | tracked | The large-data claim needs a credible path from overview to exact visible points. | visible-query latency, tier-switch latency, exact-point recovery, badge accuracy | `benchmarks/test_codspeed_kernels.py::test_adaptive_drilldown_cycle` | Exact points when visible count is under budget; sampled/density with explicit counts otherwise. | | `huge_line_time_series` | Huge line / time series | tracked | Common observability and finance workload; Plotly-resampler sets the bar here. | decimation time, zoom re-decimation latency, TTFR, extrema preservation | `benchmarks/bench.py`, `bench_native.py`, `bench_interaction.py`, `test_decimate_view` | Screen-bounded line payloads with extrema-preserving decimation and fast zoom refresh. | -| `many_chart_dashboards` | Many-chart dashboards | tracked | Plotly-class apps often fail from total page weight and many live canvases, not one chart. | payload prep, navigation readiness, JS heap, steady redraw, chart-count ceiling | `benchmarks/bench_dashboard.py` | Measure the 10-50 chart scaling curve and expose browser context ceilings. | +| `many_chart_dashboards` | Many-chart dashboards | tracked | Plotly-class apps often fail from total page weight and many live canvases, not one chart. | payload prep, navigation readiness, JS heap, redraw submission, scroll visibility, context loss/restore, stable chart-count ceiling | `benchmarks/bench_dashboard.py` | Measure the 10-50 chart scaling curve and expose LRU context eviction without discarding partial-row metrics. | | `interaction_smoothness` | Interaction smoothness | tracked | Users judge performance by pan/zoom/hover, not just export time. | pan/zoom FPS, wheel latency, hover latency, tooltip stability, selection latency, frame color delta | `benchmarks/bench_interaction.py` | Stay responsive during interaction, avoid blank/flickering frames, then refine view after interaction settles. | | `payload_export_size` | Payload/export size | tracked | Notebooks, static HTML, docs, and dashboards pay for every byte shipped. | standalone HTML bytes, binary payload bytes, bundle bytes | `bench_vs.py`, `bench_scatter_native.py`, `test_first_payload_density_large`, `test_memory_report_density_medium`, example app asset sizes | Keep data payloads binary and screen-bounded where possible; warn when exact export would be huge. | | `core_2d_chart_breadth` | Core 2D chart breadth | tracked | The library needs to stay fast beyond the scatter wedge: bars, histograms, areas, and heatmaps are everyday chart workloads. | payload-prep time, payload bytes, standalone HTML bytes, TTFR | `benchmarks/bench_2d_charts.py` smoke/standard profiles vs Plotly and Seaborn; `bench_interaction.py`; CodSpeed core-2D payload rows | Beat Plotly on user-visible first paint for common 2D charts while tracking Seaborn raster baselines where applicable. | @@ -142,8 +142,9 @@ also declares `tooltip_sample_count`, and eligible rows must report that exact `tooltip_visible_samples` count so a tooltip that appears once and then flickers away fails verification. `bench_dashboard.py` attempts 10/20/50 mixed dashboards and reports payload prep, -navigation readiness, JS heap, steady-redraw p95, nonblank canvases, and the largest -successful count. `bench_workflows.py` covers ingestion, streaming/pyramid +navigation readiness, JS heap, redraw-submission p95, per-chart context loss and +restore events, initial/scrolled nonblank IDs, and the largest stable loss-free +count. Partial rows retain their timing and memory metrics. `bench_workflows.py` covers ingestion, streaming/pyramid invalidation, and separate HTML/SVG/native-PNG/Chromium-PNG export rows. All three emit schema-versioned JSON with environment metadata and benchmark category IDs. diff --git a/docs/design/benchmark-methodology.md b/docs/design/benchmark-methodology.md index e6339d33..6c6da20c 100644 --- a/docs/design/benchmark-methodology.md +++ b/docs/design/benchmark-methodology.md @@ -118,8 +118,10 @@ reported). `benchmarks/bench_2d_charts.py` stays the Plotly/Seaborn chart-to-pixels comparison. 7. `dashboard_scale`: `benchmarks/bench_dashboard.py` attempts 10/20/50 mixed - charts, checks every canvas, and records payload prep, navigation readiness, - JS heap, steady-redraw p95, and the successful chart-count ceiling. + charts, checks every canvas initially and while scrolling, and records payload + prep, navigation readiness, JS heap, redraw-submission p95, per-chart context + loss/restoration events, and the stable loss-free chart-count ceiling. Partial + dashboards remain successful measurement rows rather than losing their metrics. 8. `install_import`: lower-bound distribution size plus opt-in fresh-venv total site-packages, transitive distribution count, install time, and cold import. 9. `public_workflows`: `benchmarks/bench_workflows.py` tracks ingestion shapes, diff --git a/python/fastcharts/_png.py b/python/fastcharts/_png.py index 5f043db0..7d407d6a 100644 --- a/python/fastcharts/_png.py +++ b/python/fastcharts/_png.py @@ -27,6 +27,7 @@ def _chunk(tag: bytes, data: bytes) -> bytes: _SIG = b"\x89PNG\r\n\x1a\n" +_COMPRESSION_LEVEL = 6 def png_truecolor(w: int, h: int, rgba: bytes) -> bytes: @@ -35,7 +36,10 @@ def png_truecolor(w: int, h: int, rgba: bytes) -> bytes: stride = w * 4 raw = b"".join(b"\x00" + rgba[y * stride : (y + 1) * stride] for y in range(h)) return ( - _SIG + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(raw, 9)) + _chunk(b"IEND", b"") + _SIG + + _chunk(b"IHDR", ihdr) + + _chunk(b"IDAT", zlib.compress(raw, _COMPRESSION_LEVEL)) + + _chunk(b"IEND", b"") ) @@ -51,7 +55,7 @@ def _png_indexed(w: int, h: int, idx: np.ndarray, palette: np.ndarray) -> bytes: out = _SIG + _chunk(b"IHDR", ihdr) + _chunk(b"PLTE", plte) # tRNS may omit trailing opaque (255) entries; keep it simple and always emit. out += _chunk(b"tRNS", trns) - out += _chunk(b"IDAT", zlib.compress(rows.tobytes(), 9)) + _chunk(b"IEND", b"") + out += _chunk(b"IDAT", zlib.compress(rows.tobytes(), _COMPRESSION_LEVEL)) + _chunk(b"IEND", b"") return out diff --git a/scripts/verify_benchmark_report.py b/scripts/verify_benchmark_report.py index 401e02af..664446dc 100644 --- a/scripts/verify_benchmark_report.py +++ b/scripts/verify_benchmark_report.py @@ -116,6 +116,16 @@ def _require_positive_integer(obj: dict[str, Any], key: str, path: str, errors: errors.append(f"{path}.{key} must be > 0") +def _require_nonnegative_integer( + obj: dict[str, Any], key: str, path: str, errors: list[str] +) -> None: + value = obj.get(key) + if isinstance(value, bool) or not isinstance(value, int): + errors.append(f"{path}.{key} must be a nonnegative integer") + elif value < 0: + errors.append(f"{path}.{key} must be >= 0") + + def _require_optional_nonnegative_number( obj: dict[str, Any], key: str, path: str, errors: list[str] ) -> None: @@ -1156,22 +1166,64 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No errors.append(f"{path}.status has unknown value {row.get('status')!r}") _validate_browser_category_list(row, path, category_ids, errors) if status == "ok": + _require_keys( + row, + { + "render_status", + "fully_nonblank", + "render_ms", + "ms_per_chart", + "payload_prep_ms", + "navigation_ready_ms", + "scroll_pass_ms", + "steady_redraw_p95_ms", + "steady_redraw_active_charts", + "created_charts", + "creation_failed_charts", + "creation_failure_ids", + "nonblank_charts", + "initial_nonblank_charts", + "initial_nonblank_chart_ids", + "initial_blank_chart_ids", + "scroll_nonblank_charts", + "scroll_nonblank_chart_ids", + "scroll_blank_chart_ids", + "context_lost_events", + "context_restored_events", + "context_lost_chart_ids", + "context_restored_chart_ids", + "currently_lost_chart_ids", + "context_events", + }, + path, + errors, + ) for key in ( "render_ms", "ms_per_chart", "payload_prep_ms", "navigation_ready_ms", + "scroll_pass_ms", "steady_redraw_p95_ms", ): _require_nonnegative_number(row, key, path, errors) - _require_positive_number(row, "nonblank_charts", path, errors) - if row.get("nonblank_charts") != row.get("chart_count"): - errors.append(f"{path}.nonblank_charts must equal chart_count") + for key in ( + "steady_redraw_active_charts", + "created_charts", + "creation_failed_charts", + "nonblank_charts", + "initial_nonblank_charts", + "scroll_nonblank_charts", + "context_lost_events", + "context_restored_events", + ): + _require_nonnegative_integer(row, key, path, errors) for key in ("js_heap_before_bytes", "js_heap_bytes"): _require_optional_nonnegative_number(row, key, path, errors) delta = row.get("js_heap_delta_bytes") if delta is not None and not _is_number(delta): errors.append(f"{path}.js_heap_delta_bytes must be a finite number or null") + _validate_dashboard_telemetry(row, path, errors) attempted_counts = {row.get("chart_count") for row in rows if isinstance(row, dict)} missing_counts = sorted(DASHBOARD_REQUIRED_COUNTS - attempted_counts) if missing_counts: @@ -1182,7 +1234,9 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No ok_counts = { row.get("chart_count") for row in rows - if isinstance(row, dict) and _status_kind(row.get("status")) == "ok" + if isinstance(row, dict) + and _status_kind(row.get("status")) == "ok" + and row.get("fully_nonblank") is True } expected_ceiling = max(ok_counts) if ok_counts else None if report.get("chart_count_ceiling") != expected_ceiling: @@ -1192,6 +1246,132 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No ) +def _dashboard_id_list( + row: dict[str, Any], key: str, path: str, expected_ids: set[str], errors: list[str] +) -> set[str]: + value = row.get(key) + if not isinstance(value, list): + errors.append(f"{path}.{key} must be a list") + return set() + if any(not isinstance(item, str) for item in value): + errors.append(f"{path}.{key} must contain only strings") + return set() + ids = set(value) + if len(ids) != len(value): + errors.append(f"{path}.{key} must not contain duplicate chart IDs") + unknown = sorted(ids - expected_ids) + if unknown: + errors.append(f"{path}.{key} contains unknown chart IDs: {unknown}") + return ids + + +def _validate_dashboard_telemetry(row: dict[str, Any], path: str, errors: list[str]) -> None: + chart_count = row.get("chart_count") + if isinstance(chart_count, bool) or not isinstance(chart_count, int) or chart_count <= 0: + return + expected_ids = {f"chart-{i}" for i in range(chart_count)} + id_keys = ( + "creation_failure_ids", + "initial_nonblank_chart_ids", + "initial_blank_chart_ids", + "scroll_nonblank_chart_ids", + "scroll_blank_chart_ids", + "context_lost_chart_ids", + "context_restored_chart_ids", + "currently_lost_chart_ids", + ) + ids = {key: _dashboard_id_list(row, key, path, expected_ids, errors) for key in id_keys} + + count_keys = ( + "steady_redraw_active_charts", + "created_charts", + "creation_failed_charts", + "nonblank_charts", + "initial_nonblank_charts", + "scroll_nonblank_charts", + ) + for key in count_keys: + value = row.get(key) + if isinstance(value, int) and not isinstance(value, bool) and value > chart_count: + errors.append(f"{path}.{key} must be <= chart_count") + + expected_pairs = ( + ("creation_failed_charts", "creation_failure_ids"), + ("initial_nonblank_charts", "initial_nonblank_chart_ids"), + ("scroll_nonblank_charts", "scroll_nonblank_chart_ids"), + ) + for count_key, ids_key in expected_pairs: + if row.get(count_key) != len(ids[ids_key]): + errors.append(f"{path}.{count_key} must equal len({ids_key})") + if row.get("nonblank_charts") != row.get("initial_nonblank_charts"): + errors.append(f"{path}.nonblank_charts must equal initial_nonblank_charts") + if row.get("created_charts") != chart_count - len(ids["creation_failure_ids"]): + errors.append(f"{path}.created_charts must equal chart_count minus creation failures") + if row.get("steady_redraw_active_charts") != row.get("created_charts", 0) - len( + ids["currently_lost_chart_ids"] + ): + errors.append( + f"{path}.steady_redraw_active_charts must equal created charts minus currently lost" + ) + + for prefix in ("initial", "scroll"): + lit = ids[f"{prefix}_nonblank_chart_ids"] + blank = ids[f"{prefix}_blank_chart_ids"] + if lit & blank: + errors.append(f"{path}.{prefix} nonblank and blank chart IDs must be disjoint") + if lit | blank != expected_ids: + errors.append(f"{path}.{prefix} chart IDs must cover every attempted chart") + + events = row.get("context_events") + lost_event_ids: list[str] = [] + restored_event_ids: list[str] = [] + if not isinstance(events, list): + errors.append(f"{path}.context_events must be a list") + events = [] + for index, event in enumerate(events): + event_path = f"{path}.context_events[{index}]" + _require_keys(event, {"id", "type", "phase", "at_ms"}, event_path, errors) + if not isinstance(event, dict): + continue + event_id = event.get("id") + if event_id not in expected_ids: + errors.append(f"{event_path}.id must identify an attempted chart") + event_type = event.get("type") + if event_type not in {"lost", "restored"}: + errors.append(f"{event_path}.type must be 'lost' or 'restored'") + elif isinstance(event_id, str): + (lost_event_ids if event_type == "lost" else restored_event_ids).append(event_id) + if event.get("phase") not in {"create", "initial", "scroll", "redraw", "report"}: + errors.append(f"{event_path}.phase has an unknown value") + _require_nonnegative_number(event, "at_ms", event_path, errors) + + if row.get("context_lost_events") != len(lost_event_ids): + errors.append(f"{path}.context_lost_events must equal lost context event count") + if row.get("context_restored_events") != len(restored_event_ids): + errors.append(f"{path}.context_restored_events must equal restored context event count") + if ids["context_lost_chart_ids"] != set(lost_event_ids): + errors.append(f"{path}.context_lost_chart_ids must match context_events") + if ids["context_restored_chart_ids"] != set(restored_event_ids): + errors.append(f"{path}.context_restored_chart_ids must match context_events") + if not ids["currently_lost_chart_ids"] <= ids["context_lost_chart_ids"]: + errors.append(f"{path}.currently_lost_chart_ids must be a subset of lost chart IDs") + + expected_complete = ( + row.get("created_charts") == chart_count + and row.get("initial_nonblank_charts") == chart_count + and row.get("scroll_nonblank_charts") == chart_count + and row.get("context_lost_events") == 0 + and not ids["currently_lost_chart_ids"] + ) + if not isinstance(row.get("fully_nonblank"), bool): + errors.append(f"{path}.fully_nonblank must be a boolean") + elif row.get("fully_nonblank") != expected_complete: + errors.append(f"{path}.fully_nonblank is inconsistent with dashboard telemetry") + expected_status = "complete" if expected_complete else "partial" + if row.get("render_status") != expected_status: + errors.append(f"{path}.render_status must be {expected_status!r}") + + def _validate_workflow_native(report: dict[str, Any], errors: list[str]) -> None: _require_native_backend(report, "workflow-native", errors) _require_keys( diff --git a/tests/test_benchmark_environment.py b/tests/test_benchmark_environment.py index 7eb5d29b..615c52f3 100644 --- a/tests/test_benchmark_environment.py +++ b/tests/test_benchmark_environment.py @@ -146,6 +146,35 @@ def test_interaction_browser_gates_cover_scatter_and_core_chart_families() -> No assert marker in bench +def test_interaction_benchmark_completes_gpu_warmup_before_timing() -> None: + bench = (ROOT / "benchmarks" / "bench_interaction.py").read_text(encoding="utf-8") + warm_start = bench.index("// Warm shader compilation") + timing_start = bench.index("let viewChanged = false", warm_start) + settle_start = bench.index("function settlePixels()", timing_start) + settle_end = bench.index("function measure(", settle_start) + + assert "settlePixels();" in bench[warm_start:timing_start] + assert "cancelAnimationFrame(view._raf);" in bench[settle_start:settle_end] + assert "gl.readPixels(" in bench[settle_start:settle_end] + + +def test_dashboard_benchmark_reports_eviction_and_scroll_telemetry() -> None: + bench = (ROOT / "benchmarks" / "bench_dashboard.py").read_text(encoding="utf-8") + + for marker in ( + 'addEventListener("webglcontextlost"', + 'addEventListener("webglcontextrestored"', + "scrollIntoView", + "context_lost_chart_ids", + "context_restored_chart_ids", + "initial_nonblank_chart_ids", + "scroll_nonblank_chart_ids", + 'render_status: fullyNonblank ? "complete" : "partial"', + ): + assert marker in bench + assert "blank dashboard chart" not in bench + + def test_benchmark_categories_track_core_hardening_metrics() -> None: medium_scatter_metrics = CATEGORY_BY_ID["medium_direct_scatter"]["metrics"] interaction_metrics = CATEGORY_BY_ID["interaction_smoothness"]["metrics"] diff --git a/tests/test_png_export.py b/tests/test_png_export.py index 88bff5c7..99fd7d94 100644 --- a/tests/test_png_export.py +++ b/tests/test_png_export.py @@ -111,6 +111,24 @@ def test_png_encoder_selects_indexed_for_few_colors() -> None: assert _ihdr(_png.encode(many))[2] == 6 +def test_png_encoder_uses_balanced_compression_level(monkeypatch) -> None: + levels: list[int] = [] + compress = _png.zlib.compress + + def recording_compress(data: bytes, level: int) -> bytes: + levels.append(level) + return compress(data, level) + + monkeypatch.setattr(_png.zlib, "compress", recording_compress) + few = np.zeros((10, 10, 4), np.uint8) + many = (np.random.default_rng(4).random((20, 20, 4)) * 255).astype(np.uint8) + + _png.encode(few) + _png.encode(many) + + assert levels == [6, 6] + + def test_native_and_svg_share_layout() -> None: # Both exporters compute the same plot rect / tick labels from one spec. from fastcharts import _svg diff --git a/tests/test_verify_benchmark_report.py b/tests/test_verify_benchmark_report.py index 6c037059..12aa05b3 100644 --- a/tests/test_verify_benchmark_report.py +++ b/tests/test_verify_benchmark_report.py @@ -422,6 +422,50 @@ def _dashboard_browser_report() -> dict: "small_data_startup", "payload_export_size", ) + + def row(count: int) -> dict: + chart_ids = [f"chart-{i}" for i in range(count)] + return { + "scenario": f"dashboard_{count}", + "chart_count": count, + "benchmark_categories": [ + "many_chart_dashboards", + "small_data_startup", + "payload_export_size", + ], + "total_payload_bytes": 262_144, + "html_bytes": 524_288, + "status": "ok", + "render_status": "complete", + "fully_nonblank": True, + "render_ms": 7.0 * count, + "ms_per_chart": 7.0, + "payload_prep_ms": 3.0 * count, + "navigation_ready_ms": 12.0 * count, + "scroll_pass_ms": 5.0 * count, + "steady_redraw_p95_ms": 16.7, + "steady_redraw_active_charts": count, + "js_heap_before_bytes": 1_000_000, + "js_heap_bytes": 2_000_000, + "js_heap_delta_bytes": 1_000_000, + "created_charts": count, + "creation_failed_charts": 0, + "creation_failure_ids": [], + "nonblank_charts": count, + "initial_nonblank_charts": count, + "initial_nonblank_chart_ids": chart_ids, + "initial_blank_chart_ids": [], + "scroll_nonblank_charts": count, + "scroll_nonblank_chart_ids": chart_ids, + "scroll_blank_chart_ids": [], + "context_lost_events": 0, + "context_restored_events": 0, + "context_lost_chart_ids": [], + "context_restored_chart_ids": [], + "currently_lost_chart_ids": [], + "context_events": [], + } + return { **_base(), "kind": "dashboard-browser", @@ -429,33 +473,40 @@ def _dashboard_browser_report() -> dict: "tracked_categories": tracked, "attempted_chart_counts": [10, 20, 50], "chart_count_ceiling": 50, - "rows": [ - { - "scenario": f"dashboard_{count}", - "chart_count": count, - "benchmark_categories": [ - "many_chart_dashboards", - "small_data_startup", - "payload_export_size", - ], - "total_payload_bytes": 262_144, - "html_bytes": 524_288, - "status": "ok", - "render_ms": 7.0 * count, - "ms_per_chart": 7.0, - "payload_prep_ms": 3.0 * count, - "navigation_ready_ms": 12.0 * count, - "steady_redraw_p95_ms": 16.7, - "js_heap_before_bytes": 1_000_000, - "js_heap_bytes": 2_000_000, - "js_heap_delta_bytes": 1_000_000, - "nonblank_charts": count, - } - for count in (10, 20, 50) - ], + "rows": [row(count) for count in (10, 20, 50)], } +def _set_dashboard_partial(row: dict, nonblank: int = 16) -> None: + count = row["chart_count"] + blank_ids = [f"chart-{i}" for i in range(count - nonblank)] + lit_ids = [f"chart-{i}" for i in range(count - nonblank, count)] + events = [ + {"id": chart_id, "type": "lost", "phase": "create", "at_ms": 10.0 + i} + for i, chart_id in enumerate(blank_ids) + ] + row.update( + { + "render_status": "partial", + "fully_nonblank": False, + "steady_redraw_active_charts": nonblank, + "nonblank_charts": nonblank, + "initial_nonblank_charts": nonblank, + "initial_nonblank_chart_ids": lit_ids, + "initial_blank_chart_ids": blank_ids, + "scroll_nonblank_charts": nonblank, + "scroll_nonblank_chart_ids": lit_ids, + "scroll_blank_chart_ids": blank_ids, + "context_lost_events": len(events), + "context_restored_events": 0, + "context_lost_chart_ids": blank_ids, + "context_restored_chart_ids": [], + "currently_lost_chart_ids": blank_ids, + "context_events": events, + } + ) + + def _workflow_native_report() -> dict: categories, tracked = _category_registry( "input_ingestion", "streaming_updates", "static_export" @@ -581,7 +632,7 @@ def test_scatter_report_requires_mode_target_and_oracle(tmp_path: Path) -> None: def test_dashboard_report_rejects_overstated_ceiling(tmp_path: Path) -> None: payload = _dashboard_browser_report() - payload["rows"][-1]["status"] = "failed(context limit)" + _set_dashboard_partial(payload["rows"][-1]) path = _write_report(tmp_path, payload) errors = verify_benchmark_report.validate_report(path, kind="dashboard-browser") @@ -589,6 +640,18 @@ def test_dashboard_report_rejects_overstated_ceiling(tmp_path: Path) -> None: assert any("chart_count_ceiling" in error for error in errors) +def test_dashboard_report_accepts_partial_rows_with_context_telemetry(tmp_path: Path) -> None: + payload = _dashboard_browser_report() + _set_dashboard_partial(payload["rows"][1]) + _set_dashboard_partial(payload["rows"][2]) + payload["chart_count_ceiling"] = 10 + path = _write_report(tmp_path, payload) + + errors = verify_benchmark_report.validate_report(path, kind="dashboard-browser") + + assert errors == [] + + def test_workflow_report_rejects_missing_required_scenario(tmp_path: Path) -> None: payload = _workflow_native_report() payload["rows"] = payload["rows"][1:] From d79cf000d18b8edb0af6044b92475ab8347c1bf7 Mon Sep 17 00:00:00 2001 From: Alek Date: Thu, 9 Jul 2026 13:37:02 -0700 Subject: [PATCH 2/3] Clarify benchmark artifact byte comparisons --- docs/benchmark.md | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/benchmark.md b/docs/benchmark.md index f0b71079..faf5ac50 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -201,21 +201,45 @@ PYTHONPATH=python .venv/bin/python benchmarks/bench_vs.py \ --sizes 1e3,1e4,1e5 --budget 20 ``` -| Library | Render target | 100k total | Peak memory | Output bytes | Points/sec | +| Library | Render target | 100k total ¶ | Peak memory | Output bytes | Points/sec ¶ | |---|---|---:|---:|---:|---:| -| fastcharts | binary payload (native) | **2 ms** | **2 MB** | 781 KB | 51,702,966 | -| matplotlib | Agg PNG | 83 ms | 6 MB | 53 KB | 1,206,226 | -| seaborn | matplotlib PNG | 152 ms | 11 MB | 39 KB | 657,728 | -| Plotly `Scattergl` | Kaleido PNG | 2,991 ms | 22 MB | 61 KB | 33,434 | -| Plotly `Scatter` | Kaleido PNG | 6,281 ms | 22 MB | 107 KB | 15,921 | -| Bokeh canvas ¹ | standalone HTML | 75 ms | 14 MB | 2 MB | 1,327,770 | -| Bokeh WebGL ¹ | standalone HTML | 73 ms | 14 MB | 2 MB | 1,360,995 | -| Altair / Vega-Lite ¹ | standalone HTML | 1,846 ms | 35 MB | 5 MB | 54,171 | -| Datashader ¹ | PNG raster | 13 ms | 15 MB | 58 KB | 7,502,931 | -| hvPlot / HoloViews ¹ | Bokeh HTML | 95 ms | 17 MB | 2 MB | 1,052,353 | +| fastcharts | binary payload (native) * | **2 ms** | **2 MB** | 781 KB | 51,702,966 | +| matplotlib | Agg PNG † | 83 ms | 6 MB | 53 KB | 1,206,226 | +| seaborn | matplotlib PNG † | 152 ms | 11 MB | 39 KB | 657,728 | +| Plotly `Scattergl` | Kaleido PNG † | 2,991 ms | 22 MB | 61 KB | 33,434 | +| Plotly `Scatter` | Kaleido PNG † | 6,281 ms | 22 MB | 107 KB | 15,921 | +| Bokeh canvas ¹ | standalone HTML ‡ | 75 ms | 14 MB | 2 MB | 1,327,770 | +| Bokeh WebGL ¹ | standalone HTML ‡ | 73 ms | 14 MB | 2 MB | 1,360,995 | +| Altair / Vega-Lite ¹ | standalone HTML ‡ | 1,846 ms | 35 MB | 5 MB | 54,171 | +| Datashader ¹ | PNG raster § | 13 ms | 15 MB | 58 KB | 7,502,931 | +| hvPlot / HoloViews ¹ | Bokeh HTML ‡ | 95 ms | 17 MB | 2 MB | 1,052,353 | ¹ Earlier local run, not re-measured in the 2026-07-09 CI refresh. +\* **Exact interactive wire payload.** The fastcharts row counts compact spec +metadata plus the GPU-ready x/y float32 buffers, excluding the JavaScript runtime +and HTML wrapper. At 100k direct points, two 4-byte coordinates account for about +781 KiB (8 bytes/point); exact hover, selection, and view updates remain possible. + +† **Static raster output.** These byte counts are compressed finished pixels, not +the source-point transport. The original coordinates cannot be recovered for +exact hover, selection, or client-side re-rendering, so these values should not be +compared directly with `*` or `‡` payload sizes. + +‡ **Interactive HTML output.** These artifacts retain chart data/specification and +an HTML wrapper. Adapter resource modes differ (for example, a CDN reference may +exclude the library runtime), so they are the closest interactive comparison to +`*`, but not identical package/bundle measurements. + +§ **Aggregated raster output.** Datashader's PNG retains screen-sized density/count +pixels rather than 100k exact points. Its compact size represents aggregation as +well as PNG compression. + +¶ `total` and `points/sec` measure production of the target named in each row. +They are useful for scaling within a target class, not as same-render-target +speedups. Compare output bytes only between artifacts with equivalent retained +information and runtime-inclusion rules. + --- ## Core 2D chart benchmark — fastcharts vs Plotly and Seaborn From 8d9d51ec997746c498b55df1095bcb29125d1950 Mon Sep 17 00:00:00 2001 From: Alek Date: Thu, 9 Jul 2026 13:42:03 -0700 Subject: [PATCH 3/3] Label creation-phase context evictions correctly; add changelog entry webglcontextlost dispatches as a task, so evictions triggered by the creation loop only fire on the next yield. The probe flipped phase to "initial" before yielding, so every creation eviction was mislabeled "initial" and the "create" phase could never appear in real reports. Yield first, then advance the phase; guard the ordering in the telemetry marker test. Also add the missing Unreleased changelog entries for the dashboard telemetry rework and the PNG compression-level change. --- CHANGELOG.md | 12 ++++++++++++ benchmarks/bench_dashboard.py | 5 ++++- tests/test_benchmark_environment.py | 10 ++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b53e86d..b357dbc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,18 @@ in the README). - `LICENSE` (Apache-2.0), `CHANGELOG.md`, `SECURITY.md`, root `CONTRIBUTING.md`. ### Changed +- **Native PNG export compression** dropped from zlib level 9 to level 6: a + 1M-point line export goes from ~298 ms to ~64 ms (reference hardware) for + ~2.65% larger output. Regression tests pin the level for both truecolor + and indexed encoders. +- **Dashboard benchmark telemetry:** `bench_dashboard.py` no longer discards + metrics when Chrome evicts WebGL contexts. Partial dashboards stay + measurement rows with per-chart `webglcontextlost`/`webglcontextrestored` + events (id, phase, timestamp), creation-failure vs eviction distinction, + initial and scrolled nonblank chart IDs, live-context redraw submission, + and a stable loss-free chart-count ceiling; the report verifier + cross-checks all of it. The interaction benchmark warm-up now completes + GPU work (draw + readback) before the first timed sample. - **Performance:** WebGL client now uses vertex-array objects (no per-frame attribute re-binding), lazily compiles shader programs on first use, and ships a compacted bundle (193 KB → 154 KB) that every `to_html()` inlines. diff --git a/benchmarks/bench_dashboard.py b/benchmarks/bench_dashboard.py index 61c4b1a7..6c561b35 100644 --- a/benchmarks/bench_dashboard.py +++ b/benchmarks/bench_dashboard.py @@ -145,8 +145,11 @@ def _probe_js() -> str: slots.push({id: payload.id, cell, view: null, state: {lost: true}}); } } - phase = "initial"; + // webglcontextlost dispatches as a task, so evictions triggered by the + // creation loop above only fire during this yield — keep phase "create" + // until they have drained. await new Promise((resolve) => setTimeout(resolve, 0)); + phase = "initial"; function contextLost(slot) { if (!slot.view || !slot.view.gl) return true; diff --git a/tests/test_benchmark_environment.py b/tests/test_benchmark_environment.py index 615c52f3..5f02d661 100644 --- a/tests/test_benchmark_environment.py +++ b/tests/test_benchmark_environment.py @@ -174,6 +174,16 @@ def test_dashboard_benchmark_reports_eviction_and_scroll_telemetry() -> None: assert marker in bench assert "blank dashboard chart" not in bench + # webglcontextlost dispatches as a task, so the probe must yield before + # leaving the "create" phase or creation-loop evictions get mislabeled + # with whatever phase is current when the queued events finally fire. + creation_loop = bench.index('addEventListener("webglcontextlost"') + first_yield = bench.index( + "await new Promise((resolve) => setTimeout(resolve, 0));", creation_loop + ) + phase_initial = bench.index('phase = "initial";', creation_loop) + assert first_yield < phase_initial + def test_benchmark_categories_track_core_hardening_metrics() -> None: medium_scatter_metrics = CATEGORY_BY_ID["medium_direct_scatter"]["metrics"]