diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b53e86d..b357dbc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,18 @@ in the README). - `LICENSE` (Apache-2.0), `CHANGELOG.md`, `SECURITY.md`, root `CONTRIBUTING.md`. ### Changed +- **Native PNG export compression** dropped from zlib level 9 to level 6: a + 1M-point line export goes from ~298 ms to ~64 ms (reference hardware) for + ~2.65% larger output. Regression tests pin the level for both truecolor + and indexed encoders. +- **Dashboard benchmark telemetry:** `bench_dashboard.py` no longer discards + metrics when Chrome evicts WebGL contexts. Partial dashboards stay + measurement rows with per-chart `webglcontextlost`/`webglcontextrestored` + events (id, phase, timestamp), creation-failure vs eviction distinction, + initial and scrolled nonblank chart IDs, live-context redraw submission, + and a stable loss-free chart-count ceiling; the report verifier + cross-checks all of it. The interaction benchmark warm-up now completes + GPU work (draw + readback) before the first timed sample. - **Performance:** WebGL client now uses vertex-array objects (no per-frame attribute re-binding), lazily compiles shader programs on first use, and ships a compacted bundle (193 KB → 154 KB) that every `to_html()` inlines. diff --git a/README.md b/README.md index 460f76e3..9c9b6f5e 100644 --- a/README.md +++ b/README.md @@ -501,8 +501,9 @@ fails if any of those required interaction rows disappear. `bench_workflows.py` covers contiguous/converted/strided/datetime/list/Arrow ingestion, line append, density append followed by a real 2M+ pyramid rebuild, and HTML/SVG/native-PNG/Chromium-PNG export. The dashboard benchmark attempts 10, 20, and 50 charts, -records JS heap and steady-redraw pacing, and reports the largest fully nonblank -count rather than assuming every attempted dashboard succeeded. +records per-chart context loss/restoration, initial and scrolled visibility, JS +heap, and redraw submission pacing, then reports the largest loss-free nonblank +count without discarding partial-dashboard metrics. ### 10M-point native benchmark diff --git a/benchmarks/README.md b/benchmarks/README.md index 3f34d4a1..18e3d9af 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -67,7 +67,8 @@ JSON artifacts, retain failed/over-budget rows, and label the table - Interactive TTFR is build + HTML serialization + chart-ready time. - Interaction browser rows are standalone client input-to-pixel-readback; backend LOD work is in CodSpeed and workflow rows. -- Dashboard rows attempt 10/20/50 charts and publish the largest fully nonblank - count, including context-limit failures. +- Dashboard rows attempt 10/20/50 charts, retain timings for partial dashboards, + record per-chart context loss/restoration plus initial/scrolled nonblank IDs, + and publish the largest stable loss-free count. - Density rows must include a count-conservation oracle and explicit aggregate dimensions. A density result is not an exact-marker result. diff --git a/benchmarks/bench_dashboard.py b/benchmarks/bench_dashboard.py index bcfd7895..6c561b35 100644 --- a/benchmarks/bench_dashboard.py +++ b/benchmarks/bench_dashboard.py @@ -116,43 +116,131 @@ def _probe_js() -> str: (async () => { try { const root = document.getElementById("root"); - const views = []; + const slots = []; + const contextEvents = []; + const creationFailureIds = []; + let phase = "create"; const heapBefore = performance.memory ? performance.memory.usedJSHeapSize : null; const t0 = performance.now(); for (const payload of FC_CHARTS) { const cell = document.createElement("div"); cell.className = "chart-cell"; + cell.dataset.chartId = payload.id; root.appendChild(cell); - const view = fastcharts.renderStandalone(cell, payload.spec, fcBytesFromB64(payload.b64)); - view._drawNow(); - views.push(view); + try { + const view = fastcharts.renderStandalone(cell, payload.spec, fcBytesFromB64(payload.b64)); + const state = {lost: false}; + view.canvas.addEventListener("webglcontextlost", () => { + state.lost = true; + contextEvents.push({id: payload.id, type: "lost", phase, at_ms: performance.now() - t0}); + }); + view.canvas.addEventListener("webglcontextrestored", () => { + state.lost = false; + contextEvents.push({id: payload.id, type: "restored", phase, at_ms: performance.now() - t0}); + }); + view._drawNow(); + slots.push({id: payload.id, cell, view, state}); + } catch (err) { + creationFailureIds.push(payload.id); + slots.push({id: payload.id, cell, view: null, state: {lost: true}}); + } } - await fcRaf(); - let nonblank = 0; - for (const view of views) { - if (fcNonblankPixels(view) > 0) nonblank++; + // webglcontextlost dispatches as a task, so evictions triggered by the + // creation loop above only fire during this yield — keep phase "create" + // until they have drained. + await new Promise((resolve) => setTimeout(resolve, 0)); + phase = "initial"; + + function contextLost(slot) { + if (!slot.view || !slot.view.gl) return true; + return slot.state.lost || slot.view._glLost || slot.view.gl.isContextLost(); + } + function nonblankPixels(slot) { + if (contextLost(slot)) return 0; + try { + return fcNonblankPixels(slot.view); + } catch (_err) { + return 0; + } + } + function sampleNonblank() { + const lit = []; + for (const slot of slots) if (nonblankPixels(slot) > 0) lit.push(slot.id); + return lit; } - if (nonblank !== views.length) { - throw new Error(`blank dashboard chart: ${nonblank}/${views.length} nonblank`); + function complement(ids) { + const present = new Set(ids); + return slots.filter((slot) => !present.has(slot.id)).map((slot) => slot.id); } + + const initialNonblankIds = sampleNonblank(); const renderMs = performance.now() - t0; const navigationReadyMs = performance.now(); + + phase = "scroll"; + const scrollStart = performance.now(); + const scrollNonblankIds = []; + for (const slot of slots) { + slot.cell.scrollIntoView({block: "center", inline: "center"}); + await new Promise((resolve) => setTimeout(resolve, 0)); + if (nonblankPixels(slot) > 0) scrollNonblankIds.push(slot.id); + } + window.scrollTo(0, 0); + await new Promise((resolve) => setTimeout(resolve, 0)); + const scrollPassMs = performance.now() - scrollStart; + + phase = "redraw"; const steadyRedraws = []; for (let i = 0; i < 12; i++) { const redrawStart = performance.now(); - for (const view of views) view._drawNow(); + for (const slot of slots) if (!contextLost(slot)) slot.view._drawNow(); steadyRedraws.push(performance.now() - redrawStart); } + await new Promise((resolve) => setTimeout(resolve, 0)); + phase = "report"; + const heapAfter = performance.memory ? performance.memory.usedJSHeapSize : null; + const lostEvents = contextEvents.filter((event) => event.type === "lost"); + const restoredEvents = contextEvents.filter((event) => event.type === "restored"); + const uniqueIds = (events) => Array.from(new Set(events.map((event) => event.id))); + const currentlyLostIds = slots + .filter((slot) => slot.view && contextLost(slot)) + .map((slot) => slot.id); + const createdCharts = slots.length - creationFailureIds.length; + const fullyNonblank = + createdCharts === slots.length && + initialNonblankIds.length === slots.length && + scrollNonblankIds.length === slots.length && + lostEvents.length === 0 && + currentlyLostIds.length === 0; fcReport("FC_DASHBOARD", { status: "ok", + render_status: fullyNonblank ? "complete" : "partial", + fully_nonblank: fullyNonblank, render_ms: renderMs, navigation_ready_ms: navigationReadyMs, + scroll_pass_ms: scrollPassMs, steady_redraw_p95_ms: fcPercentile(steadyRedraws, 95), + steady_redraw_active_charts: slots.filter((slot) => slot.view && !contextLost(slot)).length, js_heap_before_bytes: heapBefore, js_heap_bytes: heapAfter, js_heap_delta_bytes: heapBefore == null || heapAfter == null ? null : heapAfter - heapBefore, - nonblank_charts: nonblank, + created_charts: createdCharts, + creation_failed_charts: creationFailureIds.length, + creation_failure_ids: creationFailureIds, + nonblank_charts: initialNonblankIds.length, + initial_nonblank_charts: initialNonblankIds.length, + initial_nonblank_chart_ids: initialNonblankIds, + initial_blank_chart_ids: complement(initialNonblankIds), + scroll_nonblank_charts: scrollNonblankIds.length, + scroll_nonblank_chart_ids: scrollNonblankIds, + scroll_blank_chart_ids: complement(scrollNonblankIds), + context_lost_events: lostEvents.length, + context_restored_events: restoredEvents.length, + context_lost_chart_ids: uniqueIds(lostEvents), + context_restored_chart_ids: uniqueIds(restoredEvents), + currently_lost_chart_ids: currentlyLostIds, + context_events: contextEvents, }); } catch (err) { fcFail("FC_DASHBOARD", err); @@ -196,20 +284,44 @@ def run(*, chart_counts: list[int], chromium: str | None = None) -> dict[str, An "status": result.get("status", "failed(no status)"), } if row["status"] == "ok": - row["render_ms"] = result.get("render_ms") + metric_keys = ( + "render_status", + "fully_nonblank", + "render_ms", + "navigation_ready_ms", + "scroll_pass_ms", + "steady_redraw_p95_ms", + "steady_redraw_active_charts", + "js_heap_before_bytes", + "js_heap_bytes", + "js_heap_delta_bytes", + "created_charts", + "creation_failed_charts", + "creation_failure_ids", + "nonblank_charts", + "initial_nonblank_charts", + "initial_nonblank_chart_ids", + "initial_blank_chart_ids", + "scroll_nonblank_charts", + "scroll_nonblank_chart_ids", + "scroll_blank_chart_ids", + "context_lost_events", + "context_restored_events", + "context_lost_chart_ids", + "context_restored_chart_ids", + "currently_lost_chart_ids", + "context_events", + ) + row.update({key: result.get(key) for key in metric_keys}) row["ms_per_chart"] = ( result.get("render_ms") / count if result.get("render_ms") is not None else None ) - row["nonblank_charts"] = result.get("nonblank_charts") - row["navigation_ready_ms"] = result.get("navigation_ready_ms") - row["steady_redraw_p95_ms"] = result.get("steady_redraw_p95_ms") - row["js_heap_before_bytes"] = result.get("js_heap_before_bytes") - row["js_heap_bytes"] = result.get("js_heap_bytes") - row["js_heap_delta_bytes"] = result.get("js_heap_delta_bytes") rows.append(row) successful_counts = [ - row["chart_count"] for row in rows if str(row.get("status", "")).startswith("ok") + row["chart_count"] + for row in rows + if str(row.get("status", "")).startswith("ok") and row.get("fully_nonblank") is True ] return { "schema_version": SCHEMA_VERSION, @@ -249,34 +361,42 @@ def to_markdown(report: dict[str, Any]) -> str: "Tracked in this run: " + ", ".join(f"`{category['id']}`" for category in report["tracked_categories"]), "", - f"Successful nonblank chart-count ceiling: `{report.get('chart_count_ceiling')}`.", + f"Stable loss-free chart-count ceiling: `{report.get('chart_count_ceiling')}`.", "", "## Results", "", - "| scenario | charts | prep | navigation ready | render | redraw p95 | JS heap | payload | html | nonblank charts | status |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|", + "| scenario | charts | prep | navigation ready | render | scroll pass | redraw submit p95 | active | JS heap | payload | html | initial/scroll nonblank | loss/restore | health |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] for row in report["rows"]: lines.append( - "| {scenario} | {count} | {prep} | {navigation} | {render} | {idle} | {heap} | {payload} | {html} | {nonblank} | {status} |".format( + "| {scenario} | {count} | {prep} | {navigation} | {render} | {scroll} | {idle} | {active} | {heap} | {payload} | {html} | {nonblank}/{scroll_nonblank} | {lost}/{restored} | {health} |".format( scenario=row["scenario"], count=row["chart_count"], prep=_fmt_ms(row.get("payload_prep_ms")), navigation=_fmt_ms(row.get("navigation_ready_ms")), render=_fmt_ms(row.get("render_ms")), + scroll=_fmt_ms(row.get("scroll_pass_ms")), idle=_fmt_ms(row.get("steady_redraw_p95_ms")), + active=row.get("steady_redraw_active_charts", "—"), heap=_fmt_bytes(row.get("js_heap_bytes")), payload=_fmt_bytes(row.get("total_payload_bytes")), html=_fmt_bytes(row.get("html_bytes")), nonblank=row.get("nonblank_charts", "—"), - status=row.get("status", "unknown"), + scroll_nonblank=row.get("scroll_nonblank_charts", "—"), + lost=row.get("context_lost_events", "—"), + restored=row.get("context_restored_events", "—"), + health=row.get("render_status", row.get("status", "unknown")), ) ) lines += [ "", "Notes:", "", - "- `render` is measured inside the page from first chart decode to post-RAF WebGL readback.", + "- `render` is measured inside the page from first chart decode through a context-event task yield and WebGL readback.", + "- `redraw submit p95` measures synchronous command submission for every currently live context; `active` states how many charts contributed.", + "- Partial rows retain startup, heap, redraw, context-event, and scrolling metrics; they do not raise the loss-free ceiling.", + "- Context event IDs and phases are retained in JSON so LRU eviction and restoration churn are directly observable.", "- The chart mix cycles through line, scatter, histogram, grouped bar, and heatmap.", ] return "\n".join(lines) diff --git a/benchmarks/bench_interaction.py b/benchmarks/bench_interaction.py index 26386659..b70ba414 100644 --- a/benchmarks/bench_interaction.py +++ b/benchmarks/bench_interaction.py @@ -251,7 +251,7 @@ def _probe_js(reps: int) -> str: const warmY1 = before.y0 + (before.y1 - before.y0) * 0.65; view._selectLocal(warmX0, warmX1, warmY0, warmY1); view._clearSelection(); - view._drawNow(); + settlePixels(); let viewChanged = false; function noteViewChanged() {{ @@ -275,6 +275,10 @@ def _probe_js(reps: int) -> str: view._cancelViewAnimation(); view.view = target; }} + if (view._raf) {{ + cancelAnimationFrame(view._raf); + view._raf = null; + }} const gl = view.gl; view._drawNow(); const pixel = new Uint8Array(4); diff --git a/benchmarks/categories.py b/benchmarks/categories.py index 41386a8f..49db082a 100644 --- a/benchmarks/categories.py +++ b/benchmarks/categories.py @@ -67,10 +67,10 @@ "id": "many_chart_dashboards", "name": "Many-chart dashboards", "why": "Plotly-class apps often fail from total page weight and many live canvases, not one chart.", - "metrics": "payload prep, navigation readiness, JS heap, steady redraw, chart count ceiling", + "metrics": "payload prep, navigation readiness, JS heap, redraw submission, scroll visibility, context loss/restore, stable chart-count ceiling", "harness": "benchmarks/bench_dashboard.py", "status": "tracked", - "goal": "Measure the 10-50 chart scaling curve and expose browser context ceilings.", + "goal": "Measure the 10-50 chart scaling curve and expose LRU context eviction without discarding partial-row metrics.", }, { "id": "interaction_smoothness", diff --git a/docs/benchmark.md b/docs/benchmark.md index a34c1342..faf5ac50 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -55,7 +55,7 @@ commit so CI artifacts are quick to inspect from logs. | `huge_scatter_overview` | Huge scatter overview | tracked | Proves screen-bounded rendering for datasets larger than the browser should draw point-for-point. | ingest/bin time, density payload size, peak memory, TTFR | `bench_scatter_native.py`, `bench_vs.py`, `test_first_payload_density_large`, example app assets | Keep resident/render payload flat in N while showing truthful density summaries. | | `adaptive_scatter_drilldown` | Adaptive scatter drilldown | tracked | The large-data claim needs a credible path from overview to exact visible points. | visible-query latency, tier-switch latency, exact-point recovery, badge accuracy | `benchmarks/test_codspeed_kernels.py::test_adaptive_drilldown_cycle` | Exact points when visible count is under budget; sampled/density with explicit counts otherwise. | | `huge_line_time_series` | Huge line / time series | tracked | Common observability and finance workload; Plotly-resampler sets the bar here. | decimation time, zoom re-decimation latency, TTFR, extrema preservation | `benchmarks/bench.py`, `bench_native.py`, `bench_interaction.py`, `test_decimate_view` | Screen-bounded line payloads with extrema-preserving decimation and fast zoom refresh. | -| `many_chart_dashboards` | Many-chart dashboards | tracked | Plotly-class apps often fail from total page weight and many live canvases, not one chart. | payload prep, navigation readiness, JS heap, steady redraw, chart-count ceiling | `benchmarks/bench_dashboard.py` | Measure the 10-50 chart scaling curve and expose browser context ceilings. | +| `many_chart_dashboards` | Many-chart dashboards | tracked | Plotly-class apps often fail from total page weight and many live canvases, not one chart. | payload prep, navigation readiness, JS heap, redraw submission, scroll visibility, context loss/restore, stable chart-count ceiling | `benchmarks/bench_dashboard.py` | Measure the 10-50 chart scaling curve and expose LRU context eviction without discarding partial-row metrics. | | `interaction_smoothness` | Interaction smoothness | tracked | Users judge performance by pan/zoom/hover, not just export time. | pan/zoom FPS, wheel latency, hover latency, tooltip stability, selection latency, frame color delta | `benchmarks/bench_interaction.py` | Stay responsive during interaction, avoid blank/flickering frames, then refine view after interaction settles. | | `payload_export_size` | Payload/export size | tracked | Notebooks, static HTML, docs, and dashboards pay for every byte shipped. | standalone HTML bytes, binary payload bytes, bundle bytes | `bench_vs.py`, `bench_scatter_native.py`, `test_first_payload_density_large`, `test_memory_report_density_medium`, example app asset sizes | Keep data payloads binary and screen-bounded where possible; warn when exact export would be huge. | | `core_2d_chart_breadth` | Core 2D chart breadth | tracked | The library needs to stay fast beyond the scatter wedge: bars, histograms, areas, and heatmaps are everyday chart workloads. | payload-prep time, payload bytes, standalone HTML bytes, TTFR | `benchmarks/bench_2d_charts.py` smoke/standard profiles vs Plotly and Seaborn; `bench_interaction.py`; CodSpeed core-2D payload rows | Beat Plotly on user-visible first paint for common 2D charts while tracking Seaborn raster baselines where applicable. | @@ -142,8 +142,9 @@ also declares `tooltip_sample_count`, and eligible rows must report that exact `tooltip_visible_samples` count so a tooltip that appears once and then flickers away fails verification. `bench_dashboard.py` attempts 10/20/50 mixed dashboards and reports payload prep, -navigation readiness, JS heap, steady-redraw p95, nonblank canvases, and the largest -successful count. `bench_workflows.py` covers ingestion, streaming/pyramid +navigation readiness, JS heap, redraw-submission p95, per-chart context loss and +restore events, initial/scrolled nonblank IDs, and the largest stable loss-free +count. Partial rows retain their timing and memory metrics. `bench_workflows.py` covers ingestion, streaming/pyramid invalidation, and separate HTML/SVG/native-PNG/Chromium-PNG export rows. All three emit schema-versioned JSON with environment metadata and benchmark category IDs. @@ -200,21 +201,45 @@ PYTHONPATH=python .venv/bin/python benchmarks/bench_vs.py \ --sizes 1e3,1e4,1e5 --budget 20 ``` -| Library | Render target | 100k total | Peak memory | Output bytes | Points/sec | +| Library | Render target | 100k total ¶ | Peak memory | Output bytes | Points/sec ¶ | |---|---|---:|---:|---:|---:| -| fastcharts | binary payload (native) | **2 ms** | **2 MB** | 781 KB | 51,702,966 | -| matplotlib | Agg PNG | 83 ms | 6 MB | 53 KB | 1,206,226 | -| seaborn | matplotlib PNG | 152 ms | 11 MB | 39 KB | 657,728 | -| Plotly `Scattergl` | Kaleido PNG | 2,991 ms | 22 MB | 61 KB | 33,434 | -| Plotly `Scatter` | Kaleido PNG | 6,281 ms | 22 MB | 107 KB | 15,921 | -| Bokeh canvas ¹ | standalone HTML | 75 ms | 14 MB | 2 MB | 1,327,770 | -| Bokeh WebGL ¹ | standalone HTML | 73 ms | 14 MB | 2 MB | 1,360,995 | -| Altair / Vega-Lite ¹ | standalone HTML | 1,846 ms | 35 MB | 5 MB | 54,171 | -| Datashader ¹ | PNG raster | 13 ms | 15 MB | 58 KB | 7,502,931 | -| hvPlot / HoloViews ¹ | Bokeh HTML | 95 ms | 17 MB | 2 MB | 1,052,353 | +| fastcharts | binary payload (native) * | **2 ms** | **2 MB** | 781 KB | 51,702,966 | +| matplotlib | Agg PNG † | 83 ms | 6 MB | 53 KB | 1,206,226 | +| seaborn | matplotlib PNG † | 152 ms | 11 MB | 39 KB | 657,728 | +| Plotly `Scattergl` | Kaleido PNG † | 2,991 ms | 22 MB | 61 KB | 33,434 | +| Plotly `Scatter` | Kaleido PNG † | 6,281 ms | 22 MB | 107 KB | 15,921 | +| Bokeh canvas ¹ | standalone HTML ‡ | 75 ms | 14 MB | 2 MB | 1,327,770 | +| Bokeh WebGL ¹ | standalone HTML ‡ | 73 ms | 14 MB | 2 MB | 1,360,995 | +| Altair / Vega-Lite ¹ | standalone HTML ‡ | 1,846 ms | 35 MB | 5 MB | 54,171 | +| Datashader ¹ | PNG raster § | 13 ms | 15 MB | 58 KB | 7,502,931 | +| hvPlot / HoloViews ¹ | Bokeh HTML ‡ | 95 ms | 17 MB | 2 MB | 1,052,353 | ¹ Earlier local run, not re-measured in the 2026-07-09 CI refresh. +\* **Exact interactive wire payload.** The fastcharts row counts compact spec +metadata plus the GPU-ready x/y float32 buffers, excluding the JavaScript runtime +and HTML wrapper. At 100k direct points, two 4-byte coordinates account for about +781 KiB (8 bytes/point); exact hover, selection, and view updates remain possible. + +† **Static raster output.** These byte counts are compressed finished pixels, not +the source-point transport. The original coordinates cannot be recovered for +exact hover, selection, or client-side re-rendering, so these values should not be +compared directly with `*` or `‡` payload sizes. + +‡ **Interactive HTML output.** These artifacts retain chart data/specification and +an HTML wrapper. Adapter resource modes differ (for example, a CDN reference may +exclude the library runtime), so they are the closest interactive comparison to +`*`, but not identical package/bundle measurements. + +§ **Aggregated raster output.** Datashader's PNG retains screen-sized density/count +pixels rather than 100k exact points. Its compact size represents aggregation as +well as PNG compression. + +¶ `total` and `points/sec` measure production of the target named in each row. +They are useful for scaling within a target class, not as same-render-target +speedups. Compare output bytes only between artifacts with equivalent retained +information and runtime-inclusion rules. + --- ## Core 2D chart benchmark — fastcharts vs Plotly and Seaborn diff --git a/docs/design/benchmark-methodology.md b/docs/design/benchmark-methodology.md index e6339d33..6c6da20c 100644 --- a/docs/design/benchmark-methodology.md +++ b/docs/design/benchmark-methodology.md @@ -118,8 +118,10 @@ reported). `benchmarks/bench_2d_charts.py` stays the Plotly/Seaborn chart-to-pixels comparison. 7. `dashboard_scale`: `benchmarks/bench_dashboard.py` attempts 10/20/50 mixed - charts, checks every canvas, and records payload prep, navigation readiness, - JS heap, steady-redraw p95, and the successful chart-count ceiling. + charts, checks every canvas initially and while scrolling, and records payload + prep, navigation readiness, JS heap, redraw-submission p95, per-chart context + loss/restoration events, and the stable loss-free chart-count ceiling. Partial + dashboards remain successful measurement rows rather than losing their metrics. 8. `install_import`: lower-bound distribution size plus opt-in fresh-venv total site-packages, transitive distribution count, install time, and cold import. 9. `public_workflows`: `benchmarks/bench_workflows.py` tracks ingestion shapes, diff --git a/python/fastcharts/_png.py b/python/fastcharts/_png.py index 5f043db0..7d407d6a 100644 --- a/python/fastcharts/_png.py +++ b/python/fastcharts/_png.py @@ -27,6 +27,7 @@ def _chunk(tag: bytes, data: bytes) -> bytes: _SIG = b"\x89PNG\r\n\x1a\n" +_COMPRESSION_LEVEL = 6 def png_truecolor(w: int, h: int, rgba: bytes) -> bytes: @@ -35,7 +36,10 @@ def png_truecolor(w: int, h: int, rgba: bytes) -> bytes: stride = w * 4 raw = b"".join(b"\x00" + rgba[y * stride : (y + 1) * stride] for y in range(h)) return ( - _SIG + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(raw, 9)) + _chunk(b"IEND", b"") + _SIG + + _chunk(b"IHDR", ihdr) + + _chunk(b"IDAT", zlib.compress(raw, _COMPRESSION_LEVEL)) + + _chunk(b"IEND", b"") ) @@ -51,7 +55,7 @@ def _png_indexed(w: int, h: int, idx: np.ndarray, palette: np.ndarray) -> bytes: out = _SIG + _chunk(b"IHDR", ihdr) + _chunk(b"PLTE", plte) # tRNS may omit trailing opaque (255) entries; keep it simple and always emit. out += _chunk(b"tRNS", trns) - out += _chunk(b"IDAT", zlib.compress(rows.tobytes(), 9)) + _chunk(b"IEND", b"") + out += _chunk(b"IDAT", zlib.compress(rows.tobytes(), _COMPRESSION_LEVEL)) + _chunk(b"IEND", b"") return out diff --git a/scripts/verify_benchmark_report.py b/scripts/verify_benchmark_report.py index 401e02af..664446dc 100644 --- a/scripts/verify_benchmark_report.py +++ b/scripts/verify_benchmark_report.py @@ -116,6 +116,16 @@ def _require_positive_integer(obj: dict[str, Any], key: str, path: str, errors: errors.append(f"{path}.{key} must be > 0") +def _require_nonnegative_integer( + obj: dict[str, Any], key: str, path: str, errors: list[str] +) -> None: + value = obj.get(key) + if isinstance(value, bool) or not isinstance(value, int): + errors.append(f"{path}.{key} must be a nonnegative integer") + elif value < 0: + errors.append(f"{path}.{key} must be >= 0") + + def _require_optional_nonnegative_number( obj: dict[str, Any], key: str, path: str, errors: list[str] ) -> None: @@ -1156,22 +1166,64 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No errors.append(f"{path}.status has unknown value {row.get('status')!r}") _validate_browser_category_list(row, path, category_ids, errors) if status == "ok": + _require_keys( + row, + { + "render_status", + "fully_nonblank", + "render_ms", + "ms_per_chart", + "payload_prep_ms", + "navigation_ready_ms", + "scroll_pass_ms", + "steady_redraw_p95_ms", + "steady_redraw_active_charts", + "created_charts", + "creation_failed_charts", + "creation_failure_ids", + "nonblank_charts", + "initial_nonblank_charts", + "initial_nonblank_chart_ids", + "initial_blank_chart_ids", + "scroll_nonblank_charts", + "scroll_nonblank_chart_ids", + "scroll_blank_chart_ids", + "context_lost_events", + "context_restored_events", + "context_lost_chart_ids", + "context_restored_chart_ids", + "currently_lost_chart_ids", + "context_events", + }, + path, + errors, + ) for key in ( "render_ms", "ms_per_chart", "payload_prep_ms", "navigation_ready_ms", + "scroll_pass_ms", "steady_redraw_p95_ms", ): _require_nonnegative_number(row, key, path, errors) - _require_positive_number(row, "nonblank_charts", path, errors) - if row.get("nonblank_charts") != row.get("chart_count"): - errors.append(f"{path}.nonblank_charts must equal chart_count") + for key in ( + "steady_redraw_active_charts", + "created_charts", + "creation_failed_charts", + "nonblank_charts", + "initial_nonblank_charts", + "scroll_nonblank_charts", + "context_lost_events", + "context_restored_events", + ): + _require_nonnegative_integer(row, key, path, errors) for key in ("js_heap_before_bytes", "js_heap_bytes"): _require_optional_nonnegative_number(row, key, path, errors) delta = row.get("js_heap_delta_bytes") if delta is not None and not _is_number(delta): errors.append(f"{path}.js_heap_delta_bytes must be a finite number or null") + _validate_dashboard_telemetry(row, path, errors) attempted_counts = {row.get("chart_count") for row in rows if isinstance(row, dict)} missing_counts = sorted(DASHBOARD_REQUIRED_COUNTS - attempted_counts) if missing_counts: @@ -1182,7 +1234,9 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No ok_counts = { row.get("chart_count") for row in rows - if isinstance(row, dict) and _status_kind(row.get("status")) == "ok" + if isinstance(row, dict) + and _status_kind(row.get("status")) == "ok" + and row.get("fully_nonblank") is True } expected_ceiling = max(ok_counts) if ok_counts else None if report.get("chart_count_ceiling") != expected_ceiling: @@ -1192,6 +1246,132 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No ) +def _dashboard_id_list( + row: dict[str, Any], key: str, path: str, expected_ids: set[str], errors: list[str] +) -> set[str]: + value = row.get(key) + if not isinstance(value, list): + errors.append(f"{path}.{key} must be a list") + return set() + if any(not isinstance(item, str) for item in value): + errors.append(f"{path}.{key} must contain only strings") + return set() + ids = set(value) + if len(ids) != len(value): + errors.append(f"{path}.{key} must not contain duplicate chart IDs") + unknown = sorted(ids - expected_ids) + if unknown: + errors.append(f"{path}.{key} contains unknown chart IDs: {unknown}") + return ids + + +def _validate_dashboard_telemetry(row: dict[str, Any], path: str, errors: list[str]) -> None: + chart_count = row.get("chart_count") + if isinstance(chart_count, bool) or not isinstance(chart_count, int) or chart_count <= 0: + return + expected_ids = {f"chart-{i}" for i in range(chart_count)} + id_keys = ( + "creation_failure_ids", + "initial_nonblank_chart_ids", + "initial_blank_chart_ids", + "scroll_nonblank_chart_ids", + "scroll_blank_chart_ids", + "context_lost_chart_ids", + "context_restored_chart_ids", + "currently_lost_chart_ids", + ) + ids = {key: _dashboard_id_list(row, key, path, expected_ids, errors) for key in id_keys} + + count_keys = ( + "steady_redraw_active_charts", + "created_charts", + "creation_failed_charts", + "nonblank_charts", + "initial_nonblank_charts", + "scroll_nonblank_charts", + ) + for key in count_keys: + value = row.get(key) + if isinstance(value, int) and not isinstance(value, bool) and value > chart_count: + errors.append(f"{path}.{key} must be <= chart_count") + + expected_pairs = ( + ("creation_failed_charts", "creation_failure_ids"), + ("initial_nonblank_charts", "initial_nonblank_chart_ids"), + ("scroll_nonblank_charts", "scroll_nonblank_chart_ids"), + ) + for count_key, ids_key in expected_pairs: + if row.get(count_key) != len(ids[ids_key]): + errors.append(f"{path}.{count_key} must equal len({ids_key})") + if row.get("nonblank_charts") != row.get("initial_nonblank_charts"): + errors.append(f"{path}.nonblank_charts must equal initial_nonblank_charts") + if row.get("created_charts") != chart_count - len(ids["creation_failure_ids"]): + errors.append(f"{path}.created_charts must equal chart_count minus creation failures") + if row.get("steady_redraw_active_charts") != row.get("created_charts", 0) - len( + ids["currently_lost_chart_ids"] + ): + errors.append( + f"{path}.steady_redraw_active_charts must equal created charts minus currently lost" + ) + + for prefix in ("initial", "scroll"): + lit = ids[f"{prefix}_nonblank_chart_ids"] + blank = ids[f"{prefix}_blank_chart_ids"] + if lit & blank: + errors.append(f"{path}.{prefix} nonblank and blank chart IDs must be disjoint") + if lit | blank != expected_ids: + errors.append(f"{path}.{prefix} chart IDs must cover every attempted chart") + + events = row.get("context_events") + lost_event_ids: list[str] = [] + restored_event_ids: list[str] = [] + if not isinstance(events, list): + errors.append(f"{path}.context_events must be a list") + events = [] + for index, event in enumerate(events): + event_path = f"{path}.context_events[{index}]" + _require_keys(event, {"id", "type", "phase", "at_ms"}, event_path, errors) + if not isinstance(event, dict): + continue + event_id = event.get("id") + if event_id not in expected_ids: + errors.append(f"{event_path}.id must identify an attempted chart") + event_type = event.get("type") + if event_type not in {"lost", "restored"}: + errors.append(f"{event_path}.type must be 'lost' or 'restored'") + elif isinstance(event_id, str): + (lost_event_ids if event_type == "lost" else restored_event_ids).append(event_id) + if event.get("phase") not in {"create", "initial", "scroll", "redraw", "report"}: + errors.append(f"{event_path}.phase has an unknown value") + _require_nonnegative_number(event, "at_ms", event_path, errors) + + if row.get("context_lost_events") != len(lost_event_ids): + errors.append(f"{path}.context_lost_events must equal lost context event count") + if row.get("context_restored_events") != len(restored_event_ids): + errors.append(f"{path}.context_restored_events must equal restored context event count") + if ids["context_lost_chart_ids"] != set(lost_event_ids): + errors.append(f"{path}.context_lost_chart_ids must match context_events") + if ids["context_restored_chart_ids"] != set(restored_event_ids): + errors.append(f"{path}.context_restored_chart_ids must match context_events") + if not ids["currently_lost_chart_ids"] <= ids["context_lost_chart_ids"]: + errors.append(f"{path}.currently_lost_chart_ids must be a subset of lost chart IDs") + + expected_complete = ( + row.get("created_charts") == chart_count + and row.get("initial_nonblank_charts") == chart_count + and row.get("scroll_nonblank_charts") == chart_count + and row.get("context_lost_events") == 0 + and not ids["currently_lost_chart_ids"] + ) + if not isinstance(row.get("fully_nonblank"), bool): + errors.append(f"{path}.fully_nonblank must be a boolean") + elif row.get("fully_nonblank") != expected_complete: + errors.append(f"{path}.fully_nonblank is inconsistent with dashboard telemetry") + expected_status = "complete" if expected_complete else "partial" + if row.get("render_status") != expected_status: + errors.append(f"{path}.render_status must be {expected_status!r}") + + def _validate_workflow_native(report: dict[str, Any], errors: list[str]) -> None: _require_native_backend(report, "workflow-native", errors) _require_keys( diff --git a/tests/test_benchmark_environment.py b/tests/test_benchmark_environment.py index 7eb5d29b..5f02d661 100644 --- a/tests/test_benchmark_environment.py +++ b/tests/test_benchmark_environment.py @@ -146,6 +146,45 @@ def test_interaction_browser_gates_cover_scatter_and_core_chart_families() -> No assert marker in bench +def test_interaction_benchmark_completes_gpu_warmup_before_timing() -> None: + bench = (ROOT / "benchmarks" / "bench_interaction.py").read_text(encoding="utf-8") + warm_start = bench.index("// Warm shader compilation") + timing_start = bench.index("let viewChanged = false", warm_start) + settle_start = bench.index("function settlePixels()", timing_start) + settle_end = bench.index("function measure(", settle_start) + + assert "settlePixels();" in bench[warm_start:timing_start] + assert "cancelAnimationFrame(view._raf);" in bench[settle_start:settle_end] + assert "gl.readPixels(" in bench[settle_start:settle_end] + + +def test_dashboard_benchmark_reports_eviction_and_scroll_telemetry() -> None: + bench = (ROOT / "benchmarks" / "bench_dashboard.py").read_text(encoding="utf-8") + + for marker in ( + 'addEventListener("webglcontextlost"', + 'addEventListener("webglcontextrestored"', + "scrollIntoView", + "context_lost_chart_ids", + "context_restored_chart_ids", + "initial_nonblank_chart_ids", + "scroll_nonblank_chart_ids", + 'render_status: fullyNonblank ? "complete" : "partial"', + ): + assert marker in bench + assert "blank dashboard chart" not in bench + + # webglcontextlost dispatches as a task, so the probe must yield before + # leaving the "create" phase or creation-loop evictions get mislabeled + # with whatever phase is current when the queued events finally fire. + creation_loop = bench.index('addEventListener("webglcontextlost"') + first_yield = bench.index( + "await new Promise((resolve) => setTimeout(resolve, 0));", creation_loop + ) + phase_initial = bench.index('phase = "initial";', creation_loop) + assert first_yield < phase_initial + + def test_benchmark_categories_track_core_hardening_metrics() -> None: medium_scatter_metrics = CATEGORY_BY_ID["medium_direct_scatter"]["metrics"] interaction_metrics = CATEGORY_BY_ID["interaction_smoothness"]["metrics"] diff --git a/tests/test_png_export.py b/tests/test_png_export.py index 88bff5c7..99fd7d94 100644 --- a/tests/test_png_export.py +++ b/tests/test_png_export.py @@ -111,6 +111,24 @@ def test_png_encoder_selects_indexed_for_few_colors() -> None: assert _ihdr(_png.encode(many))[2] == 6 +def test_png_encoder_uses_balanced_compression_level(monkeypatch) -> None: + levels: list[int] = [] + compress = _png.zlib.compress + + def recording_compress(data: bytes, level: int) -> bytes: + levels.append(level) + return compress(data, level) + + monkeypatch.setattr(_png.zlib, "compress", recording_compress) + few = np.zeros((10, 10, 4), np.uint8) + many = (np.random.default_rng(4).random((20, 20, 4)) * 255).astype(np.uint8) + + _png.encode(few) + _png.encode(many) + + assert levels == [6, 6] + + def test_native_and_svg_share_layout() -> None: # Both exporters compute the same plot rect / tick labels from one spec. from fastcharts import _svg diff --git a/tests/test_verify_benchmark_report.py b/tests/test_verify_benchmark_report.py index 6c037059..12aa05b3 100644 --- a/tests/test_verify_benchmark_report.py +++ b/tests/test_verify_benchmark_report.py @@ -422,6 +422,50 @@ def _dashboard_browser_report() -> dict: "small_data_startup", "payload_export_size", ) + + def row(count: int) -> dict: + chart_ids = [f"chart-{i}" for i in range(count)] + return { + "scenario": f"dashboard_{count}", + "chart_count": count, + "benchmark_categories": [ + "many_chart_dashboards", + "small_data_startup", + "payload_export_size", + ], + "total_payload_bytes": 262_144, + "html_bytes": 524_288, + "status": "ok", + "render_status": "complete", + "fully_nonblank": True, + "render_ms": 7.0 * count, + "ms_per_chart": 7.0, + "payload_prep_ms": 3.0 * count, + "navigation_ready_ms": 12.0 * count, + "scroll_pass_ms": 5.0 * count, + "steady_redraw_p95_ms": 16.7, + "steady_redraw_active_charts": count, + "js_heap_before_bytes": 1_000_000, + "js_heap_bytes": 2_000_000, + "js_heap_delta_bytes": 1_000_000, + "created_charts": count, + "creation_failed_charts": 0, + "creation_failure_ids": [], + "nonblank_charts": count, + "initial_nonblank_charts": count, + "initial_nonblank_chart_ids": chart_ids, + "initial_blank_chart_ids": [], + "scroll_nonblank_charts": count, + "scroll_nonblank_chart_ids": chart_ids, + "scroll_blank_chart_ids": [], + "context_lost_events": 0, + "context_restored_events": 0, + "context_lost_chart_ids": [], + "context_restored_chart_ids": [], + "currently_lost_chart_ids": [], + "context_events": [], + } + return { **_base(), "kind": "dashboard-browser", @@ -429,33 +473,40 @@ def _dashboard_browser_report() -> dict: "tracked_categories": tracked, "attempted_chart_counts": [10, 20, 50], "chart_count_ceiling": 50, - "rows": [ - { - "scenario": f"dashboard_{count}", - "chart_count": count, - "benchmark_categories": [ - "many_chart_dashboards", - "small_data_startup", - "payload_export_size", - ], - "total_payload_bytes": 262_144, - "html_bytes": 524_288, - "status": "ok", - "render_ms": 7.0 * count, - "ms_per_chart": 7.0, - "payload_prep_ms": 3.0 * count, - "navigation_ready_ms": 12.0 * count, - "steady_redraw_p95_ms": 16.7, - "js_heap_before_bytes": 1_000_000, - "js_heap_bytes": 2_000_000, - "js_heap_delta_bytes": 1_000_000, - "nonblank_charts": count, - } - for count in (10, 20, 50) - ], + "rows": [row(count) for count in (10, 20, 50)], } +def _set_dashboard_partial(row: dict, nonblank: int = 16) -> None: + count = row["chart_count"] + blank_ids = [f"chart-{i}" for i in range(count - nonblank)] + lit_ids = [f"chart-{i}" for i in range(count - nonblank, count)] + events = [ + {"id": chart_id, "type": "lost", "phase": "create", "at_ms": 10.0 + i} + for i, chart_id in enumerate(blank_ids) + ] + row.update( + { + "render_status": "partial", + "fully_nonblank": False, + "steady_redraw_active_charts": nonblank, + "nonblank_charts": nonblank, + "initial_nonblank_charts": nonblank, + "initial_nonblank_chart_ids": lit_ids, + "initial_blank_chart_ids": blank_ids, + "scroll_nonblank_charts": nonblank, + "scroll_nonblank_chart_ids": lit_ids, + "scroll_blank_chart_ids": blank_ids, + "context_lost_events": len(events), + "context_restored_events": 0, + "context_lost_chart_ids": blank_ids, + "context_restored_chart_ids": [], + "currently_lost_chart_ids": blank_ids, + "context_events": events, + } + ) + + def _workflow_native_report() -> dict: categories, tracked = _category_registry( "input_ingestion", "streaming_updates", "static_export" @@ -581,7 +632,7 @@ def test_scatter_report_requires_mode_target_and_oracle(tmp_path: Path) -> None: def test_dashboard_report_rejects_overstated_ceiling(tmp_path: Path) -> None: payload = _dashboard_browser_report() - payload["rows"][-1]["status"] = "failed(context limit)" + _set_dashboard_partial(payload["rows"][-1]) path = _write_report(tmp_path, payload) errors = verify_benchmark_report.validate_report(path, kind="dashboard-browser") @@ -589,6 +640,18 @@ def test_dashboard_report_rejects_overstated_ceiling(tmp_path: Path) -> None: assert any("chart_count_ceiling" in error for error in errors) +def test_dashboard_report_accepts_partial_rows_with_context_telemetry(tmp_path: Path) -> None: + payload = _dashboard_browser_report() + _set_dashboard_partial(payload["rows"][1]) + _set_dashboard_partial(payload["rows"][2]) + payload["chart_count_ceiling"] = 10 + path = _write_report(tmp_path, payload) + + errors = verify_benchmark_report.validate_report(path, kind="dashboard-browser") + + assert errors == [] + + def test_workflow_report_rejects_missing_required_scenario(tmp_path: Path) -> None: payload = _workflow_native_report() payload["rows"] = payload["rows"][1:]