From b552c8126e14ca5cb0ad0942c1a405fce8bb1c19 Mon Sep 17 00:00:00 2001 From: Alek Petuskey Date: Sun, 26 Jul 2026 01:36:24 -0700 Subject: [PATCH 1/3] Set XY's competitive product north star --- .github/workflows/ci.yml | 3 - CLAUDE.md | 28 +- Makefile | 10 +- README.md | 95 ++++--- docs/advanced/index.md | 3 +- docs/api-reference/contributing.md | 6 +- .../large-data-and-performance.md | 5 +- docs/overview/benchmarks.md | 29 +-- scripts/check_claim_guardrails.py | 243 ------------------ scripts/verify_ci_workflow.py | 1 - scripts/verify_local.py | 9 +- scripts/verify_sdist.py | 5 +- spec/README.md | 4 +- spec/api/chart-roadmap.md | 33 +-- spec/benchmarks/methodology.md | 8 +- spec/benchmarks/results.md | 61 ++--- spec/process/contributing.md | 40 ++- spec/process/production-readiness.md | 84 +----- tests/test_claim_guardrails.py | 175 ------------- tests/test_verify_ci_workflow.py | 17 -- tests/test_verify_local.py | 18 +- tests/test_verify_sdist.py | 4 +- 22 files changed, 202 insertions(+), 679 deletions(-) delete mode 100644 scripts/check_claim_guardrails.py delete mode 100644 tests/test_claim_guardrails.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee01f8ed..94fd8806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,9 +160,6 @@ jobs: - name: Public API coherence run: .venv/bin/python scripts/check_public_api.py - - name: Claim guardrails - run: .venv/bin/python scripts/check_claim_guardrails.py - - name: Lint (ruff) run: | .venv/bin/ruff check . diff --git a/CLAUDE.md b/CLAUDE.md index f3baeb12..a19fc500 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,6 +11,32 @@ release change. A change is incomplete while its affected specification is missing, stale, or inconsistent with the implementation; resolve discrepancies instead of treating the implementation alone as authoritative. +## Product North Star + +XY is being built to outperform every competing charting library and become +the best overall charting system for Python. That goal spans every chart type +and every data scale, from a handful of values to billions of rows, across the +two dimensions users should not have to trade off: performance and +customization. + +Treat every competitor lead as a concrete product gap. Work that affects a +user-visible capability should: + +- compare XY with the relevant leaders, including Matplotlib, Seaborn, Plotly, + Bokeh, Altair, Datashader, HoloViews/hvPlot, and emerging alternatives; +- add or extend reproducible evidence across small, medium, large, and massive + data, covering startup, build and render time, interaction, memory, payload + and export size, and multi-chart applications where applicable; +- update the capability matrix and visual examples when the improvement is + about chart breadth or customization rather than timing; and +- commit the environment, raw results, output contracts, and reproduction + commands needed to inspect the win and catch regressions. + +The goal is not to win one large-scatter benchmark. XY should become the +library users choose for ordinary charts, massive data, every chart family, +notebooks, applications, static output, performance, and complete design +control. + ## Layout - `src/` — Rust core, **minimal external crates** (C ABI; one cdylib per @@ -107,5 +133,3 @@ PRs, or code. Set `git config user.name/user.email` to the human author cache (§27). NaN never reaches vertex buffers (§19). - f32 uploads are offset-encoded; tick/hover math stays f64 (§4/§16). - Every decimation/tier decision is recorded in the spec, never silent (§28). -- Claims are mode-scoped and benchmarked (§2); update README numbers from - `scripts/bench.py`, don't invent them. diff --git a/Makefile b/Makefile index 53167a6f..f5aedcde 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ WHEEL ?= BENCHMARK_JSON ?= benchmark.json BENCHMARK_KIND ?= auto -.PHONY: help setup setup-browser check check-full check-browser check-conformance check-docs check-examples check-security check-errors check-api check-import check-ci check-claims check-benchmark-harness check-pyplot check-pyplot-speed check-sdist check-wheel check-artifacts check-benchmark-report list-checks test lint format typecheck public-api python-floor js-check rust-check abi-smoke +.PHONY: help setup setup-browser check check-full check-browser check-conformance check-docs check-examples check-security check-errors check-api check-import check-ci check-benchmark-harness check-pyplot check-pyplot-speed check-sdist check-wheel check-artifacts check-benchmark-report list-checks test lint format typecheck public-api python-floor js-check rust-check abi-smoke help: @printf '%s\n' \ @@ -19,14 +19,13 @@ help: ' make check-full run JS, Rust, and ABI gates too' \ ' make check-browser run browser smokes (set CHROMIUM=/path/to/chrome)' \ ' make check-conformance run accessibility + Chromium/Firefox/WebKit conformance' \ - ' make check-docs run docs examples and public claim guardrails' \ + ' make check-docs run documentation examples' \ ' make check-examples run canonical API examples and Reflex asset registry checks' \ ' make check-security run standalone HTML safety and client text-sink checks' \ ' make check-errors run public error, LOD, and mutation-safety tests' \ ' make check-api run lazy public API and type-surface checks' \ ' make check-import run import-time and dependency-boundary checks' \ ' make check-ci run CI/release workflow invariant checks' \ - ' make check-claims run public performance-claim guardrails' \ ' make check-benchmark-harness run benchmark metadata/report/regression tests' \ ' make check-pyplot run the matplotlib-shim suite and compatibility corpus' \ ' make check-pyplot-speed enforce the per-family 10x static-PNG target (requires .[bench])' \ @@ -76,7 +75,7 @@ check-conformance: node scripts/browser_conformance.mjs check-docs: - $(PYTHON) scripts/verify_local.py --only examples,claim_guardrails + $(PYTHON) scripts/verify_local.py --only examples check-examples: $(PYTHON) scripts/verify_local.py --only examples @@ -99,9 +98,6 @@ check-import: check-ci: $(PYTHON) scripts/verify_local.py --only ci_workflow -check-claims: - $(PYTHON) scripts/verify_local.py --only claim_guardrails - check-benchmark-harness: $(PYTHON) scripts/verify_local.py --only benchmark_harness diff --git a/README.md b/README.md index 2bfcd053..80b53e7a 100644 --- a/README.md +++ b/README.md @@ -13,25 +13,37 @@ Docs

-XY is an actively evolving, early-alpha Python charting library for large, -interactive datasets. Its Rust core and WebGL2 renderer keep work bounded by -what the screen can show; find guides, API reference, and examples in the -[documentation](https://reflex.dev/docs/xy/). +

+ Small data should be effortless. Large data should stay interactive.
+ Neither should limit the design. +

+ +XY is an actively evolving, early-alpha project with one ambition: become the +fastest and most customizable charting library for Python — across every chart +type and every scale, from a handful of values to billions of rows. + +Small datasets render directly and exactly. As data grows, XY's Rust core, +binary transport, and WebGL2 renderer reduce work to what the screen can show, +then bring detail back as you zoom. The same composable API controls marks, +axes, annotations, legends, tooltips, interactions, themes, responsive layout, +and HTML, raster, and vector exports. Find guides, API reference, and examples +in the [documentation](https://reflex.dev/docs/xy/). ## Highlights -- **Built for large data.** Reduces long lines and dense scatters to what the screen can show, and brings detail back as you zoom. +- **Designed for every scale.** Ordinary datasets take the direct path; large datasets automatically use decimated or density representations. +- **Customize every layer.** Control marks and composition in Python, then style chart chrome through stable CSS and Tailwind hooks. - **Declarative interface.** Compose marks and guides, or use the familiar `xy.pyplot`. - **Interactive by default.** Pan, zoom, hover, select, and inspect exact source rows. - **One chart, many outputs.** Use notebooks or export HTML, raster, and vector formats. -- **Built for apps.** Embed responsive charts and style them with CSS or Tailwind. +- **Built for apps.** Embed responsive charts and style them with Python, CSS, or Tailwind. ## Is XY for me? -XY is a great fit for teams that want to explore large 2D datasets in Python, -share interactive notebook results, or ship self-contained charts on the web. -Build charts once, then display them in notebooks and apps or export them as -HTML, images, and vector graphics. +XY is for Python users who want one charting system for everyday plots, deeply +custom application visuals, and data too large for conventional browser chart +stacks. Build charts once, then display them in notebooks and apps or export +them as self-contained HTML, images, and vector graphics. ## Installation @@ -42,7 +54,7 @@ pip install xy uv add xy ``` -## Getting started +## Getting started: one API, every scale A chart is a container plus the marks inside it. Any sequence works — plain Python lists need no NumPy: @@ -57,7 +69,8 @@ chart = xy.line_chart(xy.line([1, 2, 3, 4, 5], [120, 180, 165, 240, 310])) chart # notebooks render it ``` -The same API scales. Chart a hundred million points as a density surface: +The data size changes; the chart model does not. The same API scales to a +hundred million points as a density surface:

@@ -117,25 +130,20 @@ plt.show() The shim intentionally covers common plotting workflows rather than every matplotlib feature. See the [compatibility guide](spec/matplotlib/compat.md). -## Benchmarks +## Customize every layer -

- Cold-render time for a 10-million-point chart in XY, Matplotlib, and Plotly. Lower is better. -

- -In the recorded 10-million-point baseline, XY produced a static PNG in 0.023 s -versus 2.8 s for Matplotlib and 9.6 s for Plotly, and reached first interactive -render 16–20× sooner. - -The committed launch baseline uses identical seeded data, a 900×420 output, -and three isolated cold runs. See the -[launch report](benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/report.md) -and [benchmark runbook](benchmarks/README.md) for the environment, -methodology, and raw results. +Performance is not a preset. Use Python to control data marks and chart +composition, then style browser chrome through stable CSS and Tailwind hooks. -## Styling +- **Marks:** data-driven color and size, custom colormaps, opacity, symbols, + gradients, strokes, and curves. +- **Guides:** axes, ticks, grids, annotations, legends, colorbars, and tooltips. +- **Interaction:** pan, zoom, hover, selection, crosshairs, callbacks, and + linked charts. +- **Layout and output:** layering, facets, responsive dimensions, themes, and + HTML, raster, or vector export. -Customize marks and chart chrome with Python, CSS, or Tailwind. See the [styling guide](docs/styling/index.md). +See the [styling guide](docs/styling/index.md) for patterns and examples. What each mechanism reaches — per property, per chrome slot, per renderer, and where it stops — is the [capability matrix](spec/api/capability-matrix.md), @@ -150,6 +158,30 @@ chart = xy.line_chart( ) ``` +## Benchmarks + +

+ Cold-render time for a 10-million-point chart in XY, Matplotlib, and Plotly. Lower is better. +

+ +In the recorded 10-million-point baseline, XY produced a static PNG in 0.023 s +versus 2.8 s for Matplotlib and 9.6 s for Plotly, and reached first interactive +render 16–20× sooner. + +That baseline is one proof point, not the finish line. XY is being built to +outperform every competing charting library across small-data startup, +massive-data rendering, interaction, memory, payload and export size, +multi-chart applications, chart breadth, and customization. The committed +benchmark program, raw results, capability matrix, and visual examples are how +the project measures and proves that goal. + +The committed launch baseline uses identical seeded data, a 900×420 output, +and three isolated cold runs. See the +[launch report](benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/report.md), +[benchmark runbook](benchmarks/README.md), and +[competitive benchmark specification](spec/benchmarks/results.md) for the +environment, methodology, raw results, and full evidence program. + ## Embed XY in a Reflex app With the `reflex-xy` adapter, any XY chart becomes a regular Reflex component. @@ -266,8 +298,10 @@ the chart gallery, guides, and API reference. The repository also includes ## Roadmap -XY is 2D-first: broad chart coverage on top of the binary transport and -screen-bounded rendering, before any 3D work. Queued next, no dates implied: +XY is headed toward every chart family. The current implementation sequence +builds broad 2D coverage on top of the binary transport and screen-bounded +renderer, then extends the same performance and customization goals to +geographic, 3D, and volume visualization. Queued next, no dates implied: - **Categorical distributions** — strip, swarm, beeswarm, boxen, rug - **Regression diagnostics** — trendline, residual, QQ, PP @@ -278,6 +312,7 @@ screen-bounded rendering, before any 3D work. Queued next, no dates implied: - **Treemap, sunburst, and icicle** - **Radar / polar and gauge** — needs polar axes first - **Slope, bump, and dumbbell** +- **3D and volume** — scatter, surfaces, meshes, isosurfaces, and volumetric views The full ranked backlog is in the [chart roadmap](spec/api/chart-roadmap.md). Want a chart or feature that isn't listed? diff --git a/docs/advanced/index.md b/docs/advanced/index.md index 03402dae..ed897713 100644 --- a/docs/advanced/index.md +++ b/docs/advanced/index.md @@ -103,8 +103,7 @@ does not automatically become a browser object: - Retained GPU buffers and local interaction avoid rebuilding the chart on every pointer movement. -This is a focused design, not a claim that every chart is faster. Compare the -same data, output, and rendering mode. The +This focused design is the foundation for making every chart faster. The [benchmark snapshot](/docs/xy/overview/benchmarks/) publishes those contracts and records when XY uses a reduced representation. diff --git a/docs/api-reference/contributing.md b/docs/api-reference/contributing.md index 63d3a743..67a1f9ab 100644 --- a/docs/api-reference/contributing.md +++ b/docs/api-reference/contributing.md @@ -38,17 +38,13 @@ import error rather than silently selecting another backend. | --- | --- | | Fast local verification | `make check` | | Production-facing change | `make check-full` | -| Public docs, examples, or claims | `make check-docs` | +| Public docs and examples | `make check-docs` | | Public exports or annotations | `make check-api` | | Lazy imports and dependency boundaries | `make check-import` | | Validation and mutation behavior | `make check-errors` | | Standalone HTML and text safety | `make check-security` | | Browser lifecycle and interaction | `make check-browser CHROMIUM=/path/to/chrome` | -Run `make check-claims` before publishing performance prose. Every comparison -must name the chart type, data size and shape, representation mode, backend, -render target, and whether browser time-to-first-render is included. - ## Adding a Chart Type A complete chart-family contribution normally spans: diff --git a/docs/core-concepts/large-data-and-performance.md b/docs/core-concepts/large-data-and-performance.md index 119ca434..7ebb2fdb 100644 --- a/docs/core-concepts/large-data-and-performance.md +++ b/docs/core-concepts/large-data-and-performance.md @@ -78,9 +78,8 @@ There are two different scaling regimes: 2. The resulting wire payload, WebGL geometry, and static SVG/native-PNG scene are bounded by the chosen viewport representation. -This is why a “cost scales with pixels, not points” slogan needs qualification: -it describes the output side after reduction, not the complete data-to-pixels -pipeline. +The output side after reduction scales with pixels; the complete data-to-pixels +pipeline still includes row-dependent ingest and reduction work. ## Inspect memory and tier decisions diff --git a/docs/overview/benchmarks.md b/docs/overview/benchmarks.md index 803bffaa..08538a1e 100644 --- a/docs/overview/benchmarks.md +++ b/docs/overview/benchmarks.md @@ -5,18 +5,18 @@ description: Inspect XY's recorded launch benchmark with its exact output contra # Benchmarks -XY's large-data claim is about the representation sent to a fixed-size output, -not about drawing every source row as an individual marker. The committed 0.1.0 -launch baseline measures identical seeded scatter data at 900×420 pixels on an -Apple M5 Pro with 64 GiB RAM. Each successful cell below is the mean of three -isolated cold runs. The machine name and memory are copied verbatim from the -committed environment record. +XY's large-data architecture reduces source rows to the representation useful +for a fixed-size output instead of drawing every row as an individual marker. +The committed 0.1.0 launch baseline measures identical seeded scatter data at +900×420 pixels on an Apple M5 Pro with 64 GiB RAM. Each successful cell below +is the mean of three isolated cold runs. The machine name and memory are copied +verbatim from the committed environment record. > **How to read this comparison.** XY switches dense scatter output to a > screen-bounded density representation, while the default Plotly and > Matplotlib paths retain every marker. These results compare each library's -> default user-visible outcome at the same output size; they do not claim that -> the libraries send identical geometry to the renderer. +> default user-visible outcome at the same output size; the libraries send +> different geometry to the renderer. ## Snapshot at 10 million points @@ -121,8 +121,9 @@ frozen: - **Release and hardware tracking:** immutable release directories plus clearly separated macOS hardware-WebGL and CI SwiftShader results. -Until those artifacts are published, this page intentionally keeps its -headline claims scoped to the committed launch scatter. +The launch scatter is the first committed proof point. The competitive +benchmark program expands it across these workloads, chart families, +competitors, and environments. ## Inspect and reproduce the evidence @@ -148,9 +149,7 @@ uv run --project "$BASELINE" --frozen python benchmarks/bench_launch_scatter.py --chrome "$CHROME" --out launch-scatter-default.json ``` -One machine and three runs describe that recorded environment, not every -machine or workload. New comparisons should retain chart type, data size, -representation, backend, output target, and browser-TTFR status rather than -shortening these results to a universal “faster than” claim. For the rendering -model behind the numbers, read +Each new comparison records its chart type, data size, representation, backend, +output target, and browser-TTFR status so the result can be reproduced and +improved. For the rendering model behind the numbers, read [Large data and performance](/docs/xy/core-concepts/large-data-and-performance/). diff --git a/scripts/check_claim_guardrails.py b/scripts/check_claim_guardrails.py deleted file mode 100644 index f7d4f4d2..00000000 --- a/scripts/check_claim_guardrails.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 -"""Check public docs for broad, unqualified performance claims. - -This is intentionally a small guardrail, not a full natural-language judge. It -flags phrases that are easy to post publicly and hard to defend, while allowing -measured benchmark rows and policy text that names mode/data/backend context. -""" - -from __future__ import annotations - -import argparse -import re -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Optional - -ROOT = Path(__file__).resolve().parents[1] -DEFAULT_DOCS = ( - "README.md", - "CLAUDE.md", - "pyproject.toml", - "SECURITY.md", - "CONTRIBUTING.md", - "spec/api/api-examples.md", - "spec/benchmarks/results.md", - "spec/api/chart-roadmap.md", - "spec/process/contributing.md", - "spec/process/production-readiness.md", -) - -BROAD_SUPERLATIVE_RE = re.compile( - r"\b(" - r"faster\s+charting\s+engine|" - r"fastest(?:\s+charting\s+library)?|" - r"best\s+(?:python\s+)?(?:charting|plotting|graphing|data)\s+(?:library|engine)|" - r"best\s+(?:at|for)\s+rendering|" - r"most\s+performant(?:\s+(?:python\s+)?(?:charting|plotting|graphing|data)\s+" - r"(?:library|engine))?|" - r"better\s+than\s+(?:all|every|any)|" - r"faster\s+than\s+(?:all|every|any)|" - r"faster\s+than\s+everything|" - r"more\s+performant\s+than\s+(?:all|every|any|everything)|" - r"blow\s+(?:them|it|everyone)\s+away" - r")\b", - re.IGNORECASE, -) -# Customization is the second axis people overclaim on, and it went unguarded -# while performance was fenced in. These shapes are unqualifiable: the styling -# surface is a bounded, enumerated subset (`spec/api/capability-matrix.md`), so -# "anything" and "most" are wrong however the sentence is framed. -CUSTOMIZATION_SUPERLATIVE_RE = re.compile( - r"\b(" - r"most\s+(?:customi[sz]able|themeable|styl(?:e)?able|extensible|flexible)|" - r"(?:fully|completely|infinitely|endlessly|totally)\s+customi[sz]able|" - r"customi[sz]e\s+(?:everything|anything)|" - r"style\s+(?:everything|anything)|" - r"unlimited\s+(?:styling|customi[sz]ation)|" - r"(?:more|as)\s+(?:customi[sz]able|themeable|extensible)\s+than\s+" - r"(?:all|every|any|everything|anything)" - r")\b", - re.IGNORECASE, -) -COMPARATIVE_RE = re.compile( - r"\b(?:faster\s+than|beats?|outperforms?)\s+" - r"(?:plotly|matplotlib|bokeh|altair|datashader|holoviews|hvplot|seaborn)\b", - re.IGNORECASE, -) -LARGE_RENDER_RE = re.compile( - r"\brenders?\s+(?:\d[\d,]*(?:\.\d+)?\s*)?(?:k|m|b|million|billion)?[- ]?points\b", - re.IGNORECASE, -) -NUMERIC_PERFORMANCE_RE = re.compile( - r"\b\d+(?:\.\d+)?\s*x\s+(?:faster|slower|smaller|larger|less|more)\b", - re.IGNORECASE, -) -# The project has lived under earlier names (charts-exp, personal forks). -# Public docs must point at the canonical repository — a stale URL here means -# security reports, clones, and badges land on the wrong repo. -STALE_REPO_RE = re.compile(r"github\.com/Alek99|app\.codspeed\.io/Alek99|charts-exp", re.IGNORECASE) - -POLICY_WORDS = re.compile( - r"\b(do not|don't|never|no amount of|must|should|guardrail|policy|claim|goal|planned|target|" - r"blurry|rather than|not\s+(?:a|the|safe|same|one)|without naming|" - r"needs qualification)\b", - re.IGNORECASE, -) -QUALIFIER_GROUPS = ( - re.compile(r"\b(?:\d[\d,]*(?:\.\d+)?\s*)?(?:k|m|b|million|billion|points|rows|n=)\b", re.I), - re.compile( - r"\b(?:direct|density|decimated|adaptive|sampled|screen-bounded|exact markers?)\b", re.I - ), - re.compile( - r"\b(?:native|numpy|backend|browser|chromium|swiftshader|render target|" - r"webgl|svg|png|to pixels)\b", - re.I, - ), - re.compile( - r"\b(?:benchmark|measured|documented|ttfr|payload|memory|ms|mb|gb|artifact)\b", re.I - ), - re.compile(r"\b(?:chart type|data size|mode|row|table|spec/benchmarks/results\.md)\b", re.I), -) - - -@dataclass(frozen=True) -class Finding: - path: Path - line: int - message: str - text: str - - def format(self) -> str: - rel = self.path.relative_to(ROOT) if self.path.is_relative_to(ROOT) else self.path - return f"{rel}:{self.line}: {self.message}: {self.text.strip()}" - - -def _line_window(lines: list[str], index: int, radius: int = 2) -> str: - start = max(0, index - radius) - end = min(len(lines), index + radius + 1) - return "\n".join(lines[start:end]) - - -def _is_policy_or_negative_context(text: str) -> bool: - return bool(POLICY_WORDS.search(text)) - - -def _has_claim_qualifiers(text: str) -> bool: - return sum(1 for pattern in QUALIFIER_GROUPS if pattern.search(text)) >= 3 - - -def _findings_for_file(path: Path) -> list[Finding]: - lines = path.read_text(encoding="utf-8").splitlines() - findings: list[Finding] = [] - for index, line in enumerate(lines): - window = _line_window(lines, index) - if STALE_REPO_RE.search(line): - # No policy-context exemption: a stale repo identity is always wrong. - findings.append( - Finding( - path, - index + 1, - "stale repository identity; use github.com/reflex-dev/xy", - line, - ) - ) - if BROAD_SUPERLATIVE_RE.search(line) and not _is_policy_or_negative_context(window): - findings.append( - Finding( - path, - index + 1, - "broad superlative needs measured scope or policy framing", - line, - ) - ) - if CUSTOMIZATION_SUPERLATIVE_RE.search(line) and not _is_policy_or_negative_context(window): - findings.append( - Finding( - path, - index + 1, - "customization superlative is not defensible from the capability matrix", - line, - ) - ) - if COMPARATIVE_RE.search(line) and not ( - _is_policy_or_negative_context(window) or _has_claim_qualifiers(window) - ): - findings.append( - Finding( - path, - index + 1, - "comparative performance claim needs chart/data/backend/render context", - line, - ) - ) - if LARGE_RENDER_RE.search(line) and not ( - _is_policy_or_negative_context(window) or _has_claim_qualifiers(window) - ): - findings.append( - Finding( - path, - index + 1, - "large-point rendering claim needs mode context", - line, - ) - ) - if NUMERIC_PERFORMANCE_RE.search(line): - # Benchmark tables often carry the measurement context a few lines - # above the rows; give this rule a wider view than sentence-level - # comparative claims while still catching slogan-style multipliers. - metric_window = _line_window(lines, index, radius=10) - if not ( - _is_policy_or_negative_context(metric_window) - or _has_claim_qualifiers(metric_window) - ): - findings.append( - Finding( - path, - index + 1, - "numeric performance multiplier needs measured benchmark context", - line, - ) - ) - return findings - - -def check_claims(paths: list[Path]) -> list[Finding]: - findings: list[Finding] = [] - for path in paths: - if not path.exists(): - findings.append(Finding(path, 0, "document does not exist", "")) - continue - findings.extend(_findings_for_file(path)) - return findings - - -def _default_paths() -> list[Path]: - paths = [ROOT / item for item in DEFAULT_DOCS] - public_docs = ( - path - for path in sorted((ROOT / "docs").rglob("*.md")) - if "app" not in path.relative_to(ROOT / "docs").parts - ) - return list(dict.fromkeys((*paths, *public_docs))) - - -def main(argv: Optional[list[str]] = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="*", type=Path, help="docs to scan") - args = parser.parse_args(argv) - - paths = args.paths or _default_paths() - findings = check_claims([path if path.is_absolute() else ROOT / path for path in paths]) - if findings: - print("performance claim guardrail failed:", file=sys.stderr) - for finding in findings: - print(f"- {finding.format()}", file=sys.stderr) - return 1 - print(f"performance claim guardrail OK: {len(paths)} document(s)") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index e1b4bf28..d23b1d94 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -329,7 +329,6 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "hard production gates", "scripts/verify_ci_workflow.py", "scripts/check_public_api.py", - "scripts/check_claim_guardrails.py", "ruff check .", "scripts/smoke_render.py", "Browser lifecycle smoke", diff --git a/scripts/verify_local.py b/scripts/verify_local.py index b59303fe..052d883e 100644 --- a/scripts/verify_local.py +++ b/scripts/verify_local.py @@ -75,11 +75,6 @@ def _base_checks( (py, "-m", "pytest", "-q", "tests/test_import.py", "tests/test_dependencies.py"), requires_modules=("pytest",), ), - Check( - "claim_guardrails", - "public performance-claim guardrails", - (py, "scripts/check_claim_guardrails.py"), - ), Check( "capability_matrix", "generated capability matrix is current", @@ -87,7 +82,7 @@ def _base_checks( ), Check( "benchmark_harness", - "benchmark metadata, report, regression, and claim guardrail tests", + "benchmark metadata, report, and regression tests", ( py, "-m", @@ -97,7 +92,6 @@ def _base_checks( "tests/test_bench_pyplot_vs_matplotlib.py", "tests/test_verify_benchmark_report.py", "tests/test_check_regressions.py", - "tests/test_claim_guardrails.py", ), requires_modules=("pytest",), ), @@ -288,7 +282,6 @@ def _base_checks( QUICK_CHECKS = ( "python_floor", "public_api", - "claim_guardrails", "capability_matrix", "ci_workflow", "ruff_check", diff --git a/scripts/verify_sdist.py b/scripts/verify_sdist.py index e0fdb2ab..51d0eb86 100644 --- a/scripts/verify_sdist.py +++ b/scripts/verify_sdist.py @@ -104,7 +104,6 @@ "examples/reflex/xy_reflex_demo/__init__.py", "examples/reflex/xy_reflex_demo/xy_reflex_demo.py", "scripts/check_public_api.py", - "scripts/check_claim_guardrails.py", "scripts/gen_capability_matrix.py", "scripts/check_python_floor.py", "scripts/check_regressions.py", @@ -120,7 +119,6 @@ "src/kernels.rs", "src/lib.rs", "tests/test_public_api.py", - "tests/test_claim_guardrails.py", "tests/test_benchmark_environment.py", "tests/test_bench_pyplot_vs_matplotlib.py", "tests/test_check_regressions.py", @@ -397,7 +395,8 @@ def verify_sdist(path: str) -> None: "make check-sdist", "make check-examples", "make check-benchmark-report", - "Performance Claims", + "Competitive Evidence", + "outperform every competing charting library", }, ) _require_file_contains( diff --git a/spec/README.md b/spec/README.md index bde52d7d..669ca289 100644 --- a/spec/README.md +++ b/spec/README.md @@ -29,8 +29,8 @@ The public surface: what callers can build, style, export, and interact with. `python/xy/styling/capabilities.py`. Regenerate with `scripts/gen_capability_matrix.py --write`; the test suite fails if it is stale. -- [`chart-roadmap.md`](api/chart-roadmap.md) — the single 2D-first chart-type - coverage backlog, prioritized by popularity and primitive reuse. +- [`chart-roadmap.md`](api/chart-roadmap.md) — the staged chart-type coverage + backlog, from core 2D through geographic, 3D, and volume visualization. - [`export.md`](api/export.md) — how a figure becomes bytes: one entry point across five image formats, deterministic engine choice, browser-free default. - [`interaction.md`](api/interaction.md) — the authority on which browser diff --git a/spec/api/chart-roadmap.md b/spec/api/chart-roadmap.md index 25f5e51d..25ffb8f0 100644 --- a/spec/api/chart-roadmap.md +++ b/spec/api/chart-roadmap.md @@ -1,11 +1,13 @@ # Chart Type Roadmap -This is the single chart-type roadmap for xy. It is intentionally -**2D-first**: the library should become broad enough for Plotly-class analytics, -finance, science, operations, and dashboard use cases before spending product -energy on 3D/volume rendering. Nothing in this section implies immediate -implementation; it is the coverage backlog we will pull from as the primitives -land. +This is the single chart-type roadmap for xy. The destination is every chart +family: analytical and statistical charts, finance, science, operations, +dashboards, geographic views, 3D, and volume rendering. Delivery is staged: +the current implementation builds broad 2D coverage first because those shared +primitives unlock the most use cases, then carries the same performance and +customization goals into additional coordinate systems. Nothing in this +section implies immediate implementation; it is the coverage backlog we will +pull from as the primitives land. The roadmap prioritizes chart types by two signals: @@ -14,10 +16,10 @@ The roadmap prioritizes chart types by two signals: 2. **xy fit:** chart types where the engine can win on large data, binary transport, WebGL rendering, and screen-bounded aggregation. -The near-term wedge is large data, but the long-term product goal is broader: -xy should become a **Plotly-class general-purpose charting library** for -analytics, science, finance, operations, and dashboards. Performance is the -entry point, not the boundary of the product. +The near-term wedge is large data, but the product goal is broader: xy should +outperform every competing charting library across chart breadth, data scale, +performance, and customization. Performance is the entry point, not the +boundary of the product. The current implemented surface is **line**, **scatter**, **area**, **histogram**, **bar/column**, **heatmap**, **error bars/bands**, @@ -83,7 +85,7 @@ These sources are used as popularity proxies, not as exact usage telemetry: | [Bokeh gallery](https://docs.bokeh.org/en/latest/docs/gallery.html) | line, scatter, bar, patches, image, heatmap, contour, candlestick, maps, network, linked brushing, streaming | | Internal design dossier, section 28 and 24 | bar/histogram, heatmap, box/violin, candlestick/OHLC as important compatibility targets | -## 2D-First Priority Roadmap +## Staged Chart Coverage Roadmap ### P0 - Already in place @@ -209,18 +211,19 @@ depth: strip/swarm/boxen/rug distributions, regression diagnostics, richer | 32 | Packed bubble / Venn / Euler | Compatibility and presentation charts. | Layout algorithms and label placement dominate. | | 33 | Pictorial bar / item chart / image markers / text marks | ECharts/Highcharts compatibility polish. | Symbol systems and asset handling. | -### P6 - 2D domains to defer until core breadth is strong +### P6 - Additional domains after core breadth is strong | Chart family | Reason to defer | |---|---| | Maps / choropleth / geo scatter / routes / density maps | 2D, but requires projection, tile, geography data, and viewport semantics; valuable as a later domain package. | -| 3D scatter / surface / volume | Explicitly outside the 2D-first roadmap; different rendering and interaction model. | +| 3D scatter / surface / mesh / volume | Requires dedicated camera, projection, lighting, depth, picking, and interaction primitives; part of the product destination after the shared 2D foundations are broad. | ### P7 - Long-term breadth target The long-term target is not only "fast large scatter." It is an expansive -library with enough chart breadth to be used across industries the way Plotly is -used today, while keeping xy' transport and rendering model underneath. +library that leads across industries and chart families, including geographic, +3D, and volume visualization, while keeping xy's transport and rendering model +underneath. | Industry / use case | Chart coverage needed | |---|---| diff --git a/spec/benchmarks/methodology.md b/spec/benchmarks/methodology.md index 94cb341b..04c49c9b 100644 --- a/spec/benchmarks/methodology.md +++ b/spec/benchmarks/methodology.md @@ -10,10 +10,10 @@ are published. ## 0. The three rules -1. **Mode-scoped claims only** (§2 of the dossier, already policy): "10M - density in X ms" is a different claim from "10M individually-styled - markers." Output labels every row `direct | decimated | density | sampled` - — a reader can never mistake an aggregate result for a raw-marker result. +1. **Record the rendering mode.** "10M density in X ms" and "10M + individually-styled markers in X ms" are different workloads. Output labels + every row `direct | decimated | density | sampled`, preserving the result + needed to compare and improve each path. 2. **Same-work comparisons.** Each competitor renders the *same visual contract*, not the same API call: if XY aggregates at 10M, the fair Plotly comparison is Plotly failing/succeeding at raw markers AND diff --git a/spec/benchmarks/results.md b/spec/benchmarks/results.md index 1b92238c..93c1812c 100644 --- a/spec/benchmarks/results.md +++ b/spec/benchmarks/results.md @@ -1,8 +1,9 @@ -# Scatter benchmark: xy vs Python charting libraries +# Competitive benchmark program -The same scatter (a correlated 2-D cloud) at growing point counts, across -popular Python charting libraries, on four factors: **how many points each can -render, how fast, how much memory, and the render payload size.** +The recorded launch comparison uses the same scatter (a correlated 2-D cloud) +at growing point counts across popular Python charting libraries, measuring +four factors: **how many points each can render, how fast, how much memory, and +the render payload size.** The cross-library harness lives in `benchmarks/bench_vs.py` and includes optional adapters for xy, matplotlib, seaborn, Plotly, Bokeh, Altair, @@ -25,12 +26,32 @@ python benchmarks/bench_vs.py The xy-only arm also runs with no dependencies via `benchmarks/bench_scatter_native.py`. +## Competitive product goal + +XY's product goal is to outperform every competing charting library across +user-visible performance, chart breadth, and customization. The benchmark +program is the evidence system for that goal, not a showcase for one favorable +workload. + +Every major competitor should be represented wherever it leads or defines +user expectations. Comparisons should cover small through massive datasets, +exact and aggregated rendering, static and interactive output, cold and warm +paths, interaction latency, memory, transport and export size, and multi-chart +applications. Environments, raw machine-readable results, output contracts, +and reproduction commands belong in the repository so every result can be +inspected and rerun. + +Timing alone cannot prove chart breadth or design control. Those dimensions +use the generated capability matrix, matched visual examples, renderer +coverage, and compatibility corpora as evidence. When another library is +better on a relevant workload or capability, record the loss as a product gap, +add a reproducible comparison, and keep it visible until XY wins it. + ## Benchmark categories and goals -The performance story should be measured by mode, not with one blanket -"fastest charting library" number. A small exact scatter, a 10M density view, a -large line, and a 30-chart dashboard stress different parts of the system. These -are the categories we track or plan to add to CI. +Performance is tracked by mode and workload. A small exact scatter, a 10M +density view, a large line, and a 30-chart dashboard stress different parts of +the system. These are the categories we track or plan to add to CI. The stable category IDs live in `benchmarks/categories.py`. CI's benchmark JSON artifacts (`benchmark.json`, `line.json`, `install.json`, `install-fresh.json`, @@ -191,25 +212,6 @@ count. Partial rows retain their timing and memory metrics. `bench_workflows.py` updates, and separate HTML/SVG/native-PNG/Chromium-PNG export rows. All three emit schema-versioned JSON with environment metadata and benchmark category IDs. -## Copyable claim taxonomy - -Use these shapes when turning benchmark rows into public prose, release notes, -or posts. The goal is to make every public claim reproducible from a row in -this document or from a verified JSON artifact. - -| Claim shape | Safe wording pattern | Required context | -|---|---|---| -| Payload/prep comparison | "In the native backend benchmark, histogram payload prep for 10k values / 200 bins was 17.3x faster than Plotly." | chart type, workload, backend, compared library, metric | -| Browser first paint | "For the measured Chrome TTFR row, the 10k-value histogram first painted 5.0x faster than Plotly." | browser/render target, workload, chart type, TTFR included | -| Large scatter overview | "The 100M scatter overview uses density mode with a 258 KB wire payload; it is not drawing 100M exact markers." | mode, point count, payload, exact-vs-aggregate wording | -| Line decimation | "The 10M line benchmark ships an M4-decimated ~60 KB payload while preserving the extrema oracle." | mode, point count, payload, correctness oracle | -| Install/import footprint | "In the install-footprint benchmark, cold import was 6.4 ms for the measured distribution." | benchmark name, metric, measured distribution | - -Do not shorten those into broad slogans such as "xy is faster than -Plotly" or "renders 10M points" without the row context. If a sentence does not -name the chart type, workload, mode, backend, metric, and render target where -they matter, it is not ready to publish. - ## `xy.pyplot` versus Matplotlib/Agg `benchmarks/bench_pyplot_vs_matplotlib.py` runs the same Matplotlib-style calls @@ -1010,6 +1012,5 @@ falls over. This is the path xy exists to replace. - xy memory is Python-side `tracemalloc` peak; its GPU/native bytes are separate and itemized by `Figure.memory_report()` (§27). The payload column is the transport cost across the kernel→browser boundary. -- Single CI machine, one run; treat as order-of-magnitude, not a spec. Re-run in - CI (`benchmark` job) or locally. No universal claims — every number is - mode-scoped (dossier §2/§31). +- Single CI machine, one run; re-run in CI (`benchmark` job) or locally and + expand the committed baselines across hardware, workloads, and competitors. diff --git a/spec/process/contributing.md b/spec/process/contributing.md index 4d710a5c..1a5a8488 100644 --- a/spec/process/contributing.md +++ b/spec/process/contributing.md @@ -14,6 +14,22 @@ the hard-won production invariants while the chart surface grows. the asset. Do not patch only the generated HTML unless you are applying an emergency local fix and immediately back-porting it to the generator. +## Competitive Evidence + +XY's product north star is to outperform every competing charting library +across performance, chart breadth, customization, and data scale. Feature work +should move that comparison forward and leave evidence behind: + +- benchmark the affected workload against the relevant best-in-class + alternatives at representative small and large data sizes; +- preserve the benchmark environment, raw report, and reproduction command; +- update the generated capability matrix, compatibility corpus, or matched + visual examples for breadth and customization work; and +- add regression coverage for the user-visible result that created the win. + +See `spec/benchmarks/results.md` for the competitive evidence program and +tracked benchmark categories. + ## Local Checks Install the dev environment and build the required native core: @@ -97,13 +113,6 @@ regression comparison scripts, or report-schema validation, run: make check-benchmark-harness ``` -When you edit public docs, package metadata, benchmark summaries, or anything -that could become a public performance claim, run: - -```bash -make check-claims -``` - When you edit `spec/api/api-examples.md` or the Reflex example dashboard registry/assets, run: @@ -260,20 +269,3 @@ Start with the smallest reusable primitive surface: - Composition API wrapper if the chart is user-facing. - Example app card with normal-size data, not only a stress demo. - Benchmark row only when the comparison methodology is honest. - -## Performance Claims - -Do not write broad claims like "faster than Plotly" without naming: - -- chart type -- data size and shape -- backend (always `native`; there is no NumPy fallback) -- render target -- whether browser time-to-first-render is included -- whether the result is exact markers, decimated geometry, density, or adaptive - -Numeric multipliers such as "10x faster" or "5x smaller" need the same measured -benchmark context. - -When in doubt, phrase it as a measured row from `spec/benchmarks/results.md`, not as a -universal product claim, and run `make check-claims` before publishing. diff --git a/spec/process/production-readiness.md b/spec/process/production-readiness.md index 9ee5049a..35e5c73b 100644 --- a/spec/process/production-readiness.md +++ b/spec/process/production-readiness.md @@ -1,8 +1,8 @@ # Production Readiness This is the release bar for xy while the core renderer is still moving. -It separates hard gates from advisory measurements so performance claims, -packaging promises, and API stability do not depend on memory or vibes. +It separates hard gates from advisory measurements so packaging promises and +API stability do not depend on memory or vibes. ## Current Contract @@ -46,12 +46,11 @@ integration are still experimental and may change before a 1.0 release. ## Accessibility and Cross-Browser Conformance Status -The current conformance tier is intentionally narrower than a claim of full -WCAG parity or pixel-identical output across browsers. The browser client now -ships a parallel semantic chart region and generated trace/axis summary, a -polite live region for hover and keyboard readouts, focusable direct-point -navigation with Arrow/Home/End keys, named toolbar controls with toggle state, -visible focus styling, reduced-motion behavior, and forced-colors affordances. +The current conformance tier covers a parallel semantic chart region and +generated trace/axis summary, a polite live region for hover and keyboard +readouts, focusable direct-point navigation with Arrow/Home/End keys, named +toolbar controls with toggle state, visible focus styling, reduced-motion +behavior, and forced-colors affordances. CI runs the same focused chart in Playwright Chromium, Firefox, and WebKit. It checks those semantics and interactions in every engine, compares WebGL output @@ -59,22 +58,19 @@ with a coarse per-channel perceptual signature, and compares DOM chrome through layout boxes rather than browser-font glyph pixels. The gate does **not** yet cover aggregated-bin keyboard navigation, a view-as-table escape hatch, screen-reader/OS combinations, every chart family, or full-page screenshot -parity. Until those surfaces have dedicated evidence, neither full -accessibility parity nor broad perceptual cross-browser consistency is a safe -public claim. Run the focused tier locally with `make check-conformance` after +parity. Run the focused tier locally with `make check-conformance` after installing all three engines with `npx playwright install chromium firefox webkit`. ## Release-Blocking Gates -These must pass before publishing or making a broad performance claim. +These must pass before publishing. | Area | Gate | Command or evidence | |---|---|---| | Python floor | `pyproject.toml`, Ruff, docs, syntax, and annotations stay on the Python 3.11+ floor | `python scripts/check_python_floor.py` | | Public API | `__all__`, lazy exports, `__version__`, the source `py.typed` marker, focused type-surface tests, and fresh-process import-time budget stay coherent | `make check-api` | | Import-time budget | `xy.__init__`, `dir(xy)`, export helpers, chart construction, and `.widget()` keep their lazy import boundaries | `make check-import` | -| Claim guardrails | Public docs and package metadata avoid broad, unqualified performance claims | `make check-claims` | | CI/release workflows | Hard gates, non-blocking benchmarks, best-effort benchmark artifact upload/download, trusted publishing, and no-Rust clear-error jobs stay wired | `make check-ci` | | HTML export safety | Inline JSON/script escaping, atomic path writes, hostile user strings, and browser client text-node insertion stay protected | `make check-security` | | Python tests | Native backend passes | `pytest -q` | @@ -260,8 +256,8 @@ dependency boundaries, widget/export boundaries, or backend import setup: make check-import ``` -Use this before turning a generated benchmark JSON file into docs, release -notes, or a public claim: +Use this to validate generated benchmark JSON before publication or downstream +analysis: ```bash make check-benchmark-report BENCHMARK_JSON=benchmark.json BENCHMARK_KIND=scatter-vs @@ -275,13 +271,6 @@ tests: make check-benchmark-harness ``` -Use this after editing public docs, package metadata, benchmark summaries, -release notes, or other performance-claim surfaces: - -```bash -make check-claims -``` - Browser smoke and package artifact verification need a built bundle, Chromium, and wheel/sdist outputs. The interaction gate's real-wall-clock worker probe also uses the pinned development-only Playwright driver; install it once with @@ -406,11 +395,6 @@ Before tagging a release: are sdist-only. - Confirm the wheel size budget is still below 15 MB. - Confirm `spec/api/api-examples.md` runs against the tagged API. -- Confirm package metadata uses measured, scoped language rather than broad - "faster/best" positioning. -- Confirm performance claims mention chart type, mode, backend, data size, and - browser TTFR status. - ### reflex-xy releases The adapter is a pure-Python distribution — no native core, no JS build, no @@ -447,52 +431,6 @@ Before the first adapter release (and after any change to its pipeline): carry a copy of the render client — the client links out of the installed xy package at app compile so it can never drift from the kernel. -## Claim Guardrails - -Safe claims: - -- xy avoids JSON-number payloads for core chart data. -- Large scatter overview rendering is screen-bounded when using density mode. -- The native backend is substantially faster than pure Python/NumPy for kernel - work covered by the benchmarks. -- Current 2D core chart benchmarks beat Plotly on the measured payload-prep, - payload-size, and TTFR rows documented in `spec/benchmarks/results.md`. - -Claims that need qualification: - -- "Renders 10M points" must say whether the chart is drawing exact markers or a - density/adaptive representation. -- "Faster than Plotly" must name the chart type, data size, render target, - backend, and whether browser TTFR is included. -- "Install without Rust" means a published wheel (which carries the native core) - installs with no toolchain; a source build still requires Rust, and a - platform with no native core raises a clear ImportError rather than degrading. - -Not yet safe: - -- Plotly-level chart breadth. -- Full accessibility parity. -- Cross-browser/perceptual rendering conformance. -- Exact-marker interaction for every possible 100M-point zoom path. -- Production Reflex state integration as a first-class API. -- More than ~12 charts *simultaneously in view* holding live WebGL contexts. - Browsers cap live contexts per page (~16 in Chrome); the render client's - context governor keeps xy inside a budget (default 12) by having - the least-recently-visible off-screen chart release its context and - re-acquire on scroll-into-view. Measured (`benchmarks/bench_dashboard.py`, - 2026-07-09, Chrome/macOS): 10/20/50-chart dashboards are all fully usable — - every chart nonblank when visited, recovery p95 ~8 ms, heap sublinear - (28 MB at 50 charts) — but a layout keeping more than the budget visible - at once can still hit browser-side eviction, so do not claim unbounded - simultaneous live charts. The browser cap is process-wide (shared across a - tab's iframes), so the governor shares one budget across **same-origin** - frames over a `BroadcastChannel` (§18): a chart-per-iframe page (the - `examples/fastapi` gallery) stays under the cap instead of flooding the - console with "Too many active WebGL contexts". Cross-origin and - `sandbox`-without-`allow-same-origin` frames cannot share the channel and - fall back to per-document budgeting — many such isolated frames in one tab - can still collectively exceed the cap. - ## Hardening Backlog Keep pushing these in low-conflict increments: diff --git a/tests/test_claim_guardrails.py b/tests/test_claim_guardrails.py deleted file mode 100644 index 3ae5024b..00000000 --- a/tests/test_claim_guardrails.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path - - -def _load_claim_module(): - path = Path(__file__).resolve().parents[1] / "scripts" / "check_claim_guardrails.py" - spec = importlib.util.spec_from_file_location("check_claim_guardrails", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -check_claim_guardrails = _load_claim_module() - - -def _write(tmp_path: Path, text: str) -> Path: - path = tmp_path / "doc.md" - path.write_text(text, encoding="utf-8") - return path - - -def test_claim_guardrail_rejects_broad_fastest_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is the fastest charting library.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_broad_best_charting_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is the best Python charting library.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_most_performant_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is the most performant graphing engine.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_broad_more_performant_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is more performant than every data library.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_broad_package_metadata_claim(tmp_path: Path) -> None: - path = _write( - tmp_path, - 'description = "A faster charting engine: native Rust core, binary transport"\n', - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_unqualified_plotly_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is faster than Plotly.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("comparative performance" in finding.message for finding in findings) - - -def test_claim_guardrail_accepts_measured_qualified_plotly_claim(tmp_path: Path) -> None: - path = _write( - tmp_path, - ( - "In the smoke benchmark, the native backend is faster than Plotly " - "for the 100k histogram payload-prep row with TTFR measured.\n" - ), - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_rejects_unqualified_numeric_multiplier(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is 10x faster and 5x smaller.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("numeric performance multiplier" in finding.message for finding in findings) - - -def test_claim_guardrail_accepts_measured_numeric_multiplier(tmp_path: Path) -> None: - path = _write( - tmp_path, - ( - "Measured in the native backend benchmark with Chrome TTFR.\n\n" - "| Chart | Workload | Payload-prep vs Plotly | Payload reduction |\n" - "|---|---:|---:|---:|\n" - "| Histogram | 100k values / 200 bins | 303x faster | 348x smaller |\n" - ), - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_accepts_policy_text(tmp_path: Path) -> None: - path = _write( - tmp_path, - 'Do not write broad claims like "faster than Plotly" without naming chart type.\n', - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_requires_mode_for_large_point_rendering(tmp_path: Path) -> None: - path = _write(tmp_path, "xy renders 100M points.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("large-point rendering" in finding.message for finding in findings) - - -def test_claim_guardrail_accepts_mode_scoped_large_point_rendering(tmp_path: Path) -> None: - path = _write( - tmp_path, - "The benchmark measured 100M points in density mode on the native backend.\n", - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_rejects_stale_repo_identity(tmp_path: Path) -> None: - path = _write( - tmp_path, - "Report issues at https://github.com/Alek99/charts-exp/security/advisories/new\n", - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert any("stale repository identity" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_customization_superlatives(tmp_path: Path) -> None: - # The styling surface is a bounded, enumerated subset — see - # spec/api/capability-matrix.md — so these shapes are wrong however the - # sentence is framed, and no amount of shipping earns them. - for line in ( - "XY is the most customizable charting library.\n", - "XY is fully customizable.\n", - "Style anything you like.\n", - "Customize everything about your chart.\n", - "More extensible than any Python plotting library.\n", - ): - findings = check_claim_guardrails.check_claims([_write(tmp_path, line)]) - assert any("customization superlative" in f.message for f in findings), line - - -def test_claim_guardrail_scans_the_readme(tmp_path: Path) -> None: - # The README was outside the default set, which is where a slogan is most - # likely to be written and least likely to be reviewed. - assert "README.md" in check_claim_guardrails.DEFAULT_DOCS diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py index 2220aca7..8e5725e5 100644 --- a/tests/test_verify_ci_workflow.py +++ b/tests/test_verify_ci_workflow.py @@ -346,23 +346,6 @@ def test_codspeed_workflow_rejects_non_strict_native_install(tmp_path: Path) -> ) -def test_ci_workflow_rejects_missing_claim_guardrail_gate(tmp_path: Path) -> None: - workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") - path = tmp_path / "ci.yml" - path.write_text( - workflow.replace( - " - name: Claim guardrails\n" - " run: .venv/bin/python scripts/check_claim_guardrails.py\n\n", - "", - ), - encoding="utf-8", - ) - - errors = verify_ci_workflow.validate_workflow(path) - - assert any("test job" in error and "check_claim_guardrails" in error for error in errors) - - def test_ci_workflow_rejects_missing_interaction_stress_smoke(tmp_path: Path) -> None: workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") path = tmp_path / "ci.yml" diff --git a/tests/test_verify_local.py b/tests/test_verify_local.py index 7e41a0b5..cc1547d7 100644 --- a/tests/test_verify_local.py +++ b/tests/test_verify_local.py @@ -161,7 +161,6 @@ def test_benchmark_harness_check_is_known_as_targeted_gate() -> None: assert "tests/test_benchmark_environment.py" in command assert "tests/test_verify_benchmark_report.py" in command assert "tests/test_check_regressions.py" in command - assert "tests/test_claim_guardrails.py" in command assert selected[0].requires_modules == ("pytest",) @@ -337,7 +336,7 @@ def test_list_output_includes_check_requirements(capsys: pytest.CaptureFixture[s def test_dry_run_prints_selected_commands_without_running(capsys) -> None: - rc = verify_local.main(["--dry-run", "--only", "python_floor,public_api,claim_guardrails"]) + rc = verify_local.main(["--dry-run", "--only", "python_floor,public_api"]) out = capsys.readouterr().out assert rc == 0 @@ -345,8 +344,6 @@ def test_dry_run_prints_selected_commands_without_running(capsys) -> None: assert "scripts/check_python_floor.py" in out assert "public_api" in out assert "scripts/check_public_api.py" in out - assert "claim_guardrails" in out - assert "scripts/check_claim_guardrails.py" in out def test_dry_run_includes_ci_workflow_gate(capsys) -> None: @@ -532,9 +529,9 @@ def test_makefile_exposes_docs_verification_shortcut() -> None: makefile = (ROOT / "Makefile").read_text(encoding="utf-8") assert "check-docs:" in makefile - assert "scripts/verify_local.py --only examples,claim_guardrails" in makefile + assert "scripts/verify_local.py --only examples" in makefile assert "make check-docs" in makefile - assert "docs examples and public claim guardrails" in makefile + assert "run documentation examples" in makefile def test_makefile_exposes_security_verification_shortcut() -> None: @@ -582,15 +579,6 @@ def test_makefile_exposes_ci_workflow_verification_shortcut() -> None: assert "CI/release workflow invariant checks" in makefile -def test_makefile_exposes_claim_guardrail_shortcut() -> None: - makefile = (ROOT / "Makefile").read_text(encoding="utf-8") - - assert "check-claims:" in makefile - assert "scripts/verify_local.py --only claim_guardrails" in makefile - assert "make check-claims" in makefile - assert "public performance-claim guardrails" in makefile - - def test_node_version_parser_accepts_v_prefixed_versions() -> None: assert verify_local._node_major("v18.19.1\n") == 18 assert verify_local._node_major("20.11.0") == 20 diff --git a/tests/test_verify_sdist.py b/tests/test_verify_sdist.py index 8a52bf4b..462da9ea 100644 --- a/tests/test_verify_sdist.py +++ b/tests/test_verify_sdist.py @@ -60,8 +60,8 @@ "make check-benchmark-report.\n" "Run make check-examples for spec/api/api-examples.md and " "the example apps.\n\n" - "## Performance Claims\n\n" - "Claims need benchmark context.\n" + ("contributing padding\n" * 100) + "## Competitive Evidence\n\n" + "XY aims to outperform every competing charting library.\n" + ("contributing padding\n" * 100) ) CI_YML = ( "name: CI\n" From d5b556caf22c12138e64b944b40f9c534dec44fa Mon Sep 17 00:00:00 2001 From: Alek Petuskey Date: Sun, 26 Jul 2026 12:06:48 -0700 Subject: [PATCH 2/3] Remove documentation sentence from README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 80b53e7a..fe6797b8 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,7 @@ Small datasets render directly and exactly. As data grows, XY's Rust core, binary transport, and WebGL2 renderer reduce work to what the screen can show, then bring detail back as you zoom. The same composable API controls marks, axes, annotations, legends, tooltips, interactions, themes, responsive layout, -and HTML, raster, and vector exports. Find guides, API reference, and examples -in the [documentation](https://reflex.dev/docs/xy/). +and HTML, raster, and vector exports. ## Highlights From efeea9f3afb48dda87c7c4a41baa0057d8e19959 Mon Sep 17 00:00:00 2001 From: Alek Petuskey Date: Sun, 26 Jul 2026 12:09:31 -0700 Subject: [PATCH 3/3] Remove live demo link from README hero --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index fe6797b8..38ad7434 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,6 @@ XY-shaped probability field shown as a binned scatter chart.

-

- Try it live: a million points in your browser → -

-

CI CodSpeed