diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee01f8ed..94fd8806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,9 +160,6 @@ jobs: - name: Public API coherence run: .venv/bin/python scripts/check_public_api.py - - name: Claim guardrails - run: .venv/bin/python scripts/check_claim_guardrails.py - - name: Lint (ruff) run: | .venv/bin/ruff check . diff --git a/CLAUDE.md b/CLAUDE.md index f3baeb12..a19fc500 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,6 +11,32 @@ release change. A change is incomplete while its affected specification is missing, stale, or inconsistent with the implementation; resolve discrepancies instead of treating the implementation alone as authoritative. +## Product North Star + +XY is being built to outperform every competing charting library and become +the best overall charting system for Python. That goal spans every chart type +and every data scale, from a handful of values to billions of rows, across the +two dimensions users should not have to trade off: performance and +customization. + +Treat every competitor lead as a concrete product gap. Work that affects a +user-visible capability should: + +- compare XY with the relevant leaders, including Matplotlib, Seaborn, Plotly, + Bokeh, Altair, Datashader, HoloViews/hvPlot, and emerging alternatives; +- add or extend reproducible evidence across small, medium, large, and massive + data, covering startup, build and render time, interaction, memory, payload + and export size, and multi-chart applications where applicable; +- update the capability matrix and visual examples when the improvement is + about chart breadth or customization rather than timing; and +- commit the environment, raw results, output contracts, and reproduction + commands needed to inspect the win and catch regressions. + +The goal is not to win one large-scatter benchmark. XY should become the +library users choose for ordinary charts, massive data, every chart family, +notebooks, applications, static output, performance, and complete design +control. + ## Layout - `src/` — Rust core, **minimal external crates** (C ABI; one cdylib per @@ -107,5 +133,3 @@ PRs, or code. Set `git config user.name/user.email` to the human author cache (§27). NaN never reaches vertex buffers (§19). - f32 uploads are offset-encoded; tick/hover math stays f64 (§4/§16). - Every decimation/tier decision is recorded in the spec, never silent (§28). -- Claims are mode-scoped and benchmarked (§2); update README numbers from - `scripts/bench.py`, don't invent them. diff --git a/Makefile b/Makefile index 53167a6f..f5aedcde 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ WHEEL ?= BENCHMARK_JSON ?= benchmark.json BENCHMARK_KIND ?= auto -.PHONY: help setup setup-browser check check-full check-browser check-conformance check-docs check-examples check-security check-errors check-api check-import check-ci check-claims check-benchmark-harness check-pyplot check-pyplot-speed check-sdist check-wheel check-artifacts check-benchmark-report list-checks test lint format typecheck public-api python-floor js-check rust-check abi-smoke +.PHONY: help setup setup-browser check check-full check-browser check-conformance check-docs check-examples check-security check-errors check-api check-import check-ci check-benchmark-harness check-pyplot check-pyplot-speed check-sdist check-wheel check-artifacts check-benchmark-report list-checks test lint format typecheck public-api python-floor js-check rust-check abi-smoke help: @printf '%s\n' \ @@ -19,14 +19,13 @@ help: ' make check-full run JS, Rust, and ABI gates too' \ ' make check-browser run browser smokes (set CHROMIUM=/path/to/chrome)' \ ' make check-conformance run accessibility + Chromium/Firefox/WebKit conformance' \ - ' make check-docs run docs examples and public claim guardrails' \ + ' make check-docs run documentation examples' \ ' make check-examples run canonical API examples and Reflex asset registry checks' \ ' make check-security run standalone HTML safety and client text-sink checks' \ ' make check-errors run public error, LOD, and mutation-safety tests' \ ' make check-api run lazy public API and type-surface checks' \ ' make check-import run import-time and dependency-boundary checks' \ ' make check-ci run CI/release workflow invariant checks' \ - ' make check-claims run public performance-claim guardrails' \ ' make check-benchmark-harness run benchmark metadata/report/regression tests' \ ' make check-pyplot run the matplotlib-shim suite and compatibility corpus' \ ' make check-pyplot-speed enforce the per-family 10x static-PNG target (requires .[bench])' \ @@ -76,7 +75,7 @@ check-conformance: node scripts/browser_conformance.mjs check-docs: - $(PYTHON) scripts/verify_local.py --only examples,claim_guardrails + $(PYTHON) scripts/verify_local.py --only examples check-examples: $(PYTHON) scripts/verify_local.py --only examples @@ -99,9 +98,6 @@ check-import: check-ci: $(PYTHON) scripts/verify_local.py --only ci_workflow -check-claims: - $(PYTHON) scripts/verify_local.py --only claim_guardrails - check-benchmark-harness: $(PYTHON) scripts/verify_local.py --only benchmark_harness diff --git a/README.md b/README.md index 2bfcd053..38ad7434 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,6 @@ XY-shaped probability field shown as a binned scatter chart.

-

- Try it live: a million points in your browser → -

-

CI CodSpeed @@ -13,25 +9,36 @@ Docs

-XY is an actively evolving, early-alpha Python charting library for large, -interactive datasets. Its Rust core and WebGL2 renderer keep work bounded by -what the screen can show; find guides, API reference, and examples in the -[documentation](https://reflex.dev/docs/xy/). +

+ Small data should be effortless. Large data should stay interactive.
+ Neither should limit the design. +

+ +XY is an actively evolving, early-alpha project with one ambition: become the +fastest and most customizable charting library for Python — across every chart +type and every scale, from a handful of values to billions of rows. + +Small datasets render directly and exactly. As data grows, XY's Rust core, +binary transport, and WebGL2 renderer reduce work to what the screen can show, +then bring detail back as you zoom. The same composable API controls marks, +axes, annotations, legends, tooltips, interactions, themes, responsive layout, +and HTML, raster, and vector exports. ## Highlights -- **Built for large data.** Reduces long lines and dense scatters to what the screen can show, and brings detail back as you zoom. +- **Designed for every scale.** Ordinary datasets take the direct path; large datasets automatically use decimated or density representations. +- **Customize every layer.** Control marks and composition in Python, then style chart chrome through stable CSS and Tailwind hooks. - **Declarative interface.** Compose marks and guides, or use the familiar `xy.pyplot`. - **Interactive by default.** Pan, zoom, hover, select, and inspect exact source rows. - **One chart, many outputs.** Use notebooks or export HTML, raster, and vector formats. -- **Built for apps.** Embed responsive charts and style them with CSS or Tailwind. +- **Built for apps.** Embed responsive charts and style them with Python, CSS, or Tailwind. ## Is XY for me? -XY is a great fit for teams that want to explore large 2D datasets in Python, -share interactive notebook results, or ship self-contained charts on the web. -Build charts once, then display them in notebooks and apps or export them as -HTML, images, and vector graphics. +XY is for Python users who want one charting system for everyday plots, deeply +custom application visuals, and data too large for conventional browser chart +stacks. Build charts once, then display them in notebooks and apps or export +them as self-contained HTML, images, and vector graphics. ## Installation @@ -42,7 +49,7 @@ pip install xy uv add xy ``` -## Getting started +## Getting started: one API, every scale A chart is a container plus the marks inside it. Any sequence works — plain Python lists need no NumPy: @@ -57,7 +64,8 @@ chart = xy.line_chart(xy.line([1, 2, 3, 4, 5], [120, 180, 165, 240, 310])) chart # notebooks render it ``` -The same API scales. Chart a hundred million points as a density surface: +The data size changes; the chart model does not. The same API scales to a +hundred million points as a density surface:

@@ -117,25 +125,20 @@ plt.show() The shim intentionally covers common plotting workflows rather than every matplotlib feature. See the [compatibility guide](spec/matplotlib/compat.md). -## Benchmarks - -

- Cold-render time for a 10-million-point chart in XY, Matplotlib, and Plotly. Lower is better. -

- -In the recorded 10-million-point baseline, XY produced a static PNG in 0.023 s -versus 2.8 s for Matplotlib and 9.6 s for Plotly, and reached first interactive -render 16–20× sooner. +## Customize every layer -The committed launch baseline uses identical seeded data, a 900×420 output, -and three isolated cold runs. See the -[launch report](benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/report.md) -and [benchmark runbook](benchmarks/README.md) for the environment, -methodology, and raw results. +Performance is not a preset. Use Python to control data marks and chart +composition, then style browser chrome through stable CSS and Tailwind hooks. -## Styling +- **Marks:** data-driven color and size, custom colormaps, opacity, symbols, + gradients, strokes, and curves. +- **Guides:** axes, ticks, grids, annotations, legends, colorbars, and tooltips. +- **Interaction:** pan, zoom, hover, selection, crosshairs, callbacks, and + linked charts. +- **Layout and output:** layering, facets, responsive dimensions, themes, and + HTML, raster, or vector export. -Customize marks and chart chrome with Python, CSS, or Tailwind. See the [styling guide](docs/styling/index.md). +See the [styling guide](docs/styling/index.md) for patterns and examples. What each mechanism reaches — per property, per chrome slot, per renderer, and where it stops — is the [capability matrix](spec/api/capability-matrix.md), @@ -150,6 +153,30 @@ chart = xy.line_chart( ) ``` +## Benchmarks + +

+ Cold-render time for a 10-million-point chart in XY, Matplotlib, and Plotly. Lower is better. +

+ +In the recorded 10-million-point baseline, XY produced a static PNG in 0.023 s +versus 2.8 s for Matplotlib and 9.6 s for Plotly, and reached first interactive +render 16–20× sooner. + +That baseline is one proof point, not the finish line. XY is being built to +outperform every competing charting library across small-data startup, +massive-data rendering, interaction, memory, payload and export size, +multi-chart applications, chart breadth, and customization. The committed +benchmark program, raw results, capability matrix, and visual examples are how +the project measures and proves that goal. + +The committed launch baseline uses identical seeded data, a 900×420 output, +and three isolated cold runs. See the +[launch report](benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/report.md), +[benchmark runbook](benchmarks/README.md), and +[competitive benchmark specification](spec/benchmarks/results.md) for the +environment, methodology, raw results, and full evidence program. + ## Embed XY in a Reflex app With the `reflex-xy` adapter, any XY chart becomes a regular Reflex component. @@ -266,8 +293,10 @@ the chart gallery, guides, and API reference. The repository also includes ## Roadmap -XY is 2D-first: broad chart coverage on top of the binary transport and -screen-bounded rendering, before any 3D work. Queued next, no dates implied: +XY is headed toward every chart family. The current implementation sequence +builds broad 2D coverage on top of the binary transport and screen-bounded +renderer, then extends the same performance and customization goals to +geographic, 3D, and volume visualization. Queued next, no dates implied: - **Categorical distributions** — strip, swarm, beeswarm, boxen, rug - **Regression diagnostics** — trendline, residual, QQ, PP @@ -278,6 +307,7 @@ screen-bounded rendering, before any 3D work. Queued next, no dates implied: - **Treemap, sunburst, and icicle** - **Radar / polar and gauge** — needs polar axes first - **Slope, bump, and dumbbell** +- **3D and volume** — scatter, surfaces, meshes, isosurfaces, and volumetric views The full ranked backlog is in the [chart roadmap](spec/api/chart-roadmap.md). Want a chart or feature that isn't listed? diff --git a/docs/advanced/index.md b/docs/advanced/index.md index 03402dae..ed897713 100644 --- a/docs/advanced/index.md +++ b/docs/advanced/index.md @@ -103,8 +103,7 @@ does not automatically become a browser object: - Retained GPU buffers and local interaction avoid rebuilding the chart on every pointer movement. -This is a focused design, not a claim that every chart is faster. Compare the -same data, output, and rendering mode. The +This focused design is the foundation for making every chart faster. The [benchmark snapshot](/docs/xy/overview/benchmarks/) publishes those contracts and records when XY uses a reduced representation. diff --git a/docs/api-reference/contributing.md b/docs/api-reference/contributing.md index 63d3a743..67a1f9ab 100644 --- a/docs/api-reference/contributing.md +++ b/docs/api-reference/contributing.md @@ -38,17 +38,13 @@ import error rather than silently selecting another backend. | --- | --- | | Fast local verification | `make check` | | Production-facing change | `make check-full` | -| Public docs, examples, or claims | `make check-docs` | +| Public docs and examples | `make check-docs` | | Public exports or annotations | `make check-api` | | Lazy imports and dependency boundaries | `make check-import` | | Validation and mutation behavior | `make check-errors` | | Standalone HTML and text safety | `make check-security` | | Browser lifecycle and interaction | `make check-browser CHROMIUM=/path/to/chrome` | -Run `make check-claims` before publishing performance prose. Every comparison -must name the chart type, data size and shape, representation mode, backend, -render target, and whether browser time-to-first-render is included. - ## Adding a Chart Type A complete chart-family contribution normally spans: diff --git a/docs/core-concepts/large-data-and-performance.md b/docs/core-concepts/large-data-and-performance.md index 119ca434..7ebb2fdb 100644 --- a/docs/core-concepts/large-data-and-performance.md +++ b/docs/core-concepts/large-data-and-performance.md @@ -78,9 +78,8 @@ There are two different scaling regimes: 2. The resulting wire payload, WebGL geometry, and static SVG/native-PNG scene are bounded by the chosen viewport representation. -This is why a “cost scales with pixels, not points” slogan needs qualification: -it describes the output side after reduction, not the complete data-to-pixels -pipeline. +The output side after reduction scales with pixels; the complete data-to-pixels +pipeline still includes row-dependent ingest and reduction work. ## Inspect memory and tier decisions diff --git a/docs/overview/benchmarks.md b/docs/overview/benchmarks.md index 803bffaa..08538a1e 100644 --- a/docs/overview/benchmarks.md +++ b/docs/overview/benchmarks.md @@ -5,18 +5,18 @@ description: Inspect XY's recorded launch benchmark with its exact output contra # Benchmarks -XY's large-data claim is about the representation sent to a fixed-size output, -not about drawing every source row as an individual marker. The committed 0.1.0 -launch baseline measures identical seeded scatter data at 900×420 pixels on an -Apple M5 Pro with 64 GiB RAM. Each successful cell below is the mean of three -isolated cold runs. The machine name and memory are copied verbatim from the -committed environment record. +XY's large-data architecture reduces source rows to the representation useful +for a fixed-size output instead of drawing every row as an individual marker. +The committed 0.1.0 launch baseline measures identical seeded scatter data at +900×420 pixels on an Apple M5 Pro with 64 GiB RAM. Each successful cell below +is the mean of three isolated cold runs. The machine name and memory are copied +verbatim from the committed environment record. > **How to read this comparison.** XY switches dense scatter output to a > screen-bounded density representation, while the default Plotly and > Matplotlib paths retain every marker. These results compare each library's -> default user-visible outcome at the same output size; they do not claim that -> the libraries send identical geometry to the renderer. +> default user-visible outcome at the same output size; the libraries send +> different geometry to the renderer. ## Snapshot at 10 million points @@ -121,8 +121,9 @@ frozen: - **Release and hardware tracking:** immutable release directories plus clearly separated macOS hardware-WebGL and CI SwiftShader results. -Until those artifacts are published, this page intentionally keeps its -headline claims scoped to the committed launch scatter. +The launch scatter is the first committed proof point. The competitive +benchmark program expands it across these workloads, chart families, +competitors, and environments. ## Inspect and reproduce the evidence @@ -148,9 +149,7 @@ uv run --project "$BASELINE" --frozen python benchmarks/bench_launch_scatter.py --chrome "$CHROME" --out launch-scatter-default.json ``` -One machine and three runs describe that recorded environment, not every -machine or workload. New comparisons should retain chart type, data size, -representation, backend, output target, and browser-TTFR status rather than -shortening these results to a universal “faster than” claim. For the rendering -model behind the numbers, read +Each new comparison records its chart type, data size, representation, backend, +output target, and browser-TTFR status so the result can be reproduced and +improved. For the rendering model behind the numbers, read [Large data and performance](/docs/xy/core-concepts/large-data-and-performance/). diff --git a/scripts/check_claim_guardrails.py b/scripts/check_claim_guardrails.py deleted file mode 100644 index f7d4f4d2..00000000 --- a/scripts/check_claim_guardrails.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 -"""Check public docs for broad, unqualified performance claims. - -This is intentionally a small guardrail, not a full natural-language judge. It -flags phrases that are easy to post publicly and hard to defend, while allowing -measured benchmark rows and policy text that names mode/data/backend context. -""" - -from __future__ import annotations - -import argparse -import re -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Optional - -ROOT = Path(__file__).resolve().parents[1] -DEFAULT_DOCS = ( - "README.md", - "CLAUDE.md", - "pyproject.toml", - "SECURITY.md", - "CONTRIBUTING.md", - "spec/api/api-examples.md", - "spec/benchmarks/results.md", - "spec/api/chart-roadmap.md", - "spec/process/contributing.md", - "spec/process/production-readiness.md", -) - -BROAD_SUPERLATIVE_RE = re.compile( - r"\b(" - r"faster\s+charting\s+engine|" - r"fastest(?:\s+charting\s+library)?|" - r"best\s+(?:python\s+)?(?:charting|plotting|graphing|data)\s+(?:library|engine)|" - r"best\s+(?:at|for)\s+rendering|" - r"most\s+performant(?:\s+(?:python\s+)?(?:charting|plotting|graphing|data)\s+" - r"(?:library|engine))?|" - r"better\s+than\s+(?:all|every|any)|" - r"faster\s+than\s+(?:all|every|any)|" - r"faster\s+than\s+everything|" - r"more\s+performant\s+than\s+(?:all|every|any|everything)|" - r"blow\s+(?:them|it|everyone)\s+away" - r")\b", - re.IGNORECASE, -) -# Customization is the second axis people overclaim on, and it went unguarded -# while performance was fenced in. These shapes are unqualifiable: the styling -# surface is a bounded, enumerated subset (`spec/api/capability-matrix.md`), so -# "anything" and "most" are wrong however the sentence is framed. -CUSTOMIZATION_SUPERLATIVE_RE = re.compile( - r"\b(" - r"most\s+(?:customi[sz]able|themeable|styl(?:e)?able|extensible|flexible)|" - r"(?:fully|completely|infinitely|endlessly|totally)\s+customi[sz]able|" - r"customi[sz]e\s+(?:everything|anything)|" - r"style\s+(?:everything|anything)|" - r"unlimited\s+(?:styling|customi[sz]ation)|" - r"(?:more|as)\s+(?:customi[sz]able|themeable|extensible)\s+than\s+" - r"(?:all|every|any|everything|anything)" - r")\b", - re.IGNORECASE, -) -COMPARATIVE_RE = re.compile( - r"\b(?:faster\s+than|beats?|outperforms?)\s+" - r"(?:plotly|matplotlib|bokeh|altair|datashader|holoviews|hvplot|seaborn)\b", - re.IGNORECASE, -) -LARGE_RENDER_RE = re.compile( - r"\brenders?\s+(?:\d[\d,]*(?:\.\d+)?\s*)?(?:k|m|b|million|billion)?[- ]?points\b", - re.IGNORECASE, -) -NUMERIC_PERFORMANCE_RE = re.compile( - r"\b\d+(?:\.\d+)?\s*x\s+(?:faster|slower|smaller|larger|less|more)\b", - re.IGNORECASE, -) -# The project has lived under earlier names (charts-exp, personal forks). -# Public docs must point at the canonical repository — a stale URL here means -# security reports, clones, and badges land on the wrong repo. -STALE_REPO_RE = re.compile(r"github\.com/Alek99|app\.codspeed\.io/Alek99|charts-exp", re.IGNORECASE) - -POLICY_WORDS = re.compile( - r"\b(do not|don't|never|no amount of|must|should|guardrail|policy|claim|goal|planned|target|" - r"blurry|rather than|not\s+(?:a|the|safe|same|one)|without naming|" - r"needs qualification)\b", - re.IGNORECASE, -) -QUALIFIER_GROUPS = ( - re.compile(r"\b(?:\d[\d,]*(?:\.\d+)?\s*)?(?:k|m|b|million|billion|points|rows|n=)\b", re.I), - re.compile( - r"\b(?:direct|density|decimated|adaptive|sampled|screen-bounded|exact markers?)\b", re.I - ), - re.compile( - r"\b(?:native|numpy|backend|browser|chromium|swiftshader|render target|" - r"webgl|svg|png|to pixels)\b", - re.I, - ), - re.compile( - r"\b(?:benchmark|measured|documented|ttfr|payload|memory|ms|mb|gb|artifact)\b", re.I - ), - re.compile(r"\b(?:chart type|data size|mode|row|table|spec/benchmarks/results\.md)\b", re.I), -) - - -@dataclass(frozen=True) -class Finding: - path: Path - line: int - message: str - text: str - - def format(self) -> str: - rel = self.path.relative_to(ROOT) if self.path.is_relative_to(ROOT) else self.path - return f"{rel}:{self.line}: {self.message}: {self.text.strip()}" - - -def _line_window(lines: list[str], index: int, radius: int = 2) -> str: - start = max(0, index - radius) - end = min(len(lines), index + radius + 1) - return "\n".join(lines[start:end]) - - -def _is_policy_or_negative_context(text: str) -> bool: - return bool(POLICY_WORDS.search(text)) - - -def _has_claim_qualifiers(text: str) -> bool: - return sum(1 for pattern in QUALIFIER_GROUPS if pattern.search(text)) >= 3 - - -def _findings_for_file(path: Path) -> list[Finding]: - lines = path.read_text(encoding="utf-8").splitlines() - findings: list[Finding] = [] - for index, line in enumerate(lines): - window = _line_window(lines, index) - if STALE_REPO_RE.search(line): - # No policy-context exemption: a stale repo identity is always wrong. - findings.append( - Finding( - path, - index + 1, - "stale repository identity; use github.com/reflex-dev/xy", - line, - ) - ) - if BROAD_SUPERLATIVE_RE.search(line) and not _is_policy_or_negative_context(window): - findings.append( - Finding( - path, - index + 1, - "broad superlative needs measured scope or policy framing", - line, - ) - ) - if CUSTOMIZATION_SUPERLATIVE_RE.search(line) and not _is_policy_or_negative_context(window): - findings.append( - Finding( - path, - index + 1, - "customization superlative is not defensible from the capability matrix", - line, - ) - ) - if COMPARATIVE_RE.search(line) and not ( - _is_policy_or_negative_context(window) or _has_claim_qualifiers(window) - ): - findings.append( - Finding( - path, - index + 1, - "comparative performance claim needs chart/data/backend/render context", - line, - ) - ) - if LARGE_RENDER_RE.search(line) and not ( - _is_policy_or_negative_context(window) or _has_claim_qualifiers(window) - ): - findings.append( - Finding( - path, - index + 1, - "large-point rendering claim needs mode context", - line, - ) - ) - if NUMERIC_PERFORMANCE_RE.search(line): - # Benchmark tables often carry the measurement context a few lines - # above the rows; give this rule a wider view than sentence-level - # comparative claims while still catching slogan-style multipliers. - metric_window = _line_window(lines, index, radius=10) - if not ( - _is_policy_or_negative_context(metric_window) - or _has_claim_qualifiers(metric_window) - ): - findings.append( - Finding( - path, - index + 1, - "numeric performance multiplier needs measured benchmark context", - line, - ) - ) - return findings - - -def check_claims(paths: list[Path]) -> list[Finding]: - findings: list[Finding] = [] - for path in paths: - if not path.exists(): - findings.append(Finding(path, 0, "document does not exist", "")) - continue - findings.extend(_findings_for_file(path)) - return findings - - -def _default_paths() -> list[Path]: - paths = [ROOT / item for item in DEFAULT_DOCS] - public_docs = ( - path - for path in sorted((ROOT / "docs").rglob("*.md")) - if "app" not in path.relative_to(ROOT / "docs").parts - ) - return list(dict.fromkeys((*paths, *public_docs))) - - -def main(argv: Optional[list[str]] = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="*", type=Path, help="docs to scan") - args = parser.parse_args(argv) - - paths = args.paths or _default_paths() - findings = check_claims([path if path.is_absolute() else ROOT / path for path in paths]) - if findings: - print("performance claim guardrail failed:", file=sys.stderr) - for finding in findings: - print(f"- {finding.format()}", file=sys.stderr) - return 1 - print(f"performance claim guardrail OK: {len(paths)} document(s)") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index e1b4bf28..d23b1d94 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -329,7 +329,6 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "hard production gates", "scripts/verify_ci_workflow.py", "scripts/check_public_api.py", - "scripts/check_claim_guardrails.py", "ruff check .", "scripts/smoke_render.py", "Browser lifecycle smoke", diff --git a/scripts/verify_local.py b/scripts/verify_local.py index b59303fe..052d883e 100644 --- a/scripts/verify_local.py +++ b/scripts/verify_local.py @@ -75,11 +75,6 @@ def _base_checks( (py, "-m", "pytest", "-q", "tests/test_import.py", "tests/test_dependencies.py"), requires_modules=("pytest",), ), - Check( - "claim_guardrails", - "public performance-claim guardrails", - (py, "scripts/check_claim_guardrails.py"), - ), Check( "capability_matrix", "generated capability matrix is current", @@ -87,7 +82,7 @@ def _base_checks( ), Check( "benchmark_harness", - "benchmark metadata, report, regression, and claim guardrail tests", + "benchmark metadata, report, and regression tests", ( py, "-m", @@ -97,7 +92,6 @@ def _base_checks( "tests/test_bench_pyplot_vs_matplotlib.py", "tests/test_verify_benchmark_report.py", "tests/test_check_regressions.py", - "tests/test_claim_guardrails.py", ), requires_modules=("pytest",), ), @@ -288,7 +282,6 @@ def _base_checks( QUICK_CHECKS = ( "python_floor", "public_api", - "claim_guardrails", "capability_matrix", "ci_workflow", "ruff_check", diff --git a/scripts/verify_sdist.py b/scripts/verify_sdist.py index e0fdb2ab..51d0eb86 100644 --- a/scripts/verify_sdist.py +++ b/scripts/verify_sdist.py @@ -104,7 +104,6 @@ "examples/reflex/xy_reflex_demo/__init__.py", "examples/reflex/xy_reflex_demo/xy_reflex_demo.py", "scripts/check_public_api.py", - "scripts/check_claim_guardrails.py", "scripts/gen_capability_matrix.py", "scripts/check_python_floor.py", "scripts/check_regressions.py", @@ -120,7 +119,6 @@ "src/kernels.rs", "src/lib.rs", "tests/test_public_api.py", - "tests/test_claim_guardrails.py", "tests/test_benchmark_environment.py", "tests/test_bench_pyplot_vs_matplotlib.py", "tests/test_check_regressions.py", @@ -397,7 +395,8 @@ def verify_sdist(path: str) -> None: "make check-sdist", "make check-examples", "make check-benchmark-report", - "Performance Claims", + "Competitive Evidence", + "outperform every competing charting library", }, ) _require_file_contains( diff --git a/spec/README.md b/spec/README.md index bde52d7d..669ca289 100644 --- a/spec/README.md +++ b/spec/README.md @@ -29,8 +29,8 @@ The public surface: what callers can build, style, export, and interact with. `python/xy/styling/capabilities.py`. Regenerate with `scripts/gen_capability_matrix.py --write`; the test suite fails if it is stale. -- [`chart-roadmap.md`](api/chart-roadmap.md) — the single 2D-first chart-type - coverage backlog, prioritized by popularity and primitive reuse. +- [`chart-roadmap.md`](api/chart-roadmap.md) — the staged chart-type coverage + backlog, from core 2D through geographic, 3D, and volume visualization. - [`export.md`](api/export.md) — how a figure becomes bytes: one entry point across five image formats, deterministic engine choice, browser-free default. - [`interaction.md`](api/interaction.md) — the authority on which browser diff --git a/spec/api/chart-roadmap.md b/spec/api/chart-roadmap.md index 25f5e51d..25ffb8f0 100644 --- a/spec/api/chart-roadmap.md +++ b/spec/api/chart-roadmap.md @@ -1,11 +1,13 @@ # Chart Type Roadmap -This is the single chart-type roadmap for xy. It is intentionally -**2D-first**: the library should become broad enough for Plotly-class analytics, -finance, science, operations, and dashboard use cases before spending product -energy on 3D/volume rendering. Nothing in this section implies immediate -implementation; it is the coverage backlog we will pull from as the primitives -land. +This is the single chart-type roadmap for xy. The destination is every chart +family: analytical and statistical charts, finance, science, operations, +dashboards, geographic views, 3D, and volume rendering. Delivery is staged: +the current implementation builds broad 2D coverage first because those shared +primitives unlock the most use cases, then carries the same performance and +customization goals into additional coordinate systems. Nothing in this +section implies immediate implementation; it is the coverage backlog we will +pull from as the primitives land. The roadmap prioritizes chart types by two signals: @@ -14,10 +16,10 @@ The roadmap prioritizes chart types by two signals: 2. **xy fit:** chart types where the engine can win on large data, binary transport, WebGL rendering, and screen-bounded aggregation. -The near-term wedge is large data, but the long-term product goal is broader: -xy should become a **Plotly-class general-purpose charting library** for -analytics, science, finance, operations, and dashboards. Performance is the -entry point, not the boundary of the product. +The near-term wedge is large data, but the product goal is broader: xy should +outperform every competing charting library across chart breadth, data scale, +performance, and customization. Performance is the entry point, not the +boundary of the product. The current implemented surface is **line**, **scatter**, **area**, **histogram**, **bar/column**, **heatmap**, **error bars/bands**, @@ -83,7 +85,7 @@ These sources are used as popularity proxies, not as exact usage telemetry: | [Bokeh gallery](https://docs.bokeh.org/en/latest/docs/gallery.html) | line, scatter, bar, patches, image, heatmap, contour, candlestick, maps, network, linked brushing, streaming | | Internal design dossier, section 28 and 24 | bar/histogram, heatmap, box/violin, candlestick/OHLC as important compatibility targets | -## 2D-First Priority Roadmap +## Staged Chart Coverage Roadmap ### P0 - Already in place @@ -209,18 +211,19 @@ depth: strip/swarm/boxen/rug distributions, regression diagnostics, richer | 32 | Packed bubble / Venn / Euler | Compatibility and presentation charts. | Layout algorithms and label placement dominate. | | 33 | Pictorial bar / item chart / image markers / text marks | ECharts/Highcharts compatibility polish. | Symbol systems and asset handling. | -### P6 - 2D domains to defer until core breadth is strong +### P6 - Additional domains after core breadth is strong | Chart family | Reason to defer | |---|---| | Maps / choropleth / geo scatter / routes / density maps | 2D, but requires projection, tile, geography data, and viewport semantics; valuable as a later domain package. | -| 3D scatter / surface / volume | Explicitly outside the 2D-first roadmap; different rendering and interaction model. | +| 3D scatter / surface / mesh / volume | Requires dedicated camera, projection, lighting, depth, picking, and interaction primitives; part of the product destination after the shared 2D foundations are broad. | ### P7 - Long-term breadth target The long-term target is not only "fast large scatter." It is an expansive -library with enough chart breadth to be used across industries the way Plotly is -used today, while keeping xy' transport and rendering model underneath. +library that leads across industries and chart families, including geographic, +3D, and volume visualization, while keeping xy's transport and rendering model +underneath. | Industry / use case | Chart coverage needed | |---|---| diff --git a/spec/benchmarks/methodology.md b/spec/benchmarks/methodology.md index 94cb341b..04c49c9b 100644 --- a/spec/benchmarks/methodology.md +++ b/spec/benchmarks/methodology.md @@ -10,10 +10,10 @@ are published. ## 0. The three rules -1. **Mode-scoped claims only** (§2 of the dossier, already policy): "10M - density in X ms" is a different claim from "10M individually-styled - markers." Output labels every row `direct | decimated | density | sampled` - — a reader can never mistake an aggregate result for a raw-marker result. +1. **Record the rendering mode.** "10M density in X ms" and "10M + individually-styled markers in X ms" are different workloads. Output labels + every row `direct | decimated | density | sampled`, preserving the result + needed to compare and improve each path. 2. **Same-work comparisons.** Each competitor renders the *same visual contract*, not the same API call: if XY aggregates at 10M, the fair Plotly comparison is Plotly failing/succeeding at raw markers AND diff --git a/spec/benchmarks/results.md b/spec/benchmarks/results.md index 1b92238c..93c1812c 100644 --- a/spec/benchmarks/results.md +++ b/spec/benchmarks/results.md @@ -1,8 +1,9 @@ -# Scatter benchmark: xy vs Python charting libraries +# Competitive benchmark program -The same scatter (a correlated 2-D cloud) at growing point counts, across -popular Python charting libraries, on four factors: **how many points each can -render, how fast, how much memory, and the render payload size.** +The recorded launch comparison uses the same scatter (a correlated 2-D cloud) +at growing point counts across popular Python charting libraries, measuring +four factors: **how many points each can render, how fast, how much memory, and +the render payload size.** The cross-library harness lives in `benchmarks/bench_vs.py` and includes optional adapters for xy, matplotlib, seaborn, Plotly, Bokeh, Altair, @@ -25,12 +26,32 @@ python benchmarks/bench_vs.py The xy-only arm also runs with no dependencies via `benchmarks/bench_scatter_native.py`. +## Competitive product goal + +XY's product goal is to outperform every competing charting library across +user-visible performance, chart breadth, and customization. The benchmark +program is the evidence system for that goal, not a showcase for one favorable +workload. + +Every major competitor should be represented wherever it leads or defines +user expectations. Comparisons should cover small through massive datasets, +exact and aggregated rendering, static and interactive output, cold and warm +paths, interaction latency, memory, transport and export size, and multi-chart +applications. Environments, raw machine-readable results, output contracts, +and reproduction commands belong in the repository so every result can be +inspected and rerun. + +Timing alone cannot prove chart breadth or design control. Those dimensions +use the generated capability matrix, matched visual examples, renderer +coverage, and compatibility corpora as evidence. When another library is +better on a relevant workload or capability, record the loss as a product gap, +add a reproducible comparison, and keep it visible until XY wins it. + ## Benchmark categories and goals -The performance story should be measured by mode, not with one blanket -"fastest charting library" number. A small exact scatter, a 10M density view, a -large line, and a 30-chart dashboard stress different parts of the system. These -are the categories we track or plan to add to CI. +Performance is tracked by mode and workload. A small exact scatter, a 10M +density view, a large line, and a 30-chart dashboard stress different parts of +the system. These are the categories we track or plan to add to CI. The stable category IDs live in `benchmarks/categories.py`. CI's benchmark JSON artifacts (`benchmark.json`, `line.json`, `install.json`, `install-fresh.json`, @@ -191,25 +212,6 @@ count. Partial rows retain their timing and memory metrics. `bench_workflows.py` updates, and separate HTML/SVG/native-PNG/Chromium-PNG export rows. All three emit schema-versioned JSON with environment metadata and benchmark category IDs. -## Copyable claim taxonomy - -Use these shapes when turning benchmark rows into public prose, release notes, -or posts. The goal is to make every public claim reproducible from a row in -this document or from a verified JSON artifact. - -| Claim shape | Safe wording pattern | Required context | -|---|---|---| -| Payload/prep comparison | "In the native backend benchmark, histogram payload prep for 10k values / 200 bins was 17.3x faster than Plotly." | chart type, workload, backend, compared library, metric | -| Browser first paint | "For the measured Chrome TTFR row, the 10k-value histogram first painted 5.0x faster than Plotly." | browser/render target, workload, chart type, TTFR included | -| Large scatter overview | "The 100M scatter overview uses density mode with a 258 KB wire payload; it is not drawing 100M exact markers." | mode, point count, payload, exact-vs-aggregate wording | -| Line decimation | "The 10M line benchmark ships an M4-decimated ~60 KB payload while preserving the extrema oracle." | mode, point count, payload, correctness oracle | -| Install/import footprint | "In the install-footprint benchmark, cold import was 6.4 ms for the measured distribution." | benchmark name, metric, measured distribution | - -Do not shorten those into broad slogans such as "xy is faster than -Plotly" or "renders 10M points" without the row context. If a sentence does not -name the chart type, workload, mode, backend, metric, and render target where -they matter, it is not ready to publish. - ## `xy.pyplot` versus Matplotlib/Agg `benchmarks/bench_pyplot_vs_matplotlib.py` runs the same Matplotlib-style calls @@ -1010,6 +1012,5 @@ falls over. This is the path xy exists to replace. - xy memory is Python-side `tracemalloc` peak; its GPU/native bytes are separate and itemized by `Figure.memory_report()` (§27). The payload column is the transport cost across the kernel→browser boundary. -- Single CI machine, one run; treat as order-of-magnitude, not a spec. Re-run in - CI (`benchmark` job) or locally. No universal claims — every number is - mode-scoped (dossier §2/§31). +- Single CI machine, one run; re-run in CI (`benchmark` job) or locally and + expand the committed baselines across hardware, workloads, and competitors. diff --git a/spec/process/contributing.md b/spec/process/contributing.md index 4d710a5c..1a5a8488 100644 --- a/spec/process/contributing.md +++ b/spec/process/contributing.md @@ -14,6 +14,22 @@ the hard-won production invariants while the chart surface grows. the asset. Do not patch only the generated HTML unless you are applying an emergency local fix and immediately back-porting it to the generator. +## Competitive Evidence + +XY's product north star is to outperform every competing charting library +across performance, chart breadth, customization, and data scale. Feature work +should move that comparison forward and leave evidence behind: + +- benchmark the affected workload against the relevant best-in-class + alternatives at representative small and large data sizes; +- preserve the benchmark environment, raw report, and reproduction command; +- update the generated capability matrix, compatibility corpus, or matched + visual examples for breadth and customization work; and +- add regression coverage for the user-visible result that created the win. + +See `spec/benchmarks/results.md` for the competitive evidence program and +tracked benchmark categories. + ## Local Checks Install the dev environment and build the required native core: @@ -97,13 +113,6 @@ regression comparison scripts, or report-schema validation, run: make check-benchmark-harness ``` -When you edit public docs, package metadata, benchmark summaries, or anything -that could become a public performance claim, run: - -```bash -make check-claims -``` - When you edit `spec/api/api-examples.md` or the Reflex example dashboard registry/assets, run: @@ -260,20 +269,3 @@ Start with the smallest reusable primitive surface: - Composition API wrapper if the chart is user-facing. - Example app card with normal-size data, not only a stress demo. - Benchmark row only when the comparison methodology is honest. - -## Performance Claims - -Do not write broad claims like "faster than Plotly" without naming: - -- chart type -- data size and shape -- backend (always `native`; there is no NumPy fallback) -- render target -- whether browser time-to-first-render is included -- whether the result is exact markers, decimated geometry, density, or adaptive - -Numeric multipliers such as "10x faster" or "5x smaller" need the same measured -benchmark context. - -When in doubt, phrase it as a measured row from `spec/benchmarks/results.md`, not as a -universal product claim, and run `make check-claims` before publishing. diff --git a/spec/process/production-readiness.md b/spec/process/production-readiness.md index 9ee5049a..35e5c73b 100644 --- a/spec/process/production-readiness.md +++ b/spec/process/production-readiness.md @@ -1,8 +1,8 @@ # Production Readiness This is the release bar for xy while the core renderer is still moving. -It separates hard gates from advisory measurements so performance claims, -packaging promises, and API stability do not depend on memory or vibes. +It separates hard gates from advisory measurements so packaging promises and +API stability do not depend on memory or vibes. ## Current Contract @@ -46,12 +46,11 @@ integration are still experimental and may change before a 1.0 release. ## Accessibility and Cross-Browser Conformance Status -The current conformance tier is intentionally narrower than a claim of full -WCAG parity or pixel-identical output across browsers. The browser client now -ships a parallel semantic chart region and generated trace/axis summary, a -polite live region for hover and keyboard readouts, focusable direct-point -navigation with Arrow/Home/End keys, named toolbar controls with toggle state, -visible focus styling, reduced-motion behavior, and forced-colors affordances. +The current conformance tier covers a parallel semantic chart region and +generated trace/axis summary, a polite live region for hover and keyboard +readouts, focusable direct-point navigation with Arrow/Home/End keys, named +toolbar controls with toggle state, visible focus styling, reduced-motion +behavior, and forced-colors affordances. CI runs the same focused chart in Playwright Chromium, Firefox, and WebKit. It checks those semantics and interactions in every engine, compares WebGL output @@ -59,22 +58,19 @@ with a coarse per-channel perceptual signature, and compares DOM chrome through layout boxes rather than browser-font glyph pixels. The gate does **not** yet cover aggregated-bin keyboard navigation, a view-as-table escape hatch, screen-reader/OS combinations, every chart family, or full-page screenshot -parity. Until those surfaces have dedicated evidence, neither full -accessibility parity nor broad perceptual cross-browser consistency is a safe -public claim. Run the focused tier locally with `make check-conformance` after +parity. Run the focused tier locally with `make check-conformance` after installing all three engines with `npx playwright install chromium firefox webkit`. ## Release-Blocking Gates -These must pass before publishing or making a broad performance claim. +These must pass before publishing. | Area | Gate | Command or evidence | |---|---|---| | Python floor | `pyproject.toml`, Ruff, docs, syntax, and annotations stay on the Python 3.11+ floor | `python scripts/check_python_floor.py` | | Public API | `__all__`, lazy exports, `__version__`, the source `py.typed` marker, focused type-surface tests, and fresh-process import-time budget stay coherent | `make check-api` | | Import-time budget | `xy.__init__`, `dir(xy)`, export helpers, chart construction, and `.widget()` keep their lazy import boundaries | `make check-import` | -| Claim guardrails | Public docs and package metadata avoid broad, unqualified performance claims | `make check-claims` | | CI/release workflows | Hard gates, non-blocking benchmarks, best-effort benchmark artifact upload/download, trusted publishing, and no-Rust clear-error jobs stay wired | `make check-ci` | | HTML export safety | Inline JSON/script escaping, atomic path writes, hostile user strings, and browser client text-node insertion stay protected | `make check-security` | | Python tests | Native backend passes | `pytest -q` | @@ -260,8 +256,8 @@ dependency boundaries, widget/export boundaries, or backend import setup: make check-import ``` -Use this before turning a generated benchmark JSON file into docs, release -notes, or a public claim: +Use this to validate generated benchmark JSON before publication or downstream +analysis: ```bash make check-benchmark-report BENCHMARK_JSON=benchmark.json BENCHMARK_KIND=scatter-vs @@ -275,13 +271,6 @@ tests: make check-benchmark-harness ``` -Use this after editing public docs, package metadata, benchmark summaries, -release notes, or other performance-claim surfaces: - -```bash -make check-claims -``` - Browser smoke and package artifact verification need a built bundle, Chromium, and wheel/sdist outputs. The interaction gate's real-wall-clock worker probe also uses the pinned development-only Playwright driver; install it once with @@ -406,11 +395,6 @@ Before tagging a release: are sdist-only. - Confirm the wheel size budget is still below 15 MB. - Confirm `spec/api/api-examples.md` runs against the tagged API. -- Confirm package metadata uses measured, scoped language rather than broad - "faster/best" positioning. -- Confirm performance claims mention chart type, mode, backend, data size, and - browser TTFR status. - ### reflex-xy releases The adapter is a pure-Python distribution — no native core, no JS build, no @@ -447,52 +431,6 @@ Before the first adapter release (and after any change to its pipeline): carry a copy of the render client — the client links out of the installed xy package at app compile so it can never drift from the kernel. -## Claim Guardrails - -Safe claims: - -- xy avoids JSON-number payloads for core chart data. -- Large scatter overview rendering is screen-bounded when using density mode. -- The native backend is substantially faster than pure Python/NumPy for kernel - work covered by the benchmarks. -- Current 2D core chart benchmarks beat Plotly on the measured payload-prep, - payload-size, and TTFR rows documented in `spec/benchmarks/results.md`. - -Claims that need qualification: - -- "Renders 10M points" must say whether the chart is drawing exact markers or a - density/adaptive representation. -- "Faster than Plotly" must name the chart type, data size, render target, - backend, and whether browser TTFR is included. -- "Install without Rust" means a published wheel (which carries the native core) - installs with no toolchain; a source build still requires Rust, and a - platform with no native core raises a clear ImportError rather than degrading. - -Not yet safe: - -- Plotly-level chart breadth. -- Full accessibility parity. -- Cross-browser/perceptual rendering conformance. -- Exact-marker interaction for every possible 100M-point zoom path. -- Production Reflex state integration as a first-class API. -- More than ~12 charts *simultaneously in view* holding live WebGL contexts. - Browsers cap live contexts per page (~16 in Chrome); the render client's - context governor keeps xy inside a budget (default 12) by having - the least-recently-visible off-screen chart release its context and - re-acquire on scroll-into-view. Measured (`benchmarks/bench_dashboard.py`, - 2026-07-09, Chrome/macOS): 10/20/50-chart dashboards are all fully usable — - every chart nonblank when visited, recovery p95 ~8 ms, heap sublinear - (28 MB at 50 charts) — but a layout keeping more than the budget visible - at once can still hit browser-side eviction, so do not claim unbounded - simultaneous live charts. The browser cap is process-wide (shared across a - tab's iframes), so the governor shares one budget across **same-origin** - frames over a `BroadcastChannel` (§18): a chart-per-iframe page (the - `examples/fastapi` gallery) stays under the cap instead of flooding the - console with "Too many active WebGL contexts". Cross-origin and - `sandbox`-without-`allow-same-origin` frames cannot share the channel and - fall back to per-document budgeting — many such isolated frames in one tab - can still collectively exceed the cap. - ## Hardening Backlog Keep pushing these in low-conflict increments: diff --git a/tests/test_claim_guardrails.py b/tests/test_claim_guardrails.py deleted file mode 100644 index 3ae5024b..00000000 --- a/tests/test_claim_guardrails.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path - - -def _load_claim_module(): - path = Path(__file__).resolve().parents[1] / "scripts" / "check_claim_guardrails.py" - spec = importlib.util.spec_from_file_location("check_claim_guardrails", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -check_claim_guardrails = _load_claim_module() - - -def _write(tmp_path: Path, text: str) -> Path: - path = tmp_path / "doc.md" - path.write_text(text, encoding="utf-8") - return path - - -def test_claim_guardrail_rejects_broad_fastest_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is the fastest charting library.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_broad_best_charting_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is the best Python charting library.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_most_performant_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is the most performant graphing engine.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_broad_more_performant_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is more performant than every data library.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_broad_package_metadata_claim(tmp_path: Path) -> None: - path = _write( - tmp_path, - 'description = "A faster charting engine: native Rust core, binary transport"\n', - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert any("broad superlative" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_unqualified_plotly_claim(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is faster than Plotly.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("comparative performance" in finding.message for finding in findings) - - -def test_claim_guardrail_accepts_measured_qualified_plotly_claim(tmp_path: Path) -> None: - path = _write( - tmp_path, - ( - "In the smoke benchmark, the native backend is faster than Plotly " - "for the 100k histogram payload-prep row with TTFR measured.\n" - ), - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_rejects_unqualified_numeric_multiplier(tmp_path: Path) -> None: - path = _write(tmp_path, "xy is 10x faster and 5x smaller.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("numeric performance multiplier" in finding.message for finding in findings) - - -def test_claim_guardrail_accepts_measured_numeric_multiplier(tmp_path: Path) -> None: - path = _write( - tmp_path, - ( - "Measured in the native backend benchmark with Chrome TTFR.\n\n" - "| Chart | Workload | Payload-prep vs Plotly | Payload reduction |\n" - "|---|---:|---:|---:|\n" - "| Histogram | 100k values / 200 bins | 303x faster | 348x smaller |\n" - ), - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_accepts_policy_text(tmp_path: Path) -> None: - path = _write( - tmp_path, - 'Do not write broad claims like "faster than Plotly" without naming chart type.\n', - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_requires_mode_for_large_point_rendering(tmp_path: Path) -> None: - path = _write(tmp_path, "xy renders 100M points.\n") - - findings = check_claim_guardrails.check_claims([path]) - - assert any("large-point rendering" in finding.message for finding in findings) - - -def test_claim_guardrail_accepts_mode_scoped_large_point_rendering(tmp_path: Path) -> None: - path = _write( - tmp_path, - "The benchmark measured 100M points in density mode on the native backend.\n", - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert findings == [] - - -def test_claim_guardrail_rejects_stale_repo_identity(tmp_path: Path) -> None: - path = _write( - tmp_path, - "Report issues at https://github.com/Alek99/charts-exp/security/advisories/new\n", - ) - - findings = check_claim_guardrails.check_claims([path]) - - assert any("stale repository identity" in finding.message for finding in findings) - - -def test_claim_guardrail_rejects_customization_superlatives(tmp_path: Path) -> None: - # The styling surface is a bounded, enumerated subset — see - # spec/api/capability-matrix.md — so these shapes are wrong however the - # sentence is framed, and no amount of shipping earns them. - for line in ( - "XY is the most customizable charting library.\n", - "XY is fully customizable.\n", - "Style anything you like.\n", - "Customize everything about your chart.\n", - "More extensible than any Python plotting library.\n", - ): - findings = check_claim_guardrails.check_claims([_write(tmp_path, line)]) - assert any("customization superlative" in f.message for f in findings), line - - -def test_claim_guardrail_scans_the_readme(tmp_path: Path) -> None: - # The README was outside the default set, which is where a slogan is most - # likely to be written and least likely to be reviewed. - assert "README.md" in check_claim_guardrails.DEFAULT_DOCS diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py index 2220aca7..8e5725e5 100644 --- a/tests/test_verify_ci_workflow.py +++ b/tests/test_verify_ci_workflow.py @@ -346,23 +346,6 @@ def test_codspeed_workflow_rejects_non_strict_native_install(tmp_path: Path) -> ) -def test_ci_workflow_rejects_missing_claim_guardrail_gate(tmp_path: Path) -> None: - workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") - path = tmp_path / "ci.yml" - path.write_text( - workflow.replace( - " - name: Claim guardrails\n" - " run: .venv/bin/python scripts/check_claim_guardrails.py\n\n", - "", - ), - encoding="utf-8", - ) - - errors = verify_ci_workflow.validate_workflow(path) - - assert any("test job" in error and "check_claim_guardrails" in error for error in errors) - - def test_ci_workflow_rejects_missing_interaction_stress_smoke(tmp_path: Path) -> None: workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") path = tmp_path / "ci.yml" diff --git a/tests/test_verify_local.py b/tests/test_verify_local.py index 7e41a0b5..cc1547d7 100644 --- a/tests/test_verify_local.py +++ b/tests/test_verify_local.py @@ -161,7 +161,6 @@ def test_benchmark_harness_check_is_known_as_targeted_gate() -> None: assert "tests/test_benchmark_environment.py" in command assert "tests/test_verify_benchmark_report.py" in command assert "tests/test_check_regressions.py" in command - assert "tests/test_claim_guardrails.py" in command assert selected[0].requires_modules == ("pytest",) @@ -337,7 +336,7 @@ def test_list_output_includes_check_requirements(capsys: pytest.CaptureFixture[s def test_dry_run_prints_selected_commands_without_running(capsys) -> None: - rc = verify_local.main(["--dry-run", "--only", "python_floor,public_api,claim_guardrails"]) + rc = verify_local.main(["--dry-run", "--only", "python_floor,public_api"]) out = capsys.readouterr().out assert rc == 0 @@ -345,8 +344,6 @@ def test_dry_run_prints_selected_commands_without_running(capsys) -> None: assert "scripts/check_python_floor.py" in out assert "public_api" in out assert "scripts/check_public_api.py" in out - assert "claim_guardrails" in out - assert "scripts/check_claim_guardrails.py" in out def test_dry_run_includes_ci_workflow_gate(capsys) -> None: @@ -532,9 +529,9 @@ def test_makefile_exposes_docs_verification_shortcut() -> None: makefile = (ROOT / "Makefile").read_text(encoding="utf-8") assert "check-docs:" in makefile - assert "scripts/verify_local.py --only examples,claim_guardrails" in makefile + assert "scripts/verify_local.py --only examples" in makefile assert "make check-docs" in makefile - assert "docs examples and public claim guardrails" in makefile + assert "run documentation examples" in makefile def test_makefile_exposes_security_verification_shortcut() -> None: @@ -582,15 +579,6 @@ def test_makefile_exposes_ci_workflow_verification_shortcut() -> None: assert "CI/release workflow invariant checks" in makefile -def test_makefile_exposes_claim_guardrail_shortcut() -> None: - makefile = (ROOT / "Makefile").read_text(encoding="utf-8") - - assert "check-claims:" in makefile - assert "scripts/verify_local.py --only claim_guardrails" in makefile - assert "make check-claims" in makefile - assert "public performance-claim guardrails" in makefile - - def test_node_version_parser_accepts_v_prefixed_versions() -> None: assert verify_local._node_major("v18.19.1\n") == 18 assert verify_local._node_major("20.11.0") == 20 diff --git a/tests/test_verify_sdist.py b/tests/test_verify_sdist.py index 8a52bf4b..462da9ea 100644 --- a/tests/test_verify_sdist.py +++ b/tests/test_verify_sdist.py @@ -60,8 +60,8 @@ "make check-benchmark-report.\n" "Run make check-examples for spec/api/api-examples.md and " "the example apps.\n\n" - "## Performance Claims\n\n" - "Claims need benchmark context.\n" + ("contributing padding\n" * 100) + "## Competitive Evidence\n\n" + "XY aims to outperform every competing charting library.\n" + ("contributing padding\n" * 100) ) CI_YML = ( "name: CI\n"