diff --git a/.github/workflows/_build-docs-images.yml b/.github/workflows/_build-docs-images.yml index 579df74a..549a6fb0 100644 --- a/.github/workflows/_build-docs-images.yml +++ b/.github/workflows/_build-docs-images.yml @@ -17,6 +17,13 @@ on: description: "Exact reflex-dev/xy commit or tag to build" required: true type: string + outputs: + frontend_digest: + description: "Immutable ECR digest for the frontend image" + value: ${{ jobs.build-and-push.outputs.frontend_digest }} + backend_digest: + description: "Immutable ECR digest for the backend image" + value: ${{ jobs.build-and-push.outputs.backend_digest }} permissions: contents: read @@ -33,6 +40,9 @@ jobs: timeout-minutes: 60 permissions: contents: read + outputs: + frontend_digest: ${{ steps.digests.outputs.frontend_digest }} + backend_digest: ${{ steps.digests.outputs.backend_digest }} steps: - name: Validate inputs env: @@ -123,7 +133,7 @@ jobs: --tag "${CADDY_IMAGE}:${IMAGE_TAG}" \ --push \ --platform linux/amd64,linux/arm64 \ - --provenance=false \ + --provenance=mode=max \ ./docs/app - name: Build and push backend image @@ -139,20 +149,46 @@ jobs: --tag "${BACKEND_IMAGE}:${IMAGE_TAG}" \ --push \ --platform linux/amd64,linux/arm64 \ - --provenance=false \ + --provenance=mode=max \ . + - name: Resolve immutable image digests + id: digests + env: + IMAGE_TAG: ${{ inputs.image_tag }} + run: | + FRONTEND_DIGEST=$(aws ecr describe-images \ + --repository-name xy/frontend \ + --image-ids "imageTag=${IMAGE_TAG}" \ + --query 'imageDetails[0].imageDigest' \ + --output text) + BACKEND_DIGEST=$(aws ecr describe-images \ + --repository-name xy/backend \ + --image-ids "imageTag=${IMAGE_TAG}" \ + --query 'imageDetails[0].imageDigest' \ + --output text) + for DIGEST in "$FRONTEND_DIGEST" "$BACKEND_DIGEST"; do + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::ECR returned an invalid image digest: $DIGEST" + exit 1 + fi + done + echo "frontend_digest=$FRONTEND_DIGEST" >> "$GITHUB_OUTPUT" + echo "backend_digest=$BACKEND_DIGEST" >> "$GITHUB_OUTPUT" + - name: Summary env: IMAGE_TAG: ${{ inputs.image_tag }} CADDY_IMAGE: ${{ env.CADDY_IMAGE }} BACKEND_IMAGE: ${{ env.BACKEND_IMAGE }} + FRONTEND_DIGEST: ${{ steps.digests.outputs.frontend_digest }} + BACKEND_DIGEST: ${{ steps.digests.outputs.backend_digest }} run: | { echo "### XY docs images built" echo "" echo "**Tag:** \`${IMAGE_TAG}\`" echo "" - echo "- Frontend: \`${CADDY_IMAGE}:${IMAGE_TAG}\`" - echo "- Backend: \`${BACKEND_IMAGE}:${IMAGE_TAG}\`" + echo "- Frontend: \`${CADDY_IMAGE}:${IMAGE_TAG}@${FRONTEND_DIGEST}\`" + echo "- Backend: \`${BACKEND_IMAGE}:${IMAGE_TAG}@${BACKEND_DIGEST}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/_helm-docs-pr.yml b/.github/workflows/_helm-docs-pr.yml index 7b58de11..885d293d 100644 --- a/.github/workflows/_helm-docs-pr.yml +++ b/.github/workflows/_helm-docs-pr.yml @@ -19,6 +19,14 @@ on: description: "Source commit SHA from reflex-dev/xy" required: true type: string + frontend_digest: + description: "Immutable sha256 digest for the frontend image" + required: true + type: string + backend_digest: + description: "Immutable sha256 digest for the backend image" + required: true + type: string auto_merge: description: "Merge the Helm PR immediately" required: false @@ -46,6 +54,8 @@ jobs: IMAGE_TAG: ${{ inputs.image_tag }} ENVIRONMENT: ${{ inputs.environment }} SOURCE_REF: ${{ inputs.source_ref }} + FRONTEND_DIGEST: ${{ inputs.frontend_digest }} + BACKEND_DIGEST: ${{ inputs.backend_digest }} run: | if [[ ! "$IMAGE_TAG" =~ ^[A-Za-z0-9._-]+$ ]] || [[ "${#IMAGE_TAG}" -gt 128 ]]; then echo "::error::image_tag must match [A-Za-z0-9._-]+ and be <=128 characters" @@ -59,6 +69,12 @@ jobs: echo "::error::source_ref contains unsafe characters" exit 1 fi + for DIGEST in "$FRONTEND_DIGEST" "$BACKEND_DIGEST"; do + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::image digests must be sha256 digests" + exit 1 + fi + done - name: Generate GitHub App token id: app-token @@ -132,11 +148,15 @@ jobs: env: BRANCH: ${{ steps.check.outputs.branch }} IMAGE_TAG: ${{ inputs.image_tag }} + FRONTEND_DIGEST: ${{ inputs.frontend_digest }} + BACKEND_DIGEST: ${{ inputs.backend_digest }} VALUES_FILE: ${{ steps.values.outputs.file }} run: | git checkout -b "$BRANCH" - IMAGE_TAG="$IMAGE_TAG" yq -i '.apps.xy.frontend.image.tag = strenv(IMAGE_TAG)' "$VALUES_FILE" - IMAGE_TAG="$IMAGE_TAG" yq -i '.apps.xy.backend.image.tag = strenv(IMAGE_TAG)' "$VALUES_FILE" + FRONTEND_REF="${IMAGE_TAG}@${FRONTEND_DIGEST}" \ + yq -i '.apps.xy.frontend.image.tag = strenv(FRONTEND_REF)' "$VALUES_FILE" + BACKEND_REF="${IMAGE_TAG}@${BACKEND_DIGEST}" \ + yq -i '.apps.xy.backend.image.tag = strenv(BACKEND_REF)' "$VALUES_FILE" # ChartVersion reconcile strategy: Flux only repackages the chart # when Chart.yaml's version changes, so bump patch every deploy. CHART_FILE="$(dirname "$VALUES_FILE")/Chart.yaml" @@ -176,6 +196,8 @@ jobs: VALUES_FILE: ${{ steps.values.outputs.file }} FRONTEND_IMAGE: ${{ env.FRONTEND_IMAGE }} BACKEND_IMAGE: ${{ env.BACKEND_IMAGE }} + FRONTEND_DIGEST: ${{ inputs.frontend_digest }} + BACKEND_DIGEST: ${{ inputs.backend_digest }} REPO: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} RUN_ID: ${{ github.run_id }} @@ -187,10 +209,10 @@ jobs: echo "**Values file:** \`${VALUES_FILE}\`" echo "**Image tag:** \`${IMAGE_TAG}\`" echo "" - echo "**Frontend:** \`${FRONTEND_IMAGE}:${IMAGE_TAG}\`" - echo "**Backend:** \`${BACKEND_IMAGE}:${IMAGE_TAG}\`" + echo "**Frontend:** \`${FRONTEND_IMAGE}:${IMAGE_TAG}@${FRONTEND_DIGEST}\`" + echo "**Backend:** \`${BACKEND_IMAGE}:${IMAGE_TAG}@${BACKEND_DIGEST}\`" echo "" - echo "Updates \`apps.xy.frontend.image.tag\` and \`apps.xy.backend.image.tag\`." + echo "Pins \`apps.xy.frontend.image.tag\` and \`apps.xy.backend.image.tag\` to immutable digests." echo "" echo "---" echo "**Source commit:** https://github.com/${REPO}/commit/${SOURCE_REF}" @@ -229,6 +251,8 @@ jobs: ENVIRONMENT: ${{ inputs.environment }} IMAGE_TAG: ${{ inputs.image_tag }} VALUES_FILE: ${{ steps.values.outputs.file }} + FRONTEND_DIGEST: ${{ inputs.frontend_digest }} + BACKEND_DIGEST: ${{ inputs.backend_digest }} run: | { echo "### XY docs Helm PR created" @@ -236,4 +260,6 @@ jobs: echo "**Environment:** ${ENVIRONMENT}" echo "**Values file:** \`${VALUES_FILE}\`" echo "**Image tag:** \`${IMAGE_TAG}\`" + echo "**Frontend digest:** \`${FRONTEND_DIGEST}\`" + echo "**Backend digest:** \`${BACKEND_DIGEST}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4eefa83..d14ea312 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,10 +3,9 @@ name: CI on: push: branches: ["main", "claude/**"] + # Keep this unfiltered: `required_ci` is the stable branch-protection result + # and must exist even for documentation- or specification-only pull requests. pull_request: - paths-ignore: - - "docs/**" - - "spec/**" workflow_dispatch: # Multiple sessions push to two branches in bursts; without this the runner @@ -17,6 +16,197 @@ concurrency: cancel-in-progress: true jobs: + dependency_audit: + name: Multi-ecosystem dependency audit + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate dependency audit policy + run: python3 scripts/dependency_audit.py validate + - name: Install pinned OSV-Scanner + env: + OSV_SCANNER_URL: "https://github.com/google/osv-scanner/releases/download/v2.4.0/osv-scanner_linux_amd64" + OSV_SCANNER_SHA256: "15314940c10d26af9c6649f150b8a47c1262e8fc7e17b1d1029b0e479e8ed8a0" + run: | + mkdir -p .dependency-audit/bin dependency-audit + curl --fail --location --retry 3 --retry-all-errors \ + "$OSV_SCANNER_URL" --output .dependency-audit/bin/osv-scanner + printf '%s %s\n' \ + "$OSV_SCANNER_SHA256" .dependency-audit/bin/osv-scanner \ + | sha256sum --check --strict + chmod +x .dependency-audit/bin/osv-scanner + - name: Audit every active dependency environment + run: >- + python3 scripts/dependency_audit.py scan + --scanner .dependency-audit/bin/osv-scanner + --output-dir dependency-audit + - name: Retain dependency audit evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dependency-audit + if-no-files-found: error + retention-days: 30 + path: | + dependency-audit/*.json + dependency-audit/*.txt + + rust_release: + name: Rust release-profile tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - name: Inventory release-only regression coverage + run: | + cargo test --locked --release -- --list | tee release-tests.txt + grep -Fqx \ + 'tiles::tests::compose_window_astronomically_past_domain_is_empty_not_panic: test' \ + release-tests.txt + - name: Run locked release-profile suite + run: cargo test --locked --release + - name: Upload release-profile test inventory + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rust-release-test-inventory + if-no-files-found: error + path: release-tests.txt + + native_parity: + name: Native parity (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + default: avx2 + - os: ubuntu-24.04-arm + arch: aarch64 + default: aarch64 + - os: windows-latest + arch: x86_64 + default: avx2 + - os: macos-14 + arch: aarch64 + default: aarch64 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - name: Build locked native core + run: cargo build --locked --release + - name: Run common kernel, FFI, and raster parity probe + run: >- + python scripts/native_parity.py + --expect-arch ${{ matrix.arch }} + --expect-default ${{ matrix.default }} + --report native-parity-${{ matrix.os }}.json + - name: Retain native parity report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-parity-${{ matrix.os }} + if-no-files-found: error + path: native-parity-${{ matrix.os }}.json + + javascript_semantics: + name: JavaScript semantic unit suite + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Run JavaScript semantic unit suite + shell: bash + run: | + mkdir -p test-results/javascript coverage/javascript + set -o pipefail + NODE_V8_COVERAGE=coverage/javascript \ + node --test --experimental-test-coverage \ + --test-coverage-include=python/xy/static/index.js \ + --test-coverage-lines=15 --test-coverage-branches=60 \ + --test-coverage-functions=10 \ + --test-reporter=spec --test-reporter-destination=stdout \ + --test-reporter=junit \ + --test-reporter-destination=test-results/javascript/junit.xml \ + js/test/*.test.mjs \ + | tee test-results/javascript/coverage.txt + - name: Upload JavaScript semantic evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: javascript-semantic-evidence + if-no-files-found: error + path: | + test-results/javascript/ + coverage/javascript/ + + python_coverage: + name: Python branch and diff coverage + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: false + - name: Install locked coverage environment + run: | + uv sync --frozen --project python/reflex-xy --extra dev + UV_PROJECT_ENVIRONMENT=python/reflex-xy/.venv \ + uv sync --frozen --extra dev --inexact + - name: Measure core and pyplot branch coverage + run: | + python/reflex-xy/.venv/bin/coverage erase + python/reflex-xy/.venv/bin/coverage run --branch --source=python/xy \ + -m pytest -q tests + - name: Measure Reflex adapter branch coverage + if: ${{ !cancelled() }} + run: | + python/reflex-xy/.venv/bin/coverage run --branch --append \ + --source=python/reflex-xy/reflex_xy \ + scripts/run_pytest_no_skips.py -q \ + python/reflex-xy/tests + - name: Enforce reviewed package, module, and changed-line floors + if: ${{ !cancelled() }} + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p coverage/python + python/reflex-xy/.venv/bin/coverage json -o coverage/python/coverage.json + python/reflex-xy/.venv/bin/coverage xml -o coverage/python/coverage.xml + python/reflex-xy/.venv/bin/python scripts/coverage_ratchet.py \ + --coverage-json coverage/python/coverage.json \ + --policy spec/testing/coverage-policy.json \ + --base "$BASE_SHA" --head "$HEAD_SHA" \ + --report coverage/python/ratchet.json + - name: Upload Python coverage evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-coverage-evidence + if-no-files-found: error + retention-days: 30 + path: | + .coverage + coverage/python/ + matplotlib_reference: name: Matplotlib 3.11 reference compatibility runs-on: ubuntu-latest @@ -96,6 +286,20 @@ jobs: run: | npm ci npx playwright install --with-deps chromium + + - name: Rendered label and formatter oracle (Chromium) + env: + XY_LABEL_EVIDENCE: rendered-label-evidence.json + run: npm run test:labels + + - name: Upload rendered label evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rendered-label-evidence + if-no-files-found: error + path: rendered-label-evidence.json + - name: Headless render smoke (stdlib + Chromium) run: | CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") @@ -142,6 +346,17 @@ jobs: - name: Public API coherence run: .venv/bin/python scripts/check_public_api.py + - name: Pyplot option contract + run: .venv/bin/python scripts/check_pyplot_options.py --report pyplot-option-contract.json + + - name: Upload pyplot option contract evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyplot-option-contract + if-no-files-found: error + path: pyplot-option-contract.json + - name: Claim guardrails run: .venv/bin/python scripts/check_claim_guardrails.py @@ -157,6 +372,22 @@ jobs: - name: Typecheck (ty, advisory) run: .venv/bin/ty check python || echo "::warning::ty reported type findings (advisory)" + - name: Protocol catalog and byte-mutation conformance + run: | + mkdir -p test-results/protocol + .venv/bin/pytest -q \ + tests/test_protocol_catalog.py tests/test_framing.py tests/test_framing_property.py \ + --junitxml=test-results/protocol/junit.xml + + - name: Upload protocol conformance evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: protocol-conformance-evidence + if-no-files-found: error + retention-days: 30 + path: test-results/protocol/junit.xml + - name: Python tests (native core) # Point PNG-export tests at the Playwright Chromium (not on PATH) so the # full Figure.to_png path runs here rather than skipping. @@ -164,6 +395,23 @@ jobs: export XY_CHROMIUM="$(node -e "console.log(require('playwright').chromium.executablePath())")" .venv/bin/pytest -q + # Runtime page-content isolation is distinct from the browser process + # sandbox policy. This trusted repository fixture opts out explicitly on + # the constrained runner; the public export path never retries this way. + - name: Runtime standalone security smoke (Chromium) + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + .venv/bin/python scripts/runtime_security_smoke.py "$CHROME" --no-sandbox \ + --evidence runtime-security-evidence.json + + - name: Upload runtime security evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runtime-security-evidence + if-no-files-found: error + path: runtime-security-evidence.json + # End-to-end: a real Figure (line decimation + scatter) through to_html's # exact payload path, rendered in Chromium, pixels read back. Complements # the stdlib smoke, which hand-builds payloads. (Was dead code — audit @@ -178,20 +426,104 @@ jobs: CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") .venv/bin/python scripts/reflex_lifecycle_smoke.py "$CHROME" - - name: Browser visual regression smoke (Chromium) + - name: Browser visual health smoke (Chromium) + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + .venv/bin/python scripts/visual_health_smoke.py "$CHROME" + + - name: Reviewed visual baseline (Chromium) + if: ${{ !cancelled() }} + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + .venv/bin/python scripts/visual_baseline.py "$CHROME" \ + --baseline spec/visual-baselines/v1.json \ + --artifacts visual-baseline-artifacts \ + --evidence visual-baseline-evidence.json + + - name: Upload visual baseline evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: visual-baseline-evidence + if-no-files-found: error + path: | + visual-baseline-evidence.json + visual-baseline-artifacts/ + + - name: Every chart-kind render matrix (Chromium) run: | CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") - .venv/bin/python scripts/visual_regression_smoke.py "$CHROME" + .venv/bin/python scripts/chart_kind_matrix.py "$CHROME" --no-sandbox \ + --evidence chart-kind-matrix-evidence.json + + - name: Upload chart-kind matrix evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: chart-kind-matrix-evidence + if-no-files-found: error + path: chart-kind-matrix-evidence.json - name: Step tier-update smoke (Chromium) run: | CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") .venv/bin/python scripts/step_tier_smoke.py "$CHROME" + - name: Animation smoke (Chromium) + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + .venv/bin/python scripts/animation_smoke.py "$CHROME" \ + --evidence animation-browser-evidence.json + + - name: Upload animation browser evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: animation-browser-evidence + if-no-files-found: error + path: animation-browser-evidence.json + + - name: Pick boundary smoke (Chromium) + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + .venv/bin/python scripts/pick_boundary_smoke.py "$CHROME" \ + --evidence pick-boundary-evidence.json + + - name: Upload pick boundary evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pick-boundary-evidence + if-no-files-found: error + path: pick-boundary-evidence.json + - name: Browser interaction stress smoke (Chromium) run: | CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") - .venv/bin/python scripts/interaction_stress_smoke.py "$CHROME" + .venv/bin/python scripts/interaction_stress_smoke.py "$CHROME" \ + --json interaction-worker-evidence.json + + - name: Upload interaction worker evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: interaction-worker-evidence + if-no-files-found: error + path: interaction-worker-evidence.json + + - name: Pan/zoom acceptance matrix (Chromium) + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + node scripts/pan_zoom_matrix.mjs --profile full --browsers chromium \ + --executable-path "$CHROME" --evidence pan-zoom-matrix-evidence.json + + - name: Upload pan/zoom matrix evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pan-zoom-matrix-evidence + if-no-files-found: error + path: pan-zoom-matrix-evidence.json - name: Browser dashboard reliability smoke (Chromium) run: | @@ -199,14 +531,217 @@ jobs: .venv/bin/python benchmarks/bench_dashboard.py \ --chart-counts 10,20,50 --chromium "$CHROME" --json dashboard-smoke.json .venv/bin/python scripts/verify_benchmark_report.py \ - dashboard-smoke.json --kind dashboard-browser + dashboard-smoke.json --kind dashboard-browser --profile strict + + - name: Upload dashboard health evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dashboard-health-evidence + if-no-files-found: error + path: dashboard-smoke.json - name: Benchmark smoke (1e5/1e6 — §12 harness runs every phase) run: .venv/bin/python scripts/bench.py --sizes 1e5,1e6 + reflex_adapter: + name: Reflex adapter E2E (${{ matrix.name }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - name: floor + reflex: reflex==0.9.6 + - name: latest + reflex: reflex>=0.9.6,<1 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Build native core and client + run: | + cargo build --release + npm ci + node js/build.mjs --check + - name: Install xy, Reflex adapter, and selected Reflex version + env: + XY_REQUIRE_CARGO: "1" + run: | + uv venv .venv + uv pip install -p .venv/bin/python -e . + uv pip install -p .venv/bin/python -e "python/reflex-xy[dev]" "${{ matrix.reflex }}" + - name: Verify selected host versions + run: | + .venv/bin/python -c "from importlib.metadata import version; print('reflex', version('reflex'))" + .venv/bin/python -c "import reflex_xy, xy; assert reflex_xy and xy" + mkdir -p examples/reflex + .venv/bin/python scripts/host_integration_policy.py validate + .venv/bin/python scripts/host_integration_policy.py report \ + --profile ${{ matrix.name }} --hosts reflex \ + --output examples/reflex/reflex-host-versions.json + - name: Run adapter suite with zero skips + run: >- + .venv/bin/python scripts/run_pytest_no_skips.py -q python/reflex-xy/tests + --junitxml=examples/reflex/reflex-adapter-junit.xml + - name: Compile real Reflex example + working-directory: examples/reflex + run: ../../.venv/bin/reflex compile --dry + - name: Install Chromium (Playwright) + run: npx playwright install --with-deps chromium + - name: Run real Reflex browser E2E + shell: bash + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + cd examples/reflex + ../../.venv/bin/reflex run --env prod --single-port --frontend-port 3100 \ + > reflex-e2e.log 2>&1 & + server_pid=$! + cleanup() { + kill -INT "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + } + trap cleanup EXIT + ready=0 + for _attempt in $(seq 1 120); do + if curl --fail --silent http://localhost:3100/ >/dev/null; then + ready=1 + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + break + fi + sleep 1 + done + if [[ "$ready" != "1" ]]; then + cat reflex-e2e.log + exit 1 + fi + matrix_status=0 + node ../../scripts/pan_zoom_matrix.mjs --profile reflex --browsers chromium \ + --url http://localhost:3100 --executable-path "$CHROME" \ + --evidence pan-zoom-reflex-evidence.json || matrix_status=$? + smoke_status=0 + ../../.venv/bin/python ../../scripts/reflex_ws_smoke.py \ + --frontend http://localhost:3100 --chromium "$CHROME" \ + --screenshot reflex-e2e.png || smoke_status=$? + if [[ "$matrix_status" != "0" || "$smoke_status" != "0" ]]; then + exit 1 + fi + - name: Upload Reflex E2E evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: reflex-e2e-${{ matrix.name }} + if-no-files-found: error + retention-days: 30 + path: | + examples/reflex/reflex-host-versions.json + examples/reflex/reflex-adapter-junit.xml + examples/reflex/reflex-e2e.log + examples/reflex/reflex-e2e.png + + - name: Upload Reflex pan/zoom evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: reflex-pan-zoom-${{ matrix.name }} + if-no-files-found: error + path: examples/reflex/pan-zoom-reflex-evidence.json + + host_integration: + name: Host integration (${{ matrix.name }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - name: floor + anywidget: anywidget==0.9.0 + traitlets: traitlets==5.14.0 + fastapi: fastapi==0.110.0 + starlette: starlette==0.36.3 + uvicorn: uvicorn==0.29.0 + httpx: httpx==0.27.0 + - name: latest + anywidget: anywidget>=0.9,<1 + traitlets: traitlets>=5.14,<6 + fastapi: fastapi>=0.110,<1 + starlette: starlette>=0.36.3,<1 + uvicorn: uvicorn>=0.29,<1 + httpx: httpx>=0.27,<1 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Build locked native core and verify client + run: | + cargo build --locked --release + npm ci + node js/build.mjs --check + - name: Install selected package-owned host stack + env: + XY_REQUIRE_CARGO: "1" + run: | + uv venv .host-venv + uv pip install -p .host-venv/bin/python -e ".[dev]" \ + "${{ matrix.anywidget }}" "${{ matrix.traitlets }}" \ + "${{ matrix.fastapi }}" "${{ matrix.starlette }}" \ + "${{ matrix.uvicorn }}" "${{ matrix.httpx }}" + - name: Validate and retain selected host versions + run: | + mkdir -p host-integration + .host-venv/bin/python scripts/host_integration_policy.py validate + .host-venv/bin/python scripts/host_integration_policy.py report \ + --profile ${{ matrix.name }} --hosts anywidget fastapi \ + --output host-integration/versions.json + - name: Run focused compile, mount, and transport suite with zero skips + env: + XY_HOST_PROFILE: ${{ matrix.name }} + run: >- + .host-venv/bin/python scripts/run_pytest_no_skips.py -q + tests/host_integration --junitxml=host-integration/junit.xml + - name: Install Chromium (Playwright) + run: npx playwright install --with-deps chromium + - name: Run FastAPI browser mount and transport probe + run: | + CHROME=$(node -e "console.log(require('playwright').chromium.executablePath())") + .host-venv/bin/python scripts/fastapi_host_smoke.py \ + --chromium "$CHROME" --no-sandbox \ + --evidence host-integration/fastapi-browser.json \ + --screenshot host-integration/fastapi-browser.png \ + --server-log host-integration/fastapi-server.log + - name: Upload host integration evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: host-integration-${{ matrix.name }} + if-no-files-found: error + retention-days: 30 + path: host-integration/ + browser_conformance: name: Accessibility + cross-browser conformance runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -230,7 +765,36 @@ jobs: env: MOZ_WEBRENDER: "1" XY_CONFORMANCE_HEADFUL: "1" - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" node scripts/browser_conformance.mjs + run: >- + xvfb-run --auto-servernum --server-args="-screen 0 1920x1200x24" + node scripts/browser_conformance.mjs + --evidence browser-conformance-evidence.json + + - name: Upload browser conformance evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: browser-conformance-evidence + if-no-files-found: error + path: browser-conformance-evidence.json + + - name: Focused pan/zoom matrix (three engines) + env: + MOZ_WEBRENDER: "1" + XY_PAN_ZOOM_HEADFUL: "1" + run: | + xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \ + node scripts/pan_zoom_matrix.mjs --profile focused \ + --browsers chromium,firefox,webkit \ + --evidence pan-zoom-cross-engine-evidence.json + + - name: Upload cross-engine pan/zoom evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pan-zoom-cross-engine-evidence + if-no-files-found: error + path: pan-zoom-cross-engine-evidence.json python_floor: name: Python 3.11 floor @@ -723,3 +1287,30 @@ jobs: else: raise SystemExit("expected ImportError without the native core") PY + + required_ci: + name: Required CI + if: always() + needs: + - browser_conformance + - dependency_audit + - install_without_rust + - javascript_semantics + - matplotlib_reference + - native_parity + - python_coverage + - python_floor + - host_integration + - reflex_adapter + - rust_release + - sdist + - test + - wheels + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Require every hard dependency to succeed + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: python3 scripts/check_required_jobs.py diff --git a/.github/workflows/deploy-docs-dev.yml b/.github/workflows/deploy-docs-dev.yml index 424a7e76..836f32a5 100644 --- a/.github/workflows/deploy-docs-dev.yml +++ b/.github/workflows/deploy-docs-dev.yml @@ -9,6 +9,7 @@ on: workflow_dispatch: permissions: + actions: read contents: read concurrency: @@ -50,9 +51,39 @@ jobs: echo "should_build=true" >> "$GITHUB_OUTPUT" fi + qualify: + name: Qualify Exact Source SHA + needs: prepare + if: needs.prepare.outputs.should_build == 'true' + runs-on: ubuntu-latest + timeout-minutes: 65 + permissions: + actions: read + contents: read + steps: + - name: Checkout exact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.prepare.outputs.source_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Fetch main ancestry + run: git fetch --no-tags origin main:refs/remotes/origin/main + + - name: Require hard CI for exact source + env: + GITHUB_TOKEN: ${{ github.token }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + run: >- + python3 scripts/verify_source_qualification.py + --sha "$SOURCE_SHA" + --repository "$GITHUB_REPOSITORY" + --wait-seconds 3600 + build: name: Build Dev Images - needs: prepare + needs: [prepare, qualify] if: needs.prepare.outputs.should_build == 'true' permissions: contents: read @@ -73,6 +104,8 @@ jobs: image_tag: ${{ needs.prepare.outputs.image_tag }} environment: dev source_ref: ${{ needs.prepare.outputs.source_sha }} + frontend_digest: ${{ needs.build.outputs.frontend_digest }} + backend_digest: ${{ needs.build.outputs.backend_digest }} auto_merge: true secrets: inherit diff --git a/.github/workflows/deploy-docs-stg.yml b/.github/workflows/deploy-docs-stg.yml index bca2ad2d..b3d4f6b1 100644 --- a/.github/workflows/deploy-docs-stg.yml +++ b/.github/workflows/deploy-docs-stg.yml @@ -18,6 +18,7 @@ on: - "20[0-9][0-9].[0-9]*" permissions: + actions: read contents: read concurrency: @@ -58,9 +59,40 @@ jobs: echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "source_sha=$SHA" >> "$GITHUB_OUTPUT" + qualify: + name: Qualify Exact Tagged Source + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 65 + permissions: + actions: read + contents: read + steps: + - name: Checkout exact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.prepare.outputs.source_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Fetch main and tags + run: git fetch --force origin main:refs/remotes/origin/main 'refs/tags/*:refs/tags/*' + + - name: Require hard CI and exact tag + env: + GITHUB_TOKEN: ${{ github.token }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + VERSION: ${{ needs.prepare.outputs.version }} + run: >- + python3 scripts/verify_source_qualification.py + --sha "$SOURCE_SHA" + --repository "$GITHUB_REPOSITORY" + --tag "$VERSION" + --wait-seconds 3600 + build: name: Build Versioned Images - needs: prepare + needs: [prepare, qualify] permissions: contents: read uses: ./.github/workflows/_build-docs-images.yml @@ -79,6 +111,8 @@ jobs: image_tag: ${{ needs.prepare.outputs.version }} environment: stg source_ref: ${{ needs.prepare.outputs.source_sha }} + frontend_digest: ${{ needs.build.outputs.frontend_digest }} + backend_digest: ${{ needs.build.outputs.backend_digest }} auto_merge: true secrets: inherit @@ -93,9 +127,50 @@ jobs: steps: - run: echo "Approved for production promotion" + verify-prod-artifacts: + name: Verify Production Image Digests + needs: [prepare, build, await-prod-approval] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + mask-aws-account-id: true + + - name: Verify promoted images are the built artifacts + env: + IMAGE_TAG: ${{ needs.prepare.outputs.version }} + EXPECTED_FRONTEND: ${{ needs.build.outputs.frontend_digest }} + EXPECTED_BACKEND: ${{ needs.build.outputs.backend_digest }} + run: | + ACTUAL_FRONTEND=$(aws ecr describe-images \ + --repository-name xy/frontend \ + --image-ids "imageTag=${IMAGE_TAG}" \ + --query 'imageDetails[0].imageDigest' \ + --output text) + ACTUAL_BACKEND=$(aws ecr describe-images \ + --repository-name xy/backend \ + --image-ids "imageTag=${IMAGE_TAG}" \ + --query 'imageDetails[0].imageDigest' \ + --output text) + if [[ "$ACTUAL_FRONTEND" != "$EXPECTED_FRONTEND" ]]; then + echo "::error::frontend tag changed after the qualified build" + exit 1 + fi + if [[ "$ACTUAL_BACKEND" != "$EXPECTED_BACKEND" ]]; then + echo "::error::backend tag changed after the qualified build" + exit 1 + fi + release: name: Create GitHub Release - needs: [prepare, await-prod-approval] + needs: [prepare, build, verify-prod-artifacts] runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -106,13 +181,15 @@ jobs: GH_TOKEN: ${{ github.token }} VERSION: ${{ needs.prepare.outputs.version }} SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + FRONTEND_DIGEST: ${{ needs.build.outputs.frontend_digest }} + BACKEND_DIGEST: ${{ needs.build.outputs.backend_digest }} REPO: ${{ github.repository }} run: | if gh release view "$VERSION" --repo "$REPO" >/dev/null 2>&1; then echo "::notice::Release $VERSION already exists" exit 0 fi - NOTES=$(printf "Production deployment of XY docs.\n\n**Source SHA:** \`%s\`\n**Image tag:** \`%s\`" "$SOURCE_SHA" "$VERSION") + NOTES=$(printf "Production deployment of XY docs.\n\n**Source SHA:** \`%s\`\n**Image tag:** \`%s\`\n**Frontend digest:** \`%s\`\n**Backend digest:** \`%s\`" "$SOURCE_SHA" "$VERSION" "$FRONTEND_DIGEST" "$BACKEND_DIGEST") gh release create "$VERSION" \ --repo "$REPO" \ --title "Release $VERSION" \ @@ -121,7 +198,7 @@ jobs: helm-pr-prod: name: Update Production Helm Values - needs: [prepare, await-prod-approval, release] + needs: [prepare, build, verify-prod-artifacts, release] permissions: contents: read uses: ./.github/workflows/_helm-docs-pr.yml @@ -129,5 +206,7 @@ jobs: image_tag: ${{ needs.prepare.outputs.version }} environment: prod source_ref: ${{ needs.prepare.outputs.source_sha }} + frontend_digest: ${{ needs.build.outputs.frontend_digest }} + backend_digest: ${{ needs.build.outputs.backend_digest }} auto_merge: true secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a802593d..c7bd3028 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,69 @@ on: type: boolean default: true +permissions: + contents: read + actions: read + jobs: + qualify: + name: Qualify exact source SHA + runs-on: ubuntu-latest + timeout-minutes: 65 + outputs: + source_sha: ${{ steps.source.outputs.source_sha }} + release_tag: ${{ steps.source.outputs.release_tag }} + release_metadata: ${{ steps.source.outputs.release_metadata }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Fetch qualification refs + run: git fetch --force origin main:refs/remotes/origin/main 'refs/tags/*:refs/tags/*' + - name: Resolve immutable release source + id: source + env: + DRY_RUN: ${{ github.event.inputs.dry_run }} + EVENT_NAME: ${{ github.event_name }} + EVENT_SHA: ${{ github.sha }} + REF_NAME: ${{ github.ref_name }} + run: | + source_sha=$(git rev-parse "${EVENT_SHA}^{commit}") + release_tag="$REF_NAME" + release_metadata=true + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + release_tag="v${version}" + if [[ "$DRY_RUN" == "true" ]]; then + release_metadata=false + fi + fi + echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT" + echo "release_tag=$release_tag" >> "$GITHUB_OUTPUT" + echo "release_metadata=$release_metadata" >> "$GITHUB_OUTPUT" + - name: Require exact-SHA hard-suite success + env: + GITHUB_TOKEN: ${{ github.token }} + RELEASE_METADATA: ${{ steps.source.outputs.release_metadata }} + RELEASE_TAG: ${{ steps.source.outputs.release_tag }} + SOURCE_SHA: ${{ steps.source.outputs.source_sha }} + run: | + args=( + --sha "$SOURCE_SHA" + --repository "$GITHUB_REPOSITORY" + --tag "$RELEASE_TAG" + --wait-seconds 3600 + ) + if [[ "$RELEASE_METADATA" == "true" ]]; then + args+=(--release-metadata) + else + # A pre-tag dry run still requires main ancestry and exact hard CI; + # only publication metadata is deferred because nothing publishes. + args=(--sha "$SOURCE_SHA" --repository "$GITHUB_REPOSITORY" --wait-seconds 3600) + fi + python3 scripts/verify_source_qualification.py "${args[@]}" + # One wheel per platform (the C-ABI core is CPython-version-independent, so # each entry produces a single py3-none- wheel). Coverage matches # what the native core can target: Linux glibc + musl across x86-64/aarch64/ @@ -28,13 +90,14 @@ jobs: # the platform tag (XY_CARGO_TARGET / XY_WHEEL_PLATFORM). wheels: name: Wheel ${{ matrix.plat }} + needs: qualify runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: # Linux glibc (manylinux_2_17 floor) — cross-compiled with zig. - - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, zigtarget: x86_64-unknown-linux-gnu.2.17, plat: manylinux_2_17_x86_64, zig: true, native: true } + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, zigtarget: x86_64-unknown-linux-gnu.2.17, plat: manylinux_2_17_x86_64, zig: true, native: true, runtime: manylinux } - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, zigtarget: aarch64-unknown-linux-gnu.2.17, plat: manylinux_2_17_aarch64, zig: true, native: false } - { os: ubuntu-latest, target: armv7-unknown-linux-gnueabihf, zigtarget: armv7-unknown-linux-gnueabihf.2.17, plat: manylinux_2_17_armv7l, zig: true, native: false } # Linux musl / Alpine. crt-static is musl's default, and rustc @@ -42,7 +105,7 @@ jobs: # since a fully-static binary can't also be a shared library — # -C target-feature=-crt-static switches to dynamic linking against # musl libc so a real cdylib gets produced. - - { os: ubuntu-latest, target: x86_64-unknown-linux-musl, zigtarget: x86_64-unknown-linux-musl, plat: musllinux_1_2_x86_64, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } + - { os: ubuntu-latest, target: x86_64-unknown-linux-musl, zigtarget: x86_64-unknown-linux-musl, plat: musllinux_1_2_x86_64, zig: true, native: false, rustflags: "-C target-feature=-crt-static", runtime: musllinux } - { os: ubuntu-latest, target: aarch64-unknown-linux-musl, zigtarget: aarch64-unknown-linux-musl, plat: musllinux_1_2_aarch64, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - { os: ubuntu-latest, target: armv7-unknown-linux-musleabihf, zigtarget: armv7-unknown-linux-musleabihf, plat: musllinux_1_2_armv7l, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } # macOS. Both build on the arm64 runner (macos-14): Apple Silicon @@ -59,6 +122,9 @@ jobs: - { os: windows-latest, target: aarch64-pc-windows-msvc, plat: win_arm64, zig: false, native: false } steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.qualify.outputs.source_sha }} + persist-credentials: false - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable with: targets: ${{ matrix.target }} @@ -120,6 +186,35 @@ jobs: uv pip install -p smoke dist/*.whl numpy anywidget ./smoke/bin/python -c "import xy.kernels as k; assert k.BACKEND=='native', k.BACKEND; print('native', k.__file__)" \ || ./smoke/Scripts/python.exe -c "import xy.kernels as k; assert k.BACKEND=='native', k.BACKEND; print('native')" + - name: Run manylinux wheel parity in glibc 2.17 + if: matrix.runtime == 'manylinux' + shell: bash + run: | + wheel=$(basename "$(ls dist/*.whl)") + docker run --rm --platform linux/amd64 \ + --volume "$PWD:/io" --workdir /io \ + quay.io/pypa/manylinux2014_x86_64@sha256:0d25b049964b2549b83384036abdff06789a8c0b1e9ff003ec80f0d531f79e50 \ + /opt/python/cp313-cp313/bin/python scripts/native_parity.py \ + --wheel "dist/$wheel" --expect-arch x86_64 --expect-default avx2 \ + --report native-parity-${{ matrix.plat }}.json + - name: Run musllinux wheel parity in musl 1.2 + if: matrix.runtime == 'musllinux' + shell: bash + run: | + wheel=$(basename "$(ls dist/*.whl)") + docker run --rm --platform linux/amd64 \ + --volume "$PWD:/io" --workdir /io \ + docker.io/library/python:3.13-alpine3.22@sha256:e81548ac35b07a3bd4805f275107592ef458b1e893c0e04d45aedaa19416cca5 \ + python scripts/native_parity.py \ + --wheel "dist/$wheel" --expect-arch x86_64 --expect-default avx2 \ + --report native-parity-${{ matrix.plat }}.json + - name: Retain native artifact runtime report + if: always() && (matrix.runtime == 'manylinux' || matrix.runtime == 'musllinux') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-parity-${{ matrix.plat }} + if-no-files-found: error + path: native-parity-${{ matrix.plat }}.json - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dist-${{ matrix.plat }} @@ -130,9 +225,13 @@ jobs: # runtime probe pinned as one tested set (see spec/process/production-readiness.md). wasm: name: Wheel Pyodide (runtime verified) + needs: qualify runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.qualify.outputs.source_sha }} + persist-credentials: false - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable with: toolchain: 1.97.0 @@ -190,9 +289,13 @@ jobs: sdist: name: sdist + needs: qualify runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.qualify.outputs.source_sha }} + persist-credentials: false - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: false @@ -242,30 +345,98 @@ jobs: name: dist-sdist path: dist/*.tar.gz + provenance: + name: Hash release artifacts and record provenance + needs: [qualify, wheels, sdist, wasm] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.qualify.outputs.source_sha }} + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: release-inputs/dist + pattern: dist-* + merge-multiple: true + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pyodide-wheel + path: release-inputs/pyodide + - name: Create and self-verify artifact provenance + env: + RELEASE_TAG: ${{ needs.qualify.outputs.release_tag }} + SOURCE_SHA: ${{ needs.qualify.outputs.source_sha }} + run: | + python3 scripts/release_provenance.py create release-inputs \ + --output release-provenance.json \ + --source-sha "$SOURCE_SHA" \ + --repository "$GITHUB_REPOSITORY" \ + --workflow-run-id "$GITHUB_RUN_ID" \ + --tag "$RELEASE_TAG" + mapfile -t artifacts < <(find release-inputs -type f -print | sort) + python3 scripts/release_provenance.py verify release-provenance.json \ + "${artifacts[@]}" \ + --source-sha "$SOURCE_SHA" \ + --repository "$GITHUB_REPOSITORY" \ + --workflow-run-id "$GITHUB_RUN_ID" \ + --tag "$RELEASE_TAG" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-provenance + if-no-files-found: error + path: release-provenance.json + publish: name: Publish to PyPI # Publishing still requires the runtime-verified Pyodide job to pass, but # only native wheels and the sdist use dist-* artifact names. PyPI does not # accept Pyodide platform tags, so its wheel remains a separate artifact. - needs: [wheels, sdist, wasm] + needs: [qualify, wheels, sdist, wasm, provenance] runs-on: ubuntu-latest environment: pypi permissions: + contents: read + actions: read id-token: write # PyPI trusted publishing (OIDC) — no API token stored steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.qualify.outputs.source_sha }} + persist-credentials: false # The artifact verifiers check everything inside the wheels/sdist, but # only this gate stops a mismatched tag from publishing whatever version # sits in pyproject.toml: the tag must equal v and # CHANGELOG.md must carry a dated entry for it. - name: Release version gate (tag == pyproject == CHANGELOG) - if: github.event_name == 'push' - run: python3 scripts/check_release_version.py + if: github.event_name == 'push' || github.event.inputs.dry_run != 'true' + run: python3 scripts/check_release_version.py --tag "${{ needs.qualify.outputs.release_tag }}" - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist pattern: dist-* merge-multiple: true + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pyodide-wheel + path: dist-pyodide + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-provenance + path: provenance + - name: Verify immutable package hashes + env: + RELEASE_TAG: ${{ needs.qualify.outputs.release_tag }} + SOURCE_SHA: ${{ needs.qualify.outputs.source_sha }} + run: | + python3 scripts/release_provenance.py verify \ + provenance/release-provenance.json \ + dist/*.whl dist/*.tar.gz dist-pyodide/*.whl \ + --source-sha "$SOURCE_SHA" \ + --repository "$GITHUB_REPOSITORY" \ + --workflow-run-id "$GITHUB_RUN_ID" \ + --tag "$RELEASE_TAG" - name: List artifacts run: ls -la dist/ # A manual workflow_dispatch defaults to dry_run=true: every wheel/sdist/ @@ -288,12 +459,16 @@ jobs: publish-pyodide: name: Publish Pyodide wheel if: github.event_name != 'workflow_dispatch' || github.event.inputs.dry_run != 'true' - needs: [wheels, sdist, wasm] + needs: [qualify, wheels, sdist, wasm, provenance] runs-on: ubuntu-latest permissions: contents: write + actions: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.qualify.outputs.source_sha }} + persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "22" @@ -301,6 +476,30 @@ jobs: with: name: pyodide-wheel path: dist-pyodide + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: dist-release + pattern: dist-* + merge-multiple: true + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-provenance + path: provenance + - name: Verify immutable Pyodide wheel hash + env: + RELEASE_TAG: ${{ needs.qualify.outputs.release_tag }} + SOURCE_SHA: ${{ needs.qualify.outputs.source_sha }} + shell: bash + run: | + wheel=$(find dist-pyodide -maxdepth 1 -name '*.whl' -print -quit) + test -n "$wheel" + python3 scripts/release_provenance.py verify \ + provenance/release-provenance.json \ + dist-release/*.whl dist-release/*.tar.gz "$wheel" \ + --source-sha "$SOURCE_SHA" \ + --repository "$GITHUB_REPOSITORY" \ + --workflow-run-id "$GITHUB_RUN_ID" \ + --tag "$RELEASE_TAG" # PyPI rejects Pyodide platform tags. Publish the runtime-verified wheel # as a release asset instead; micropip supports installing binary WASM # wheels directly from HTTPS URLs. @@ -323,9 +522,10 @@ jobs: wheel=$(find dist-pyodide -maxdepth 1 -name '*.whl' -print -quit) test -n "$wheel" if gh release view "$tag" >/dev/null 2>&1; then - gh release upload "$tag" "$wheel" --clobber + gh release upload "$tag" "$wheel" provenance/release-provenance.json --clobber else - gh release create "$tag" "$wheel" --verify-tag --generate-notes --title "$tag" + gh release create "$tag" "$wheel" provenance/release-provenance.json \ + --verify-tag --generate-notes --title "$tag" fi url="https://github.com/${GH_REPO}/releases/download/${tag}/$(basename "$wheel")" echo "wheel_url=$url" >> "$GITHUB_OUTPUT" diff --git a/.gitignore b/.gitignore index 1e7098a4..0daa8e4c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,9 @@ __pycache__/ *.pyc .pytest_cache/ .ruff_cache/ +.coverage +coverage/ node_modules/ *.egg-info/ +examples/reflex/reflex-e2e.log +examples/reflex/reflex-e2e.png diff --git a/Makefile b/Makefile index 540b7ba6..f116370a 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,14 @@ SDIST ?= WHEEL ?= BENCHMARK_JSON ?= benchmark.json BENCHMARK_KIND ?= auto +BENCHMARK_PROFILE ?= baseline +COVERAGE_JSON ?= coverage/python/coverage.json +COVERAGE_BASE ?= origin/main +COVERAGE_HEAD ?= HEAD +COVERAGE_REPORT ?= coverage/python/ratchet.json +PROTOCOL_JUNIT ?= /tmp/xy-protocol-junit.xml -.PHONY: help setup setup-browser check check-full check-browser check-conformance check-docs check-examples check-security check-errors check-api check-import check-ci check-claims check-benchmark-harness check-pyplot check-pyplot-speed check-sdist check-wheel check-artifacts check-benchmark-report list-checks test lint format typecheck public-api python-floor js-check rust-check abi-smoke +.PHONY: help setup setup-browser check check-full check-browser check-labels check-pan-zoom check-conformance check-protocol check-docs check-examples check-security check-errors check-api check-import check-ci check-claims check-testing-spec check-benchmark-harness check-coverage check-pyplot check-pyplot-speed check-sdist check-wheel check-artifacts check-benchmark-report list-checks test lint format typecheck public-api python-floor js-check js-test rust-check abi-smoke help: @printf '%s\n' \ @@ -17,8 +23,11 @@ help: ' make setup-browser install the pinned Playwright browser-test driver' \ ' make check run the fast local verification gate' \ ' make check-full run JS, Rust, and ABI gates too' \ - ' make check-browser run browser smokes (set CHROMIUM=/path/to/chrome)' \ - ' make check-conformance run accessibility + Chromium/Firefox/WebKit conformance' \ + ' make check-browser run browser smokes, including every chart kind, runtime security, animation, and pick boundaries (set CHROMIUM=/path/to/chrome)' \ + ' make check-labels run strict formatter units and rendered-label DOM oracles' \ + ' make check-pan-zoom run the complete Chromium pan/zoom acceptance matrix (set CHROMIUM=/path/to/chrome)' \ + ' make check-conformance run the bounded accessibility/DPR/motion matrix in Chromium, Firefox, and WebKit' \ + ' make check-protocol run the request/reply catalog, shared golden frames, and structural byte mutations' \ ' make check-docs run docs examples and public claim guardrails' \ ' make check-examples run README/API examples and Reflex asset registry checks' \ ' make check-security run standalone HTML safety and client text-sink checks' \ @@ -27,13 +36,15 @@ help: ' make check-import run import-time and dependency-boundary checks' \ ' make check-ci run CI/release workflow invariant checks' \ ' make check-claims run public performance-claim guardrails' \ + ' make check-testing-spec validate all specifications, evidence, and public claims' \ ' make check-benchmark-harness run benchmark metadata/report/regression tests' \ + ' make check-coverage validate a branch-aware COVERAGE_JSON against package/module and COVERAGE_BASE..COVERAGE_HEAD diff floors' \ ' make check-pyplot run the matplotlib-shim suite and compatibility corpus' \ ' make check-pyplot-speed enforce the per-family 10x static-PNG target (requires .[bench])' \ ' make check-sdist build and verify the source distribution' \ ' make check-wheel build and verify a wheel (set WHEEL_EXPECT=--expect-native)' \ ' make check-artifacts verify prebuilt artifacts (set SDIST=... WHEEL=...)' \ - ' make check-benchmark-report validate BENCHMARK_JSON (scatter-vs, pyplot-vs-matplotlib, line-decimation, install-footprint, core-2d, scatter-native, heatmap-native, kernel-native, interaction-browser, dashboard-browser, workflow-native)' \ + ' make check-benchmark-report validate BENCHMARK_JSON (scatter-vs, pyplot-vs-matplotlib, line-decimation, install-footprint, core-2d, scatter-native, heatmap-native, kernel-native, interaction-browser, dashboard-browser, workflow-native); set BENCHMARK_PROFILE=strict for dashboard release health' \ ' override UV_CACHE_DIR if your uv cache lives elsewhere' \ ' make list-checks list verifier check names' \ ' make test run pytest' \ @@ -41,6 +52,7 @@ help: ' make format run ruff format --check' \ ' make typecheck run ty over the shippable package' \ ' make js-check verify committed JS bundles are fresh' \ + ' make js-test run dependency-free JS semantic units with coverage' \ ' make rust-check run cargo test and clippy' setup: @@ -50,6 +62,7 @@ setup: setup-browser: npm install + npx playwright install chromium check: $(PYTHON) scripts/verify_local.py --quick @@ -68,20 +81,48 @@ check-browser: } $(PYTHON) scripts/verify_local.py --browser --chromium "$(CHROMIUM)" +check-labels: + @node -e "require.resolve('playwright')" >/dev/null 2>&1 || { \ + echo 'Playwright is required. Run: make setup-browser' >&2; \ + exit 2; \ + } + npm run test:labels + +check-pan-zoom: + @if [ -z "$(CHROMIUM)" ]; then \ + echo 'Set CHROMIUM=/path/to/chrome for the pan/zoom matrix.' >&2; \ + exit 2; \ + fi + @node -e "require.resolve('playwright')" >/dev/null 2>&1 || { \ + echo 'Playwright is required. Run: make setup-browser' >&2; \ + exit 2; \ + } + node scripts/pan_zoom_matrix.mjs --profile full --browsers chromium \ + --executable-path "$(CHROMIUM)" --evidence /tmp/xy-pan-zoom-matrix-evidence.json + check-conformance: @node -e "require.resolve('playwright')" >/dev/null 2>&1 || { \ echo 'Playwright is required. Run: make setup-browser && npx playwright install chromium firefox webkit' >&2; \ exit 2; \ } node scripts/browser_conformance.mjs + node scripts/pan_zoom_matrix.mjs --profile focused \ + --browsers chromium,firefox,webkit \ + --evidence /tmp/xy-pan-zoom-cross-engine-evidence.json + +check-protocol: + $(PYTHON) -m pytest -q \ + tests/test_protocol_catalog.py tests/test_framing.py tests/test_framing_property.py \ + --junitxml="$(PROTOCOL_JUNIT)" check-docs: - $(PYTHON) scripts/verify_local.py --only examples,claim_guardrails + $(PYTHON) scripts/verify_local.py --only examples,claim_guardrails,testing_spec check-examples: $(PYTHON) scripts/verify_local.py --only examples check-pyplot: + $(PYTHON) scripts/check_pyplot_options.py $(PYTHON) -m pytest tests/pyplot -q check-security: @@ -102,9 +143,18 @@ check-ci: check-claims: $(PYTHON) scripts/verify_local.py --only claim_guardrails +check-testing-spec: + $(PYTHON) scripts/verify_local.py --only testing_spec,claim_guardrails + check-benchmark-harness: $(PYTHON) scripts/verify_local.py --only benchmark_harness +check-coverage: + $(PYTHON) scripts/coverage_ratchet.py \ + --coverage-json "$(COVERAGE_JSON)" \ + --base "$(COVERAGE_BASE)" --head "$(COVERAGE_HEAD)" \ + --report "$(COVERAGE_REPORT)" + check-pyplot-speed: PYTHONPATH=python $(PYTHON) benchmarks/bench_pyplot_vs_matplotlib.py \ --profile standard --reps 21 --warmups 3 --target-speedup 10 --require-target @@ -135,7 +185,7 @@ check-artifacts: $(PYTHON) scripts/verify_local.py --packaging --sdist "$(SDIST)" --wheel "$(WHEEL)" $(WHEEL_EXPECT) check-benchmark-report: - $(PYTHON) scripts/verify_benchmark_report.py "$(BENCHMARK_JSON)" --kind "$(BENCHMARK_KIND)" + $(PYTHON) scripts/verify_benchmark_report.py "$(BENCHMARK_JSON)" --kind "$(BENCHMARK_KIND)" --profile "$(BENCHMARK_PROFILE)" list-checks: $(PYTHON) scripts/verify_local.py --list @@ -161,6 +211,12 @@ python-floor: js-check: node js/build.mjs --check +js-test: + node --test --experimental-test-coverage \ + --test-coverage-include=python/xy/static/index.js \ + --test-coverage-lines=15 --test-coverage-branches=60 \ + --test-coverage-functions=10 js/test/*.test.mjs + rust-check: cargo test cargo clippy --all-targets -- -D warnings diff --git a/README.md b/README.md index 68abc23d..63cd99c6 100644 --- a/README.md +++ b/README.md @@ -232,9 +232,11 @@ including widget/export boundaries, and `make check-ci` for workflow and benchmark artifact wiring. Browser work uses `make check-browser`, which runs the Browser lifecycle smoke -(Chromium), Browser visual regression smoke (Chromium), and Browser interaction -stress smoke (Chromium) gates. The full gate additionally needs Node 18+, -`cargo`, `rustc`, and clippy (`rustup component add clippy`). +(Chromium), Browser visual health smoke (Chromium), Reviewed visual baseline +(Chromium), Every chart-kind render matrix (Chromium), and Browser interaction +stress smoke (Chromium) gates. The full +gate additionally needs Node 18+, `cargo`, `rustc`, and clippy +(`rustup component add clippy`). See [CONTRIBUTING.md](CONTRIBUTING.md) for the contributor workflow. diff --git a/benchmarks/bench_dashboard.py b/benchmarks/bench_dashboard.py index 4b4f4467..925b4821 100644 --- a/benchmarks/bench_dashboard.py +++ b/benchmarks/bench_dashboard.py @@ -39,6 +39,9 @@ "payload_export_size", ) RENDER_W, RENDER_H = 420, 280 +DASHBOARD_BASE_VIRTUAL_TIME_MS = 15_000 +DASHBOARD_SETTLE_VIRTUAL_TIME_MS_PER_CHART = 500 +DASHBOARD_PROBE_TIMEOUT_S = 180 def _parse_counts(text: str) -> list[int]: @@ -147,14 +150,14 @@ def _probe_js() -> str: try { const view = xy.renderStandalone(cell, payload.spec, xyBytesFromPayload(payload)); const state = {lost: false}; - view.canvas.addEventListener("webglcontextlost", () => { + view.root.addEventListener("xy:context_lost", (event) => { state.lost = true; contextEvents.push({ id: payload.id, type: "lost", phase, at_ms: performance.now() - t0, - governed: view.canvas.dataset.xyCtx === "released", + governed: event.detail?.governed === true, }); }); - view.canvas.addEventListener("webglcontextrestored", () => { + view.root.addEventListener("xy:context_restored", () => { state.lost = false; contextEvents.push({id: payload.id, type: "restored", phase, at_ms: performance.now() - t0}); }); @@ -210,6 +213,10 @@ def _probe_js() -> str: // recovery latency for a chart scrolled back into view. for (const slot of slots) { slot.cell.scrollIntoView({block: "center", inline: "center"}); + // Model an actual visit, not only a programmatic scroll. Pointer entry + // is the product's explicit demand signal for a governed snapshot and + // promotes the requested chart in the context-governor LRU. + slot.view?.root.dispatchEvent(new PointerEvent("pointerenter")); const arriveMs = performance.now(); let lit = nonblankPixels(slot) > 0; while (!lit && performance.now() - arriveMs < 400) { @@ -330,7 +337,20 @@ def run(*, chart_counts: list[int], chromium: str | None = None) -> dict[str, An ".chart-cell{width:420px;height:280px;border:1px solid #dde3ec;}" ), ) - result = run_json_probe(html, marker="XY_DASHBOARD", chromium=chromium) + # The page may spend up to 400 ms settling each scroll visit. A fixed + # 15-second virtual-time budget can therefore expire before a healthy + # 50-chart page reaches xyReport, yielding the misleading + # ``failed(no probe title)`` row seen in CI. Scale the virtual budget + # with the requested work while retaining a real wall-clock timeout. + result = run_json_probe( + html, + marker="XY_DASHBOARD", + chromium=chromium, + virtual_time_ms=( + DASHBOARD_BASE_VIRTUAL_TIME_MS + count * DASHBOARD_SETTLE_VIRTUAL_TIME_MS_PER_CHART + ), + timeout_s=DASHBOARD_PROBE_TIMEOUT_S, + ) row: dict[str, Any] = { "scenario": f"dashboard_{count}", "chart_count": count, diff --git a/benchmarks/bench_interaction.py b/benchmarks/bench_interaction.py index 6fee9f48..e8554b80 100644 --- a/benchmarks/bench_interaction.py +++ b/benchmarks/bench_interaction.py @@ -214,10 +214,10 @@ def _probe_js(reps: int) -> str: // virtual time forever: the first density zoom schedules the standalone // sample-rebin worker and the page deadlocks until the wall-clock timeout // (bisected: real worker + rebin hangs even when the reply is ignored; a - // stubbed worker completes). Same class the render smoke already dropped - // its worker probe for. Disable the rebin here — density interactions - // measure the stretched-overview path, all visual invariants still run; - // the worker path itself needs a non-virtual-time harness (see PR notes). + // stubbed worker completes). Disable rebin only for this virtual-time page: + // density interactions still measure the stretched-overview path and all + // visual invariants, while run_worker_probe() below provides the required + // real-wall-clock worker, re-bin, paint, and teardown evidence. view._sampleRebinDisabled = true; view._drawNow(); const before = {{...view.view}}; @@ -684,25 +684,47 @@ def _worker_probe_js() -> str: x0: before.x0 + xSpan * 0.18, x1: before.x0 + xSpan * 0.52, y0: before.y0 + ySpan * 0.21, y1: before.y0 + ySpan * 0.61, }; + const finish = (payload) => { + const worker = view._rebinWorker; + let workerTerminated = false; + if (worker) { + const terminate = worker.terminate.bind(worker); + worker.terminate = () => { + workerTerminated = true; + return terminate(); + }; + } + view.destroy(); + const workerCleared = view._rebinWorker === null; + const rootRemoved = !view.root.isConnected; + xyReport("XY_WORKER", { + ...payload, + worker_created: !!worker, + worker_terminated: workerTerminated, + worker_cleared: workerCleared, + root_removed: rootRemoved, + teardown_complete: !!worker && workerTerminated && workerCleared && rootRemoved, + }); + }; const started = performance.now(); view._setView(target, {animate: false, request: true}); const poll = () => { if (g._sampleRebinned === true) { view._drawNow(); - xyReport("XY_WORKER", { + const nonblankPixels = xyNonblankPixels(view); + const xRangeChanged = !!(g.density && g.density.xRange && + Math.abs(g.density.xRange[0] - before.x0) > xSpan * 0.05); + finish({ status: "ok", worker_rebinned: true, - nonblank_pixels: xyNonblankPixels(view), - worker_created: !!view._rebinWorker, - x_range_changed: !!(g.density && g.density.xRange && - Math.abs(g.density.xRange[0] - before.x0) > xSpan * 0.05), + nonblank_pixels: nonblankPixels, + x_range_changed: xRangeChanged, }); return; } if (performance.now() - started >= 12_000) { - xyReport("XY_WORKER", { + finish({ status: "failed(timeout)", - worker_created: !!view._rebinWorker, sample_rebinned: !!g._sampleRebinned, }); return; diff --git a/docs/app/AGENTS.md b/docs/app/AGENTS.md index b1218c1a..197dc604 100644 --- a/docs/app/AGENTS.md +++ b/docs/app/AGENTS.md @@ -121,4 +121,4 @@ repository root. For Markdown-only changes, run the docs tests and Ruff. For changes to routing, components, plugins, shared packages, or live previews, also start the production app and run every validator documented in `README.md`. Run the -adapter tests under `tests/reflex_adapter/` when changing `reflex_xy` behavior. +adapter tests under `python/reflex-xy/tests/` when changing `reflex_xy` behavior. diff --git a/docs/app/reflex.lock/bun.lock b/docs/app/reflex.lock/bun.lock index 3c0bad9d..c1315204 100644 --- a/docs/app/reflex.lock/bun.lock +++ b/docs/app/reflex.lock/bun.lock @@ -15,7 +15,7 @@ "@radix-ui/react-dialog": "1.1.15", "@radix-ui/react-form": "0.1.8", "@radix-ui/themes": "3.3.0", - "@react-router/node": "7.15.0", + "@react-router/node": "7.15.1", "clsx-for-tailwind": "1.0.0", "es-toolkit": "1.46.1", "hast-util-to-string": "3.0.1", @@ -24,8 +24,8 @@ "react": "19.2.6", "react-dom": "19.2.6", "react-helmet": "6.1.0", - "react-router": "7.15.0", - "react-router-dom": "7.15.0", + "react-router": "7.15.1", + "react-router-dom": "7.15.1", "shiki": "3.3.0", "socket.io-client": "4.8.3", "sonner": "2.0.7", @@ -36,8 +36,8 @@ }, "devDependencies": { "@emotion/react": "11.14.0", - "@react-router/dev": "7.15.0", - "@react-router/fs-routes": "7.15.0", + "@react-router/dev": "7.15.1", + "@react-router/fs-routes": "7.15.1", "@tailwindcss/postcss": "4.3.0", "@tailwindcss/typography": "0.5.19", "autoprefixer": "10.5.0", @@ -364,11 +364,11 @@ "@radix-ui/themes": ["@radix-ui/themes@3.3.0", "", { "dependencies": { "@radix-ui/colors": "^3.0.0", "classnames": "^2.3.2", "radix-ui": "^1.1.3", "react-remove-scroll-bar": "^2.3.8" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-I0/h2CRNTpYNB7Mi3xFIvSsQq5a108d7kK8dTO5zp5b9HR5QJXKag6B8tjpz2ITkVYkFdkGk45doNkSr7OxwNw=="], - "@react-router/dev": ["@react-router/dev@7.15.0", "", { "dependencies": { "@babel/core": "^7.27.7", "@babel/generator": "^7.27.5", "@babel/parser": "^7.27.7", "@babel/plugin-syntax-jsx": "^7.27.1", "@babel/preset-typescript": "^7.27.1", "@babel/traverse": "^7.27.7", "@babel/types": "^7.27.7", "@react-router/node": "7.15.0", "@remix-run/node-fetch-server": "^0.13.0", "arg": "^5.0.1", "babel-dead-code-elimination": "^1.0.6", "chokidar": "^4.0.0", "dedent": "^1.5.3", "es-module-lexer": "^1.3.1", "exit-hook": "2.2.1", "isbot": "^5.1.11", "jsesc": "3.0.2", "lodash": "^4.17.21", "p-map": "^7.0.3", "pathe": "^1.1.2", "picocolors": "^1.1.1", "pkg-types": "^2.3.0", "prettier": "^3.6.2", "react-refresh": "^0.14.0", "semver": "^7.3.7", "tinyglobby": "^0.2.14", "valibot": "^1.2.0", "vite-node": "^3.2.2" }, "peerDependencies": { "@react-router/serve": "^7.15.0", "@vitejs/plugin-rsc": "~0.5.21", "react-router": "^7.15.0", "react-server-dom-webpack": "^19.2.3", "typescript": "^5.1.0 || ^6.0.0", "vite": "^5.1.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", "wrangler": "^3.28.2 || ^4.0.0" }, "optionalPeers": ["@react-router/serve", "@vitejs/plugin-rsc", "react-server-dom-webpack", "typescript", "wrangler"], "bin": { "react-router": "bin.js" } }, "sha512-ZwUQu4KNZrViFqdeFWqh00Bk/QbLNvoWRDfjsqOp3oyuG3jSRLYnqRD3VAMK/FYMpL+s37ByT7XqqLXaF7Nw1g=="], + "@react-router/dev": ["@react-router/dev@7.15.1", "", { "dependencies": { "@babel/core": "^7.27.7", "@babel/generator": "^7.27.5", "@babel/parser": "^7.27.7", "@babel/plugin-syntax-jsx": "^7.27.1", "@babel/preset-typescript": "^7.27.1", "@babel/traverse": "^7.27.7", "@babel/types": "^7.27.7", "@react-router/node": "7.15.1", "@remix-run/node-fetch-server": "^0.13.0", "arg": "^5.0.1", "babel-dead-code-elimination": "^1.0.6", "chokidar": "^4.0.0", "dedent": "^1.5.3", "es-module-lexer": "^1.3.1", "exit-hook": "2.2.1", "isbot": "^5.1.11", "jsesc": "3.0.2", "lodash": "^4.17.21", "p-map": "^7.0.3", "pathe": "^1.1.2", "picocolors": "^1.1.1", "pkg-types": "^2.3.0", "prettier": "^3.6.2", "react-refresh": "^0.14.0", "semver": "^7.3.7", "tinyglobby": "^0.2.14", "valibot": "^1.2.0", "vite-node": "^3.2.2" }, "peerDependencies": { "@react-router/serve": "^7.15.1", "@vitejs/plugin-rsc": "~0.5.21", "react-router": "^7.15.1", "react-server-dom-webpack": "^19.2.3", "typescript": "^5.1.0 || ^6.0.0", "vite": "^5.1.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", "wrangler": "^3.28.2 || ^4.0.0" }, "optionalPeers": ["@react-router/serve", "@vitejs/plugin-rsc", "react-server-dom-webpack", "typescript", "wrangler"], "bin": { "react-router": "bin.js" } }, "sha512-BlFEU7SjPQHJDfYuw5qJU3+p4wMPEvKpf5Kj64/rRzQQjncXzhzkIJ0xreAQSYgGwJWjIXIK9swOaeE2czhulw=="], - "@react-router/fs-routes": ["@react-router/fs-routes@7.15.0", "", { "dependencies": { "minimatch": "^9.0.0" }, "peerDependencies": { "@react-router/dev": "^7.15.0", "typescript": "^5.1.0 || ^6.0.0" }, "optionalPeers": ["typescript"] }, "sha512-Bn0PvYQCpFm547UBc2hP6bI9Rv6i5ewHiVG+hZsPxkYH/txvTWwwF7b4lle7TaO/UeHA7J5rhlB2a8C4ubXO7w=="], + "@react-router/fs-routes": ["@react-router/fs-routes@7.15.1", "", { "dependencies": { "minimatch": "^9.0.0" }, "peerDependencies": { "@react-router/dev": "^7.15.1", "typescript": "^5.1.0 || ^6.0.0" }, "optionalPeers": ["typescript"] }, "sha512-rKRszgD0h9O4UrLJl3jWBBke3RghHxZp6Bc+JRc6nsuBt7hF0sT4V7Af3EoBLtUSo2hn75Vr4T++yJL4ADBNCA=="], - "@react-router/node": ["@react-router/node@7.15.0", "", { "dependencies": { "@mjackson/node-fetch-server": "^0.2.0" }, "peerDependencies": { "react-router": "7.15.0", "typescript": "^5.1.0 || ^6.0.0" }, "optionalPeers": ["typescript"] }, "sha512-SgvWaWF1n3u+bpXXZUW9BSd2p/NwkIYLz4SSeDYqoX5RkYX5rcI4cHHuNJXszPu+Dm9QIri4J9g/4EV3KfgiXQ=="], + "@react-router/node": ["@react-router/node@7.15.1", "", { "dependencies": { "@mjackson/node-fetch-server": "^0.2.0" }, "peerDependencies": { "react-router": "7.15.1", "typescript": "^5.1.0 || ^6.0.0" }, "optionalPeers": ["typescript"] }, "sha512-lv68RaqmIa/ZRlIrGcl79HimaqpU3yV1CFKnmItU+xqI+xn9g5fqsh2Vj2LdNjnlzJgVsRMEpnv00t/6RgDrgw=="], "@remix-run/node-fetch-server": ["@remix-run/node-fetch-server@0.13.3", "", {}, "sha512-UfjOXed/DQteaM5VyTfqTeGpHwyL2J5aoRGY6cydip4tt1ehNNeSwuXCC7AEGE0RWBs/7bgKxYkL/B/+UDe4AA=="], @@ -1018,9 +1018,9 @@ "react-remove-scroll-bar": ["react-remove-scroll-bar@2.3.8", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], - "react-router": ["react-router@7.15.0", "", { "dependencies": { "cookie": "^1.0.1", "set-cookie-parser": "^2.6.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" }, "optionalPeers": ["react-dom"] }, "sha512-HW9vYwuM8f4yx66Izy8xfrzCM+SBJluoZcCbww9A1TySax11S5Vgw6fi3ZjMONw9J4gQwngL7PzkyIpJJpJ7RQ=="], + "react-router": ["react-router@7.15.1", "", { "dependencies": { "cookie": "^1.0.1", "set-cookie-parser": "^2.6.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" }, "optionalPeers": ["react-dom"] }, "sha512-R8rl9HhgikFYoPJymnUtPXWbnDb3oget6lQnfIoupbt61aT9aOhRkDsY2XRhZRyX1Z/8a5sL74fXmFNm3NRK5A=="], - "react-router-dom": ["react-router-dom@7.15.0", "", { "dependencies": { "react-router": "7.15.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" } }, "sha512-VcrVg64Fo8nwBvDscajG8gRTLIuTC6N50nb22l2HOOV4PTOHgoGp8mUjy9wLiHYoYTSYI36tUnXZgasSRFZorQ=="], + "react-router-dom": ["react-router-dom@7.15.1", "", { "dependencies": { "react-router": "7.15.1" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" } }, "sha512-AzF62gjY6U9rkMq4RfP/r2EVtQ7DMfNMjyOp/flLTCrtRylLiK4wT4pSq6O8rOXZ2eXdZYJPEYe+ifomiv+Igg=="], "react-side-effect": ["react-side-effect@2.1.2", "", { "peerDependencies": { "react": "^16.3.0 || ^17.0.0 || ^18.0.0" } }, "sha512-PVjOcvVOyIILrYoyGEpDN3vmYNLdy1CajSFNt4TDsVQC5KpTijDvWVoR+/7Rz2xT978D8/ZtFceXxzsPwZEDvw=="], diff --git a/docs/app/reflex.lock/package.json b/docs/app/reflex.lock/package.json index a22c7530..05604fac 100644 --- a/docs/app/reflex.lock/package.json +++ b/docs/app/reflex.lock/package.json @@ -16,7 +16,7 @@ "@radix-ui/react-dialog": "1.1.15", "@radix-ui/react-form": "0.1.8", "@radix-ui/themes": "3.3.0", - "@react-router/node": "7.15.0", + "@react-router/node": "7.15.1", "clsx-for-tailwind": "1.0.0", "es-toolkit": "1.46.1", "hast-util-to-string": "3.0.1", @@ -25,8 +25,8 @@ "react": "19.2.6", "react-dom": "19.2.6", "react-helmet": "6.1.0", - "react-router": "7.15.0", - "react-router-dom": "7.15.0", + "react-router": "7.15.1", + "react-router-dom": "7.15.1", "shiki": "3.3.0", "socket.io-client": "4.8.3", "sonner": "2.0.7", @@ -37,8 +37,8 @@ }, "devDependencies": { "@emotion/react": "11.14.0", - "@react-router/dev": "7.15.0", - "@react-router/fs-routes": "7.15.0", + "@react-router/dev": "7.15.1", + "@react-router/fs-routes": "7.15.1", "@tailwindcss/postcss": "4.3.0", "@tailwindcss/typography": "0.5.19", "autoprefixer": "10.5.0", @@ -50,4 +50,4 @@ "overrides": { "cookie": "1.1.1" } -} \ No newline at end of file +} diff --git a/docs/app/uv.lock b/docs/app/uv.lock index 86268afa..56725c1a 100644 --- a/docs/app/uv.lock +++ b/docs/app/uv.lock @@ -1289,7 +1289,7 @@ source = { git = "https://github.com/reflex-dev/reflex?subdirectory=packages%2Fi [[package]] name = "reflex-site-shared" -version = "0.0.37.post1.dev0+f79d011d" +version = "0.0.38" source = { git = "https://github.com/reflex-dev/reflex?subdirectory=packages%2Freflex-site-shared&rev=main#f79d011d07050cfedabff5cc8b3670363617a4d8" } dependencies = [ { name = "email-validator" }, @@ -1709,6 +1709,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, { name = "pytest-codspeed", marker = "extra == 'codspeed'", specifier = ">=5,<6" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.8" }, + { name = "starlette", marker = "extra == 'dev'", specifier = ">=0.36" }, { name = "ty", marker = "extra == 'dev'" }, ] provides-extras = ["dev", "bench", "codspeed"] diff --git a/docs/components/axes.md b/docs/components/axes.md index 32843220..7f6b25aa 100644 --- a/docs/components/axes.md +++ b/docs/components/axes.md @@ -54,6 +54,12 @@ let the renderer adapt the axis to a dashboard or publication layout. Exact option names and defaults are in the [generated component reference](/docs/xy/api-reference/marks-and-components/). +Numeric formats use fixed precision with optional locale grouping (`.2f`, +`,.0f`), a literal currency prefix (`$,.2f`), a unit after `f` (`.3f GiB`), +or percent scaling (`.1%`). Time formats use the UTC-only token subset `%Y`, +`%m`, `%d`, `%H`, `%M`, `%S`, `%b`, and `%B`. Unsupported formats raise; +category axes reject `format=` and use `tick_labels` for replacements. + ## Bind Marks to Named Axes Use named axes for different units in one panel. An x-axis identifier must diff --git a/docs/components/tooltips.md b/docs/components/tooltips.md index 50e19708..c6a06dfb 100644 --- a/docs/components/tooltips.md +++ b/docs/components/tooltips.md @@ -40,9 +40,11 @@ chart = xy.scatter_chart( ~~~ Braced field names in `title` are replaced from the hovered row. `format` maps -field names to the client's numeric format strings. A source column that is not -bound to a rendered channel is not shipped merely because its name appears in -`fields`. +field names to the same strict value formats as axes: fixed/grouped/currency/ +unit/percent numeric forms, or `%Y %m %d %H %M %S %b %B` for UTC time fields. +Unsupported formats and formats applied to the wrong value kind raise instead +of silently falling back. A source column that is not bound to a rendered +channel is not shipped merely because its name appears in `fields`. ## Exact and Resident Readout diff --git a/docs/core-concepts/axes-and-scales.md b/docs/core-concepts/axes-and-scales.md index d2badaec..9a2f1a3a 100644 --- a/docs/core-concepts/axes-and-scales.md +++ b/docs/core-concepts/axes-and-scales.md @@ -76,7 +76,10 @@ def tick_controls_demo(): return reflex_xy.chart(chart, height="360px") ~~~ -`format` controls numeric labels. For crowded labels, use +`format` uses the strict axis value grammar: fixed/grouped/currency/unit/percent +forms for numeric and log axes, or the UTC tokens `%Y`, `%m`, `%d`, `%H`, `%M`, +`%S`, `%b`, and `%B` for time axes. Unsupported formats raise rather than +falling back. For crowded labels, use `tick_label_angle`, `tick_label_min_gap`, and `tick_label_strategy`. `label_position`, `label_offset`, and `label_angle` position the axis title. diff --git a/examples/fastapi/pyproject.toml b/examples/fastapi/pyproject.toml index 4258f388..8639e6f2 100644 --- a/examples/fastapi/pyproject.toml +++ b/examples/fastapi/pyproject.toml @@ -4,8 +4,13 @@ version = "0" requires-python = ">=3.11" dependencies = [ "xy", - "fastapi>=0.110", - "uvicorn>=0.29", + "fastapi>=0.110,<1", + # TestClient is the package-owned HTTP transport oracle. + "httpx>=0.27,<1", + # live_drilldown imports Starlette directly, so it is not merely an + # implementation detail inherited from FastAPI. + "starlette>=0.36.3,<1", + "uvicorn>=0.29,<1", "numpy>=1.24", ] diff --git a/js/build.mjs b/js/build.mjs index 282e8fee..b2edc7e8 100644 --- a/js/build.mjs +++ b/js/build.mjs @@ -81,16 +81,16 @@ const staticDir = join(root, "python", "xy", "static"); const { build } = await import("vite"); -/** Build both bundles into outDir. One entry, two formats: the ESM build keeps - * 60_entries' export shape verbatim; the IIFE build assigns the same namespace - * to a top-level `var xy`, which is `window.xy` when the bundle runs as the - * classic inline """ + try: + executable = _chrome(args.chromium) + except RuntimeError as exc: + title = f"XY_ANIM_ERROR {exc}" + evidence = _evidence(title=title, chromium=args.chromium, returncode=None) + if args.evidence is not None: + _write_evidence(args.evidence, evidence) + print(title) + return 1 + with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "animation.html" - path.write_text(page) - result = subprocess.run( - [ - _chrome(), - "--headless=new", - "--no-sandbox", - "--use-angle=swiftshader", - "--enable-unsafe-swiftshader", - "--force-prefers-reduced-motion=reduce", - "--virtual-time-budget=1400", - "--dump-dom", - path.as_uri(), - ], - capture_output=True, - text=True, - timeout=60, - ) + path.write_text(page, encoding="utf-8") + command = [ + executable, + "--headless=new", + "--no-sandbox", + "--use-angle=swiftshader", + "--enable-unsafe-swiftshader", + "--force-prefers-reduced-motion=reduce", + "--virtual-time-budget=1400", + "--dump-dom", + path.as_uri(), + ] + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=60, + ) + except subprocess.TimeoutExpired as exc: + stderr = exc.stderr or "" + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + title = "XY_ANIM_ERROR browser timeout after 60s" + evidence = _evidence( + title=title, + chromium=executable, + returncode=None, + stderr=stderr, + timed_out=True, + ) + if args.evidence is not None: + _write_evidence(args.evidence, evidence) + print(title) + if stderr: + print(stderr[-2000:]) + return 1 + except OSError as exc: + title = f"XY_ANIM_ERROR browser startup failed: {exc}" + evidence = _evidence(title=title, chromium=executable, returncode=None) + if args.evidence is not None: + _write_evidence(args.evidence, evidence) + print(title) + return 1 title_match = re.search(r"([^<]+)", result.stdout) title = title_match.group(1) if title_match else "(none)" + evidence = _evidence( + title=title, + chromium=executable, + returncode=result.returncode, + stderr=result.stderr, + ) + if args.evidence is not None: + _write_evidence(args.evidence, evidence) print(title) - expected = [ - "XY_ANIM_OK", - "match=[[1,0],[0,1]]", - "scratch=1", - "ghost=1", - "pixel=1", - "partial=1", - "missing=1", - "active=1", - "pick=1", - "bounded=1", - "done=1", - "reduced=1", - "destroy=1", - "errorbar=1", - "bar=1", - "hbar=1", - ] - if not all(value in title for value in expected): + if evidence["status"] != "ok": print(result.stderr[-2000:]) - raise SystemExit("animation smoke failed") + return 1 + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/scripts/browser_conformance.mjs b/scripts/browser_conformance.mjs index 6b092f41..fe2ef2eb 100644 --- a/scripts/browser_conformance.mjs +++ b/scripts/browser_conformance.mjs @@ -1,13 +1,15 @@ #!/usr/bin/env node -/** Accessibility and perceptual cross-browser smoke. +/** Bounded semantic, layout, DPR, motion, and perceptual browser conformance. * - * This deliberately compares WebGL pixels through a coarse per-channel - * signature and compares DOM chrome by layout boxes. Browser text glyphs are - * not expected to be byte-identical (§21). + * Every catalog case runs unchanged in Chromium, Firefox, and WebKit. WebGL + * pixels are compared through a coarse per-channel signature and DOM chrome + * through CSS-pixel layout boxes. Browser text glyphs are not expected to be + * byte-identical (§21). */ -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { chromium, firefox, webkit } from "playwright"; @@ -17,45 +19,417 @@ const protocolSource = readFileSync(join(ROOT, "python", "xy", "config.py"), "ut const protocolMatch = protocolSource.match(/^PROTOCOL_VERSION\s*=\s*(\d+)\s*$/m); if (!protocolMatch) throw new Error("could not read PROTOCOL_VERSION from python/xy/config.py"); const PROTOCOL_VERSION = Number(protocolMatch[1]); +const packageJson = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8")); + const ENGINES = { chromium, firefox, webkit }; -const headless = process.env.XY_CONFORMANCE_HEADFUL !== "1"; -const selected = process.argv.find((arg) => arg.startsWith("--browsers="))?.split("=", 2)[1] - ?.split(",").filter(Boolean) || Object.keys(ENGINES); -for (const name of selected) { - if (!ENGINES[name]) throw new Error(`unknown browser ${name}; use chromium,firefox,webkit`); +const VIEWPORT = { width: 760, height: 480 }; +const MAX_LAYOUT_DELTA = 4; +const MAX_SIGNATURE_MAE = 12; +const MIN_LIT_RATIO = 0.8; +const MAX_LIT_RATIO = 1.2; +const MIN_LIT_PIXELS = 80; +const REQUIRED_TIERS = ["direct", "decimated", "density"]; +const REQUIRED_FAMILIES = ["scatter", "line", "bar", "heatmap", "mesh"]; +const REQUIRED_DPRS = [1, 2]; +const REQUIRED_MOTIONS = ["reduce", "no-preference"]; +const REQUIRED_AXES = ["linear", "log", "category", "named"]; +const EXPECTED_CASE_IDS = [ + "direct-linear-scatter-dpr1-reduced", + "decimated-log-line-dpr2-motion", + "direct-category-bar-dpr1-motion", + "direct-linear-heatmap-dpr2-reduced", + "direct-named-mesh-dpr1-motion", + "density-linear-scatter-dpr2-reduced", +]; + +function axis(id, kind, label, range, extra = {}) { + return { id, kind, label, range, side: id.startsWith("x") ? "bottom" : "left", ...extra }; } -const spec = { - protocol: PROTOCOL_VERSION, - width: 640, - height: 360, - title: "Cross-browser conformance", - x_axis: { kind: "linear", label: "time", range: [-2, 2] }, - y_axis: { kind: "linear", label: "value", range: [-1, 3] }, - traces: [{ - id: 0, - kind: "scatter", - name: "observations", +function encodeColumn(definition) { + if (definition.dtype === "u8") return Uint8Array.from(definition.values); + const bytes = new Uint8Array(definition.values.length * 4); + const view = new DataView(bytes.buffer); + definition.values.forEach((value, index) => view.setFloat32(index * 4, value, true)); + return bytes; +} + +function packColumns(definitions) { + const encoded = definitions.map(encodeColumn); + let length = 0; + const columns = definitions.map((definition, index) => { + length = Math.ceil(length / 4) * 4; + const byteOffset = length; + length += encoded[index].byteLength; + if (definition.dtype === "u8") { + return { byte_offset: byteOffset, len: definition.values.length, dtype: "u8" }; + } + return { + byte_offset: byteOffset, + len: definition.values.length, + offset: definition.offset ?? 0, + scale: definition.scale ?? 1, + kind: definition.kind || "float", + }; + }); + const bytes = new Uint8Array(length); + columns.forEach((column, index) => bytes.set(encoded[index], column.byte_offset)); + return { columns, bytes: [...bytes] }; +} + +function fixture({ title, axes, traces, columns }) { + const packed = packColumns(columns); + return { + spec: { + protocol: PROTOCOL_VERSION, + width: 640, + height: 360, + title, + x_axis: axes.x, + y_axis: axes.y, + axes, + traces, + columns: packed.columns, + interaction: { hover: true }, + backend: "none", + show_legend: true, + view: { ranges: Object.fromEntries(Object.entries(axes).map(([id, value]) => [id, value.range])) }, + }, + bytes: packed.bytes, + }; +} + +function caseDefinition(metadata, build) { + return { ...metadata, build }; +} + +const CASES = [ + caseDefinition({ + id: EXPECTED_CASE_IDS[0], + tier: "direct", + family: "scatter", + dpr: 1, + motion: "reduce", + axisClasses: ["linear"], + anchor: true, + expectedKind: "scatter", + expectedTraceAxes: ["x", "y"], + expectedAxisTitles: ["time", "value"], + expectedSummary: ["observations", "X axis (time)"], + }, () => fixture({ + title: "Cross-browser conformance", + axes: { + x: axis("x", "linear", "time", [-2, 2]), + y: axis("y", "linear", "value", [-1, 3]), + }, + traces: [{ + id: 0, + kind: "scatter", + name: "observations", + tier: "direct", + n_points: 7, + n_marks: 7, + style: { opacity: 0.9 }, + color: { mode: "constant", color: "#2563eb" }, + size: { mode: "constant", size: 14 }, + x: 0, + y: 1, + x_axis: "x", + y_axis: "y", + }], + columns: [ + { values: [-1.8, -1.2, -0.5, 0, 0.55, 1.2, 1.8] }, + { values: [0.2, 1.4, 0.7, 2.6, 1.1, 2.2, 0.4] }, + ], + })), + caseDefinition({ + id: EXPECTED_CASE_IDS[1], + tier: "decimated", + family: "line", + dpr: 2, + motion: "no-preference", + axisClasses: ["log"], + expectedKind: "line", + expectedTraceAxes: ["x", "y"], + expectedAxisTitles: ["log x", "line y"], + expectedSummary: ["decimated line", "X axis (log x)"], + }, () => { + const xs = Array.from({ length: 64 }, (_, index) => 10 ** (2 * index / 63)); + const ys = xs.map((_, index) => 1.2 + 0.65 * Math.sin(index * 0.42)); + return fixture({ + title: "Decimated log line", + axes: { + x: axis("x", "linear", "log x", [1, 100], { scale: "log" }), + y: axis("y", "linear", "line y", [0.4, 2]), + }, + traces: [{ + id: 0, + kind: "line", + name: "decimated line", + tier: "decimated", + n_points: 20_000, + n_marks: xs.length, + style: { color: "#dc2626", width: 3, opacity: 1 }, + x: 0, + y: 1, + x_axis: "x", + y_axis: "y", + }], + columns: [{ values: xs }, { values: ys }], + }); + }), + caseDefinition({ + id: EXPECTED_CASE_IDS[2], + tier: "direct", + family: "bar", + dpr: 1, + motion: "no-preference", + axisClasses: ["category"], + expectedKind: "bar", + expectedTraceAxes: ["x", "y"], + expectedAxisTitles: ["category", "amount"], + expectedTickLabels: ["alpha", "beta", "gamma"], + expectedSummary: ["category bars", "X axis (category)"], + }, () => fixture({ + title: "Category bar", + axes: { + x: axis("x", "category", "category", [-0.5, 2.5], { + categories: ["alpha", "beta", "gamma"], + }), + y: axis("y", "linear", "amount", [0, 2.2]), + }, + traces: [{ + id: 0, + kind: "bar", + name: "category bars", + tier: "direct", + n_points: 3, + n_marks: 3, + style: { + color: "#4f46e5", + opacity: 0.85, + role: "bar", + orientation: "vertical", + }, + x_axis: "x", + y_axis: "y", + bar: { + orientation: "vertical", + value_axis: "y", + pos: 0, + value1: 1, + width: 0.8, + value0_const: 0, + }, + }], + columns: [{ values: [0, 1, 2] }, { values: [1, 2, 1.5] }], + })), + caseDefinition({ + id: EXPECTED_CASE_IDS[3], tier: "direct", - n_points: 7, - style: { opacity: 0.9 }, - color: { mode: "constant", color: "#2563eb" }, - size: { mode: "constant", size: 14 }, - x: 0, - y: 1, - }], - columns: [ - { byte_offset: 0, len: 7, offset: 0, scale: 1, kind: "float" }, - { byte_offset: 28, len: 7, offset: 0, scale: 1, kind: "float" }, - ], - interaction: { hover: true }, - backend: "none", -}; -const values = [ - -1.8, -1.2, -0.5, 0, 0.55, 1.2, 1.8, - 0.2, 1.4, 0.7, 2.6, 1.1, 2.2, 0.4, + family: "heatmap", + dpr: 2, + motion: "reduce", + axisClasses: ["linear"], + expectedKind: "heatmap", + expectedTraceAxes: ["x", "y"], + expectedAxisTitles: ["heat x", "heat y"], + expectedSummary: ["heat values", "X axis (heat x)"], + }, () => fixture({ + title: "Linear heatmap", + axes: { + x: axis("x", "linear", "heat x", [-0.5, 3.5]), + y: axis("y", "linear", "heat y", [-0.5, 2.5]), + }, + traces: [{ + id: 0, + kind: "heatmap", + name: "heat values", + tier: "direct", + n_points: 12, + n_marks: 12, + x_axis: "x", + y_axis: "y", + style: { + opacity: 0.95, + role: "heatmap", + colormap: "viridis", + domain: [0, 5], + truecolor: false, + x_range: [-0.5, 3.5], + y_range: [-0.5, 2.5], + }, + heatmap: { + buf: 0, + w: 4, + h: 3, + x_range: [-0.5, 3.5], + y_range: [-0.5, 2.5], + colormap: "viridis", + domain: [0, 5], + }, + color: { mode: "continuous", colormap: "viridis", domain: [0, 5] }, + }], + columns: [{ values: [0, 1, 3, 5, 1, 4, 2, 0, 3, 2, 5, 1] }], + })), + caseDefinition({ + id: EXPECTED_CASE_IDS[4], + tier: "direct", + family: "mesh", + dpr: 1, + motion: "no-preference", + axisClasses: ["named"], + expectedKind: "triangle_mesh", + expectedTraceAxes: ["x2", "y2"], + expectedAxisTitles: ["base x", "base y", "named x", "named y"], + expectedSummary: ["named mesh", "X axis (base x)"], + }, () => fixture({ + title: "Named-axis mesh", + axes: { + x: axis("x", "linear", "base x", [0, 1]), + y: axis("y", "linear", "base y", [0, 1]), + x2: axis("x2", "linear", "named x", [0, 1], { side: "top" }), + y2: axis("y2", "linear", "named y", [0, 1], { side: "right" }), + }, + traces: [{ + id: 0, + kind: "triangle_mesh", + name: "named mesh", + tier: "direct", + n_points: 2, + n_marks: 2, + x_axis: "x2", + y_axis: "y2", + x0: 0, + x1: 1, + x2: 2, + y0: 3, + y1: 4, + y2: 5, + style: { opacity: 1, role: "triangle-mesh", stroke: "#334155", stroke_width: 1 }, + color: { mode: "continuous", colormap: "viridis", domain: [0.2, 0.8], buf: 6 }, + }], + columns: [ + { values: [0, 1] }, + { values: [1, 1] }, + { values: [0, 0] }, + { values: [0, 0] }, + { values: [0, 1] }, + { values: [1, 1] }, + { values: [0.2, 0.8] }, + ], + })), + caseDefinition({ + id: EXPECTED_CASE_IDS[5], + tier: "density", + family: "scatter", + dpr: 2, + motion: "reduce", + axisClasses: ["linear"], + expectedKind: "scatter", + expectedTraceAxes: ["x", "y"], + expectedAxisTitles: ["density x", "density y"], + expectedSummary: ["density cloud", "X axis (density x)"], + }, () => { + const width = 32; + const height = 20; + const grid = []; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const dx = (x - width * 0.5) / (width * 0.22); + const dy = (y - height * 0.5) / (height * 0.28); + grid.push(Math.max(0, Math.round(255 * Math.exp(-(dx * dx + dy * dy))))); + } + } + return fixture({ + title: "Density tier", + axes: { + x: axis("x", "linear", "density x", [-2, 2]), + y: axis("y", "linear", "density y", [-1, 1]), + }, + traces: [{ + id: 0, + kind: "scatter", + name: "density cloud", + tier: "density", + n_points: 250_000, + n_marks: width * height, + visible: 250_000, + x_axis: "x", + y_axis: "y", + style: { opacity: 0.9 }, + density: { + buf: 0, + w: width, + h: height, + max: 40, + enc: "log-u8", + colormap: "viridis", + x_range: [-2, 2], + y_range: [-1, 1], + channels_dropped: false, + dropped_channels: [], + }, + }], + columns: [{ dtype: "u8", values: grid }], + }); + }), ]; +function optionValue(name) { + const exact = process.argv.indexOf(name); + if (exact >= 0) return process.argv[exact + 1]; + return process.argv.find((argument) => argument.startsWith(`${name}=`))?.slice(name.length + 1); +} + +const headless = process.env.XY_CONFORMANCE_HEADFUL !== "1"; +const selected = (optionValue("--browsers") || Object.keys(ENGINES).join(",")) + .split(",").filter(Boolean); +const evidencePathValue = optionValue("--evidence") || process.env.XY_CONFORMANCE_EVIDENCE; +const evidencePath = evidencePathValue ? resolve(evidencePathValue) : null; + +function metadataOf(caseItem) { + const { + build: _build, + expectedSummary: _expectedSummary, + expectedAxisTitles: _expectedAxisTitles, + expectedTickLabels: _expectedTickLabels, + expectedKind: _expectedKind, + expectedTraceAxes: _expectedTraceAxes, + anchor: _anchor, + ...metadata + } = caseItem; + return metadata; +} + +function requireCoverage(cases, field, required) { + const observed = new Set(cases.flatMap((caseItem) => caseItem[field])); + const missing = required.filter((value) => !observed.has(value)); + if (missing.length) throw new Error(`browser conformance catalog missing ${field}: ${missing.join(", ")}`); +} + +function validateCatalog(cases = CASES) { + const ids = cases.map((caseItem) => caseItem.id); + if (new Set(ids).size !== ids.length) throw new Error("browser conformance case IDs must be unique"); + if (ids.join("\n") !== EXPECTED_CASE_IDS.join("\n")) { + throw new Error(`browser conformance catalog IDs differ: ${ids.join(", ")}`); + } + requireCoverage(cases, "tier", REQUIRED_TIERS); + requireCoverage(cases, "family", REQUIRED_FAMILIES); + requireCoverage(cases, "dpr", REQUIRED_DPRS); + requireCoverage(cases, "motion", REQUIRED_MOTIONS); + requireCoverage(cases, "axisClasses", REQUIRED_AXES); + for (const caseItem of cases) { + if (typeof caseItem.build !== "function") throw new Error(`${caseItem.id}: missing fixture builder`); + const built = caseItem.build(); + if (built.spec.traces.length !== 1) throw new Error(`${caseItem.id}: expected one trace`); + const trace = built.spec.traces[0]; + if (trace.tier !== caseItem.tier || trace.kind !== caseItem.expectedKind) { + throw new Error(`${caseItem.id}: fixture metadata does not match its trace`); + } + } +} + function maxLayoutDelta(base, candidate) { let worst = 0; for (const key of ["root", "canvas", "toolbar", "title"]) { @@ -83,26 +457,123 @@ function signatureMae(base, candidate) { return error / base.signature.length; } -async function probe(name) { - let browser; +function compareResults(reference, candidate) { + const layoutDelta = maxLayoutDelta(reference, candidate); + const mae = signatureMae(reference, candidate); + const litRatio = candidate.lit / reference.lit; + const failures = []; + if (layoutDelta > MAX_LAYOUT_DELTA) { + failures.push(`DOM layout delta ${layoutDelta.toFixed(2)}px exceeds ${MAX_LAYOUT_DELTA}px`); + } + if (mae > MAX_SIGNATURE_MAE) { + failures.push(`perceptual raster MAE ${mae.toFixed(2)} exceeds ${MAX_SIGNATURE_MAE}`); + } + if (litRatio < MIN_LIT_RATIO || litRatio > MAX_LIT_RATIO) { + failures.push(`lit-pixel ratio ${litRatio.toFixed(3)} is outside ${MIN_LIT_RATIO}..${MAX_LIT_RATIO}`); + } + return { layoutDelta, rasterMae: mae, litRatio, failures }; +} + +function staticNegativeControls() { + let catalogGapRejected = false; try { - const launchOptions = { headless }; - if (name === "firefox") { - // Firefox's Linux software-rendering blocklist can otherwise disable - // WebGL in virtual displays even when Mesa is available. - launchOptions.firefoxUserPrefs = { - "webgl.disabled": false, - "webgl.force-enabled": true, - }; + validateCatalog(CASES.slice(0, -1)); + } catch { + catalogGapRejected = true; + } + if (!catalogGapRejected) throw new Error("catalog-gap negative control was not rejected"); + + const base = { + layout: { + root: { x: 0, y: 0, width: 10, height: 10 }, + canvas: { x: 1, y: 1, width: 8, height: 8 }, + toolbar: { x: 1, y: 1, width: 2, height: 2 }, + title: { x: 3, y: 1, width: 4, height: 1 }, + labelPositions: [[1, 9]], + }, + signature: Array(32).fill(0), + lit: 100, + }; + const corruptedSignature = { + ...base, + signature: Array(32).fill(255), + }; + const corruptedLayout = { + ...base, + layout: { + ...base.layout, + root: { ...base.layout.root, x: 10 }, + }, + }; + const corruptedSignatureRejected = compareResults(base, corruptedSignature).failures + .some((failure) => failure.includes("raster MAE")); + const corruptedLayoutRejected = compareResults(base, corruptedLayout).failures + .some((failure) => failure.includes("layout delta")); + if (!corruptedSignatureRejected || !corruptedLayoutRejected) { + throw new Error("cross-browser comparator negative controls were not rejected"); + } + return { catalogGapRejected, corruptedSignatureRejected, corruptedLayoutRejected }; +} + +function semanticFailures(caseItem, result) { + const s = result.semantics; + const failures = []; + if (s.regionRole !== "region" || !s.regionLabel.includes(result.title)) failures.push("chart region"); + if (caseItem.expectedSummary.some((token) => !s.summary.includes(token))) failures.push("summary"); + if (s.canvasRole !== "img" || s.canvasTabIndex !== 0) failures.push("focusable plot image"); + if (s.toolbarRole !== "toolbar" || s.toolbarLabel !== "Chart controls") failures.push("toolbar"); + if (s.buttonLabels.length === 0 || s.buttonLabels.some((label) => !label)) failures.push("button names"); + if (s.pressed.length !== 1 || !s.pressed[0]) failures.push("toggle state"); + if (s.mediaReduced !== (caseItem.motion === "reduce")) failures.push("media preference"); + if (s.animationActive !== (caseItem.motion === "no-preference")) failures.push("view animation preference"); + if (result.observedDpr !== caseItem.dpr || !result.backingStoreMatchesDpr) failures.push("DPR backing store"); + if (result.gpu.kind !== caseItem.expectedKind || result.gpu.tier !== caseItem.tier) failures.push("GPU trace contract"); + if (result.gpu.xAxis !== caseItem.expectedTraceAxes[0] + || result.gpu.yAxis !== caseItem.expectedTraceAxes[1]) failures.push("trace axis binding"); + for (const title of caseItem.expectedAxisTitles) { + if (!result.axisTitles.includes(title)) failures.push(`axis title ${title}`); + } + for (const label of caseItem.expectedTickLabels || []) { + if (!result.tickLabels.includes(label)) failures.push(`tick label ${label}`); + } + if (caseItem.anchor) { + if (s.keyboardIndex !== 1 || !s.keyboardLive.startsWith("Point 2 of 7.")) { + failures.push("keyboard/live readout"); + } + if (!s.exactPreserved) failures.push("exact reply announcement"); + if (!s.transitionSuppressed) failures.push("transition suppression"); + if (s.closedLive !== "Readout closed." || s.hoverId !== -1 || s.leaveCount !== 1) { + failures.push("Escape dismissal"); } - browser = await ENGINES[name].launch(launchOptions); + if (!s.staleExactIgnored) failures.push("stale exact reply"); + } + if (result.layout.root.width <= 0 || result.layout.canvas.width <= 0) failures.push("nonzero layout"); + if (result.lit < MIN_LIT_PIXELS) failures.push(`only ${result.lit} lit WebGL pixels`); + return failures; +} + +async function launchEngine(name) { + const launchOptions = { headless }; + if (name === "firefox") { + // Firefox's Linux software-rendering blocklist can otherwise disable + // WebGL in virtual displays even when Mesa is available. + launchOptions.firefoxUserPrefs = { + "webgl.disabled": false, + "webgl.force-enabled": true, + }; + } + try { + return await ENGINES[name].launch(launchOptions); } catch (error) { throw new Error(`${name} could not launch; run npx playwright install chromium firefox webkit\n${error}`); } +} + +async function probeCase(browser, engineName, caseItem) { const context = await browser.newContext({ - viewport: { width: 760, height: 480 }, - deviceScaleFactor: 1, - reducedMotion: "reduce", + viewport: VIEWPORT, + deviceScaleFactor: caseItem.dpr, + reducedMotion: caseItem.motion, }); const page = await context.newPage(); const errors = []; @@ -110,191 +581,318 @@ async function probe(name) { page.on("console", (message) => { if (message.type() === "error") errors.push(message.text()); }); - await page.setContent("
"); - const hasWebGl2 = await page.evaluate(() => { - const canvas = document.createElement("canvas"); - const gl = canvas.getContext("webgl2"); - gl?.getExtension("WEBGL_lose_context")?.loseContext(); - return Boolean(gl); - }); - if (!hasWebGl2) { - await context.close(); - await browser.close(); - const mode = headless ? "headless" : "headful"; - throw new Error(`${name}: WebGL2 unavailable in ${mode} mode`); - } - await page.addScriptTag({ path: BUNDLE }); - await page.evaluate(({ spec, values }) => { - // The shipped client correctly uses the faster discardable default - // framebuffer. Preserve it only inside this probe so readPixels observes - // the same completed frame after browser compositing in every engine. - const originalGetContext = HTMLCanvasElement.prototype.getContext; - HTMLCanvasElement.prototype.getContext = function(type, attributes) { - if (type === "webgl2") { - return originalGetContext.call(this, type, { ...attributes, preserveDrawingBuffer: true }); - } - return originalGetContext.call(this, type, attributes); - }; - const buffer = new Float32Array(values).buffer; - window.xyConformanceView = window.xy.renderStandalone( - document.getElementById("chart"), spec, buffer, - ); - window.xyConformanceView.comm = { send() {} }; - window.xyConformanceLeaves = 0; - window.xyConformanceView.root.addEventListener("xy:leave", () => { - window.xyConformanceLeaves += 1; + try { + await page.setContent("
"); + const hasWebGl2 = await page.evaluate(() => { + const canvas = document.createElement("canvas"); + const gl = canvas.getContext("webgl2"); + gl?.getExtension("WEBGL_lose_context")?.loseContext(); + return Boolean(gl); }); - HTMLCanvasElement.prototype.getContext = originalGetContext; - window.xyConformanceView._drawNow(); - }, { spec, values }); - await page.locator('[data-xy-slot="canvas"]').focus(); - await page.keyboard.press("Home"); - await page.keyboard.press("ArrowRight"); - await page.evaluate(() => { - const v = window.xyConformanceView; - window.xyConformanceKeyboardLive = v.a11yLive.textContent; - v._onKernelMsg({ - type: "pick_result", - seq: v._pickSeq, - row: { trace: 0, index: 1, x: -1.2, y: 1.4 }, - }, []); - window.xyConformanceExactPreserved = - v.a11yLive.textContent === window.xyConformanceKeyboardLive; - const index = v._a11yPointIndex; - const live = v.a11yLive.textContent; - v._viewAnim = {}; - v._onA11yKey({ key: "ArrowRight", preventDefault() {} }); - window.xyConformanceTransitionSuppressed = - v._a11yPointIndex === index && v.a11yLive.textContent === live; - v._viewAnim = null; - }); - await page.keyboard.press("Escape"); - const result = await page.evaluate(() => { - const v = window.xyConformanceView; - const closedLive = v.a11yLive.textContent; - v._onKernelMsg({ - type: "pick_result", - seq: v._pickSeq - 1, - row: { trace: 0, index: 1, x: -1.2, y: 1.4 }, - }, []); - const staleExactIgnored = - v.tooltip.style.display === "none" && v.a11yLive.textContent === closedLive; - v._drawNow(); - const rect = (el) => { - const r = el.getBoundingClientRect(); - const root = v.root.getBoundingClientRect(); - return { - x: Math.round((r.x - root.x) * 10) / 10, - y: Math.round((r.y - root.y) * 10) / 10, - width: Math.round(r.width * 10) / 10, - height: Math.round(r.height * 10) / 10, - }; - }; - const gl = v.gl; - const width = gl.drawingBufferWidth; - const height = gl.drawingBufferHeight; - const pixels = new Uint8Array(width * height * 4); - gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels); - const cols = 32; - const rows = 20; - const signature = []; - let lit = 0; - for (let i = 3; i < pixels.length; i += 4) if (pixels[i] > 8) lit += 1; - for (let gy = 0; gy < rows; gy++) { - for (let gx = 0; gx < cols; gx++) { - const sums = [0, 0, 0, 0]; - let count = 0; - const x0 = Math.floor(gx * width / cols), x1 = Math.floor((gx + 1) * width / cols); - const y0 = Math.floor(gy * height / rows), y1 = Math.floor((gy + 1) * height / rows); - for (let y = y0; y < y1; y++) for (let x = x0; x < x1; x++) { - const p = (y * width + x) * 4; - for (let c = 0; c < 4; c++) sums[c] += pixels[p + c]; - count += 1; + if (!hasWebGl2) { + const mode = headless ? "headless" : "headful"; + throw new Error(`${engineName}: WebGL2 unavailable in ${mode} mode`); + } + await page.addScriptTag({ path: BUNDLE }); + const built = caseItem.build(); + await page.evaluate(({ spec, bytes }) => { + // The shipped client correctly uses the faster discardable default + // framebuffer. Preserve it only inside this probe so readPixels observes + // the same completed frame after browser compositing in every engine. + const originalGetContext = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function(type, attributes) { + if (type === "webgl2") { + return originalGetContext.call(this, type, { ...attributes, preserveDrawingBuffer: true }); } - signature.push(...sums.map((sum) => Math.round(sum / Math.max(1, count)))); + return originalGetContext.call(this, type, attributes); + }; + try { + window.xyConformanceView = window.xy.renderStandalone( + document.getElementById("chart"), spec, new Uint8Array(bytes).buffer, + ); + window.xyConformanceView.comm = { send() {} }; + window.xyConformanceLeaves = 0; + window.xyConformanceView.root.addEventListener("xy:leave", () => { + window.xyConformanceLeaves += 1; + }); + window.xyConformanceView._drawNow(); + } finally { + HTMLCanvasElement.prototype.getContext = originalGetContext; } + }, built); + await page.evaluate(() => new Promise((done) => requestAnimationFrame(() => requestAnimationFrame(done)))); + + if (caseItem.anchor) { + await page.locator('[data-xy-slot="canvas"]').focus(); + await page.keyboard.press("Home"); + await page.keyboard.press("ArrowRight"); + await page.evaluate(() => { + const v = window.xyConformanceView; + window.xyConformanceKeyboardLive = v.a11yLive.textContent; + v._onKernelMsg({ + type: "pick_result", + seq: v._pickSeq, + row: { trace: 0, index: 1, x: -1.2, y: 1.4 }, + }, []); + window.xyConformanceExactPreserved = + v.a11yLive.textContent === window.xyConformanceKeyboardLive; + const index = v._a11yPointIndex; + const live = v.a11yLive.textContent; + v._viewAnim = {}; + v._onA11yKey({ key: "ArrowRight", preventDefault() {} }); + window.xyConformanceTransitionSuppressed = + v._a11yPointIndex === index && v.a11yLive.textContent === live; + v._viewAnim = null; + }); + await page.keyboard.press("Escape"); } - const before = { ...v.view }; - v._setView({ x0: -1, x1: 1, y0: 0, y1: 2 }, { animate: true }); - const reducedMotionSkippedAnimation = v._viewAnim === null; - v.view = before; - v.draw(); - const buttons = [...v._modebar.querySelectorAll("button")]; - return { - semantics: { - regionRole: v.root.getAttribute("role"), - regionLabel: v.root.getAttribute("aria-label"), - summary: v.a11ySummary.textContent, - canvasRole: v.canvas.getAttribute("role"), - canvasTabIndex: v.canvas.tabIndex, - keyboardLive: window.xyConformanceKeyboardLive, - closedLive, - keyboardIndex: v._a11yPointIndex, - hoverId: v._hoverId, - leaveCount: window.xyConformanceLeaves, - exactPreserved: window.xyConformanceExactPreserved, - transitionSuppressed: window.xyConformanceTransitionSuppressed, - staleExactIgnored, - toolbarRole: v._modebar.getAttribute("role"), - toolbarLabel: v._modebar.getAttribute("aria-label"), - buttonLabels: buttons.map((button) => - button.getAttribute("aria-label") || button.textContent.trim() || button.title), - pressed: buttons.filter((button) => button.getAttribute("aria-pressed") === "true") - .map((button) => + + const result = await page.evaluate(({ anchor, requestedMotion }) => { + const v = window.xyConformanceView; + let closedLive = null; + let staleExactIgnored = true; + if (anchor) { + closedLive = v.a11yLive.textContent; + v._onKernelMsg({ + type: "pick_result", + seq: v._pickSeq - 1, + row: { trace: 0, index: 1, x: -1.2, y: 1.4 }, + }, []); + staleExactIgnored = + v.tooltip.style.display === "none" && v.a11yLive.textContent === closedLive; + } + v._drawNow(); + const rect = (element) => { + const value = element.getBoundingClientRect(); + const root = v.root.getBoundingClientRect(); + return { + x: Math.round((value.x - root.x) * 10) / 10, + y: Math.round((value.y - root.y) * 10) / 10, + width: Math.round(value.width * 10) / 10, + height: Math.round(value.height * 10) / 10, + }; + }; + const gl = v.gl; + const width = gl.drawingBufferWidth; + const height = gl.drawingBufferHeight; + const pixels = new Uint8Array(width * height * 4); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels); + const cols = 32; + const rows = 20; + const signature = []; + let lit = 0; + for (let index = 3; index < pixels.length; index += 4) if (pixels[index] > 8) lit += 1; + for (let gy = 0; gy < rows; gy++) { + for (let gx = 0; gx < cols; gx++) { + const sums = [0, 0, 0, 0]; + let count = 0; + const x0 = Math.floor(gx * width / cols), x1 = Math.floor((gx + 1) * width / cols); + const y0 = Math.floor(gy * height / rows), y1 = Math.floor((gy + 1) * height / rows); + for (let y = y0; y < y1; y++) for (let x = x0; x < x1; x++) { + const pixel = (y * width + x) * 4; + for (let channel = 0; channel < 4; channel++) sums[channel] += pixels[pixel + channel]; + count += 1; + } + signature.push(...sums.map((sum) => Math.round(sum / Math.max(1, count)))); + } + } + + const before = JSON.parse(JSON.stringify(v.view)); + const ranges = Object.fromEntries(v._axisIds().map((axisId) => { + const [lo, hi] = v._axisRange(axisId); + const inset = (hi - lo) * 0.12; + return [axisId, [lo + inset, hi - inset]]; + })); + v._setView({ ranges }, { animate: true, request: false, history: false }); + const animationActive = v._viewAnim !== null; + v._cancelViewAnimation(); + v.view = before; + v._drawNow(); + + const buttons = [...v._modebar.querySelectorAll("button")]; + const canvasRect = v.canvas.getBoundingClientRect(); + const gpu = v.gpuTraces[0]; + return { + title: v.spec.title, + semantics: { + regionRole: v.root.getAttribute("role"), + regionLabel: v.root.getAttribute("aria-label"), + summary: v.a11ySummary.textContent, + canvasRole: v.canvas.getAttribute("role"), + canvasTabIndex: v.canvas.tabIndex, + keyboardLive: window.xyConformanceKeyboardLive || "", + closedLive, + keyboardIndex: v._a11yPointIndex, + hoverId: v._hoverId, + leaveCount: window.xyConformanceLeaves, + exactPreserved: window.xyConformanceExactPreserved ?? true, + transitionSuppressed: window.xyConformanceTransitionSuppressed ?? true, + staleExactIgnored, + toolbarRole: v._modebar.getAttribute("role"), + toolbarLabel: v._modebar.getAttribute("aria-label"), + buttonLabels: buttons.map((button) => button.getAttribute("aria-label") || button.textContent.trim() || button.title), - reducedMotionSkippedAnimation, - }, - layout: { - root: rect(v.root), - canvas: rect(v.canvas), - toolbar: rect(v._modebar), - title: rect(v.root.querySelector('[data-xy-slot="title"]')), - labelPositions: [...v.labels.querySelectorAll('[data-xy-slot="tick_label"], [data-xy-slot="axis_title"]')] - .map((el) => { const r = rect(el); return [r.x, r.y]; }), - }, - signature, - lit, - }; - }); - await context.close(); - await browser.close(); - if (errors.length) throw new Error(`${name} page errors:\n${errors.join("\n")}`); - const s = result.semantics; - const semanticFailures = []; - if (s.regionRole !== "region" || !s.regionLabel.includes("Cross-browser")) semanticFailures.push("chart region"); - if (!s.summary.includes("observations") || !s.summary.includes("X axis (time)")) semanticFailures.push("summary"); - if (s.canvasRole !== "img" || s.canvasTabIndex !== 0) semanticFailures.push("focusable plot image"); - if (s.keyboardIndex !== 1 || !s.keyboardLive.startsWith("Point 2 of 7.")) semanticFailures.push("keyboard/live readout"); - if (!s.exactPreserved) semanticFailures.push("exact reply announcement"); - if (!s.transitionSuppressed) semanticFailures.push("transition suppression"); - if (s.closedLive !== "Readout closed." || s.hoverId !== -1 || s.leaveCount !== 1) semanticFailures.push("Escape dismissal"); - if (!s.staleExactIgnored) semanticFailures.push("stale exact reply"); - if (s.toolbarRole !== "toolbar" || s.toolbarLabel !== "Chart controls") semanticFailures.push("toolbar"); - if (s.buttonLabels.length === 0 || s.buttonLabels.some((label) => !label)) semanticFailures.push("button names"); - if (s.pressed.length !== 1 || !s.pressed[0]) semanticFailures.push("toggle state"); - if (!s.reducedMotionSkippedAnimation) semanticFailures.push("reduced motion"); - if (semanticFailures.length) throw new Error(`${name} accessibility failures: ${semanticFailures.join(", ")}`); - if (result.lit < 200) throw new Error(`${name} rendered only ${result.lit} lit WebGL pixels`); - return result; + pressed: buttons.filter((button) => button.getAttribute("aria-pressed") === "true") + .map((button) => + button.getAttribute("aria-label") || button.textContent.trim() || button.title), + mediaReduced: window.matchMedia("(prefers-reduced-motion: reduce)").matches, + requestedMotion, + animationActive, + }, + layout: { + root: rect(v.root), + canvas: rect(v.canvas), + toolbar: rect(v._modebar), + title: rect(v.root.querySelector('[data-xy-slot="title"]')), + labelPositions: [...v.labels.querySelectorAll( + '[data-xy-slot="tick_label"], [data-xy-slot="axis_title"]', + )].map((element) => { const value = rect(element); return [value.x, value.y]; }), + }, + gpu: { + kind: gpu.trace.kind, + tier: gpu.tier, + xAxis: gpu.xAxis, + yAxis: gpu.yAxis, + }, + axisTitles: [...v.labels.querySelectorAll('[data-xy-slot="axis_title"]')] + .map((element) => element.textContent), + tickLabels: [...v.labels.querySelectorAll('[data-xy-slot="tick_label"]')] + .map((element) => element.textContent), + observedDpr: window.devicePixelRatio, + backingStoreMatchesDpr: + Math.abs(width - canvasRect.width * window.devicePixelRatio) <= 2 + && Math.abs(height - canvasRect.height * window.devicePixelRatio) <= 2, + drawingBuffer: { width, height }, + signature, + lit, + }; + }, { anchor: Boolean(caseItem.anchor), requestedMotion: caseItem.motion }); + + if (errors.length) throw new Error(`${engineName}/${caseItem.id} page errors:\n${errors.join("\n")}`); + const failures = semanticFailures(caseItem, result); + if (failures.length) { + throw new Error(`${engineName}/${caseItem.id} semantic failures: ${failures.join(", ")}`); + } + return result; + } finally { + await page.evaluate(() => window.xyConformanceView?.destroy()).catch(() => {}); + await context.close(); + } +} + +function resultForEvidence(result) { + const { signature, ...summary } = result; + return { + ...summary, + signatureSha256: createHash("sha256").update(Uint8Array.from(signature)).digest("hex"), + }; } -const results = {}; -for (const name of selected) { - results[name] = await probe(name); - console.log(`${name}: semantics OK, ${results[name].lit} lit pixels`); +function writeEvidence(evidence) { + if (!evidencePath) return; + mkdirSync(dirname(evidencePath), { recursive: true }); + writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); } -const referenceName = selected.includes("chromium") ? "chromium" : selected[0]; -const reference = results[referenceName]; -for (const name of selected) { - if (name === referenceName) continue; - const layoutDelta = maxLayoutDelta(reference, results[name]); - const mae = signatureMae(reference, results[name]); - const litRatio = results[name].lit / reference.lit; - if (layoutDelta > 4) throw new Error(`${name}: DOM layout delta ${layoutDelta.toFixed(2)}px exceeds 4px`); - if (mae > 12) throw new Error(`${name}: perceptual raster MAE ${mae.toFixed(2)} exceeds 12`); - if (litRatio < 0.8 || litRatio > 1.2) { - throw new Error(`${name}: lit-pixel ratio ${litRatio.toFixed(3)} is outside 0.8..1.2`); + +async function runConformance() { + const evidence = { + schema: "xy-browser-conformance/v1", + status: "started", + environment: { + node: process.version, + platform: process.platform, + arch: process.arch, + playwright: packageJson.devDependencies.playwright, + mode: headless ? "headless" : "headful", + viewport: VIEWPORT, + selectedEngines: selected, + skipPolicy: "none; missing selected engines or WebGL2 fail", + }, + thresholds: { + maxLayoutDeltaCssPx: MAX_LAYOUT_DELTA, + maxSignatureMae: MAX_SIGNATURE_MAE, + litPixelRatio: [MIN_LIT_RATIO, MAX_LIT_RATIO], + minLitPixels: MIN_LIT_PIXELS, + }, + matrix: CASES.map(metadataOf), + engines: {}, + comparisons: {}, + negativeControls: {}, + }; + writeEvidence(evidence); + const results = {}; + try { + validateCatalog(); + for (const name of selected) { + if (!ENGINES[name]) throw new Error(`unknown browser ${name}; use chromium,firefox,webkit`); + } + if (!selected.length) throw new Error("at least one browser engine is required"); + evidence.negativeControls = staticNegativeControls(); + for (const name of selected) { + const engineEvidence = { status: "started", version: null, cases: {} }; + evidence.engines[name] = engineEvidence; + writeEvidence(evidence); + let browser; + try { + browser = await launchEngine(name); + engineEvidence.version = browser.version(); + results[name] = {}; + for (const caseItem of CASES) { + const result = await probeCase(browser, name, caseItem); + results[name][caseItem.id] = result; + engineEvidence.cases[caseItem.id] = resultForEvidence(result); + console.log(`${name}/${caseItem.id}: semantics OK, ${result.lit} lit pixels`); + writeEvidence(evidence); + } + engineEvidence.status = "passed"; + } catch (error) { + engineEvidence.status = "failed"; + engineEvidence.error = String(error?.stack || error); + throw error; + } finally { + await browser?.close(); + } + } + + const referenceName = selected.includes("chromium") ? "chromium" : selected[0]; + for (const caseItem of CASES) { + evidence.comparisons[caseItem.id] = {}; + const reference = results[referenceName][caseItem.id]; + for (const name of selected) { + if (name === referenceName) continue; + const comparison = compareResults(reference, results[name][caseItem.id]); + evidence.comparisons[caseItem.id][name] = { + reference: referenceName, + layoutDeltaCssPx: comparison.layoutDelta, + rasterMae: comparison.rasterMae, + litPixelRatio: comparison.litRatio, + }; + if (comparison.failures.length) { + throw new Error(`${name}/${caseItem.id} vs ${referenceName}: ${comparison.failures.join(", ")}`); + } + console.log( + `${name}/${caseItem.id} vs ${referenceName}: layout Δ ${comparison.layoutDelta.toFixed(2)}px, ` + + `raster MAE ${comparison.rasterMae.toFixed(2)}, lit ratio ${comparison.litRatio.toFixed(3)}`, + ); + } + } + evidence.status = "passed"; + console.log(`browser conformance OK: ${CASES.length} cases × ${selected.length} engines`); + } catch (error) { + evidence.status = "failed"; + evidence.error = String(error?.stack || error); + throw error; + } finally { + writeEvidence(evidence); } - console.log(`${name} vs ${referenceName}: layout Δ ${layoutDelta.toFixed(2)}px, raster MAE ${mae.toFixed(2)}, lit ratio ${litRatio.toFixed(3)}`); } -console.log(`browser conformance OK: ${selected.join(", ")}`); + +if (process.argv.includes("--list-cases")) { + validateCatalog(); + console.log(JSON.stringify(CASES.map(metadataOf), null, 2)); +} else if (process.argv.includes("--self-test")) { + validateCatalog(); + console.log(JSON.stringify(staticNegativeControls(), null, 2)); +} else { + await runConformance(); +} diff --git a/scripts/chart_kind_matrix.py b/scripts/chart_kind_matrix.py new file mode 100644 index 00000000..f8bcb16d --- /dev/null +++ b/scripts/chart_kind_matrix.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Registry-complete payload and browser render matrix for every client mark.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "python")) +sys.path.insert(0, str(ROOT / "scripts")) + +import xy # noqa: E402 +from _app_smoke import ChromiumSession, Probe, decode_png, find_chromium # noqa: E402 + +VIEWPORT = (700, 420, 1.0) +MIN_COLORED_PIXELS = 20 + + +@dataclass(frozen=True) +class Case: + name: str + expected_kinds: tuple[str, ...] + mark: Callable[[], object] + + +CASES: tuple[Case, ...] = ( + Case("scatter", ("scatter",), lambda: xy.scatter([0, 1, 2], [0, 1, 0], color="#2563eb")), + Case("line", ("line",), lambda: xy.line([0, 1, 2], [0, 1, 0], color="#dc2626", width=3)), + Case("area", ("area",), lambda: xy.area([0, 1, 2], [1, 2, 1], color="#0891b2")), + Case( + "histogram", ("histogram",), lambda: xy.hist([0, 0.2, 0.4, 1, 1.2], bins=3, color="#7c3aed") + ), + Case( + "box", + ("box_whisker", "box", "box_median"), + lambda: xy.box([1, 2, 2, 3, 5, 8], show_outliers=False, color="#ea580c"), + ), + Case( + "violin", ("violin",), lambda: xy.violin([1, 2, 2, 3, 4, 5, 5, 6], bins=8, color="#16a34a") + ), + Case( + "errorbar", + ("errorbar",), + lambda: xy.errorbar([0, 1, 2], [1, 2, 1], yerr=[0.2, 0.3, 0.2], color="#be123c"), + ), + Case( + "stem", + ("stem", "scatter"), + lambda: xy.stem([0, 1, 2], [1, 2, 1], color="#0369a1", marker_size=8), + ), + Case( + "segments", + ("segments",), + lambda: xy.segments([0, 1], [0, 1], [1, 2], [1, 0], color="#9333ea", width=4), + ), + Case( + "triangle-mesh", + ("triangle_mesh",), + lambda: xy.triangle_mesh([0], [0], [1], [0], [0.5], [1], color=[0.7]), + ), + Case( + "error-band", + ("error_band",), + lambda: xy.error_band([0, 1, 2], [0.8, 1.8, 0.8], [1.2, 2.2, 1.2], color="#0284c7"), + ), + Case( + "hexbin", + ("hexbin",), + lambda: xy.hexbin([0, 0.1, 0.8, 0.9, 0.5], [0, 0.2, 0.8, 0.9, 0.5], gridsize=4, mincnt=1), + ), + Case("bar", ("bar",), lambda: xy.bar(["a", "b"], [1, 2], color="#4f46e5")), + Case("column", ("column",), lambda: xy.column(["a", "b"], [2, 1], color="#0d9488")), + Case("heatmap", ("heatmap",), lambda: xy.heatmap([[1, 2], [3, 4]])), + Case( + "contour", + ("contour",), + lambda: xy.contour([[0, 1, 0], [1, 2, 1], [0, 1, 0]], levels=3, color="#c026d3", width=3), + ), +) + + +def shipped_registry() -> set[str]: + program = ( + "import('./python/xy/static/index.js')" + ".then(m=>console.log(JSON.stringify(Object.keys(m.MARK_KINDS).sort())))" + ) + result = subprocess.run( + ["node", "-e", program], cwd=ROOT, check=True, capture_output=True, text=True, timeout=30 + ) + return set(json.loads(result.stdout)) + + +def validate_catalog(cases: Iterable[Case], registry: set[str]) -> None: + cases = tuple(cases) + names = [case.name for case in cases] + if len(names) != len(set(names)): + raise AssertionError("chart-kind case names must be unique") + covered = {kind for case in cases for kind in case.expected_kinds} + if covered != registry: + raise AssertionError( + f"chart-kind catalog mismatch: missing={sorted(registry - covered)}, " + f"unexpected={sorted(covered - registry)}" + ) + + +def build_case(case: Case) -> tuple[object, dict, bytes]: + chart = xy.scatter_chart( + case.mark(), + xy.x_axis(label="x"), + xy.y_axis(label="y"), + title=f"registry case: {case.name}", + width=640, + height=360, + ) + spec, payload = chart.figure().build_payload(px_width=640) + actual = tuple(trace["kind"] for trace in spec["traces"]) + if actual != case.expected_kinds: + raise AssertionError(f"{case.name}: payload kinds {actual} != {case.expected_kinds}") + for trace in spec["traces"]: + if trace.get("tier") not in {"direct", "decimated", "density"}: + raise AssertionError(f"{case.name}/{trace['kind']}: missing valid tier evidence") + if int(trace.get("n_points", 0)) <= 0: + raise AssertionError(f"{case.name}/{trace['kind']}: empty payload geometry") + return chart, spec, payload + + +def colored_pixels(png: bytes, rect: dict[str, float]) -> int: + width, height, channels, rgba = decode_png(png) + left = max(0, int(rect["x"])) + top = max(0, int(rect["y"])) + right = min(width, int(rect["x"] + rect["w"])) + bottom = min(height, int(rect["y"] + rect["h"])) + count = 0 + for y in range(top, bottom): + for x in range(left, right): + pos = (y * width + x) * channels + pixel = rgba[pos : pos + channels] + red = pixel[0] + green = pixel[1] if channels > 1 else red + blue = pixel[2] if channels > 2 else red + alpha = pixel[3] if channels == 4 else 255 + if alpha > 8 and max(red, green, blue) - min(red, green, blue) > 24: + count += 1 + return count + + +def require_nonblank_pixels(name: str, count: int) -> None: + if count < MIN_COLORED_PIXELS: + raise AssertionError(f"{name}: blank/flat browser render ({count} colored pixels)") + + +def render_case(session: ChromiumSession, directory: Path, case: Case, chart: object) -> dict: + document = chart.to_html() + call = 'xy.renderStandalone(document.getElementById("chart"), spec, buf);' + replacement = "window.__xyKindView = " + call + if call not in document: + raise AssertionError(f"{case.name}: standalone hydration call not found") + page = directory / f"{case.name}.html" + page.write_text(document.replace(call, replacement, 1), encoding="utf-8") + probe = Probe(session, page.as_uri(), emulate=VIEWPORT) + try: + report = probe.wait_for( + "(() => { const v=window.__xyKindView; if(!v||!v.gpuTraces) return null; " + "v._drawNow(); return {kinds:v.gpuTraces.map(g=>g.trace.kind), " + "counts:v.gpuTraces.map(g=>g.n||g.trace.n_points||0)}; })()", + timeout_s=30, + label=f"{case.name} GPU geometry", + ) + if tuple(report["kinds"]) != case.expected_kinds: + raise AssertionError( + f"{case.name}: GPU kinds {report['kinds']} != {case.expected_kinds}" + ) + if any(int(value) <= 0 for value in report["counts"]): + raise AssertionError(f"{case.name}: empty GPU geometry {report['counts']}") + pixels = colored_pixels(probe.screenshot(), probe.rect("[data-xy-slot='canvas']")) + require_nonblank_pixels(case.name, pixels) + return { + "gpu_kinds": report["kinds"], + "gpu_counts": report["counts"], + "colored_pixels": pixels, + } + finally: + probe.close() + + +def write_evidence(path: Path, evidence: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("chromium", nargs="?") + parser.add_argument("--no-sandbox", action="store_true") + parser.add_argument("--evidence", type=Path, default=Path("chart-kind-matrix-evidence.json")) + args = parser.parse_args(argv) + + registry = shipped_registry() + validate_catalog(CASES, registry) + evidence: dict = {"registry": sorted(registry), "cases": {}, "status": "failed"} + try: + built = [(case, *build_case(case)) for case in CASES] + chromium = find_chromium(args.chromium) + with ( + tempfile.TemporaryDirectory(prefix="xy-chart-kind-") as temp_dir, + ChromiumSession(chromium, gl="software", sandbox=not args.no_sandbox) as session, + ): + for case, chart, spec, payload in built: + browser = render_case(session, Path(temp_dir), case, chart) + evidence["cases"][case.name] = { + "payload_kinds": [trace["kind"] for trace in spec["traces"]], + "payload_tiers": [trace["tier"] for trace in spec["traces"]], + "payload_bytes": len(payload), + **browser, + } + evidence["status"] = "passed" + print(f"chart-kind matrix OK: {len(CASES)} cases cover {len(registry)} registry kinds") + return 0 + except Exception as exc: + evidence["error"] = str(exc) + raise + finally: + write_evidence(args.evidence, evidence) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_claim_guardrails.py b/scripts/check_claim_guardrails.py index cb5f1a07..ae6e3b2f 100644 --- a/scripts/check_claim_guardrails.py +++ b/scripts/check_claim_guardrails.py @@ -191,12 +191,13 @@ def check_claims(paths: list[Path]) -> list[Finding]: def _default_paths() -> list[Path]: paths = [ROOT / item for item in DEFAULT_DOCS] + specification = sorted((ROOT / "spec").rglob("*.md")) public_docs = ( path for path in sorted((ROOT / "docs").rglob("*.md")) if "app" not in path.relative_to(ROOT / "docs").parts ) - return list(dict.fromkeys((*paths, *public_docs))) + return list(dict.fromkeys((*paths, *specification, *public_docs))) def main(argv: Optional[list[str]] = None) -> int: diff --git a/scripts/check_pyplot_options.py b/scripts/check_pyplot_options.py new file mode 100644 index 00000000..1f5b0de0 --- /dev/null +++ b/scripts/check_pyplot_options.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Fail on accepted pyplot options that have no effect or reviewed no-op contract.""" + +from __future__ import annotations + +import argparse +import ast +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = ROOT / "python" / "xy" / "pyplot" +POLICY = ROOT / "spec" / "testing" / "pyplot-noops.json" + + +@dataclass(frozen=True, order=True) +class Noop: + path: str + function: str + option: str + + +def _qualnames(tree: ast.AST) -> dict[ast.AST, str]: + result: dict[ast.AST, str] = {} + + def visit(node: ast.AST, parents: tuple[str, ...] = ()) -> None: + next_parents = parents + if isinstance(node, ast.ClassDef): + next_parents = (*parents, node.name) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + result[node] = ".".join((*parents, node.name)) + next_parents = (*parents, node.name) + for child in ast.iter_child_nodes(node): + visit(child, next_parents) + + visit(tree) + return result + + +def _is_stub_or_rejection(function: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + if any( + (isinstance(decorator, ast.Name) and decorator.id == "overload") + or (isinstance(decorator, ast.Attribute) and decorator.attr == "overload") + for decorator in function.decorator_list + ): + return True + statements = list(function.body) + if ( + statements + and isinstance(statements[0], ast.Expr) + and isinstance(statements[0].value, ast.Constant) + and isinstance(statements[0].value.value, str) + ): + statements = statements[1:] + if len(statements) != 1: + return False + statement = statements[0] + return isinstance(statement, (ast.Pass, ast.Raise)) or ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and statement.value.value is Ellipsis + ) + + +def _function_nodes(function: ast.FunctionDef | ast.AsyncFunctionDef) -> list[ast.AST]: + """Walk one function without crediting nested definitions for parameter use.""" + nodes: list[ast.AST] = [] + + def visit(node: ast.AST) -> None: + nodes.append(node) + for child in ast.iter_child_nodes(node): + if child is not function and isinstance( + child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef) + ): + # Keep the definition node so callers can decide whether its + # closure is reachable, but do not credit its body yet. + nodes.append(child) + continue + visit(child) + + visit(function) + return nodes + + +def _loaded_names(function: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + """Names read by a function, including only reachable local closures. + + A nested helper may legitimately consume an outer option (``findobj``'s + predicate is one example), but an unreferenced nested definition must not + be enough to disguise an unused public parameter. Follow local function + definitions only when their name is read by the containing scope. + """ + nodes = _function_nodes(function) + loaded = { + node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } + closures = { + node.name: node + for node in nodes + if node is not function and isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + pending = [closures[name] for name in sorted(loaded & closures.keys())] + followed: set[str] = set() + while pending: + closure = pending.pop() + if closure.name in followed: + continue + followed.add(closure.name) + closure_nodes = _function_nodes(closure) + closure_loads = { + node.id + for node in closure_nodes + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } + loaded.update(closure_loads) + nested = { + node.name: node + for node in closure_nodes + if node is not closure and isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + pending.extend(nested[name] for name in sorted(closure_loads & nested.keys())) + return loaded + + +def discover_noops(root: Path = SOURCE_ROOT, *, project_root: Path = ROOT) -> set[Noop]: + found: set[Noop] = set() + for path in sorted(root.glob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + qualnames = _qualnames(tree) + parents: dict[ast.AST, ast.AST] = { + child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent) + } + relpath = path.relative_to(project_root).as_posix() + for function in ( + node + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + ): + parent = parents.get(function) + nested = False + while parent is not None: + if isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef)): + nested = True + break + parent = parents.get(parent) + if nested: + continue + leaf_name = qualnames[function].rsplit(".", 1)[-1] + if leaf_name.startswith("_") and leaf_name not in {"__init__", "__call__"}: + continue + if _is_stub_or_rejection(function): + continue + nodes = _function_nodes(function) + loads = _loaded_names(function) + parameters = { + arg.arg + for arg in ( + *function.args.posonlyargs, + *function.args.args, + *function.args.kwonlyargs, + ) + if arg.arg not in {"self", "cls"} + } + for option in parameters - loads: + found.add(Noop(relpath, qualnames[function], option)) + + for node in nodes: + call: ast.Call | None = None + assigned: str | None = None + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + call = node.value + elif ( + isinstance(node, ast.Assign) + and isinstance(node.value, ast.Call) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + call = node.value + assigned = node.targets[0].id + if not call or not isinstance(call.func, ast.Attribute) or call.func.attr != "pop": + continue + if not isinstance(call.func.value, ast.Name) or call.func.value.id not in { + "kwargs", + "options", + }: + continue + if ( + not call.args + or not isinstance(call.args[0], ast.Constant) + or not isinstance(call.args[0].value, str) + ): + continue + if assigned is None or assigned not in loads: + found.add(Noop(relpath, qualnames[function], call.args[0].value)) + return found + + +def load_policy(path: Path = POLICY, *, project_root: Path = ROOT) -> tuple[set[Noop], list[str]]: + errors: list[str] = [] + try: + data: Any = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return set(), [f"cannot read no-op policy: {exc}"] + if not isinstance(data, dict) or set(data) != {"schema_version", "noops"}: + return set(), ["policy must contain exactly schema_version and noops"] + if data["schema_version"] != 1 or not isinstance(data["noops"], list): + return set(), ["policy schema_version must be 1 and noops must be a list"] + noops: set[Noop] = set() + for index, entry in enumerate(data["noops"]): + if not isinstance(entry, dict) or set(entry) != { + "path", + "function", + "options", + "rationale", + "test", + }: + errors.append(f"noops[{index}] has invalid fields") + continue + if not all( + isinstance(entry[field], str) and entry[field].strip() + for field in ("path", "function", "rationale", "test") + ): + errors.append(f"noops[{index}] path/function/rationale/test must be strings") + continue + if not isinstance(entry["rationale"], str) or len(entry["rationale"].strip()) < 20: + errors.append(f"noops[{index}] needs a substantive rationale") + test = entry["test"] + if not isinstance(test, str) or "::" not in test: + errors.append(f"noops[{index}] test must be path::function") + else: + test_path, test_name = test.split("::", 1) + candidate = project_root / test_path + if not candidate.is_file() or f"def {test_name}(" not in candidate.read_text( + encoding="utf-8" + ): + errors.append(f"noops[{index}] references missing test {test!r}") + options = entry["options"] + if ( + not isinstance(options, list) + or not options + or not all(isinstance(option, str) and option for option in options) + ): + errors.append(f"noops[{index}] options must be a nonempty string list") + continue + for option in options: + noop = Noop(entry["path"], entry["function"], option) + if noop in noops: + errors.append(f"noops[{index}] duplicates {noop}") + noops.add(noop) + return noops, errors + + +def validate( + root: Path = SOURCE_ROOT, policy_path: Path = POLICY, *, project_root: Path = ROOT +) -> list[str]: + discovered = discover_noops(root, project_root=project_root) + policy, errors = load_policy(policy_path, project_root=project_root) + for noop in sorted(discovered - policy): + errors.append(f"accepted option has no effect and no reviewed contract: {noop}") + for noop in sorted(policy - discovered): + errors.append(f"stale no-op policy entry now appears effectful or absent: {noop}") + return errors + + +def _record(noop: Noop) -> dict[str, str]: + return {"path": noop.path, "function": noop.function, "option": noop.option} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, help="write machine-readable JSON evidence") + args = parser.parse_args(argv) + discovered = discover_noops() + policy, policy_errors = load_policy() + errors = list(policy_errors) + errors.extend( + f"accepted option has no effect and no reviewed contract: {noop}" + for noop in sorted(discovered - policy) + ) + errors.extend( + f"stale no-op policy entry now appears effectful or absent: {noop}" + for noop in sorted(policy - discovered) + ) + if args.report is not None: + report = { + "schema_version": 1, + "status": "failed" if errors else "passed", + "reviewed_noop_count": len(discovered), + "discovered": [_record(noop) for noop in sorted(discovered)], + "errors": errors, + } + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + if errors: + print("pyplot option contract failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print(f"pyplot option contract OK: {len(discovered)} reviewed no-op options") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_required_jobs.py b/scripts/check_required_jobs.py new file mode 100644 index 00000000..00505c4e --- /dev/null +++ b/scripts/check_required_jobs.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fail unless every hard CI dependency completed successfully. + +GitHub's branch protection needs one stable check name, while the hard suite is +split across platform and package jobs. The ``required_ci`` workflow job passes +its complete ``needs`` context here. Keeping the policy in a tested script +avoids a shell expression that accidentally treats ``skipped`` or ``cancelled`` +as success. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections.abc import Mapping +from typing import Any + +HARD_JOBS = ( + "browser_conformance", + "dependency_audit", + "host_integration", + "install_without_rust", + "javascript_semantics", + "matplotlib_reference", + "native_parity", + "python_coverage", + "python_floor", + "reflex_adapter", + "rust_release", + "sdist", + "test", + "wheels", +) + + +def evaluate_required_jobs(needs: Mapping[str, Any]) -> list[str]: + """Return policy failures for a decoded GitHub ``needs`` context.""" + errors: list[str] = [] + expected = set(HARD_JOBS) + actual = set(needs) + for name in sorted(expected - actual): + errors.append(f"hard job {name!r} is missing from the aggregate needs context") + for name in sorted(actual - expected): + errors.append(f"unexpected job {name!r} is wired into the hard aggregate") + + for name in HARD_JOBS: + record = needs.get(name) + if not isinstance(record, Mapping): + if name in actual: + errors.append(f"hard job {name!r} has a malformed result record") + continue + result = record.get("result") + if result != "success": + errors.append(f"hard job {name!r} concluded {result!r}, expected 'success'") + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--needs-json", + default=os.environ.get("NEEDS_JSON", ""), + help="JSON needs context (defaults to $NEEDS_JSON)", + ) + args = parser.parse_args(argv) + if not args.needs_json: + print("required CI aggregate: no needs JSON provided", file=sys.stderr) + return 2 + try: + needs = json.loads(args.needs_json) + except json.JSONDecodeError as exc: + print(f"required CI aggregate: invalid needs JSON: {exc}", file=sys.stderr) + return 2 + if not isinstance(needs, dict): + print("required CI aggregate: needs JSON must be an object", file=sys.stderr) + return 2 + + errors = evaluate_required_jobs(needs) + if errors: + print("required CI aggregate failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("required CI aggregate OK: " + ", ".join(HARD_JOBS)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_testing_spec.py b/scripts/check_testing_spec.py new file mode 100644 index 00000000..5510063f --- /dev/null +++ b/scripts/check_testing_spec.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Check the complete specification tree for internal and repository truth. + +The catalog claims what the repository tests and what it does not. Those claims +decay silently when a script is renamed, a Make target is retired, or a workflow +job disappears, so this checker validates the mechanical parts: + +- status values and current-inventory evidence rows use the documented vocabulary; +- gap IDs are unique, sequentially numbered, and defined exactly once; +- every referenced gap ID resolves, and every defined gap is reachable from the + current inventory; +- relative Markdown links resolve, including heading anchors; and +- referenced commands, repository paths, Python test symbols, and workflow jobs exist. + +It is deliberately mechanical. Whether a row's status is *honest* stays a review +question; this only rejects claims that are checkably false. +""" + +from __future__ import annotations + +import argparse +import ast +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC_DIR = ROOT / "spec" +TESTING_DIR = ROOT / "spec" / "testing" +MAKEFILE = ROOT / "Makefile" +WORKFLOW_DIR = ROOT / ".github" / "workflows" + +STATUS_VOCABULARY = frozenset( + { + "IMPLEMENTED", + "PARTIALLY IMPLEMENTED", + "NOT IMPLEMENTED", + "OUT OF SCOPE", + } +) + +GAP_ID_RE = re.compile(r"TST-NI-(\d{3})") +GAP_HEADING_RE = re.compile(r"^### (TST-NI-\d{3}) — (.+)$") +# Backticked ALL-CAPS spans in this tree are status claims; anything else in +# caps (an acronym such as `JSON`) is not, so require at least two words or a +# known single-word status. +STATUS_CLAIM_RE = re.compile(r"`([A-Z][A-Z ]{4,})`") +MARKDOWN_LINK_RE = re.compile(r"\[[^\]]*\]\(([^)]+)\)") +MAKE_TARGET_RE = re.compile(r"`(make [a-z0-9-]+)") +REPO_PATH_RE = re.compile( + r"`((?:\.github|scripts|benchmarks|tests|python|js|src|docs|spec|examples)/[^`\s]+)`" +) +WORKFLOW_FILE_RE = re.compile(r"`(_?[a-z0-9-]+\.yml)`") +LINE_SUFFIX_RE = re.compile(r":\d+(?:-\d+)?$") +GAP_STATUS_RE = re.compile(r"^- Status: `([^`]+)`\s*$", re.M) +GENERATED_PATHS = frozenset({"docs/benchmark_ci.md"}) + + +class Findings: + """Collects failures so one run reports every problem, not just the first.""" + + def __init__(self) -> None: + self.errors: list[str] = [] + + def add(self, path: Path, message: str) -> None: + self.errors.append(f"{path.relative_to(ROOT)}: {message}") + + def ok(self) -> bool: + return not self.errors + + +def _slug(heading: str) -> str: + """Reproduce GitHub's heading-anchor slug for a Markdown heading's text.""" + text = heading.strip().lower() + text = re.sub(r"[^\w\s-]", "", text) + # GitHub replaces each space individually; a removed em dash leaves two. + return re.sub(r"\s", "-", text.strip()) + + +def _headings(text: str) -> set[str]: + anchors: set[str] = set() + for line in text.splitlines(): + if line.startswith("#"): + anchors.add(_slug(line.lstrip("#"))) + return anchors + + +def _strip_code_blocks(text: str) -> str: + return re.sub(r"```.*?```", "", text, flags=re.DOTALL) + + +def _make_targets() -> set[str]: + if not MAKEFILE.is_file(): + return set() + return { + match.group(1) + for match in re.finditer(r"^([a-z][a-z0-9-]*):", MAKEFILE.read_text(encoding="utf-8"), re.M) + } + + +def _workflow_jobs(path: Path) -> set[str]: + """Read top-level job names without a YAML dependency (stdlib-only tool).""" + jobs: set[str] = set() + in_jobs = False + for line in path.read_text(encoding="utf-8").splitlines(): + if re.match(r"^jobs:\s*$", line): + in_jobs = True + continue + if in_jobs: + if line and not line[0].isspace(): + in_jobs = False + continue + match = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) + if match: + jobs.add(match.group(1)) + return jobs + + +def check_status_vocabulary(path: Path, text: str, findings: Findings) -> None: + for match in STATUS_CLAIM_RE.finditer(text): + claim = match.group(1).strip() + # Judge any span that reads as a status claim — including near misses + # such as `MOSTLY IMPLEMENTED` — while ignoring caps acronyms. + if not any(word in claim for word in ("IMPLEMENTED", "SCOPE")): + continue + if claim not in STATUS_VOCABULARY: + findings.add(path, f"unknown status value `{claim}`") + + +def check_links(path: Path, text: str, findings: Findings) -> None: + for match in MARKDOWN_LINK_RE.finditer(text): + target = match.group(1).strip() + if target.startswith(("http://", "https://", "mailto:")): + continue + anchor = "" + if "#" in target: + target, _, anchor = target.partition("#") + if not target: + resolved = path + else: + resolved = (path.parent / target).resolve() + if not resolved.exists(): + findings.add(path, f"broken link target {target}") + continue + if ( + anchor + and resolved.suffix == ".md" + and anchor not in _headings(resolved.read_text(encoding="utf-8")) + ): + findings.add(path, f"broken link anchor #{anchor} in {target or path.name}") + + +def check_repository_references(path: Path, text: str, findings: Findings) -> None: + targets = _make_targets() + for match in MAKE_TARGET_RE.finditer(text): + target = match.group(1).removeprefix("make ") + if target not in targets: + findings.add(path, f"`make {target}` is not a Makefile target") + + for match in REPO_PATH_RE.finditer(text): + reference = match.group(1) + candidate, _, symbol = reference.partition("::") + # Trailing punctuation and pytest node ids are not part of the path. + candidate = LINE_SUFFIX_RE.sub("", candidate.rstrip(".,;:")) + # `python/reflex-xy[dev]` is an install target, not a path on disk. + candidate = re.sub(r"\[[^\]]*\]$", "", candidate) + if candidate in GENERATED_PATHS: + workflows = "\n".join( + workflow.read_text(encoding="utf-8") + for workflow in sorted(WORKFLOW_DIR.glob("*.yml")) + ) + if candidate not in workflows: + findings.add(path, f"generated path {candidate} has no workflow producer") + continue + if "*" in candidate: + if not list(ROOT.glob(candidate)): + findings.add(path, f"referenced path glob {candidate} matches nothing") + continue + resolved = ROOT / candidate + if not resolved.exists(): + findings.add(path, f"referenced path {candidate} does not exist") + continue + if symbol and resolved.suffix == ".py": + symbol = re.sub(r"\[.*\]$", "", symbol).strip(":") + if symbol and symbol not in _python_symbols(resolved): + findings.add(path, f"referenced Python symbol {candidate}::{symbol} does not exist") + + for match in WORKFLOW_FILE_RE.finditer(text): + workflow = match.group(1) + if not (WORKFLOW_DIR / workflow).is_file(): + findings.add(path, f"referenced workflow {workflow} does not exist") + + +def _python_symbols(path: Path) -> set[str]: + """Return top-level and class-qualified symbols addressable as pytest node IDs.""" + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except (OSError, SyntaxError): + return set() + symbols: set[str] = set() + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + symbols.add(node.name) + if isinstance(node, ast.ClassDef): + for child in node.body: + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbols.add(f"{node.name}::{child.name}") + return symbols + + +def check_evidence_rows(path: Path, text: str, findings: Findings) -> None: + """Require complete, vocabulary-backed rows in current.md evidence tables.""" + in_evidence_table = False + for line in text.splitlines(): + if line.startswith("| Surface |") and "| Status |" in line: + in_evidence_table = True + continue + if not in_evidence_table: + continue + if not line.startswith("|"): + in_evidence_table = False + continue + cells = [cell.strip() for cell in line.strip("|").split("|")] + if cells and all(set(cell) <= {"-", ":"} for cell in cells): + continue + if len(cells) != 5: + findings.add(path, f"evidence row must have five cells, got {len(cells)}: {line}") + continue + surface, evidence, enforcement, status_cell, boundary = cells + if not all((surface, evidence, enforcement, boundary)): + findings.add(path, f"evidence row has an empty required cell: {line}") + status_match = re.fullmatch(r"`([^`]+)`", status_cell) + if status_match is None or status_match.group(1) not in STATUS_VOCABULARY: + findings.add(path, f"evidence row has invalid status cell {status_cell!r}") + + +def check_workflow_registry(path: Path, text: str, findings: Findings) -> None: + """Validate the `| workflow.yml | job, job | role |` rows in current.md.""" + for line in text.splitlines(): + if not line.startswith("|"): + continue + cells = [cell.strip() for cell in line.strip("|").split("|")] + if len(cells) < 2: + continue + workflow_match = re.fullmatch(r"`(_?[a-z0-9-]+\.yml)`", cells[0]) + if not workflow_match: + continue + workflow_path = WORKFLOW_DIR / workflow_match.group(1) + if not workflow_path.is_file(): + continue # Reported by check_repository_references. + actual = _workflow_jobs(workflow_path) + claimed = {match.strip("`") for match in re.findall(r"`([A-Za-z0-9_-]+)`", cells[1])} + for job in sorted(claimed - actual): + findings.add(path, f"{workflow_match.group(1)} has no job `{job}`") + + +def check_gap_register(findings: Findings) -> None: + gaps_path = TESTING_DIR / "gaps.md" + if not gaps_path.is_file(): + findings.add(TESTING_DIR, "gaps.md is missing") + return + gaps_text = gaps_path.read_text(encoding="utf-8") + + defined: list[str] = [] + for line in gaps_text.splitlines(): + match = GAP_HEADING_RE.match(line) + if match: + defined.append(match.group(1)) + + seen: set[str] = set() + for gap_id in defined: + if gap_id in seen: + findings.add(gaps_path, f"duplicate definition of {gap_id}") + seen.add(gap_id) + + for index, gap_id in enumerate(defined, 1): + expected = f"TST-NI-{index:03d}" + if gap_id != expected: + findings.add( + gaps_path, f"gap IDs are not sequential: expected {expected}, got {gap_id}" + ) + break + + # Stable gap IDs stay in this register after completion. Completed entries + # need explicit executable evidence; incomplete entries stay NOT IMPLEMENTED. + p0_ids = _p0_gap_ids(gaps_text) + for gap_id, body in _gap_bodies(gaps_text).items(): + statuses = GAP_STATUS_RE.findall(body) + if len(statuses) != 1: + findings.add(gaps_path, f"{gap_id} must declare exactly one status") + status = "" + else: + status = statuses[0] + if status not in {"IMPLEMENTED", "NOT IMPLEMENTED"}: + findings.add( + gaps_path, + f"{gap_id} status must be `IMPLEMENTED` or `NOT IMPLEMENTED`, got `{status}`", + ) + if status == "IMPLEMENTED": + evidence = re.search(r"^- Evidence:\s+(.+)$", body, re.M) + if evidence is None or not evidence.group(1).strip(): + findings.add(gaps_path, f"{gap_id} is implemented but has no explicit evidence") + if "- Implemented when:" not in body: + findings.add(gaps_path, f"{gap_id} has no completion criteria") + if gap_id in p0_ids and "- Owner:" not in body: + findings.add(gaps_path, f"{gap_id} is P0 and must name an owner") + + referenced: dict[str, set[Path]] = {} + for path in sorted(SPEC_DIR.rglob("*.md")): + if path == gaps_path or not path.is_file(): + continue + for match in GAP_ID_RE.finditer(path.read_text(encoding="utf-8")): + referenced.setdefault(match.group(0), set()).add(path) + + for gap_id, sources in sorted(referenced.items()): + if gap_id not in seen: + for source in sorted(sources): + findings.add(source, f"references undefined gap {gap_id}") + + current_path = TESTING_DIR / "current.md" + current_text = current_path.read_text(encoding="utf-8") if current_path.is_file() else "" + current_refs = {match.group(0) for match in GAP_ID_RE.finditer(current_text)} + for gap_id in sorted(seen - current_refs): + findings.add( + gaps_path, + f"{gap_id} is unreachable from current.md; add the surface row that motivates it", + ) + + +def _p0_gap_ids(text: str) -> set[str]: + """Gap IDs defined under the P0 section heading.""" + match = re.search(r"^## P0[^\n]*\n(.*?)(?=^## |\Z)", text, re.M | re.S) + if not match: + return set() + return {found.group(0) for found in GAP_ID_RE.finditer(match.group(1))} + + +def _gap_bodies(text: str) -> dict[str, str]: + bodies: dict[str, str] = {} + current: str | None = None + lines: list[str] = [] + for line in text.splitlines(): + match = GAP_HEADING_RE.match(line) + if match: + if current: + bodies[current] = "\n".join(lines) + current = match.group(1) + lines = [] + elif current: + if line.startswith("## "): + bodies[current] = "\n".join(lines) + current = None + else: + lines.append(line) + if current: + bodies[current] = "\n".join(lines) + return bodies + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args(argv) + + if not SPEC_DIR.is_dir(): + print(f"error: {SPEC_DIR} is missing", file=sys.stderr) + return 2 + + findings = Findings() + documents = sorted(SPEC_DIR.rglob("*.md")) + for path in documents: + raw_text = path.read_text(encoding="utf-8") + prose = _strip_code_blocks(raw_text) + check_status_vocabulary(path, prose, findings) + check_links(path, prose, findings) + check_repository_references(path, raw_text, findings) + if path == TESTING_DIR / "current.md": + check_workflow_registry(path, raw_text, findings) + check_evidence_rows(path, raw_text, findings) + + check_gap_register(findings) + + if not findings.ok(): + print("specification contract check failed:", file=sys.stderr) + for error in findings.errors: + print(f" {error}", file=sys.stderr) + return 1 + + print(f"specification contract OK ({len(documents)} documents)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/coverage_ratchet.py b/scripts/coverage_ratchet.py new file mode 100644 index 00000000..07760e5d --- /dev/null +++ b/scripts/coverage_ratchet.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""Enforce reviewed Python package, module, and changed-line coverage floors.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections.abc import Mapping +from fnmatch import fnmatch +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +POLICY = ROOT / "spec" / "testing" / "coverage-policy.json" +_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +def _percent(covered: int, total: int) -> float: + return 100.0 if total == 0 else 100.0 * covered / total + + +def _number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def load_policy(path: Path = POLICY) -> tuple[dict[str, Any], list[str]]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {}, [f"cannot read coverage policy: {exc}"] + errors: list[str] = [] + expected = { + "schema_version", + "coverage_format", + "source_roots", + "packages", + "modules", + "diff", + "exclusions", + } + if not isinstance(data, dict) or set(data) != expected: + return {}, [f"coverage policy must contain exactly {sorted(expected)}"] + if data["schema_version"] != 1 or data["coverage_format"] != "coverage.py-json-v3": + errors.append("coverage policy must use schema 1 and coverage.py-json-v3") + roots = data["source_roots"] + if ( + not isinstance(roots, list) + or not roots + or not all(isinstance(item, str) and item and not item.startswith("/") for item in roots) + ): + errors.append("source_roots must be a nonempty list of relative paths") + packages = data["packages"] + if not isinstance(packages, list) or not packages: + errors.append("packages must be a nonempty list") + else: + names: set[str] = set() + for index, package in enumerate(packages): + fields = { + "name", + "path_prefix", + "exclude_prefixes", + "minimum_line_percent", + "minimum_branch_percent", + } + if not isinstance(package, dict) or set(package) != fields: + errors.append(f"packages[{index}] has invalid fields") + continue + if not isinstance(package["name"], str) or not package["name"]: + errors.append(f"packages[{index}] needs a name") + elif package["name"] in names: + errors.append(f"duplicate package name {package['name']!r}") + else: + names.add(package["name"]) + if not isinstance(package["path_prefix"], str) or not package["path_prefix"].endswith( + "/" + ): + errors.append(f"packages[{index}] path_prefix must end with /") + excluded = package["exclude_prefixes"] + if not isinstance(excluded, list) or not all( + isinstance(item, str) and item.endswith("/") for item in excluded + ): + errors.append(f"packages[{index}] exclude_prefixes must be path prefixes") + for field in ("minimum_line_percent", "minimum_branch_percent"): + if not _number(package[field]) or not 0 <= package[field] <= 100: + errors.append(f"packages[{index}] {field} must be in [0, 100]") + modules = data["modules"] + if not isinstance(modules, list) or not modules: + errors.append("modules must be a nonempty list") + else: + paths: set[str] = set() + for index, module in enumerate(modules): + fields = {"path", "minimum_line_percent", "minimum_branch_percent"} + if not isinstance(module, dict) or set(module) != fields: + errors.append(f"modules[{index}] has invalid fields") + continue + if not isinstance(module["path"], str) or not module["path"].endswith(".py"): + errors.append(f"modules[{index}] path must name a Python file") + elif module["path"] in paths: + errors.append(f"duplicate module path {module['path']!r}") + else: + paths.add(module["path"]) + for field in ("minimum_line_percent", "minimum_branch_percent"): + if not _number(module[field]) or not 0 <= module[field] <= 100: + errors.append(f"modules[{index}] {field} must be in [0, 100]") + diff = data["diff"] + if not isinstance(diff, dict) or set(diff) != { + "minimum_line_percent", + "missing_coverage_file", + }: + errors.append("diff must declare minimum_line_percent and missing_coverage_file") + elif ( + not _number(diff["minimum_line_percent"]) + or not 0 <= diff["minimum_line_percent"] <= 100 + or diff["missing_coverage_file"] != "fail" + ): + errors.append("diff coverage must use a [0, 100] floor and fail on missing files") + exclusions = data["exclusions"] + if not isinstance(exclusions, list): + errors.append("exclusions must be a list") + else: + for index, exclusion in enumerate(exclusions): + if not isinstance(exclusion, dict) or set(exclusion) != {"pattern", "rationale"}: + errors.append(f"exclusions[{index}] has invalid fields") + continue + if not isinstance(exclusion["pattern"], str) or not exclusion["pattern"]: + errors.append(f"exclusions[{index}] needs a pattern") + if ( + not isinstance(exclusion["rationale"], str) + or len(exclusion["rationale"].strip()) < 20 + ): + errors.append(f"exclusions[{index}] needs a substantive rationale") + return data, errors + + +def load_coverage(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {}, [f"cannot read coverage JSON: {exc}"] + errors: list[str] = [] + if not isinstance(data, dict) or not isinstance(data.get("meta"), dict): + return {}, ["coverage JSON must contain meta and files objects"] + meta = data["meta"] + if meta.get("format") != 3: + errors.append("coverage JSON format must be 3") + if meta.get("branch_coverage") is not True: + errors.append("coverage JSON must be generated with branch coverage enabled") + if not isinstance(data.get("files"), dict): + errors.append("coverage JSON files must be an object") + return data, errors + + +def parse_changed_lines(diff: str) -> dict[str, set[int]]: + """Return added line numbers from a zero-context git diff.""" + result: dict[str, set[int]] = {} + current: str | None = None + for line in diff.splitlines(): + if line.startswith("+++ "): + target = line[4:].split("\t", 1)[0] + current = None if target == "/dev/null" else target.removeprefix("b/") + continue + match = _HUNK.match(line) + if current is None or match is None: + continue + start = int(match.group(1)) + count = 1 if match.group(2) is None else int(match.group(2)) + result.setdefault(current, set()).update(range(start, start + count)) + return result + + +def git_changed_lines( + base: str, head: str, source_roots: list[str], *, project_root: Path = ROOT +) -> tuple[dict[str, set[int]], list[str]]: + command = [ + "git", + "diff", + "--unified=0", + "--no-ext-diff", + "--no-renames", + base, + head, + "--", + *source_roots, + ] + completed = subprocess.run( + command, cwd=project_root, text=True, capture_output=True, check=False + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() or "unknown git error" + return {}, [f"cannot compute changed lines for {base}..{head}: {detail}"] + return parse_changed_lines(completed.stdout), [] + + +def _source_files(policy: Mapping[str, Any], project_root: Path) -> set[str]: + files: set[str] = set() + for root in policy["source_roots"]: + path = project_root / root + if path.is_dir(): + files.update( + candidate.relative_to(project_root).as_posix() + for candidate in path.rglob("*.py") + if "__pycache__" not in candidate.parts + ) + return files + + +def _summary(files: Mapping[str, Any], paths: list[str]) -> dict[str, float | int]: + statements = sum(int(files[path]["summary"]["num_statements"]) for path in paths) + covered_lines = sum(int(files[path]["summary"]["covered_lines"]) for path in paths) + branches = sum(int(files[path]["summary"]["num_branches"]) for path in paths) + covered_branches = sum(int(files[path]["summary"]["covered_branches"]) for path in paths) + return { + "file_count": len(paths), + "covered_lines": covered_lines, + "statements": statements, + "line_percent": _percent(covered_lines, statements), + "covered_branches": covered_branches, + "branches": branches, + "branch_percent": _percent(covered_branches, branches), + } + + +def _meets( + errors: list[str], label: str, actual: Mapping[str, float | int], floor: Mapping[str, Any] +) -> None: + for metric, key in (("line", "minimum_line_percent"), ("branch", "minimum_branch_percent")): + value = float(actual[f"{metric}_percent"]) + minimum = float(floor[key]) + if value + 1e-9 < minimum: + errors.append(f"{label} {metric} coverage {value:.2f}% is below {minimum:.2f}%") + + +def evaluate( + coverage: Mapping[str, Any], + policy: Mapping[str, Any], + changed: Mapping[str, set[int]], + *, + project_root: Path = ROOT, +) -> tuple[dict[str, Any], list[str]]: + errors: list[str] = [] + raw_files = coverage.get("files", {}) + files = {str(path).removeprefix("./"): value for path, value in raw_files.items()} + sources = _source_files(policy, project_root) + exclusions = policy["exclusions"] + + def excluded(path: str) -> bool: + return any(fnmatch(path, item["pattern"]) for item in exclusions) + + governed_sources = {path for path in sources if not excluded(path)} + missing = sorted(governed_sources - files.keys()) + if missing: + errors.append(f"coverage JSON is missing shipped Python files: {missing}") + + memberships: dict[str, list[str]] = {} + for path in sorted(governed_sources): + memberships[path] = [ + package["name"] + for package in policy["packages"] + if path.startswith(package["path_prefix"]) + and not any(path.startswith(prefix) for prefix in package["exclude_prefixes"]) + ] + ambiguous = {path: names for path, names in memberships.items() if len(names) != 1} + if ambiguous: + errors.append(f"every shipped Python file must map to exactly one package: {ambiguous}") + + package_reports: dict[str, Any] = {} + for package in policy["packages"]: + paths = sorted( + path + for path, names in memberships.items() + if names == [package["name"]] and path in files + ) + if not paths: + errors.append(f"package {package['name']!r} has no covered files") + continue + report = _summary(files, paths) + report.update( + minimum_line_percent=package["minimum_line_percent"], + minimum_branch_percent=package["minimum_branch_percent"], + ) + package_reports[package["name"]] = report + _meets(errors, f"package {package['name']}", report, package) + + module_reports: dict[str, Any] = {} + for module in policy["modules"]: + path = module["path"] + if path not in governed_sources: + errors.append(f"reviewed module is absent from shipped sources: {path}") + continue + if path not in files: + continue + report = _summary(files, [path]) + report.update( + minimum_line_percent=module["minimum_line_percent"], + minimum_branch_percent=module["minimum_branch_percent"], + ) + module_reports[path] = report + _meets(errors, f"module {path}", report, module) + + diff_files: dict[str, Any] = {} + changed_executable: set[tuple[str, int]] = set() + covered_executable: set[tuple[str, int]] = set() + for path, lines in sorted(changed.items()): + if excluded(path) or path not in governed_sources: + continue + if path not in files: + errors.append(f"changed shipped Python file has no coverage data: {path}") + continue + file_data = files[path] + statements = set(file_data.get("executed_lines", ())) | set( + file_data.get("missing_lines", ()) + ) + executable = set(lines) & statements + covered_lines = executable & set(file_data.get("executed_lines", ())) + changed_executable.update((path, line) for line in executable) + covered_executable.update((path, line) for line in covered_lines) + if executable: + diff_files[path] = { + "executable_lines": sorted(executable), + "covered_lines": sorted(covered_lines), + "missing_lines": sorted(executable - covered_lines), + } + diff_percent = _percent(len(covered_executable), len(changed_executable)) + diff_floor = float(policy["diff"]["minimum_line_percent"]) + if changed_executable and diff_percent + 1e-9 < diff_floor: + errors.append(f"changed-line coverage {diff_percent:.2f}% is below {diff_floor:.2f}%") + + report = { + "schema_version": 1, + "status": "failed" if errors else "passed", + "packages": package_reports, + "modules": module_reports, + "diff": { + "minimum_line_percent": diff_floor, + "executable_line_count": len(changed_executable), + "covered_line_count": len(covered_executable), + "line_percent": diff_percent, + "files": diff_files, + }, + "exclusions": exclusions, + "errors": errors, + } + return report, errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--coverage-json", type=Path, required=True) + parser.add_argument("--base", required=True, help="base Git commit for diff coverage") + parser.add_argument("--head", required=True, help="head Git commit for diff coverage") + parser.add_argument("--policy", type=Path, default=POLICY) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args(argv) + + policy, errors = load_policy(args.policy) + coverage, coverage_errors = load_coverage(args.coverage_json) + errors.extend(coverage_errors) + changed: dict[str, set[int]] = {} + if policy: + changed, git_errors = git_changed_lines(args.base, args.head, policy["source_roots"]) + errors.extend(git_errors) + report: dict[str, Any] = { + "schema_version": 1, + "status": "failed", + "base": args.base, + "head": args.head, + "errors": errors, + } + if not errors: + report, errors = evaluate(coverage, policy, changed) + report.update(base=args.base, head=args.head) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if errors: + print("coverage ratchet failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print( + "coverage ratchet OK: " + f"{report['diff']['line_percent']:.2f}% of " + f"{report['diff']['executable_line_count']} changed executable lines" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dependency_audit.py b/scripts/dependency_audit.py new file mode 100644 index 00000000..c00c5c44 --- /dev/null +++ b/scripts/dependency_audit.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +"""Enforce the repository's multi-ecosystem vulnerability policy. + +The policy and runner are stdlib-only. CI can therefore validate its complete +lock inventory, verify the scanner binary, and produce retained evidence +without first installing any project environment that it is meant to audit. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import subprocess +import sys +import zipfile +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_POLICY = ROOT / "spec" / "testing" / "dependency-audit-policy.json" + +SEVERITIES = {"CRITICAL", "HIGH", "MODERATE", "LOW", "UNKNOWN"} +MAX_EXCEPTION_DAYS = 180 +LOCK_NAMES = {"Cargo.lock", "bun.lock", "package-lock.json", "requirements-ci.lock", "uv.lock"} +DISCOVERY_IGNORES = { + ".git", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".tox", + ".venv", + "dist", + "node_modules", + "target", + "venv", +} + +REQUIRED_ENVIRONMENTS = [ + {"subsystem": "root-python", "path": "uv.lock", "format": "uv.lock"}, + {"subsystem": "docs-python", "path": "docs/app/uv.lock", "format": "uv.lock"}, + { + "subsystem": "docs-javascript", + "path": "docs/app/reflex.lock/bun.lock", + "format": "bun.lock", + }, + { + "subsystem": "reflex-adapter-python", + "path": "python/reflex-xy/uv.lock", + "format": "uv.lock", + }, + { + "subsystem": "benchmark-python", + "path": "benchmarks/requirements-ci.lock", + "format": "requirements.txt", + }, + {"subsystem": "native-rust", "path": "Cargo.lock", "format": "Cargo.lock"}, + { + "subsystem": "browser-javascript", + "path": "package-lock.json", + "format": "package-lock.json", + }, +] +REQUIRED_EXCLUDED_LOCKS = [ + { + "path": "benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/uv.lock", + "owner": "@reflex-dev/xy", + "reason": ( + "Immutable historical launch-measurement evidence; it is never installed or " + "executed by a current subsystem." + ), + }, + { + "path": "examples/reflex/.web/bun.lock", + "owner": "@reflex-dev/xy", + "reason": "Example fixture lock is not an installed production subsystem.", + }, + { + "path": "examples/reflex/reflex.lock/bun.lock", + "owner": "@reflex-dev/xy", + "reason": "Example fixture lock is not an installed production subsystem.", + }, +] +TRUSTED_SCANNER = { + "name": "osv-scanner", + "version": "2.4.0", + "artifacts": { + "darwin-arm64": { + "url": ( + "https://github.com/google/osv-scanner/releases/download/v2.4.0/" + "osv-scanner_darwin_arm64" + ), + "sha256": "9ca3185ad63e9ab54f7cb90f46a7362be02d80e37f0123d095a54355ea202f5d", + }, + "linux-x86_64": { + "url": ( + "https://github.com/google/osv-scanner/releases/download/v2.4.0/" + "osv-scanner_linux_amd64" + ), + "sha256": "15314940c10d26af9c6649f150b8a47c1262e8fc7e17b1d1029b0e479e8ed8a0", + }, + }, +} + + +class AuditError(RuntimeError): + """Raised when policy, scanner, or report evidence is incomplete.""" + + +def _read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise AuditError(f"cannot read JSON from {path}: {exc}") from exc + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _require_exact_keys(value: Any, expected: set[str], label: str) -> list[str]: + if not isinstance(value, dict): + return [f"{label} must be an object"] + actual = set(value) + if actual == expected: + return [] + return [f"{label} keys must be exactly {sorted(expected)}; got {sorted(actual)}"] + + +def _discover_locks(root: Path) -> set[str]: + discovered: set[str] = set() + for path in root.rglob("*"): + if not path.is_file() or path.name not in LOCK_NAMES: + continue + relative = path.relative_to(root) + if any(part in DISCOVERY_IGNORES for part in relative.parts[:-1]): + continue + discovered.add(relative.as_posix()) + return discovered + + +def _validate_exception( + item: Any, index: int, *, today: date +) -> tuple[list[str], tuple[str, str, str, str] | None]: + label = f"exceptions[{index}]" + errors = _require_exact_keys( + item, {"id", "package", "environment", "owner", "reason", "expires"}, label + ) + if errors: + return errors, None + + assert isinstance(item, dict) + errors.extend(_require_exact_keys(item["package"], {"ecosystem", "name"}, f"{label}.package")) + if errors: + return errors, None + package = item["package"] + assert isinstance(package, dict) + + finding_id = item["id"] + ecosystem = package["ecosystem"] + package_name = package["name"] + environment = item["environment"] + for value, field in ( + (finding_id, "id"), + (ecosystem, "package.ecosystem"), + (package_name, "package.name"), + ): + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+:/-]*", value): + errors.append(f"{label}.{field} must be an exact non-wildcard identifier") + valid_environments = {item["subsystem"] for item in REQUIRED_ENVIRONMENTS} + if environment not in valid_environments: + errors.append(f"{label}.environment must identify one of {sorted(valid_environments)}") + + owner = item["owner"] + if not isinstance(owner, str) or not re.fullmatch( + r"(?:@[A-Za-z0-9][A-Za-z0-9_.-]*(?:/[A-Za-z0-9][A-Za-z0-9_.-]*)?" + r"|[^@\s]+@[^@\s]+\.[^@\s]+)", + owner, + ): + errors.append(f"{label}.owner must be a GitHub user/team or an email address") + + reason = item["reason"] + if not isinstance(reason, str) or len(reason.strip()) < 20: + errors.append(f"{label}.reason must contain at least 20 non-padding characters") + + expires = item["expires"] + expiry: date | None = None + if not isinstance(expires, str): + errors.append(f"{label}.expires must be an ISO 8601 date") + else: + try: + expiry = date.fromisoformat(expires) + except ValueError: + errors.append(f"{label}.expires must be an ISO 8601 date") + if expiry is not None: + remaining = (expiry - today).days + if remaining <= 0: + errors.append(f"{label} expired on {expiry.isoformat()}") + elif remaining > MAX_EXCEPTION_DAYS: + errors.append( + f"{label}.expires must be no more than {MAX_EXCEPTION_DAYS} days from review" + ) + + if errors: + return errors, None + assert isinstance(finding_id, str) + assert isinstance(ecosystem, str) + assert isinstance(package_name, str) + assert isinstance(environment, str) + return [], (finding_id, ecosystem, package_name, environment) + + +def validate_policy( + policy_path: Path = DEFAULT_POLICY, + *, + root: Path = ROOT, + today: date | None = None, +) -> dict[str, Any]: + """Return a validated policy or raise with every detected defect.""" + policy = _read_json(policy_path) + errors = _require_exact_keys( + policy, + { + "schema_version", + "scanner", + "fail_severities", + "environments", + "excluded_locks", + "exceptions", + }, + "policy", + ) + if errors: + raise AuditError("; ".join(errors)) + assert isinstance(policy, dict) + + if policy["schema_version"] != 1: + errors.append("schema_version must be 1") + if policy["scanner"] != TRUSTED_SCANNER: + errors.append("scanner must exactly match the reviewed version, URLs, and SHA-256 pins") + fail_severities = policy["fail_severities"] + if ( + not isinstance(fail_severities, list) + or not all(isinstance(item, str) for item in fail_severities) + or set(fail_severities) != SEVERITIES + ): + errors.append(f"fail_severities must contain every reviewed class: {sorted(SEVERITIES)}") + elif len(fail_severities) != len(SEVERITIES): + errors.append("fail_severities must not contain duplicate classes") + if policy["environments"] != REQUIRED_ENVIRONMENTS: + errors.append("environments must exactly inventory every required subsystem lock") + if policy["excluded_locks"] != REQUIRED_EXCLUDED_LOCKS: + errors.append("excluded_locks must exactly identify the reviewed historical fixture") + + expected_locks = {item["path"] for item in REQUIRED_ENVIRONMENTS + REQUIRED_EXCLUDED_LOCKS} + discovered_locks = _discover_locks(root) + missing = sorted(expected_locks - discovered_locks) + extra = sorted(discovered_locks - expected_locks) + if missing: + errors.append(f"inventoried locks do not exist: {missing}") + if extra: + errors.append(f"unreviewed committed-style locks are not inventoried: {extra}") + for environment in REQUIRED_ENVIRONMENTS: + lock = root / environment["path"] + if lock.exists() and lock.stat().st_size == 0: + errors.append(f"environment lock is empty: {environment['path']}") + + exceptions = policy["exceptions"] + if not isinstance(exceptions, list): + errors.append("exceptions must be a list") + else: + keys: set[tuple[str, str, str, str]] = set() + review_date = today or datetime.now(UTC).date() + for index, item in enumerate(exceptions): + item_errors, key = _validate_exception(item, index, today=review_date) + errors.extend(item_errors) + if key is not None and key in keys: + errors.append(f"exceptions[{index}] duplicates {key!r}") + elif key is not None: + keys.add(key) + + if errors: + raise AuditError("; ".join(errors)) + return policy + + +def _platform_key() -> str: + os_name = ( + "darwin" + if sys.platform == "darwin" + else "linux" + if sys.platform.startswith("linux") + else "" + ) + machine = platform.machine().lower() + arch = ( + "x86_64" + if machine in {"amd64", "x86_64"} + else "arm64" + if machine in {"aarch64", "arm64"} + else "" + ) + key = f"{os_name}-{arch}" + if not os_name or not arch: + raise AuditError(f"unsupported scanner host: {sys.platform}/{platform.machine()}") + return key + + +def _scanner_metadata(scanner: Path, policy: dict[str, Any], output_dir: Path) -> dict[str, str]: + platform_key = _platform_key() + artifact = policy["scanner"]["artifacts"].get(platform_key) + if artifact is None: + raise AuditError(f"policy has no reviewed scanner artifact for {platform_key}") + if not scanner.is_file(): + raise AuditError(f"scanner executable does not exist: {scanner}") + actual_hash = _sha256(scanner) + if actual_hash != artifact["sha256"]: + raise AuditError( + f"scanner SHA-256 mismatch for {platform_key}: got {actual_hash}, " + f"expected {artifact['sha256']}" + ) + + completed = subprocess.run( + [str(scanner), "--version"], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + version_text = (completed.stdout + completed.stderr).strip() + (output_dir / "scanner-version.txt").write_text(version_text + "\n", encoding="utf-8") + if completed.returncode: + raise AuditError(f"scanner --version exited {completed.returncode}: {version_text}") + + fields: dict[str, str] = {} + for line in version_text.splitlines(): + if ": " in line: + key, value = line.split(": ", 1) + fields[key.strip()] = value.strip() + version = fields.get("osv-scanner version") + commit = fields.get("commit", "") + built_at = fields.get("built at", "") + if version != policy["scanner"]["version"]: + raise AuditError( + f"scanner version mismatch: got {version!r}, expected {policy['scanner']['version']!r}" + ) + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise AuditError("scanner version output is missing its exact commit") + try: + built_timestamp = datetime.fromisoformat(built_at.replace("Z", "+00:00")) + except ValueError as exc: + raise AuditError("scanner version output is missing its build timestamp") from exc + if built_timestamp.utcoffset() is None: + raise AuditError("scanner build timestamp must include its UTC offset") + return { + "name": policy["scanner"]["name"], + "version": version, + "scalibr_version": fields.get("osv-scalibr version", ""), + "commit": commit, + "built_at": built_at, + "platform": platform_key, + "sha256": actual_hash, + } + + +def _severity_label(vulnerability: dict[str, Any]) -> str: + database = vulnerability.get("database_specific") + candidates: list[Any] = [] + if isinstance(database, dict): + candidates.extend((database.get("severity"), database.get("cvss_score"))) + severity = vulnerability.get("severity") + if isinstance(severity, list): + candidates.extend(item.get("score") for item in severity if isinstance(item, dict)) + for candidate in candidates: + if not isinstance(candidate, (str, int, float)): + continue + normalized = str(candidate).strip().upper() + if normalized == "MEDIUM": + return "MODERATE" + if normalized in SEVERITIES: + return normalized + try: + score = float(normalized) + except ValueError: + continue + if score >= 9: + return "CRITICAL" + if score >= 7: + return "HIGH" + if score >= 4: + return "MODERATE" + if score > 0: + return "LOW" + return "UNKNOWN" + + +def evaluate_report( + raw_report: dict[str, Any], policy: dict[str, Any], *, root: Path = ROOT +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + """Validate scanner coverage and return environments, findings, unused exceptions.""" + results = raw_report.get("results") + if not isinstance(results, list): + raise AuditError("raw OSV report is missing its results list") + + environment_by_path = {(root / item["path"]).resolve(): item for item in policy["environments"]} + seen: set[Path] = set() + environment_reports: list[dict[str, Any]] = [] + findings: list[dict[str, Any]] = [] + exception_by_key = { + ( + item["id"], + item["package"]["ecosystem"], + item["package"]["name"], + item["environment"], + ): item + for item in policy["exceptions"] + } + used_exceptions: set[tuple[str, str, str, str]] = set() + + for index, result in enumerate(results): + if not isinstance(result, dict): + raise AuditError(f"raw OSV result {index} must be an object") + source = result.get("source") + if not isinstance(source, dict) or source.get("type") != "lockfile": + raise AuditError(f"raw OSV result {index} is not sourced from a lockfile") + source_path = source.get("path") + if not isinstance(source_path, str): + raise AuditError(f"raw OSV result {index} has no source path") + source_candidate = Path(source_path) + resolved = ( + source_candidate if source_candidate.is_absolute() else root / source_candidate + ).resolve() + environment = environment_by_path.get(resolved) + if environment is None: + raise AuditError(f"raw OSV report contains an unrequested source: {source_path}") + if resolved in seen: + raise AuditError(f"raw OSV report repeats source: {environment['path']}") + seen.add(resolved) + + packages = result.get("packages") + if not isinstance(packages, list) or not packages: + raise AuditError(f"scanner found no packages in {environment['path']}") + environment_reports.append( + { + **environment, + "package_count": len(packages), + } + ) + for package_result in packages: + if not isinstance(package_result, dict): + raise AuditError(f"invalid package entry in {environment['path']}") + package = package_result.get("package") + if not isinstance(package, dict): + raise AuditError(f"package metadata missing in {environment['path']}") + name = package.get("name") + version = package.get("version") + ecosystem = package.get("ecosystem") + if not all(isinstance(value, str) and value for value in (name, version, ecosystem)): + raise AuditError(f"incomplete package identity in {environment['path']}") + vulnerabilities = package_result.get("vulnerabilities", []) + if not isinstance(vulnerabilities, list): + raise AuditError(f"invalid vulnerabilities list for {ecosystem}/{name}") + for vulnerability in vulnerabilities: + if not isinstance(vulnerability, dict) or not isinstance( + vulnerability.get("id"), str + ): + raise AuditError(f"vulnerability without an ID for {ecosystem}/{name}") + finding_id = vulnerability["id"] + key = (finding_id, ecosystem, name, environment["subsystem"]) + exception = exception_by_key.get(key) + if exception is not None: + used_exceptions.add(key) + aliases = vulnerability.get("aliases", []) + findings.append( + { + "id": finding_id, + "aliases": aliases if isinstance(aliases, list) else [], + "summary": vulnerability.get("summary", ""), + "published": vulnerability.get("published"), + "modified": vulnerability.get("modified"), + "severity": _severity_label(vulnerability), + "raw_severity": vulnerability.get("severity", []), + "package": { + "ecosystem": ecosystem, + "name": name, + "version": version, + }, + "environment": environment["subsystem"], + "source": environment["path"], + "excepted": exception is not None, + "exception": exception, + } + ) + + missing = sorted( + environment["path"] for path, environment in environment_by_path.items() if path not in seen + ) + if missing: + raise AuditError(f"raw OSV report omitted required environments: {missing}") + + unused = [item for key, item in exception_by_key.items() if key not in used_exceptions] + return environment_reports, findings, unused + + +def _database_metadata(cache_dir: Path) -> list[dict[str, Any]]: + database_root = cache_dir / "osv-scanner" + archives = sorted(database_root.glob("*/all.zip")) + expected = {"PyPI", "crates.io", "npm"} + actual = {path.parent.name for path in archives} + if actual != expected: + raise AuditError( + f"scanner database snapshot must exactly cover {sorted(expected)}; got {sorted(actual)}" + ) + reports: list[dict[str, Any]] = [] + for archive in archives: + if archive.stat().st_size == 0: + raise AuditError(f"scanner database archive is empty: {archive}") + try: + with zipfile.ZipFile(archive) as database_zip: + entries = database_zip.infolist() + except (OSError, zipfile.BadZipFile) as exc: + raise AuditError(f"invalid scanner database archive {archive}: {exc}") from exc + if not entries: + raise AuditError(f"scanner database archive has no advisories: {archive}") + latest = max(entry.date_time for entry in entries) + reports.append( + { + "ecosystem": archive.parent.name, + "retrieved_at": datetime.fromtimestamp(archive.stat().st_mtime, tz=UTC).isoformat(), + "latest_archive_entry_at": datetime(*latest, tzinfo=UTC).isoformat(), + "archive_entry_count": len(entries), + "archive_size_bytes": archive.stat().st_size, + "archive_sha256": _sha256(archive), + } + ) + return reports + + +def _write_report(output: Path, report: dict[str, Any]) -> None: + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def run_scan( + scanner: Path, + output_dir: Path, + *, + policy_path: Path = DEFAULT_POLICY, + root: Path = ROOT, +) -> int: + policy = validate_policy(policy_path, root=root) + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + scanner = scanner.resolve() + scanner_metadata = _scanner_metadata(scanner, policy, output_dir) + raw_output = output_dir / "osv-results.json" + scanner_log = output_dir / "scanner-output.txt" + cache_dir = output_dir / "osv-databases" + started = datetime.now(UTC) + + command = [ + str(scanner), + "scan", + "source", + "--offline-vulnerabilities", + "--download-offline-databases", + "--format", + "json", + "--all-packages", + "--output-file", + str(raw_output), + ] + for environment in policy["environments"]: + command.extend(["--lockfile", f"{environment['format']}:{environment['path']}"]) + environment = os.environ.copy() + environment["OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY"] = str(cache_dir) + completed = subprocess.run( + command, + cwd=root, + env=environment, + text=True, + capture_output=True, + check=False, + ) + finished = datetime.now(UTC) + scanner_text = completed.stdout + completed.stderr + scanner_log.write_text(scanner_text, encoding="utf-8") + if scanner_text: + print(scanner_text, end="" if scanner_text.endswith("\n") else "\n") + if completed.returncode not in {0, 1}: + report = { + "schema_version": 1, + "started_at": started.isoformat(), + "finished_at": finished.isoformat(), + "scanner": scanner_metadata, + "scanner_exit_code": completed.returncode, + "outcome": "scanner-error", + } + _write_report(output_dir / "dependency-audit.json", report) + raise AuditError(f"OSV-Scanner exited unexpectedly with {completed.returncode}") + if not raw_output.is_file(): + raise AuditError("OSV-Scanner did not produce its machine-readable result") + + raw_report = _read_json(raw_output) + if not isinstance(raw_report, dict): + raise AuditError("raw OSV report must be an object") + environments, findings, unused_exceptions = evaluate_report(raw_report, policy, root=root) + databases = _database_metadata(cache_dir) + blocking = [ + item + for item in findings + if not item["excepted"] and item["severity"] in policy["fail_severities"] + ] + if completed.returncode == 1 and not findings: + raise AuditError("OSV-Scanner reported vulnerabilities but emitted no findings") + + outcome = "pass" if not blocking and not unused_exceptions else "fail" + try: + report_policy_path = policy_path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + report_policy_path = str(policy_path.resolve()) + report = { + "schema_version": 1, + "started_at": started.isoformat(), + "finished_at": finished.isoformat(), + "policy": { + "path": report_policy_path, + "sha256": _sha256(policy_path), + "fail_severities": policy["fail_severities"], + "exception_count": len(policy["exceptions"]), + }, + "scanner": scanner_metadata, + "databases": databases, + "environments": environments, + "findings": findings, + "unused_exceptions": unused_exceptions, + "summary": { + "environment_count": len(environments), + "package_count": sum(item["package_count"] for item in environments), + "finding_count": len(findings), + "excepted_finding_count": sum(item["excepted"] for item in findings), + "blocking_finding_count": len(blocking), + "unused_exception_count": len(unused_exceptions), + "scanner_exit_code": completed.returncode, + "outcome": outcome, + }, + } + _write_report(output_dir / "dependency-audit.json", report) + print( + "Dependency audit " + f"{outcome}: {report['summary']['package_count']} packages across " + f"{len(environments)} environments; {len(blocking)} blocking findings; " + f"{len(unused_exceptions)} unused exceptions." + ) + return 0 if outcome == "pass" else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("validate", help="validate policy and lock inventory") + scan = subparsers.add_parser("scan", help="run the pinned scanner and enforce policy") + scan.add_argument("--scanner", type=Path, required=True) + scan.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + try: + if args.command == "validate": + policy = validate_policy(args.policy) + print( + f"Dependency audit policy valid: {len(policy['environments'])} environments, " + f"{len(policy['exceptions'])} exceptions." + ) + return 0 + return run_scan(args.scanner, args.output_dir, policy_path=args.policy) + except AuditError as exc: + print(f"dependency audit error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fastapi_host_smoke.py b/scripts/fastapi_host_smoke.py new file mode 100644 index 00000000..fb20272d --- /dev/null +++ b/scripts/fastapi_host_smoke.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Focused FastAPI/Starlette browser mount and drilldown transport probe.""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from collections.abc import Iterator +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +FASTAPI_DIR = ROOT / "examples" / "fastapi" +sys.path.insert(0, str(ROOT / "scripts")) + +from _app_smoke import ChromiumSession, Probe, decode_png, find_chromium # noqa: E402 + +MIN_COLORED_PIXELS = 20 + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as stream: + stream.bind(("127.0.0.1", 0)) + return int(stream.getsockname()[1]) + + +@contextlib.contextmanager +def serve_current_environment(log_path: Path, *, points: int = 5000) -> Iterator[str]: + """Serve the example with this interpreter, preserving matrix versions.""" + port = _free_port() + env = dict(os.environ) + env["XY_LIVE_POINTS"] = str(points) + env["PYTHONPATH"] = str(FASTAPI_DIR) + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("wb") as log: + process = subprocess.Popen( + [ + sys.executable, + "-m", + "uvicorn", + "app:app", + "--host", + "127.0.0.1", + "--port", + str(port), + "--log-level", + "info", + ], + cwd=FASTAPI_DIR, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + ) + base_url = f"http://127.0.0.1:{port}" + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"uvicorn exited with status {process.returncode}") + try: + with urllib.request.urlopen(f"{base_url}/healthz", timeout=1) as response: + if response.status == 200: + break + except (urllib.error.URLError, ConnectionError, OSError): + time.sleep(0.2) + else: + raise RuntimeError("uvicorn did not become ready within 60 seconds") + yield base_url + finally: + process.terminate() + with contextlib.suppress(subprocess.TimeoutExpired): + process.wait(timeout=10) + if process.poll() is None: + process.kill() + process.wait() + + +def colored_pixels(png: bytes, rect: dict[str, float]) -> int: + width, height, channels, pixels = decode_png(png) + left = max(0, int(rect["x"])) + top = max(0, int(rect["y"])) + right = min(width, int(rect["x"] + rect["w"])) + bottom = min(height, int(rect["y"] + rect["h"])) + count = 0 + for y in range(top, bottom): + for x in range(left, right): + offset = (y * width + x) * channels + red, green, blue = pixels[offset : offset + 3] + alpha = pixels[offset + 3] if channels == 4 else 255 + if alpha > 8 and max(red, green, blue) - min(red, green, blue) > 24: + count += 1 + return count + + +def require_ink(count: int) -> None: + if count < MIN_COLORED_PIXELS: + raise AssertionError( + f"FastAPI browser mount is blank ({count} colored pixels; need {MIN_COLORED_PIXELS})" + ) + + +def _write_evidence(path: Path, evidence: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--chromium") + parser.add_argument("--no-sandbox", action="store_true") + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--screenshot", type=Path, required=True) + parser.add_argument("--server-log", type=Path, required=True) + args = parser.parse_args(argv) + + evidence: dict[str, object] = {"schema_version": 1, "status": "failed"} + probe: Probe | None = None + try: + chromium = find_chromium(args.chromium) + with ( + serve_current_environment(args.server_log) as base_url, + ChromiumSession( + chromium, + gl="software", + sandbox=not args.no_sandbox, + ) as session, + ): + probe = Probe(session, f"{base_url}/chart/line-walk", emulate=(900, 560, 1.0)) + mount = probe.wait_for( + "(() => { const c=document.querySelector('canvas');" + " if(!c || !c.width || !c.height) return null;" + " const gl=c.getContext('webgl2');" + " return gl ? {width:c.width,height:c.height} : null; })()", + timeout_s=45, + label="FastAPI WebGL2 chart mount", + ) + rect = probe.rect("canvas") + screenshot = probe.screenshot() + args.screenshot.parent.mkdir(parents=True, exist_ok=True) + args.screenshot.write_bytes(screenshot) + ink = colored_pixels(screenshot, rect) + require_ink(ink) + + # Standalone chart exports intentionally deny network access. Use + # the host-owned gallery shell for a separate browser-originated + # transport assertion after proving the static chart mount above. + probe.close() + probe = Probe(session, base_url, emulate=(900, 560, 1.0)) + probe.wait_for( + "document.readyState === 'complete' ? true : null", + timeout_s=30, + label="FastAPI gallery shell", + ) + endpoint = json.dumps(f"{base_url}/api/xy/drilldown") + transport = probe.eval( + f"(async () => {{ const response=await fetch({endpoint},{{" + "method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({" + "type:'density_view',trace:0,x0:-1,x1:1,y0:-1,y1:1,w:64,h:48," + "seq:23,client_id:'host-browser'})});" + "const payload=await response.json();" + "return {status:response.status,type:payload.message?.type}; })()" + ) + if transport != {"status": 200, "type": "density_update"}: + raise AssertionError(f"browser drilldown transport failed: {transport}") + evidence.update( + { + "status": "passed", + "mount": mount, + "colored_pixels": ink, + "transport": transport, + } + ) + print(f"FastAPI host browser smoke OK: {ink} colored pixels") + return 0 + except Exception as exc: + evidence["error"] = str(exc) + raise + finally: + if probe is not None: + probe.close() + _write_evidence(args.evidence, evidence) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/host_integration_policy.py b/scripts/host_integration_policy.py new file mode 100644 index 00000000..ef785b43 --- /dev/null +++ b/scripts/host_integration_policy.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Validate host support metadata and record installed floor/latest versions.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import sys +import tomllib +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_POLICY = ROOT / "spec" / "testing" / "host-integration-policy.json" + +EXPECTED_PACKAGES = { + "anywidget": (">=0.9,<1", "==0.9.0"), + "traitlets": (">=5.14,<6", "==5.14.0"), + "reflex": (">=0.9.6,<1", "==0.9.6"), + "fastapi": (">=0.110,<1", "==0.110.0"), + "starlette": (">=0.36.3,<1", "==0.36.3"), + "uvicorn": (">=0.29,<1", "==0.29.0"), + "httpx": (">=0.27,<1", "==0.27.0"), +} +EXPECTED_HOSTS = { + "anywidget": ["anywidget", "traitlets"], + "reflex": ["reflex"], + "fastapi": ["fastapi", "starlette", "uvicorn", "httpx"], +} +SOURCE_REQUIREMENTS = { + "anywidget": [("pyproject.toml", "dependencies")], + "traitlets": [("pyproject.toml", "dependencies")], + "reflex": [("python/reflex-xy/pyproject.toml", "dependencies")], + "fastapi": [ + ("pyproject.toml", "dev"), + ("examples/fastapi/pyproject.toml", "dependencies"), + ], + "starlette": [ + ("pyproject.toml", "dev"), + ("examples/fastapi/pyproject.toml", "dependencies"), + ], + "uvicorn": [ + ("pyproject.toml", "dev"), + ("examples/fastapi/pyproject.toml", "dependencies"), + ], + "httpx": [ + ("pyproject.toml", "dev"), + ("examples/fastapi/pyproject.toml", "dependencies"), + ], +} + + +class PolicyError(RuntimeError): + """Raised when the declared matrix and package metadata diverge.""" + + +def load_policy(path: Path = DEFAULT_POLICY) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PolicyError(f"cannot read host policy {path}: {exc}") from exc + if not isinstance(value, dict): + raise PolicyError("host policy root must be an object") + return value + + +def _requirements(root: Path, relative: str, group: str) -> list[str]: + path = root / relative + try: + project = tomllib.loads(path.read_text(encoding="utf-8"))["project"] + except (OSError, KeyError, tomllib.TOMLDecodeError) as exc: + raise PolicyError(f"cannot read project requirements from {relative}: {exc}") from exc + if group == "dependencies": + values = project.get("dependencies", []) + else: + values = project.get("optional-dependencies", {}).get(group, []) + if not isinstance(values, list) or not all(isinstance(item, str) for item in values): + raise PolicyError(f"{relative} project requirement group {group!r} is not a string list") + return values + + +def _requirement_for(requirements: list[str], package: str) -> str | None: + canonical = package.lower().replace("_", "-") + for requirement in requirements: + name = requirement.split(";", 1)[0].split("[", 1)[0] + for separator in ("<", ">", "=", "!", "~"): + name = name.split(separator, 1)[0] + if name.strip().lower().replace("_", "-") == canonical: + return requirement.replace(" ", "") + return None + + +def validate_policy(policy: dict[str, Any], root: Path = ROOT) -> list[str]: + errors: list[str] = [] + if policy.get("schema_version") != 1: + errors.append("schema_version must equal 1") + packages = policy.get("packages") + if not isinstance(packages, dict) or set(packages) != set(EXPECTED_PACKAGES): + actual = sorted(packages) if isinstance(packages, dict) else type(packages).__name__ + errors.append(f"package inventory must be exact; got {actual}") + packages = {} + for name, (supported, floor) in EXPECTED_PACKAGES.items(): + expected = {"distribution": name, "supported": supported, "floor": floor} + if packages.get(name) != expected: + errors.append(f"{name} policy must exactly equal {expected}") + if policy.get("hosts") != EXPECTED_HOSTS: + errors.append(f"host ownership must exactly equal {EXPECTED_HOSTS}") + + for package, sources in SOURCE_REQUIREMENTS.items(): + supported = EXPECTED_PACKAGES[package][0] + expected_requirement = f"{package}{supported}" + for relative, group in sources: + try: + actual = _requirement_for(_requirements(root, relative, group), package) + except PolicyError as exc: + errors.append(str(exc)) + continue + if actual != expected_requirement: + errors.append( + f"{relative} [{group}] must own {expected_requirement}; got {actual!r}" + ) + return errors + + +def validate_installed( + policy: dict[str, Any], + profile: str, + hosts: list[str], + versions: dict[str, str] | None = None, +) -> tuple[list[str], dict[str, str]]: + if profile not in {"floor", "latest"}: + return [f"unknown profile {profile!r}"], {} + unknown = sorted(set(hosts) - set(EXPECTED_HOSTS)) + if unknown: + return [f"unknown hosts: {unknown}"], {} + package_names = sorted({name for host in hosts for name in EXPECTED_HOSTS[host]}) + installed: dict[str, str] = {} + errors: list[str] = [] + for name in package_names: + distribution = policy["packages"][name]["distribution"] + try: + installed[name] = ( + versions[name] if versions is not None else importlib.metadata.version(distribution) + ) + except (KeyError, importlib.metadata.PackageNotFoundError): + errors.append(f"required host package {distribution!r} is not installed") + + try: + from packaging.specifiers import SpecifierSet + from packaging.version import Version + except ImportError: + return [*errors, "packaging is required to validate host versions"], installed + for name, version in installed.items(): + key = "floor" if profile == "floor" else "supported" + selector = policy["packages"][name][key] + if Version(version) not in SpecifierSet(selector): + errors.append(f"{name} {version} does not satisfy {profile} selector {selector}") + return errors, installed + + +def write_installed_report( + path: Path, + *, + policy: dict[str, Any], + profile: str, + hosts: list[str], +) -> list[str]: + policy_errors = validate_policy(policy) + if policy_errors: + version_errors, installed = [], {} + else: + version_errors, installed = validate_installed(policy, profile, hosts) + errors = [*policy_errors, *version_errors] + payload = { + "schema_version": 1, + "profile": profile, + "hosts": hosts, + "installed": installed, + "selectors": { + name: policy["packages"][name]["floor" if profile == "floor" else "supported"] + for name in installed + }, + "status": "failed" if errors else "passed", + "errors": errors, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("validate") + report = subparsers.add_parser("report") + report.add_argument("--profile", choices=("floor", "latest"), required=True) + report.add_argument("--hosts", nargs="+", choices=tuple(EXPECTED_HOSTS), required=True) + report.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + policy = load_policy(args.policy) + except PolicyError as exc: + print(f"host integration policy error: {exc}", file=sys.stderr) + return 1 + if args.command == "validate": + errors = validate_policy(policy) + else: + errors = write_installed_report( + args.output, + policy=policy, + profile=args.profile, + hosts=args.hosts, + ) + if errors: + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("host integration policy OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/interaction_stress_smoke.py b/scripts/interaction_stress_smoke.py index 55e45a39..3d847d76 100644 --- a/scripts/interaction_stress_smoke.py +++ b/scripts/interaction_stress_smoke.py @@ -13,6 +13,8 @@ import argparse import importlib.util import json +import math +import shutil import sys import tempfile from pathlib import Path @@ -22,6 +24,21 @@ import verify_benchmark_report # noqa: E402 +WORKER_REQUIRED_TRUE = ( + "worker_created", + "worker_rebinned", + "x_range_changed", + "worker_terminated", + "worker_cleared", + "root_removed", + "teardown_complete", +) +WORKER_OPTIONAL_UNAVAILABLE = ( + "skipped(", + "failed(Node.js is required", + "failed(Playwright is not installed", +) + def _load_bench_interaction(): sys.path.insert(0, str(ROOT / "benchmarks")) @@ -35,6 +52,34 @@ def _load_bench_interaction(): return module +def _write_json(path: str | None, payload: dict) -> None: + if path: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + + +def _worker_errors(worker: dict, *, allow_skip: bool) -> list[str]: + status = str(worker.get("status", "")) + if allow_skip and status.startswith(WORKER_OPTIONAL_UNAVAILABLE): + return [] + if status != "ok": + return [f"status is {status!r}, expected 'ok'"] + + errors = [ + f"{field} is not true" for field in WORKER_REQUIRED_TRUE if worker.get(field) is not True + ] + nonblank = worker.get("nonblank_pixels") + if ( + isinstance(nonblank, bool) + or not isinstance(nonblank, int | float) + or not math.isfinite(nonblank) + or nonblank <= 0 + ): + errors.append(f"nonblank_pixels is not positive: {nonblank!r}") + return errors + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("chromium", nargs="?", default=None) @@ -51,9 +96,27 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--json", default=None, help="write the validated JSON report here") parser.add_argument("--markdown", default=None, help="write the Markdown report here") + parser.add_argument( + "--allow-worker-skip", + action="store_true", + help="local-only escape hatch for an unavailable Node/Playwright worker harness; " + "CI and make check-browser intentionally omit it", + ) args = parser.parse_args(argv) chromium = args.chromium_flag or args.chromium + if chromium and not (Path(chromium).is_file() or shutil.which(chromium)): + failure = { + "kind": "interaction-browser", + "status": f"failed(configured chromium not found: {chromium})", + "standalone_density_worker": {"status": "not-run"}, + } + _write_json(args.json, failure) + print( + f"interaction stress smoke FAILED: configured chromium not found: {chromium}", + file=sys.stderr, + ) + return 1 bench_interaction = _load_bench_interaction() report = bench_interaction.run( sizes=bench_interaction._parse_sizes(args.sizes), @@ -66,12 +129,10 @@ def main(argv: list[str] | None = None) -> int: report_path.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") errors = verify_benchmark_report.validate_report(report_path, kind="interaction-browser") - if args.json: - Path(args.json).write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") - if args.markdown: - Path(args.markdown).write_text(bench_interaction.to_markdown(report), encoding="utf-8") - if errors: + _write_json(args.json, report) + if args.markdown: + Path(args.markdown).write_text(bench_interaction.to_markdown(report), encoding="utf-8") print("interaction stress smoke FAILED:", file=sys.stderr) for error in errors[:20]: print(f" - {error}", file=sys.stderr) @@ -86,13 +147,16 @@ def main(argv: list[str] | None = None) -> int: return 1 worker = bench_interaction.run_worker_probe(chromium=chromium) + report["standalone_density_worker"] = worker + _write_json(args.json, report) + if args.markdown: + Path(args.markdown).write_text(bench_interaction.to_markdown(report), encoding="utf-8") worker_status = str(worker.get("status", "")) - if worker_status != "ok" and not worker_status.startswith("skipped("): - print( - "interaction stress smoke FAILED: standalone density worker probe " - f"returned {worker_status!r}", - file=sys.stderr, - ) + worker_errors = _worker_errors(worker, allow_skip=args.allow_worker_skip) + if worker_errors: + print("interaction stress smoke FAILED: standalone density worker probe", file=sys.stderr) + for error in worker_errors: + print(f" - {error}", file=sys.stderr) return 1 print( @@ -113,15 +177,17 @@ def main(argv: list[str] | None = None) -> int: overlaps=row["tick_label_overlap_count"], ) ) - if worker_status.startswith("skipped("): - print(f" standalone_density_worker: SKIPPED ({worker_status})") + if worker_status != "ok": + print(f" standalone_density_worker: SKIPPED BY EXPLICIT LOCAL OPT-IN ({worker_status})") else: print( " standalone_density_worker: rebinned={rebinned} worker={worker} " - "nonblank={nonblank}".format( + "nonblank={nonblank} terminated={terminated} teardown={teardown}".format( rebinned=worker.get("worker_rebinned"), worker=worker.get("worker_created"), nonblank=worker.get("nonblank_pixels"), + terminated=worker.get("worker_terminated"), + teardown=worker.get("teardown_complete"), ) ) return 0 diff --git a/scripts/native_parity.py b/scripts/native_parity.py new file mode 100644 index 00000000..5cd67b8b --- /dev/null +++ b/scripts/native_parity.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Exercise one native core through scalar and architecture-selected paths. + +The probe is intentionally stdlib-only so it can run against a source build, +an installed wheel, or a wheel extracted inside a manylinux/musllinux runtime. +It validates three seams through the public C ABI: a dispatched kernel, invalid +pointer handling, and an exact raster framebuffer. The parent process runs the +probe twice because ``XY_SIMD`` dispatch is cached within one loaded library. +""" + +from __future__ import annotations + +import argparse +import ctypes +import hashlib +import importlib.util +import json +import os +import platform +import struct +import subprocess +import sys +import tempfile +import zipfile +from array import array +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +ABI_VERSION = 38 +CAP_SCALAR = 1 << 0 +CAP_AVX2_AVAILABLE = 1 << 1 +CAP_AVX2_SELECTED = 1 << 2 +CAP_AARCH64 = 1 << 3 +KNOWN_CAPABILITIES = CAP_SCALAR | CAP_AVX2_AVAILABLE | CAP_AVX2_SELECTED | CAP_AARCH64 + + +def _library_name() -> str: + if sys.platform == "win32": + return "xy_core.dll" + if sys.platform == "darwin": + return "libxy_core.dylib" + return "libxy_core.so" + + +def _normalized_architecture() -> str: + machine = platform.machine().lower() + if machine in {"amd64", "x86_64"}: + return "x86_64" + if machine in {"arm64", "aarch64"}: + return "aarch64" + return machine + + +def _installed_library() -> Path | None: + spec = importlib.util.find_spec("xy") + if spec is None or not spec.submodule_search_locations: + return None + package = Path(next(iter(spec.submodule_search_locations))) + candidate = package / "_native_lib" / _library_name() + return candidate if candidate.is_file() else None + + +def _source_library() -> Path | None: + root = Path(__file__).resolve().parents[1] + name = _library_name() + for profile in ("release", "debug"): + candidate = root / "target" / profile / name + if candidate.is_file(): + return candidate + return None + + +@contextmanager +def _resolved_library(library: Path | None, wheel: Path | None) -> Iterator[Path]: + if library is not None and wheel is not None: + raise SystemExit("pass only one of --library and --wheel") + if library is not None: + resolved = library.resolve() + if not resolved.is_file(): + raise SystemExit(f"native library does not exist: {resolved}") + yield resolved + return + if wheel is not None: + wheel = wheel.resolve() + if not wheel.is_file(): + raise SystemExit(f"wheel does not exist: {wheel}") + suffixes = ("/libxy_core.so", "/libxy_core.dylib", "/xy_core.dll") + with zipfile.ZipFile(wheel) as archive: + members = [name for name in archive.namelist() if name.endswith(suffixes)] + if len(members) != 1: + raise SystemExit( + f"expected exactly one native core in {wheel.name}, found {members}" + ) + with tempfile.TemporaryDirectory(prefix="xy-native-parity-") as tmp: + extracted = Path(tmp) / Path(members[0]).name + extracted.write_bytes(archive.read(members[0])) + yield extracted + return + discovered = _installed_library() or _source_library() + if discovered is None: + raise SystemExit( + "native core not found; build it, install a native wheel, or pass --library/--wheel" + ) + yield discovered.resolve() + + +def _pointer(buffer: array | bytearray, ctype: type[ctypes._SimpleCData]) -> object: # type: ignore[name-defined] + return ctypes.cast(ctypes.addressof(ctype.from_buffer(buffer)), ctypes.POINTER(ctype)) + + +def _load(path: Path) -> ctypes.CDLL: + library = ctypes.CDLL(str(path)) + library.xy_abi_version.restype = ctypes.c_uint32 + library.xy_abi_version.argtypes = [] + library.xy_runtime_capabilities.restype = ctypes.c_uint32 + library.xy_runtime_capabilities.argtypes = [] + library.xy_min_max.restype = ctypes.c_int32 + library.xy_min_max.argtypes = [ + ctypes.POINTER(ctypes.c_double), + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_double), + ctypes.POINTER(ctypes.c_double), + ] + library.xy_rasterize.restype = ctypes.c_int32 + library.xy_rasterize.argtypes = [ + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ctypes.c_size_t, + ] + return library + + +def _capability_report(capabilities: int) -> dict[str, bool]: + unknown = capabilities & ~KNOWN_CAPABILITIES + if unknown: + raise AssertionError(f"native core reported unknown capability bits: {unknown:#x}") + report = { + "scalar": bool(capabilities & CAP_SCALAR), + "avx2_available": bool(capabilities & CAP_AVX2_AVAILABLE), + "avx2_selected": bool(capabilities & CAP_AVX2_SELECTED), + "aarch64_baseline": bool(capabilities & CAP_AARCH64), + } + if not report["scalar"]: + raise AssertionError("native core did not report the mandatory scalar path") + if report["avx2_selected"] and not report["avx2_available"]: + raise AssertionError("native core selected AVX2 without reporting it available") + return report + + +def _raster_command() -> bytearray: + command = bytearray([1]) # OP_FILL_POLY + command.extend(struct.pack(" bytearray: + expected = bytearray(8 * 8 * 4) + for y in range(1, 6): + for x in range(1, 6): + offset = (y * 8 + x) * 4 + expected[offset : offset + 4] = bytes((37, 99, 235, 255)) + return expected + + +def _probe(path: Path) -> dict[str, object]: + library = _load(path) + abi = library.xy_abi_version() + if abi != ABI_VERSION: + raise AssertionError(f"ABI mismatch: probe expects {ABI_VERSION}, library reports {abi}") + + capabilities = _capability_report(library.xy_runtime_capabilities()) + + values = array("d", (-4.5, float("nan"), 12.25, float("inf"))) + out_min = ctypes.c_double() + out_max = ctypes.c_double() + result = library.xy_min_max( + _pointer(values, ctypes.c_double), + len(values), + ctypes.byref(out_min), + ctypes.byref(out_max), + ) + if result != 1 or (out_min.value, out_max.value) != (-4.5, 12.25): + raise AssertionError( + f"min/max kernel mismatch: result={result}, values={(out_min.value, out_max.value)}" + ) + + null_result = library.xy_min_max( + None, + len(values), + ctypes.byref(out_min), + ctypes.byref(out_max), + ) + if null_result != 0: + raise AssertionError(f"invalid pointer/length pair returned {null_result}, expected 0") + + command = _raster_command() + framebuffer = bytearray(8 * 8 * 4) + raster_result = library.xy_rasterize( + _pointer(command, ctypes.c_uint8), + len(command), + _pointer(framebuffer, ctypes.c_uint8), + 8, + 8, + ) + expected = _expected_raster() + if raster_result != 1 or framebuffer != expected: + raise AssertionError( + "raster parity mismatch: " + f"result={raster_result}, actual={hashlib.sha256(framebuffer).hexdigest()}, " + f"expected={hashlib.sha256(expected).hexdigest()}" + ) + null_raster = library.xy_rasterize(None, len(command), None, 8, 8) + if null_raster != 0: + raise AssertionError(f"invalid raster pointers returned {null_raster}, expected 0") + + return { + "abi_version": abi, + "capabilities": capabilities, + "kernel": {"min": out_min.value, "max": out_max.value}, + "ffi": {"null_min_max": null_result, "null_raster": null_raster}, + "raster": { + "sha256": hashlib.sha256(framebuffer).hexdigest(), + "painted_pixels": sum( + framebuffer[offset + 3] != 0 for offset in range(0, len(framebuffer), 4) + ), + }, + } + + +def _child_probe(path: Path, *, scalar: bool) -> dict[str, object]: + env = os.environ.copy() + if scalar: + env["XY_SIMD"] = "0" + else: + env.pop("XY_SIMD", None) + command = [sys.executable, str(Path(__file__).resolve()), "--probe", "--library", str(path)] + completed = subprocess.run(command, env=env, text=True, capture_output=True, check=False) + if completed.returncode: + raise SystemExit( + f"native {'scalar' if scalar else 'default'} probe failed\n{completed.stdout}{completed.stderr}" + ) + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise SystemExit(f"native probe emitted invalid JSON: {completed.stdout!r}") from exc + + +def _validate_architecture( + architecture: str, + expected_architecture: str, + expected_default: str, + default: dict[str, object], + scalar: dict[str, object], +) -> None: + if architecture != expected_architecture: + raise AssertionError( + f"runtime architecture is {architecture!r}, expected {expected_architecture!r}" + ) + default_caps = default["capabilities"] + scalar_caps = scalar["capabilities"] + assert isinstance(default_caps, dict) and isinstance(scalar_caps, dict) + if scalar_caps["avx2_selected"]: + raise AssertionError("XY_SIMD=0 did not force the scalar dispatch path") + if default_caps["avx2_available"] != scalar_caps["avx2_available"]: + raise AssertionError("AVX2 hardware availability changed between child probes") + + if expected_default == "avx2": + if architecture != "x86_64": + raise AssertionError("AVX2 dispatch can only be required on x86_64") + if not default_caps["avx2_available"] or not default_caps["avx2_selected"]: + raise AssertionError(f"required AVX2 path was not exercised: {default_caps}") + if default_caps["aarch64_baseline"]: + raise AssertionError("x86_64 runtime incorrectly reported the aarch64 baseline") + elif expected_default == "aarch64": + if architecture != "aarch64" or not default_caps["aarch64_baseline"]: + raise AssertionError(f"required aarch64 baseline was not exercised: {default_caps}") + if default_caps["avx2_available"] or default_caps["avx2_selected"]: + raise AssertionError(f"aarch64 runtime incorrectly reported AVX2: {default_caps}") + elif default_caps["avx2_selected"] or default_caps["aarch64_baseline"]: + raise AssertionError(f"required scalar-only default was not exercised: {default_caps}") + + +def _parity_payload(report: dict[str, object]) -> dict[str, object]: + return {key: report[key] for key in ("abi_version", "kernel", "ffi", "raster")} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--library", type=Path) + parser.add_argument("--wheel", type=Path) + parser.add_argument("--expect-arch", choices=("x86_64", "aarch64")) + parser.add_argument("--expect-default", choices=("scalar", "avx2", "aarch64")) + parser.add_argument("--report", type=Path) + parser.add_argument("--probe", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + + with _resolved_library(args.library, args.wheel) as library: + if args.probe: + print(json.dumps(_probe(library), sort_keys=True)) + return + if args.expect_arch is None or args.expect_default is None: + parser.error("--expect-arch and --expect-default are required for a parity run") + + default = _child_probe(library, scalar=False) + scalar = _child_probe(library, scalar=True) + architecture = _normalized_architecture() + _validate_architecture( + architecture, + args.expect_arch, + args.expect_default, + default, + scalar, + ) + if _parity_payload(default) != _parity_payload(scalar): + raise AssertionError( + "default/scalar native outputs differ: " + f"default={_parity_payload(default)}, scalar={_parity_payload(scalar)}" + ) + + payload = { + "schema": 1, + "architecture": architecture, + "platform": platform.platform(), + "artifact": args.wheel.name if args.wheel else library.name, + "expected_default": args.expect_default, + "default": default, + "forced_scalar": scalar, + "parity": True, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if args.report is not None: + args.report.write_text(rendered, encoding="utf-8") + print(rendered, end="") + + +if __name__ == "__main__": + main() diff --git a/scripts/pan_zoom_matrix.mjs b/scripts/pan_zoom_matrix.mjs new file mode 100644 index 00000000..89e66e04 --- /dev/null +++ b/scripts/pan_zoom_matrix.mjs @@ -0,0 +1,923 @@ +#!/usr/bin/env node +/** Bounded, data-driven pan/zoom acceptance matrix (TST-NI-011). + * + * Profiles: + * full - every standalone case in Chromium (the hard CI gate) + * focused - drag/wheel/box subset in Chromium, Firefox, and WebKit + * reflex - live and static charts in the real Reflex example + * + * Every run writes machine-readable evidence, including on failure. The + * catalog and evidence validators intentionally run without a browser so the + * matrix wiring remains cheap to test and difficult to silently narrow. + */ + +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const BUNDLE = join(ROOT, "python", "xy", "static", "standalone.js"); +const protocolSource = readFileSync(join(ROOT, "python", "xy", "config.py"), "utf8"); +const protocolMatch = protocolSource.match(/^PROTOCOL_VERSION\s*=\s*(\d+)\s*$/m); +if (!protocolMatch) throw new Error("could not read PROTOCOL_VERSION from python/xy/config.py"); +const PROTOCOL_VERSION = Number(protocolMatch[1]); + +const ENGINE_NAMES = ["chromium", "firefox", "webkit"]; +let engines = null; + +async function browserEngines() { + if (engines === null) { + try { + engines = await import("playwright"); + } catch (error) { + fail(`Playwright is required for browser profiles; run npm ci\n${error}`); + } + } + return engines; +} +const REQUIRED = Object.freeze({ + actions: ["drag", "wheel", "box", "toolbar_zoom", "reset"], + axis_classes: ["linear", "log", "reversed", "category", "dual", "named"], + hosts: ["standalone", "reflex-live", "reflex-static"], + invariants: [ + "participating_ranges_change", + "nonparticipating_ranges_exact", + "bounds", + "zoom_limits", + "event_source_axes_phase", + "linked_axes_no_echo", + "reduced_motion", + "no_op_no_event_or_lod", + "semantic_layout_health", + "json_safe_reflex_payload", + ], +}); + +const CASES = Object.freeze([ + { + id: "linear-drag-bounds", + profiles: ["full", "focused"], + actions: ["drag"], + axis_classes: ["linear"], + hosts: ["standalone"], + }, + { + id: "log-wheel-link-limits", + profiles: ["full", "focused"], + actions: ["wheel"], + axis_classes: ["log"], + hosts: ["standalone"], + }, + { + id: "reversed-box-reduced-motion", + profiles: ["full", "focused"], + actions: ["box"], + axis_classes: ["reversed"], + hosts: ["standalone"], + }, + { + id: "category-toolbar-default-limit", + profiles: ["full"], + actions: ["toolbar_zoom", "reset"], + axis_classes: ["category"], + hosts: ["standalone"], + }, + { + id: "dual-named-partial-limits", + profiles: ["full"], + actions: ["toolbar_zoom", "reset"], + axis_classes: ["dual", "named"], + hosts: ["standalone"], + }, +]); + +const REFLEX_CASES = Object.freeze([ + { + id: "reflex-live-wheel", + profiles: ["reflex"], + actions: ["wheel"], + axis_classes: ["linear"], + hosts: ["reflex-live"], + }, + { + id: "reflex-static-toolbar-reset", + profiles: ["reflex"], + actions: ["toolbar_zoom", "reset"], + axis_classes: ["linear"], + hosts: ["reflex-static"], + }, +]); + +function unique(values) { + return [...new Set(values)]; +} + +function sorted(values) { + return [...values].sort(); +} + +function equalSet(actual, expected) { + return JSON.stringify(sorted(unique(actual))) === JSON.stringify(sorted(unique(expected))); +} + +function fail(message) { + throw new Error(message); +} + +function check(condition, message) { + if (!condition) fail(message); +} + +function catalog() { + return { + schema_version: 1, + requirement: "TST-NI-011", + required: REQUIRED, + cases: [...CASES, ...REFLEX_CASES], + profiles: { + full: { + browsers: ["chromium"], + cases: CASES.filter((item) => item.profiles.includes("full")).map((item) => item.id), + }, + focused: { + browsers: ["chromium", "firefox", "webkit"], + cases: CASES.filter((item) => item.profiles.includes("focused")).map((item) => item.id), + }, + reflex: { + browsers: ["chromium"], + cases: REFLEX_CASES.map((item) => item.id), + }, + }, + }; +} + +export function validateCatalog(value = catalog()) { + check(value?.schema_version === 1, "catalog schema_version must be 1"); + check(value?.requirement === "TST-NI-011", "catalog requirement must be TST-NI-011"); + const cases = value.cases || []; + check(cases.length === 7, `catalog must contain 7 bounded cases, got ${cases.length}`); + check(new Set(cases.map((item) => item.id)).size === cases.length, "catalog case IDs must be unique"); + for (const field of ["actions", "axis_classes", "hosts"]) { + const actual = cases.flatMap((item) => item[field] || []); + check(equalSet(actual, REQUIRED[field]), `catalog ${field} coverage is incomplete`); + } + check( + equalSet(value.profiles?.focused?.browsers || [], ["chromium", "firefox", "webkit"]), + "focused profile must hard-run Chromium, Firefox, and WebKit", + ); + check((value.profiles?.full?.cases || []).length === 5, "full profile must contain 5 cases"); + check((value.profiles?.focused?.cases || []).length === 3, "focused profile must contain 3 cases"); + check((value.profiles?.reflex?.cases || []).length === 2, "reflex profile must contain 2 cases"); + return true; +} + +function coverageFor(profile) { + const selected = [...CASES, ...REFLEX_CASES].filter((item) => item.profiles.includes(profile)); + return { + actions: sorted(unique(selected.flatMap((item) => item.actions))), + axis_classes: sorted(unique(selected.flatMap((item) => item.axis_classes))), + hosts: sorted(unique(selected.flatMap((item) => item.hosts))), + }; +} + +export function validateEvidence(value) { + validateCatalog(value?.catalog); + check(value?.schema_version === 1, "evidence schema_version must be 1"); + check(["full", "focused", "reflex"].includes(value?.profile), "unknown evidence profile"); + check(value?.status === "passed", `evidence status is ${value?.status || "missing"}, expected passed`); + const expectedCoverage = coverageFor(value.profile); + for (const field of ["actions", "axis_classes", "hosts"]) { + check(equalSet(value.coverage?.[field] || [], expectedCoverage[field]), `${field} evidence coverage is incomplete`); + } + const expectedBrowsers = value.catalog.profiles[value.profile].browsers; + check(equalSet(Object.keys(value.browsers || {}), expectedBrowsers), "evidence browser coverage is incomplete"); + const expectedCases = value.catalog.profiles[value.profile].cases; + for (const browserName of expectedBrowsers) { + const browser = value.browsers[browserName]; + check(browser?.status === "passed", `${browserName} evidence did not pass`); + check(equalSet((browser.cases || []).map((item) => item.id), expectedCases), `${browserName} case coverage is incomplete`); + for (const item of browser.cases || []) { + check(item.status === "passed", `${browserName}/${item.id} did not pass`); + check((item.assertions?.semantic || []).length > 0, `${browserName}/${item.id} lacks semantic evidence`); + check((item.assertions?.layout || []).length > 0, `${browserName}/${item.id} lacks layout evidence`); + } + } + return true; +} + +function parseArgs(argv) { + const args = { + profile: "full", + browsers: null, + evidence: "pan-zoom-matrix-evidence.json", + executablePath: null, + url: null, + catalogOnly: false, + verifyEvidence: null, + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--catalog") args.catalogOnly = true; + else if (arg === "--profile") args.profile = argv[++i]; + else if (arg.startsWith("--profile=")) args.profile = arg.split("=", 2)[1]; + else if (arg === "--browsers") args.browsers = argv[++i]; + else if (arg.startsWith("--browsers=")) args.browsers = arg.split("=", 2)[1]; + else if (arg === "--evidence") args.evidence = argv[++i]; + else if (arg === "--executable-path") args.executablePath = argv[++i]; + else if (arg === "--url") args.url = argv[++i]; + else if (arg === "--verify-evidence") args.verifyEvidence = argv[++i]; + else fail(`unknown argument: ${arg}`); + } + check(["full", "focused", "reflex"].includes(args.profile), `unknown profile ${args.profile}`); + return args; +} + +function writeEvidence(path, value) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function makeSpec(kind) { + let axes; + let interaction; + if (kind === "linear-drag-bounds") { + axes = { + x: { kind: "linear", label: "bounded x", range: [0, 10], bounds: [-5, 15] }, + y: { kind: "linear", label: "fixed y", range: [-5, 5], bounds: [-10, 10] }, + }; + interaction = { + navigation: true, pan: true, zoom: true, default_drag_action: "pan", + pan_axes: ["x"], zoom_axes: ["x"], reset_axes: ["x"], + }; + } else if (kind === "log-wheel-link-limits") { + axes = { + x: { kind: "linear", scale: "log", label: "log x", range: [1, 1000], bounds: [0.1, 10000] }, + y: { kind: "linear", label: "fixed y", range: [0, 10], bounds: [-10, 20] }, + }; + interaction = { + navigation: true, pan: true, zoom: true, wheel_zoom: true, + pan_axes: ["x"], zoom_axes: ["x"], reset_axes: ["x"], + zoom_limits: { x: [1, 4] }, link_group: "tst-ni-011-log", link_axes: ["x"], + }; + } else if (kind === "reversed-box-reduced-motion") { + axes = { + x: { kind: "linear", label: "reversed x", range: [10, 0], bounds: [0, 10] }, + y: { kind: "linear", label: "y", range: [0, 10], bounds: [0, 10] }, + }; + interaction = { + navigation: true, pan: true, zoom: true, box_zoom: true, + default_drag_action: "zoom", zoom_axes: ["x", "y"], reset_axes: ["x", "y"], + zoom_limits: { x: [1, 8], y: [1, 8] }, + }; + } else if (kind === "category-toolbar-default-limit") { + axes = { + x: { + kind: "category", label: "category x", range: [-0.5, 5.5], bounds: [-0.5, 5.5], + categories: ["alpha", "beta", "gamma", "delta", "epsilon", "zeta"], + }, + y: { kind: "linear", label: "fixed y", range: [0, 10], bounds: [-10, 20] }, + }; + interaction = { + navigation: true, pan: true, zoom: true, zoom_axes: ["x"], reset_axes: ["x"], + }; + } else if (kind === "dual-named-partial-limits") { + axes = { + x: { kind: "linear", label: "named x", range: [0, 10], bounds: [-10, 20] }, + y: { kind: "linear", label: "primary y", range: [-1, 1], bounds: [-2, 2] }, + y2: { kind: "linear", label: "secondary y", range: [-50, 50], bounds: [-200, 200], side: "right" }, + }; + interaction = { + navigation: true, pan: true, zoom: true, + pan_axes: ["x", "y2"], zoom_axes: ["x", "y2"], reset_axes: ["x", "y2"], + // x intentionally inherits the default (1, None); y2 exercises both + // finite sides and the one-axis-hits-a-limit multi-axis case. + zoom_limits: { y2: [0.5, 2] }, + }; + } else { + fail(`unknown matrix case ${kind}`); + } + + const columns = []; + const values = []; + const ship = (items) => { + const index = columns.length; + columns.push({ byte_offset: values.length * 4, len: items.length, offset: 0, scale: 1, kind: "float" }); + values.push(...items); + return index; + }; + const xValues = axes.x.scale === "log" + ? [1, 3, 10, 30, 100, 300, 1000] + : axes.x.kind === "category" ? [0, 1, 2, 3, 4, 5] : [0, 1.5, 3, 5, 7, 8.5, 10]; + const yValues = xValues.map((_, index) => axes.y.range[0] + + (axes.y.range[1] - axes.y.range[0]) * ((index + 1) / (xValues.length + 1))); + const traces = [{ + id: 0, kind: "scatter", name: "primary", tier: "direct", n_points: xValues.length, + style: { opacity: 0.9 }, color: { mode: "constant", color: "#2563eb" }, + size: { mode: "constant", size: 11 }, x: ship(xValues), y: ship(yValues), + }]; + if (axes.y2) { + const y2Values = xValues.map((_, index) => -40 + 80 * ((index + 1) / (xValues.length + 1))); + traces.push({ + id: 1, kind: "scatter", name: "secondary", tier: "direct", n_points: xValues.length, + x_axis: "x", y_axis: "y2", style: { opacity: 0.9 }, + color: { mode: "constant", color: "#dc2626" }, size: { mode: "constant", size: 10 }, + x: ship(xValues), y: ship(y2Values), + }); + } + return { + spec: { + protocol: PROTOCOL_VERSION, width: 680, height: 390, title: `Pan/zoom ${kind}`, + show_legend: false, axes, x_axis: axes.x, y_axis: axes.y, + traces, columns, interaction, backend: "none", + }, + values, + }; +} + +async function mountStandalone(page, caseId, peers = 1) { + const { spec, values } = makeSpec(caseId); + const mounts = Array.from({ length: peers }, (_, index) => ({ id: index === 0 ? "a" : "b" })); + await page.setContent(`${mounts.map((item) => `
`).join("")}`); + await page.addScriptTag({ path: BUNDLE }); + await page.evaluate(({ spec, values, mounts }) => { + const originalGetContext = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function(type, attributes) { + if (type === "webgl2") { + return originalGetContext.call(this, type, { ...attributes, preserveDrawingBuffer: true }); + } + return originalGetContext.call(this, type, attributes); + }; + window.xyMatrix = { views: {}, events: {}, lod: {}, initial: {} }; + for (const mount of mounts) { + const view = window.xy.renderStandalone( + document.getElementById(mount.id), structuredClone(spec), new Float32Array(values).buffer, + ); + window.xyMatrix.views[mount.id] = view; + window.xyMatrix.events[mount.id] = []; + window.xyMatrix.lod[mount.id] = 0; + window.xyMatrix.initial[mount.id] = { + ranges: Object.fromEntries(view._axisIds().map((id) => [id, [...view._axisRange(id)]])), + }; + view.root.addEventListener("xy:view_change", (event) => { + window.xyMatrix.events[mount.id].push(JSON.parse(JSON.stringify(event.detail))); + }); + const schedule = view._scheduleViewRequest.bind(view); + view._scheduleViewRequest = (...args) => { + window.xyMatrix.lod[mount.id] += 1; + return schedule(...args); + }; + view._drawNow(); + } + HTMLCanvasElement.prototype.getContext = originalGetContext; + }, { spec, values, mounts }); + await page.waitForTimeout(60); +} + +async function snapshot(page, id = "a") { + return page.evaluate((id) => { + const view = window.xyMatrix.views[id]; + view._drawNow(); + const root = view.root.getBoundingClientRect(); + const canvas = view.canvas.getBoundingClientRect(); + const gl = view.gl; + const pixels = new Uint8Array(gl.drawingBufferWidth * gl.drawingBufferHeight * 4); + gl.readPixels(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixels); + let lit = 0; + for (let index = 0; index < pixels.length; index += 4) { + if (pixels[index] + pixels[index + 1] + pixels[index + 2] > 20) lit += 1; + } + const labels = [...view.labels.querySelectorAll('[data-xy-slot="tick_label"], [data-xy-slot="axis_title"]')]; + return { + ranges: Object.fromEntries(view._axisIds().map((axisId) => [axisId, [...view._axisRange(axisId)]])), + home: Object.fromEntries(view._axisIds().map((axisId) => [axisId, [...view._axisRange(axisId, view.view0)]])), + bounds: Object.fromEntries(view._axisIds().map((axisId) => [axisId, view._axis(axisId).bounds || null])), + scales: Object.fromEntries(view._axisIds().map((axisId) => [axisId, view._axis(axisId).scale || "linear"])), + events: structuredClone(window.xyMatrix.events[id]), + lod: window.xyMatrix.lod[id], + dragMode: view.dragMode, + reducedMotion: view._prefersReducedMotion(), + animationActive: view._viewAnim !== null, + layout: { + root: { x: root.x, y: root.y, width: root.width, height: root.height }, + canvas: { x: canvas.x, y: canvas.y, width: canvas.width, height: canvas.height }, + labels: labels.map((label) => { + const rect = label.getBoundingClientRect(); + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height, text: label.textContent }; + }), + }, + lit, + zoomLabel: view._zoomMenuLabel?.dataset.xyZoomExact || view._zoomMenuLabel?.textContent || null, + }; + }, id); +} + +function exact(a, b) { + return JSON.stringify(a) === JSON.stringify(b); +} + +function near(a, b, tolerance = 1e-8) { + return Math.abs(a - b) <= tolerance * Math.max(1, Math.abs(a), Math.abs(b)); +} + +function coord(value, scale) { + return scale === "log" ? Math.log10(value) : value; +} + +function span(range, scale = "linear") { + return Math.abs(coord(range[1], scale) - coord(range[0], scale)); +} + +function magnification(state, axisId) { + return span(state.home[axisId], state.scales[axisId]) / span(state.ranges[axisId], state.scales[axisId]); +} + +function assertBounds(state) { + for (const [axisId, bounds] of Object.entries(state.bounds)) { + if (!bounds) continue; + const range = state.ranges[axisId]; + check(Math.min(...range) >= Math.min(...bounds) - 1e-8, `${axisId} range escaped lower bound`); + check(Math.max(...range) <= Math.max(...bounds) + 1e-8, `${axisId} range escaped upper bound`); + } +} + +function assertLayout(before, after) { + check(after.lit > 40, `rendered only ${after.lit} lit WebGL pixels`); + check(after.layout.canvas.width > 100 && after.layout.canvas.height > 100, "canvas has invalid layout"); + check(after.layout.labels.length >= 2, "axis labels are missing"); + check(after.layout.labels.every((item) => [item.x, item.y, item.width, item.height].every(Number.isFinite)), "axis label layout is non-finite"); + check(near(before.layout.root.width, after.layout.root.width, 1e-4), "root width changed during navigation"); + check(near(before.layout.root.height, after.layout.root.height, 1e-4), "root height changed during navigation"); +} + +function lastEvent(state, source) { + const events = state.events.filter((event) => event.source === source); + check(events.length > 0, `missing ${source} event`); + return events.at(-1); +} + +function assertEvent(state, source, axes, phase = "end") { + const event = lastEvent(state, source); + check(event.phase === phase, `${source} final phase was ${event.phase}, expected ${phase}`); + check(equalSet(event.axes || [], axes), `${source} axes were ${JSON.stringify(event.axes)}, expected ${JSON.stringify(axes)}`); + check(Number.isInteger(event.interaction_id), `${source} lacks an interaction_id`); + check(JSON.stringify(event).length > 0, `${source} payload is not JSON safe`); + return event; +} + +async function drag(page, id, from, to) { + const rect = await page.locator(`#${id} [data-xy-slot="canvas"]`).boundingBox(); + check(rect, `missing ${id} canvas`); + const point = (fraction) => ({ x: rect.x + rect.width * fraction[0], y: rect.y + rect.height * fraction[1] }); + const a = point(from), b = point(to); + await page.mouse.move(a.x, a.y); + await page.mouse.down(); + await page.mouse.move(b.x, b.y, { steps: 8 }); + await page.mouse.up(); + await page.waitForTimeout(230); +} + +async function wheel(page, id, deltaY) { + const rect = await page.locator(`#${id} [data-xy-slot="canvas"]`).boundingBox(); + check(rect, `missing ${id} canvas`); + await page.mouse.move(rect.x + rect.width / 2, rect.y + rect.height / 2); + await page.mouse.wheel(0, deltaY); + await page.waitForTimeout(230); +} + +async function toolbar(page, id, action) { + const root = page.locator(`#${id}`); + await root.locator('[data-xy-modebar-menu-trigger]').click(); + await root.locator(`[data-xy-modebar-menu-item="${action}"]`).click(); + await page.waitForTimeout(230); +} + +const CASE_ASSERTIONS = { + "linear-drag-bounds": { + semantic: ["real pointer drag", "x changed", "y exact", "bounds", "pan_drag end"], + layout: ["nonblank WebGL", "finite labels", "stable chart box"], + }, + "log-wheel-link-limits": { + semantic: ["real wheel", "log magnification limit", "linked x only", "no echo", "no-op event/LOD"], + layout: ["nonblank WebGL", "finite labels", "stable chart box"], + }, + "reversed-box-reduced-motion": { + semantic: ["real box drag", "reversal preserved", "both axes changed", "reduced motion"], + layout: ["nonblank WebGL", "finite labels", "stable chart box"], + }, + "category-toolbar-default-limit": { + semantic: ["toolbar zoom in/out", "default zoom-out no-op", "category range", "toolbar reset"], + layout: ["category labels", "nonblank WebGL", "stable chart box"], + }, + "dual-named-partial-limits": { + semantic: ["dual named axes", "partial limit map", "one-axis clamp", "finite zoom-out", "reset subset"], + layout: ["secondary-axis labels", "nonblank WebGL", "stable chart box"], + }, +}; + +async function runStandaloneCase(page, caseId) { + const peers = caseId === "log-wheel-link-limits" ? 2 : 1; + await mountStandalone(page, caseId, peers); + const before = await snapshot(page); + + if (caseId === "linear-drag-bounds") { + await drag(page, "a", [0.5, 0.5], [0.72, 0.5]); + const after = await snapshot(page); + check(!exact(after.ranges.x, before.ranges.x), "drag did not change x"); + check(exact(after.ranges.y, before.ranges.y), "drag changed nonparticipating y"); + assertBounds(after); + assertEvent(after, "pan_drag", ["x"]); + const gestureEvents = after.events.filter((event) => event.source === "pan_drag"); + check(gestureEvents.at(-1).phase === "end", "drag lacks a final end event"); + check(new Set(gestureEvents.map((event) => event.interaction_id)).size === 1, "drag event IDs were not coalesced"); + assertLayout(before, after); + return after; + } + + if (caseId === "log-wheel-link-limits") { + await wheel(page, "a", -3000); + const after = await snapshot(page); + const peer = await snapshot(page, "b"); + check(near(magnification(after, "x"), 4, 1e-6), `log x magnification was ${magnification(after, "x")}, expected 4`); + check(exact(after.ranges.y, before.ranges.y), "wheel changed nonparticipating y"); + check(exact(peer.ranges.x, after.ranges.x), "linked peer did not receive x"); + check(exact(peer.ranges.y, before.ranges.y), "linked peer changed y"); + assertEvent(after, "wheel_zoom", ["x"]); + assertEvent(peer, "linked", ["x"]); + check(!after.events.some((event) => event.source === "linked"), "linked peer echoed to origin"); + const wheelEvents = after.events.filter((event) => event.source === "wheel_zoom"); + check(wheelEvents.some((event) => event.phase === "update"), "wheel lacks an update event"); + check(wheelEvents.at(-1).phase === "end", "wheel lacks a final end event"); + check(new Set(wheelEvents.map((event) => event.interaction_id)).size === 1, "wheel event IDs were not coalesced"); + const eventCount = after.events.length; + const lodCount = after.lod; + await wheel(page, "a", -1000); + const noop = await snapshot(page); + check(exact(noop.ranges, after.ranges), "clamped wheel no-op changed ranges"); + check(noop.events.length === eventCount, "clamped wheel no-op emitted a view event"); + check(noop.lod === lodCount, "clamped wheel no-op scheduled LOD work"); + assertBounds(noop); + assertLayout(before, noop); + return noop; + } + + if (caseId === "reversed-box-reduced-motion") { + check(before.dragMode === "zoom", `default box mode resolved to ${before.dragMode}`); + await drag(page, "a", [0.2, 0.2], [0.72, 0.75]); + const after = await snapshot(page); + check(span(after.ranges.x) < span(before.ranges.x), "box did not narrow reversed x"); + check(span(after.ranges.y) < span(before.ranges.y), "box did not narrow y"); + check(after.ranges.x[0] > after.ranges.x[1], "box lost x reversal"); + check(after.reducedMotion && !after.animationActive, "reduced motion did not suppress animation"); + assertEvent(after, "box_zoom", ["x", "y"]); + assertBounds(after); + assertLayout(before, after); + return after; + } + + if (caseId === "category-toolbar-default-limit") { + const noOpEvents = before.events.length; + const noOpLod = before.lod; + await toolbar(page, "a", "zoomout"); + let current = await snapshot(page); + check(exact(current.ranges, before.ranges), "default zoom-out limit did not clamp at home"); + check(current.events.length === noOpEvents, "default-limit no-op emitted a view event"); + check(current.lod === noOpLod, "default-limit no-op scheduled LOD work"); + await toolbar(page, "a", "zoomin"); + current = await snapshot(page); + check(near(magnification(current, "x"), 2), "category toolbar zoom-in did not reach 2x"); + check(exact(current.ranges.y, before.ranges.y), "category zoom changed y"); + assertEvent(current, "zoom_in", ["x"]); + await toolbar(page, "a", "zoomout"); + current = await snapshot(page); + check(exact(current.ranges.x, before.ranges.x), "category zoom-out did not return home"); + assertEvent(current, "zoom_out", ["x"]); + await toolbar(page, "a", "zoomin"); + await toolbar(page, "a", "reset"); + current = await snapshot(page); + check(exact(current.ranges.x, before.ranges.x), "category reset did not restore x"); + check(exact(current.ranges.y, before.ranges.y), "category reset changed y"); + assertEvent(current, "reset", ["x"]); + check(current.layout.labels.some((item) => item.text === "alpha" || item.text === "zeta"), "category labels disappeared"); + assertBounds(current); + assertLayout(before, current); + return current; + } + + if (caseId === "dual-named-partial-limits") { + await toolbar(page, "a", "zoomin"); + await toolbar(page, "a", "zoomin"); + let current = await snapshot(page); + check(near(magnification(current, "x"), 4), "named x did not inherit unbounded zoom-in default"); + check(near(magnification(current, "y2"), 2), "named y2 did not clamp at 2x"); + check(exact(current.ranges.y, before.ranges.y), "dual-axis zoom changed primary y"); + const secondZoom = current.events.filter((event) => event.source === "zoom_in").at(-1); + check(equalSet(secondZoom.axes, ["x"]), "one-axis-at-limit event did not report only x"); + await toolbar(page, "a", "reset"); + current = await snapshot(page); + check(exact(current.ranges, before.ranges), "dual reset did not restore selected axes"); + assertEvent(current, "reset", ["x", "y2"]); + // Seed an over-wide candidate through the shared clamping path. Toolbar + // zoom-out is home-capped before that path, so this setup is the only way + // to exercise a configured minimum below 1 without weakening the claim + // that the user-facing command itself was driven through the real menu. + await page.evaluate(() => { + const view = window.xyMatrix.views.a; + const ranges = Object.fromEntries(view._axisIds().map((axisId) => [axisId, [...view._axisRange(axisId)]])); + ranges.y2 = [-500, 500]; + view._setView({ ranges }, { animate: false, source: "programmatic", phase: "end" }); + }); + await page.waitForTimeout(100); + current = await snapshot(page); + check(near(magnification(current, "y2"), 0.5), "y2 finite zoom-out limit was not applied"); + check(exact(current.ranges.x, before.ranges.x), "partial-map x escaped its default home limit"); + check(exact(current.ranges.y, before.ranges.y), "finite zoom-out changed primary y"); + await toolbar(page, "a", "reset"); + current = await snapshot(page); + check(exact(current.ranges, before.ranges), "dual reset after zoom-out did not restore home"); + check(current.layout.labels.some((item) => item.text === "secondary y"), "named secondary-axis label disappeared"); + assertBounds(current); + assertLayout(before, current); + return current; + } + + fail(`case implementation missing: ${caseId}`); +} + +async function launchEngine(name, executablePath = null) { + const available = await browserEngines(); + const options = { + headless: process.env.XY_PAN_ZOOM_HEADFUL !== "1" && process.env.XY_CONFORMANCE_HEADFUL !== "1", + }; + if (name === "chromium" && executablePath) options.executablePath = executablePath; + if (name === "firefox") { + options.firefoxUserPrefs = { "webgl.disabled": false, "webgl.force-enabled": true }; + } + try { + return await available[name].launch(options); + } catch (error) { + fail(`${name} could not launch; install the selected Playwright engines\n${error}`); + } +} + +async function runStandaloneBrowser(name, profile, executablePath) { + const browserResult = { status: "running", version: null, cases: [] }; + let browser; + try { + browser = await launchEngine(name, executablePath); + browserResult.version = browser.version(); + const context = await browser.newContext({ + viewport: { width: 1460, height: 900 }, deviceScaleFactor: 1, reducedMotion: "reduce", + }); + const selected = CASES.filter((item) => item.profiles.includes(profile)); + for (const item of selected) { + const row = { + id: item.id, status: "running", actions: item.actions, + axis_classes: item.axis_classes, hosts: item.hosts, + assertions: CASE_ASSERTIONS[item.id], + }; + browserResult.cases.push(row); + const page = await context.newPage(); + const errors = []; + page.on("pageerror", (error) => errors.push(String(error))); + page.on("console", (message) => { + if (message.type() === "error") errors.push(message.text()); + }); + try { + const hasWebGl2 = await page.evaluate(() => Boolean(document.createElement("canvas").getContext("webgl2"))); + check(hasWebGl2, `${name}: WebGL2 unavailable`); + const final = await runStandaloneCase(page, item.id); + check(errors.length === 0, `${name}/${item.id} page errors: ${errors.join("; ")}`); + row.status = "passed"; + row.metrics = { event_count: final.events.length, lod_request_count: final.lod, lit_pixels: final.lit }; + } catch (error) { + row.status = "failed"; + row.error = String(error?.stack || error); + } finally { + await page.close(); + } + } + await context.close(); + browserResult.status = browserResult.cases.every((item) => item.status === "passed") ? "passed" : "failed"; + } catch (error) { + browserResult.status = "failed"; + browserResult.error = String(error?.stack || error); + } finally { + await browser?.close(); + } + return browserResult; +} + +async function instrumentReflex(page) { + await page.waitForFunction(() => window.__xy_views?.has("overview") && window.__xy_views?.has("inline"), null, { timeout: 120_000 }); + await page.evaluate(() => { + window.xyReflexMatrix = {}; + for (const id of ["overview", "inline"]) { + const view = window.__xy_views.get(id); + const record = { + events: [], sends: [], home: Object.fromEntries(view._axisIds().map((axisId) => [axisId, [...view._axisRange(axisId)]])), + }; + view.root.addEventListener("xy:view_change", (event) => record.events.push(JSON.parse(JSON.stringify(event.detail)))); + if (view.comm) { + const send = view.comm.send.bind(view.comm); + view.comm.send = (message) => { + record.sends.push(JSON.parse(JSON.stringify(message))); + return send(message); + }; + } + window.xyReflexMatrix[id] = record; + } + }); +} + +async function reflexState(page, id) { + return page.evaluate((id) => { + const view = window.__xy_views.get(id); + const record = window.xyReflexMatrix[id]; + const root = view.root.getBoundingClientRect(); + const canvas = view.canvas.getBoundingClientRect(); + return { + comm: view.comm !== null, + transportViewChange: view.comm?.wantsViewChange?.() === true + || view.interaction?.view_change === true + || view.interaction?._transport_view_change === true, + ranges: Object.fromEntries(view._axisIds().map((axisId) => [axisId, [...view._axisRange(axisId)]])), + home: structuredClone(record.home), + events: structuredClone(record.events), sends: structuredClone(record.sends), + layout: { root: { width: root.width, height: root.height }, canvas: { width: canvas.width, height: canvas.height } }, + jsonSafe: (() => { try { JSON.stringify(record.events); JSON.stringify(record.sends); return true; } catch (_) { return false; } })(), + }; + }, id); +} + +async function reflexWheel(page, id, deltaY) { + await page.locator(`#${id}`).scrollIntoViewIfNeeded(); + const box = await page.locator(`#${id} [data-xy-slot="canvas"]`).boundingBox(); + check(box, `missing Reflex ${id} canvas`); + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await page.mouse.wheel(0, deltaY); + await page.waitForTimeout(500); +} + +async function reflexToolbar(page, id, action) { + await page.locator(`#${id}`).scrollIntoViewIfNeeded(); + await page.locator(`#${id} [data-xy-modebar-menu-trigger]`).click(); + await page.locator(`#${id} [data-xy-modebar-menu-item="${action}"]`).click(); + await page.waitForTimeout(300); +} + +async function runReflexBrowser(name, url, executablePath) { + const result = { status: "running", version: null, cases: [] }; + let browser; + try { + browser = await launchEngine(name, executablePath); + result.version = browser.version(); + const context = await browser.newContext({ viewport: { width: 1440, height: 900 }, reducedMotion: "reduce" }); + const page = await context.newPage(); + const pageErrors = []; + page.on("pageerror", (error) => pageErrors.push(String(error))); + await page.goto(url, { waitUntil: "domcontentloaded", timeout: 120_000 }); + await instrumentReflex(page); + + const liveRow = { + id: "reflex-live-wheel", status: "running", actions: ["wheel"], axis_classes: ["linear"], + hosts: ["reflex-live"], assertions: { + semantic: ["real wheel", "live comm", "view_change end", "LOD request", "JSON-safe buffer-free state", "backend-derived detail"], + layout: ["mounted live canvas", "stable chart box"], + }, + }; + result.cases.push(liveRow); + try { + const before = await reflexState(page, "overview"); + check(before.comm && before.transportViewChange, "overview is not a subscribed live Reflex chart"); + await reflexWheel(page, "overview", -900); + const after = await reflexState(page, "overview"); + check(!exact(after.ranges.x, before.ranges.x), "live Reflex wheel did not change x"); + check(exact(after.ranges.y, before.ranges.y), "live Reflex wheel changed nonparticipating y"); + assertEvent(after, "wheel_zoom", ["x"]); + const transportedViews = after.sends.filter((message) => message.type === "view_change"); + check(transportedViews.some((message) => message.phase === "end"), "live Reflex did not transport final view_change"); + check( + transportedViews.every((message) => !Object.keys(message).some((key) => /buffer|binary/i.test(key))), + "live Reflex view state contained binary buffers", + ); + check(after.sends.some((message) => message.type === "density_view"), "live Reflex did not request density LOD"); + check(after.jsonSafe, "live Reflex event payload was not JSON safe"); + await page.waitForFunction( + () => /x ∈ \[[^\]]+\] · [\d,]+ points/.test(document.body.innerText), + null, + { timeout: 30_000 }, + ); + check(near(before.layout.root.width, after.layout.root.width, 1e-4), "live Reflex layout width changed"); + check(after.layout.canvas.width > 100 && after.layout.canvas.height > 100, "live Reflex canvas is not laid out"); + liveRow.status = "passed"; + liveRow.metrics = { event_count: after.events.length, transport_messages: after.sends.map((item) => item.type) }; + } catch (error) { + liveRow.status = "failed"; + liveRow.error = String(error?.stack || error); + } + + const staticRow = { + id: "reflex-static-toolbar-reset", status: "running", actions: ["toolbar_zoom", "reset"], + axis_classes: ["linear"], hosts: ["reflex-static"], assertions: { + semantic: ["kernel-less static mount", "toolbar zoom", "local event", "reset home", "no transport"], + layout: ["mounted static canvas", "stable chart box"], + }, + }; + result.cases.push(staticRow); + try { + const before = await reflexState(page, "inline"); + check(!before.comm, "inline direct Chart unexpectedly has a live comm"); + await reflexToolbar(page, "inline", "zoomin"); + let after = await reflexState(page, "inline"); + check(!exact(after.ranges, before.ranges), "static Reflex toolbar zoom did not change ranges"); + assertEvent(after, "zoom_in", Object.keys(after.ranges)); + check(after.sends.length === 0 && after.jsonSafe, "static Reflex attempted transport or emitted unsafe JSON"); + await reflexToolbar(page, "inline", "reset"); + after = await reflexState(page, "inline"); + check(exact(after.ranges, before.ranges), "static Reflex reset did not restore home"); + assertEvent(after, "reset", Object.keys(after.ranges)); + check(near(before.layout.root.width, after.layout.root.width, 1e-4), "static Reflex layout width changed"); + check(after.layout.canvas.width > 100 && after.layout.canvas.height > 100, "static Reflex canvas is not laid out"); + staticRow.status = "passed"; + staticRow.metrics = { event_count: after.events.length, transport_messages: 0 }; + } catch (error) { + staticRow.status = "failed"; + staticRow.error = String(error?.stack || error); + } + if (pageErrors.length) { + result.page_errors = pageErrors; + result.cases[0].status = "failed"; + result.cases[0].error = `Reflex page errors: ${pageErrors.join("; ")}`; + } + result.status = result.cases.every((item) => item.status === "passed") ? "passed" : "failed"; + await context.close(); + } catch (error) { + result.status = "failed"; + result.error = String(error?.stack || error); + } finally { + await browser?.close(); + } + return result; +} + +async function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + const matrixCatalog = catalog(); + validateCatalog(matrixCatalog); + if (args.catalogOnly) { + process.stdout.write(`${JSON.stringify(matrixCatalog, null, 2)}\n`); + return 0; + } + if (args.verifyEvidence) { + validateEvidence(JSON.parse(readFileSync(args.verifyEvidence, "utf8"))); + process.stdout.write(`pan/zoom evidence OK: ${args.verifyEvidence}\n`); + return 0; + } + const selectedBrowsers = args.browsers + ? args.browsers.split(",").filter(Boolean) + : matrixCatalog.profiles[args.profile].browsers; + for (const name of selectedBrowsers) check(ENGINE_NAMES.includes(name), `unknown browser ${name}`); + check( + equalSet(selectedBrowsers, matrixCatalog.profiles[args.profile].browsers), + `${args.profile} profile requires browsers: ${matrixCatalog.profiles[args.profile].browsers.join(",")}`, + ); + if (args.profile === "reflex") check(args.url, "reflex profile requires --url"); + + const evidence = { + schema_version: 1, requirement: "TST-NI-011", profile: args.profile, + status: "running", generated_at: new Date().toISOString(), catalog: matrixCatalog, + coverage: coverageFor(args.profile), browsers: {}, + }; + writeEvidence(args.evidence, evidence); + try { + for (const name of selectedBrowsers) { + evidence.browsers[name] = args.profile === "reflex" + ? await runReflexBrowser(name, args.url, args.executablePath) + : await runStandaloneBrowser(name, args.profile, args.executablePath); + writeEvidence(args.evidence, evidence); + } + evidence.status = Object.values(evidence.browsers).every((item) => item.status === "passed") + ? "passed" : "failed"; + if (evidence.status === "passed") validateEvidence(evidence); + } catch (error) { + evidence.status = "failed"; + evidence.error = String(error?.stack || error); + } finally { + writeEvidence(args.evidence, evidence); + } + if (evidence.status !== "passed") { + for (const [browserName, browser] of Object.entries(evidence.browsers)) { + if (browser.error) process.stderr.write(`${browserName}: ${browser.error}\n`); + for (const item of browser.cases || []) { + if (item.status !== "passed") process.stderr.write(`${browserName}/${item.id}: ${item.error || item.status}\n`); + } + } + return 1; + } + process.stdout.write(`pan/zoom ${args.profile} matrix OK: ${args.evidence}\n`); + return 0; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + process.exitCode = await main(); +} diff --git a/scripts/pick_boundary_smoke.py b/scripts/pick_boundary_smoke.py index 36d7ce98..5ebd8fc6 100644 --- a/scripts/pick_boundary_smoke.py +++ b/scripts/pick_boundary_smoke.py @@ -21,10 +21,11 @@ from __future__ import annotations +import argparse import json +import re import shutil import subprocess -import sys import tempfile from array import array from pathlib import Path @@ -49,9 +50,12 @@ BIG_PICK_INDEX = 69_999 -def find_chromium() -> str: - if len(sys.argv) > 1: - return sys.argv[1] +def find_chromium(explicit: str | None = None) -> str: + if explicit: + resolved = explicit if Path(explicit).is_file() else shutil.which(explicit) + if resolved: + return str(resolved) + raise SystemExit(f"configured chromium not found: {explicit}") for c in CHROMIUM_CANDIDATES: if Path(c).is_file() or shutil.which(c): return c @@ -242,18 +246,22 @@ def ship2(vals): window.requestAnimationFrame = realRaf; return fail(`hover sweep rendered pick ${sweepRenders}x (want 1)`); } - // Zoom out slightly: the next pick must re-render and still resolve - // correctly in the new view. - v._setView({ x0: -0.1, x1: 1.1, y0: -0.1, y1: 1.1 }, { animate: false, request: false }); + // Zoom in slightly: the next pick must re-render and still resolve + // correctly in the new view. (Zooming out beyond the home range is + // intentionally clamped by the public navigation contract.) + v._setView({ x0: 0.05, x1: 0.95, y0: 0.05, y1: 0.95 }, { animate: false, request: false }); frame(); // the view-change frame invalidates the pick cache - const zx = ((0.5 / GRID + 0.1) / 1.2) * v.plot.w; // trace 0 in the new view - const zy = (1 - (0.5 / GRID + 0.1) / 1.2) * v.plot.h; + const zoomTarget = 136; // central point, comfortably inside the zoomed view + const zgx = (zoomTarget % GRID + 0.5) / GRID; + const zgy = (Math.floor(zoomTarget / GRID) + 0.5) / GRID; + const zx = ((zgx - 0.05) / 0.9) * v.plot.w; + const zy = (1 - (zgy - 0.05) / 0.9) * v.plot.h; const hitZoomed = v._pickAt(zx, zy); window.requestAnimationFrame = realRaf; if (pickRenders !== 2) return fail(`view change: pick rendered ${pickRenders}x total (want 2)`); - if (!hitZoomed || hitZoomed.trace !== 0) - return fail(`view change: picked ${hitZoomed && hitZoomed.trace} (want 0)`); + if (!hitZoomed || hitZoomed.trace !== zoomTarget) + return fail(`view change: picked ${hitZoomed && hitZoomed.trace} (want ${zoomTarget})`); v._renderPick = renderPick0; document.title = `XY_OK slots=${slots.join(",")} big=${hitBig.index} second=${hitSecond.trace}/${hitSecond.index} hoverPickRenders=${sweepRenders} viewRefresh=1`; @@ -265,7 +273,21 @@ def ship2(vals): """ -def main() -> None: +def _write_evidence(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("chromium", nargs="?", default=None) + parser.add_argument( + "--evidence", + type=Path, + default=None, + help="write a compact JSON diagnostic suitable for CI artifact retention", + ) + args = parser.parse_args(argv) standalone = (STATIC / "standalone.js").read_text(encoding="utf-8") spec, blob, spec2, blob2 = build_payload() import base64 @@ -283,34 +305,69 @@ def main() -> None: f'const BLOB2="{base64.b64encode(blob2).decode()}";' f"{PROBE}" ) - exe = find_chromium() + exe = find_chromium(args.chromium) with tempfile.TemporaryDirectory() as td: page = Path(td) / "pick.html" page.write_text(html, encoding="utf-8") - out = subprocess.run( - [ - exe, - "--headless=new", - "--no-sandbox", - "--disable-dev-shm-usage", - "--use-angle=swiftshader", - "--enable-unsafe-swiftshader", - "--virtual-time-budget=8000", - "--dump-dom", - page.as_uri(), - ], - capture_output=True, - text=True, - timeout=120, - ) - import re - + command = [ + exe, + "--headless=new", + "--no-sandbox", + "--disable-dev-shm-usage", + "--use-angle=swiftshader", + "--enable-unsafe-swiftshader", + "--virtual-time-budget=8000", + "--dump-dom", + page.as_uri(), + ] + try: + out = subprocess.run( + command, + capture_output=True, + text=True, + timeout=120, + ) + except subprocess.TimeoutExpired as exc: + stderr = exc.stderr or "" + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + evidence = { + "status": "failed", + "title": "XY_FAIL browser timeout after 120s", + "chromium_returncode": None, + "stderr_tail": stderr[-4000:], + "timed_out": True, + "trace_slots": [0, 127, 253, 254, 255], + "large_pick_index": BIG_PICK_INDEX, + } + if args.evidence is not None: + _write_evidence(args.evidence, evidence) + print("pick boundary smoke FAILED: browser timeout after 120s") + if stderr: + print(stderr[-2000:]) + return 1 m = re.search(r"(.*?)", out.stdout, re.S | re.I) title = m.group(1).strip() if m else "(no title)" - if not title.startswith("XY_OK"): - raise SystemExit(f"pick boundary smoke FAILED: {title}") + passed = out.returncode == 0 and title.startswith("XY_OK") + evidence = { + "status": "ok" if passed else "failed", + "title": title, + "chromium_returncode": out.returncode, + "stderr_tail": out.stderr[-4000:], + "timed_out": False, + "trace_slots": [0, 127, 253, 254, 255], + "large_pick_index": BIG_PICK_INDEX, + } + if args.evidence is not None: + _write_evidence(args.evidence, evidence) + if not passed: + print(f"pick boundary smoke FAILED: {title}") + if out.stderr: + print(out.stderr[-2000:]) + return 1 print(f"pick boundary smoke OK: {title[6:]}") + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/scripts/png_export_smoke.py b/scripts/png_export_smoke.py index 462c542e..81b3fda0 100644 --- a/scripts/png_export_smoke.py +++ b/scripts/png_export_smoke.py @@ -166,7 +166,11 @@ def main() -> None: if find_chromium() is None: print("chromium png export smoke SKIPPED (no chromium)") return - png = html_to_png(build_html(), W, H, scale=SCALE, time_budget_ms=3000) + # This CI smoke renders only the repository-owned static fixture above. + # GitHub's constrained runner has historically needed an unsandboxed + # browser, so request that security downgrade explicitly at this trusted + # call site; the public export default never retries unsandboxed. + png = html_to_png(build_html(), W, H, scale=SCALE, time_budget_ms=3000, sandbox=False) if png[:8] != b"\x89PNG\r\n\x1a\n": raise SystemExit("not a PNG") w, h = struct.unpack(">II", png[16:24]) diff --git a/scripts/reflex_ws_smoke.py b/scripts/reflex_ws_smoke.py index 2948d0d8..2ce48cd8 100644 --- a/scripts/reflex_ws_smoke.py +++ b/scripts/reflex_ws_smoke.py @@ -139,10 +139,13 @@ class Probe: def __init__(self, session: ChromiumSession, url: str) -> None: self.s = session target = session._call("Target.createTarget", {"url": "about:blank"}) + self.target_id = target["targetId"] + session._call("Target.activateTarget", {"targetId": target["targetId"]}) attached = session._call( "Target.attachToTarget", {"targetId": target["targetId"], "flatten": True} ) self.sid = attached["sessionId"] + self._call("Page.bringToFront") self._call("Network.enable") self._call("Page.enable") self._call("Runtime.enable") @@ -197,6 +200,25 @@ def sent_ws_frames(self, needle: str) -> list[str]: out.append(data) return out + def binary_ws_frames(self) -> int: + """Count received websocket binary frames (socket.io attachments).""" + self.eval("1") + return sum( + 1 + for (sid, method), events in self.s._events.items() + if sid == self.sid and method == "Network.webSocketFrameReceived" + for event in events + if event.get("response", {}).get("opcode") == 2 + ) + + def closed_websockets(self) -> int: + self.eval("1") + return sum( + len(events) + for (sid, method), events in self.s._events.items() + if sid == self.sid and method == "Network.webSocketClosed" + ) + def screenshot(self) -> bytes: shot = self._call( "Page.captureScreenshot", @@ -226,6 +248,39 @@ def mouse(self, kind: str, x: float, y: float, **extra): {"type": kind, "x": x, "y": y, **extra}, ) + def first_pick_target(self, view_id: str) -> dict | None: + """Return a client-coordinate pixel occupied in the GPU pick buffer.""" + key = json.dumps(view_id) + return self.eval( + "(() => {" + f" const v = window.__xy_views.get({key});" + " const gl = v && v.gl;" + " if (!v || !gl || !v.pickFbo) return null;" + " if (v._pickDirty) v._renderPick();" + " const w = v.canvas.width, h = v.canvas.height;" + " const pixels = new Uint8Array(w * h * 4);" + " gl.bindFramebuffer(gl.FRAMEBUFFER, v.pickFbo);" + " gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, pixels);" + " gl.bindFramebuffer(gl.FRAMEBUFFER, null);" + " let p = -1;" + " for (let i = 0; i < pixels.length; i += 4) {" + " if (pixels[i] || pixels[i + 1] || pixels[i + 2] || pixels[i + 3]) {" + " p = i / 4; break;" + " }" + " }" + " if (p < 0) return null;" + " const px = p % w, py = Math.floor(p / w);" + " const r = v.canvas.getBoundingClientRect();" + " const dpr = v.dpr || window.devicePixelRatio || 1;" + " const cssX = (px + 0.5) / dpr;" + " const cssY = (h - py - 0.5) / dpr;" + " const hit = v._pickAt(cssX, cssY);" + " if (!hit) return null;" + " return {x: r.left + cssX, y: r.top + cssY, trace: hit.trace," + " index: hit.index};" + " })()" + ) + def main() -> None: parser = argparse.ArgumentParser() @@ -256,6 +311,11 @@ def main() -> None: if len(ws) != 1: failures.append(f"expected exactly 1 backend websocket (shared transport), got {ws}") + binary_frames = probe.binary_ws_frames() + print(f"binary websocket frames received: {binary_frames}") + if binary_frames == 0: + failures.append("no binary socket.io attachments reached the browser") + # 2b) the direct-Chart mount is truly static: it never subscribes. The # six live sources (five figure vars + one inline() token) each sub. subs = probe.sent_ws_frames('"sub"') @@ -267,6 +327,9 @@ def main() -> None: # rects in page coordinates so below-the-fold charts count too) time.sleep(1.0) png = probe.screenshot() + if args.screenshot: + Path(args.screenshot).write_bytes(png) + print(f"saved initial evidence to {args.screenshot}") checks = (("cloud", 0.02), ("hist", 0.02), ("live", 0.005), ("inline", 0.005)) for chart_id, min_ink in checks: frac = ink_fraction(png, probe.rect(chart_id, page_coords=True), 1.0) @@ -275,12 +338,37 @@ def main() -> None: failures.append(f"{chart_id} looks blank ({frac:.2%} < {min_ink:.0%})") # 4) deep zoom drills density -> exact points (§16 over the socket) … - rect = probe.rect("cloud") - cx, cy = rect["x"] + rect["w"] * 0.55, rect["y"] + rect["h"] * 0.5 + # The chart is taller than Chromium's default headless viewport. Put + # its interaction surface on-screen before dispatching trusted CDP + # wheel/pointer input; off-viewport coordinates are silently ignored. + probe.scroll_to("cloud") + target = probe.eval( + "(() => { const v = window.__xy_views.get('cloud');" + " const r = v.canvas.getBoundingClientRect();" + " const x = r.left + r.width * 0.55, y = r.top + r.height * 0.5;" + " const hit = document.elementFromPoint(x, y);" + " return {x, y, rect: {x: r.x, y: r.y, w: r.width, h: r.height}," + " viewport: {w: innerWidth, h: innerHeight}," + " hit: hit && hit.tagName, hitsCanvas: hit === v.canvas}; })()" + ) + print(f"cloud input target: {target}") + if not target["hitsCanvas"]: + raise SystemExit( + "cloud interaction precondition failed: CDP coordinates do not hit " + f"the chart canvas (rect={target['rect']}, viewport={target['viewport']}, " + f"target={target['hit']!r})" + ) + cx, cy = target["x"], target["y"] probe.mouse("mouseMoved", cx, cy) for _ in range(16): probe.mouse("mouseWheel", cx, cy, deltaX=0, deltaY=-240) time.sleep(0.15) + zoom_state = probe.eval( + "(() => { const v = window.__xy_views.get('cloud');" + " const g = v.gpuTraces[0]; return {view: v.view, seq: v.seq," + " tier: g.tier, drill: !!g.drill}; })()" + ) + print(f"deep-zoom state: {zoom_state}") probe.wait_for( "(() => { const g = window.__xy_views.get('cloud').gpuTraces[0];" " return !!(g && (g.drill || g.tier !== 'density')); })()", @@ -291,10 +379,25 @@ def main() -> None: # … and hovering a drilled point closes the semantic event loop: # GPU pick -> socket pick round-trip -> reflex event -> state delta. - for dx in range(-40, 200, 8): - probe.mouse("mouseMoved", cx + dx / 4, cy + dx / 16) - time.sleep(0.12) try: + probe.wait_for( + "(() => { const v = window.__xy_views.get('cloud');" + " if (!v || !v.gpuTraces[0].drill) return false;" + " if (v._pickDirty) v._renderPick();" + " const gl = v.gl, w = v.canvas.width, h = v.canvas.height;" + " const pixels = new Uint8Array(w * h * 4);" + " gl.bindFramebuffer(gl.FRAMEBUFFER, v.pickFbo);" + " gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, pixels);" + " gl.bindFramebuffer(gl.FRAMEBUFFER, null);" + " return pixels.some((value) => value !== 0); })()", + timeout_s=15.0, + label="a drilled point in the GPU pick buffer", + ) + point = probe.first_pick_target("cloud") + if point is None: + raise SystemExit("GPU pick buffer was nonempty but yielded no pickable point") + print(f"hovering drilled GPU point: {point}") + probe.mouse("mouseMoved", point["x"], point["y"]) found_row = probe.wait_for( "(document.body.innerText.match(/x=-?[0-9.]+/) || [null])[0]", timeout_s=15.0, @@ -328,6 +431,50 @@ def main() -> None: Path(args.screenshot).write_bytes(probe.screenshot()) print(f"saved {args.screenshot}") + # 6) renderer teardown releases every resource it owns. The React + # wrapper's effect cleanup calls the same idempotent destroy method; + # direct invocation keeps the assertions observable before host + # teardown destroys the page execution context. + teardown = probe.eval( + "(() => { const views = Array.from(window.__xy_views?.values?.() || []);" + " for (const view of views) view.destroy();" + " return views.map((view) => ({" + " destroyed: view._destroyed === true, listeners: view._listeners.length," + " worker: view._rebinWorker == null, gl: view.gl === null," + " comm: view._unsubscribeComm === null, connected: view.root.isConnected" + " })); })()" + ) + print(f"destroyed views: {len(teardown)}") + if len(teardown) < 6 or any( + not item["destroyed"] + or item["listeners"] != 0 + or not item["worker"] + or not item["gl"] + or not item["comm"] + or item["connected"] + for item in teardown + ): + failures.append(f"renderer teardown leaked resources: {teardown}") + + # Reflex installs pagehide/beforeunload as its host-transport teardown + # path. Exercise that lifecycle while the CDP target remains alive so + # Network.webSocketClosed is observable (closing the target first drops + # its final target-scoped Network events). + probe.eval( + "window.dispatchEvent(new PageTransitionEvent('pagehide', {persisted: false})); true" + ) + deadline = time.monotonic() + 15.0 + closed = 0 + while time.monotonic() < deadline: + closed = probe.closed_websockets() + if closed >= 1: + break + time.sleep(0.1) + print(f"closed backend websockets: {closed}") + if closed < 1: + failures.append("backend websocket remained open after page teardown") + session._call("Target.closeTarget", {"targetId": probe.target_id}) + if failures: print("\nFAILURES:") for failure in failures: diff --git a/scripts/release_provenance.py b/scripts/release_provenance.py new file mode 100644 index 00000000..93dc342d --- /dev/null +++ b/scripts/release_provenance.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Create and verify SHA-256 provenance for immutable release artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any + +SCHEMA = "xy-release-provenance-v1" +SHA_RE = re.compile(r"[0-9a-f]{40}") +SHA256_RE = re.compile(r"[0-9a-f]{64}") +REPOSITORY_RE = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") +RUN_ID_RE = re.compile(r"[1-9][0-9]*") + + +def _digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def create_manifest( + artifact_root: Path, + *, + source_sha: str, + repository: str, + workflow_run_id: str, + tag: str, + created_at: str | None = None, +) -> dict[str, Any]: + if SHA_RE.fullmatch(source_sha) is None: + raise ValueError("source_sha must be a 40-character lowercase commit SHA") + if REPOSITORY_RE.fullmatch(repository) is None: + raise ValueError("repository must be owner/name") + if RUN_ID_RE.fullmatch(str(workflow_run_id)) is None: + raise ValueError("workflow_run_id must be a positive integer") + if TAG_RE.fullmatch(tag) is None: + raise ValueError("tag contains unsupported characters") + timestamp = created_at or datetime.now(UTC).isoformat() + _parse_timestamp(timestamp) + files = sorted(path for path in artifact_root.rglob("*") if path.is_file()) + if not files: + raise ValueError(f"no artifacts found under {artifact_root}") + names = [path.name for path in files] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"artifact basenames must be unique: {duplicates}") + return { + "schema": SCHEMA, + "source_sha": source_sha, + "repository": repository, + "workflow_run_id": str(workflow_run_id), + "tag": tag, + "created_at": timestamp, + "artifacts": [ + { + "name": path.name, + "path": path.relative_to(artifact_root).as_posix(), + "size": path.stat().st_size, + "sha256": _digest(path), + } + for path in files + ], + } + + +def _parse_timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise ValueError("created_at must be an ISO-8601 timestamp") + try: + parsed = datetime.fromisoformat(value) + except ValueError as exc: + raise ValueError("created_at must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise ValueError("created_at must include a timezone") + return parsed + + +def verify_manifest( + manifest: dict[str, Any], + artifacts: list[Path], + *, + source_sha: str | None = None, + repository: str | None = None, + workflow_run_id: str | None = None, + tag: str | None = None, +) -> list[str]: + errors: list[str] = [] + if manifest.get("schema") != SCHEMA: + errors.append(f"unsupported provenance schema {manifest.get('schema')!r}") + manifest_sha = manifest.get("source_sha") + if not isinstance(manifest_sha, str) or SHA_RE.fullmatch(manifest_sha) is None: + errors.append("provenance source_sha is not a lowercase 40-character commit SHA") + if source_sha is not None and manifest.get("source_sha") != source_sha: + errors.append( + f"provenance source SHA {manifest.get('source_sha')!r} does not match {source_sha!r}" + ) + manifest_repository = manifest.get("repository") + if ( + not isinstance(manifest_repository, str) + or REPOSITORY_RE.fullmatch(manifest_repository) is None + ): + errors.append("provenance repository is not owner/name") + if repository is not None and manifest_repository != repository: + errors.append( + f"provenance repository {manifest_repository!r} does not match {repository!r}" + ) + manifest_run_id = manifest.get("workflow_run_id") + if not isinstance(manifest_run_id, str) or RUN_ID_RE.fullmatch(manifest_run_id) is None: + errors.append("provenance workflow_run_id is not a positive integer") + if workflow_run_id is not None and manifest_run_id != str(workflow_run_id): + errors.append( + f"provenance workflow run {manifest_run_id!r} does not match {str(workflow_run_id)!r}" + ) + manifest_tag = manifest.get("tag") + if not isinstance(manifest_tag, str) or TAG_RE.fullmatch(manifest_tag) is None: + errors.append("provenance tag contains unsupported characters") + if tag is not None and manifest_tag != tag: + errors.append(f"provenance tag {manifest_tag!r} does not match {tag!r}") + try: + _parse_timestamp(manifest.get("created_at")) + except ValueError as exc: + errors.append(str(exc)) + records = manifest.get("artifacts") + if not isinstance(records, list): + return [*errors, "provenance artifacts must be a list"] + valid_records: list[dict[str, Any]] = [] + for index, record in enumerate(records): + if not isinstance(record, dict) or not isinstance(record.get("name"), str): + errors.append(f"provenance artifact record {index} has no valid name") + continue + name = record["name"] + record_path = record.get("path") + if not isinstance(record_path, str): + errors.append(f"provenance artifact {name!r} has no valid path") + else: + pure_path = PurePosixPath(record_path) + if pure_path.is_absolute() or ".." in pure_path.parts or pure_path.name != name: + errors.append(f"provenance artifact {name!r} has unsafe path {record_path!r}") + size = record.get("size") + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + errors.append(f"provenance artifact {name!r} has invalid size {size!r}") + digest = record.get("sha256") + if not isinstance(digest, str) or SHA256_RE.fullmatch(digest) is None: + errors.append(f"provenance artifact {name!r} has invalid SHA-256") + valid_records.append(record) + record_names = [record["name"] for record in valid_records] + duplicate_records = sorted({name for name in record_names if record_names.count(name) > 1}) + if duplicate_records: + errors.append(f"provenance contains duplicate artifact records: {duplicate_records}") + artifact_names = [path.name for path in artifacts] + duplicate_artifacts = sorted( + {name for name in artifact_names if artifact_names.count(name) > 1} + ) + if duplicate_artifacts: + errors.append(f"supplied artifacts have duplicate basenames: {duplicate_artifacts}") + + record_set = set(record_names) + artifact_set = set(artifact_names) + for name in sorted(record_set - artifact_set): + errors.append(f"provenance artifact {name!r} was not supplied for verification") + for name in sorted(artifact_set - record_set): + errors.append(f"artifact {name!r} is absent from provenance") + + by_name = {record["name"]: record for record in valid_records} + for path in artifacts: + record = by_name.get(path.name) + if record is None: + continue + if not path.is_file(): + errors.append(f"artifact {path} does not exist") + continue + actual_size = path.stat().st_size + if record.get("size") != actual_size: + errors.append( + f"artifact {path.name!r} size is {actual_size}, expected {record.get('size')!r}" + ) + actual_digest = _digest(path) + if record.get("sha256") != actual_digest: + errors.append(f"artifact {path.name!r} SHA-256 does not match provenance") + return errors + + +def _load_manifest(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("provenance manifest must be a JSON object") + return payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + create = subparsers.add_parser("create") + create.add_argument("artifact_root", type=Path) + create.add_argument("--output", type=Path, required=True) + create.add_argument("--source-sha", required=True) + create.add_argument("--repository", required=True) + create.add_argument("--workflow-run-id", required=True) + create.add_argument("--tag", required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("manifest", type=Path) + verify.add_argument("artifacts", nargs="+", type=Path) + verify.add_argument("--source-sha") + verify.add_argument("--repository") + verify.add_argument("--workflow-run-id") + verify.add_argument("--tag") + args = parser.parse_args(argv) + + try: + if args.command == "create": + payload = create_manifest( + args.artifact_root, + source_sha=args.source_sha, + repository=args.repository, + workflow_run_id=args.workflow_run_id, + tag=args.tag, + ) + args.output.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + f"release provenance created: {args.output} ({len(payload['artifacts'])} artifacts)" + ) + return 0 + payload = _load_manifest(args.manifest) + errors = verify_manifest( + payload, + args.artifacts, + source_sha=args.source_sha, + repository=args.repository, + workflow_run_id=args.workflow_run_id, + tag=args.tag, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"release provenance failed: {exc}", file=sys.stderr) + return 1 + if errors: + print("release provenance verification failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print(f"release provenance verified: {len(args.artifacts)} artifact(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_pytest_no_skips.py b/scripts/run_pytest_no_skips.py new file mode 100644 index 00000000..6cbe67b7 --- /dev/null +++ b/scripts/run_pytest_no_skips.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Run a focused pytest lane and fail when any item is skipped. + +Package-specific integration suites own their complete test environment. This +runner makes that contract explicit: an import failure, version incompatibility, +or any test skip cannot become a green host-integration lane. +""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass, field +from typing import Any + +import pytest + + +@dataclass +class _NoSkips: + skipped: list[str] = field(default_factory=list) + + def pytest_runtest_logreport(self, report: Any) -> None: + if report.skipped: + reason = getattr(report, "longreprtext", "skipped") + self.skipped.append(f"{report.nodeid}: {reason}") + + def pytest_collectreport(self, report: Any) -> None: + if report.skipped: + reason = getattr(report, "longreprtext", "skipped during collection") + self.skipped.append(f"{report.nodeid}: {reason}") + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not args: + print("usage: run_pytest_no_skips.py PYTEST_ARG ...", file=sys.stderr) + return 2 + + plugin = _NoSkips() + code = int(pytest.main(args, plugins=[plugin])) + if plugin.skipped: + print("\nDedicated integration lane collected skips:", file=sys.stderr) + for item in plugin.skipped: + print(f" - {item}", file=sys.stderr) + return 1 + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/runtime_security_smoke.py b/scripts/runtime_security_smoke.py new file mode 100644 index 00000000..32c6b7ea --- /dev/null +++ b/scripts/runtime_security_smoke.py @@ -0,0 +1,611 @@ +#!/usr/bin/env python3 +"""Real-browser standalone DOM-XSS, CSP, and network-isolation gate. + +This is deliberately separate from Chromium's *process sandbox* policy. It +loads a production ``Chart.to_html()`` document and observes the page-content +boundary in a browser: + +* hostile strings traverse every public text surface and remain literal text; +* no user string creates executable DOM, runs script, or opens a dialog; +* hostile author CSS is applied, while its external URL is stopped by the + shipped standalone CSP before it reaches a loopback request sentinel; and +* the browser reports the expected CSP violation and no network API attempts. + +The JSON evidence is retained by CI on success and failure. Browser launch +uses its sandbox by default; ``--no-sandbox`` is an explicit fixture-level +opt-out for constrained CI runners, never an automatic retry. +""" + +from __future__ import annotations + +import argparse +import base64 +import html +import json +import shutil +import subprocess +import sys +import tempfile +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from html.parser import HTMLParser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "python")) + +import xy # noqa: E402 + +CHROMIUM_CANDIDATES = ( + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/opt/pw-browsers/chromium", + "chromium", + "chromium-browser", + "google-chrome", + "google-chrome-stable", +) + +REPORT_ID = "xy-runtime-security-report" + + +def find_chromium(explicit: str | None = None) -> str: + """Resolve an explicit browser without silently falling back.""" + if explicit: + resolved = explicit if Path(explicit).is_file() else shutil.which(explicit) + if resolved: + return str(resolved) + raise RuntimeError(f"configured chromium not found: {explicit}") + for candidate in CHROMIUM_CANDIDATES: + resolved = candidate if Path(candidate).is_file() else shutil.which(candidate) + if resolved: + return str(resolved) + raise RuntimeError("no chromium found; pass its path as the first argument") + + +class _SentinelHandler(BaseHTTPRequestHandler): + server: "_SentinelServer" + + def _record(self) -> None: + self.server.requests.append( + {"method": self.command, "path": self.path, "client": self.client_address[0]} + ) + self.send_response(204) + self.end_headers() + + do_GET = _record + do_HEAD = _record + + def log_message(self, _format: str, *_args: object) -> None: + return + + +class _SentinelServer(ThreadingHTTPServer): + requests: list[dict[str, str]] + + +@contextmanager +def network_sentinel() -> Iterator[tuple[str, list[dict[str, str]]]]: + """Yield a loopback URL and a list populated only by real HTTP requests.""" + server = _SentinelServer(("127.0.0.1", 0), _SentinelHandler) + server.requests = [] + thread = threading.Thread(target=server.serve_forever, name="xy-security-sentinel") + thread.daemon = True + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}/xy-runtime-security-probe", server.requests + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _attack(label: str, sentinel_url: str) -> str: + """A recognizable payload that would execute, navigate, and fetch if parsed.""" + return ( + f"XY_{label}::' + '' + ) + + +def _surface_values(sentinel_url: str) -> dict[str, str]: + names = ( + "TITLE", + "X_AXIS", + "Y_AXIS", + "X_TICK", + "Y_TICK", + "TRACE_LINE", + "TRACE_COLOR", + "CATEGORY", + "ANNOTATION", + "LEGEND_TITLE", + "COLORBAR_TITLE", + "TOOLTIP_TITLE", + "FIELD_X", + "FIELD_Y", + "FIELD_CATEGORY", + "FIELD_COLOR", + ) + return {name.casefold(): _attack(name, sentinel_url) for name in names} + + +_MONITOR = r""" + +""" + + +_PROBE = rf""" + +""" + + +def build_runtime_fixture(sentinel_url: str) -> tuple[str, list[dict[str, str]]]: + """Build the instrumented production export and its DOM expectations.""" + value = _surface_values(sentinel_url) + data = { + value["field_x"]: [0.2, 0.8], + value["field_y"]: [0.3, 0.7], + value["field_category"]: [value["category"], "safe category"], + value["field_color"]: [0.1, 0.9], + } + chart = xy.chart( + xy.scatter( + x=value["field_x"], + y=value["field_y"], + color=value["field_category"], + data=data, + name="categorical points", + size=16, + ), + xy.line(x=[0.15, 0.85], y=[0.25, 0.75], name=value["trace_line"]), + xy.scatter( + x=value["field_x"], + y=value["field_y"], + color=value["field_color"], + data=data, + name=value["trace_color"], + size=10, + ), + xy.text(0.5, 0.5, value["annotation"]), + xy.x_axis( + label=value["x_axis"], + tick_values=[0.2, 0.8], + tick_labels=[value["x_tick"], "safe x tick"], + ), + xy.y_axis( + label=value["y_axis"], + tick_values=[0.3, 0.7], + tick_labels=[value["y_tick"], "safe y tick"], + ), + xy.legend(title=value["legend_title"]), + xy.tooltip( + fields=[value["field_x"], value["field_y"], value["field_category"]], + title=value["tooltip_title"], + ), + xy.colorbar(title=value["colorbar_title"]), + title=value["title"], + width=960, + height=560, + ) + custom_css = ( + '.xy[data-xy-slot="root"] {' + "--xy-runtime-security-probe: applied;" + f'background-image: url("{sentinel_url}") !important;' + "}" + ) + document = chart.to_html(custom_css=custom_css) + expectations = [ + {"name": "title", "selector": '[data-xy-slot="title"]', "text": value["title"]}, + { + "name": "x-axis title", + "selector": '[data-xy-slot="axis_title"]', + "text": value["x_axis"], + }, + { + "name": "y-axis title", + "selector": '[data-xy-slot="axis_title"]', + "text": value["y_axis"], + }, + { + "name": "x tick label", + "selector": '[data-xy-slot="tick_label"]', + "text": value["x_tick"], + }, + { + "name": "y tick label", + "selector": '[data-xy-slot="tick_label"]', + "text": value["y_tick"], + }, + { + "name": "line trace name", + "selector": '[data-xy-slot="legend"]', + "text": value["trace_line"], + }, + { + "name": "continuous trace name", + "selector": '[data-xy-slot="legend"]', + "text": value["trace_color"], + }, + { + "name": "category", + "selector": '[data-xy-slot="legend"]', + "text": value["category"], + }, + { + "name": "annotation", + "selector": '[data-xy-slot="annotation_label"]', + "text": value["annotation"], + }, + { + "name": "legend title", + "selector": '[data-xy-slot="legend"]', + "text": value["legend_title"], + }, + { + "name": "colorbar title", + "selector": '[data-xy-slot="colorbar_title"]', + "text": value["colorbar_title"], + }, + { + "name": "tooltip title", + "selector": '[data-xy-slot="tooltip"]', + "text": value["tooltip_title"], + }, + { + "name": "tooltip x field", + "selector": '[data-xy-slot="tooltip"]', + "text": value["field_x"], + }, + { + "name": "tooltip y field", + "selector": '[data-xy-slot="tooltip"]', + "text": value["field_y"], + }, + { + "name": "tooltip category field", + "selector": '[data-xy-slot="tooltip"]', + "text": value["field_category"], + }, + { + "name": "tooltip category value", + "selector": '[data-xy-slot="tooltip"]', + "text": value["category"], + }, + ] + production_call = 'xy.renderStandalone(document.getElementById("chart"), spec, buf);' + if document.count(production_call) != 1: + raise RuntimeError("production standalone render call changed; update security fixture") + document = document.replace( + production_call, + "window.__xyRuntimeSecurityView = " + production_call, + 1, + ) + if document.count("") != 1 or document.count("") != 1: + raise RuntimeError("production standalone document structure changed") + config = base64.b64encode( + json.dumps({"sentinelUrl": sentinel_url, "expectations": expectations}).encode() + ).decode("ascii") + document = document.replace("", "" + _MONITOR, 1) + document = document.replace( + "", + f'{_PROBE}', + 1, + ) + return document, expectations + + +class _ReportParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._inside = False + self.parts: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.casefold() == "pre" and dict(attrs).get("id") == REPORT_ID: + self._inside = True + + def handle_endtag(self, tag: str) -> None: + if tag.casefold() == "pre" and self._inside: + self._inside = False + + def handle_data(self, data: str) -> None: + if self._inside: + self.parts.append(data) + + +def _parse_browser_output(output: str) -> tuple[str, dict[str, Any] | None]: + parser = _ReportParser() + parser.feed(output) + title_start = output.casefold().find("") + title_end = output.casefold().find("", title_start + 7) + title = "(no title)" + if title_start >= 0 and title_end >= 0: + title = html.unescape(output[title_start + 7 : title_end]).strip() + report_text = "".join(parser.parts).strip() + if not report_text: + return title, None + try: + report = json.loads(report_text) + except json.JSONDecodeError as exc: + return title, {"failures": [f"invalid browser report JSON: {exc}"]} + return title, report + + +def _run_browser(executable: str, page: Path, *, no_sandbox: bool) -> SimpleNamespace: + command = [ + executable, + "--headless=new", + "--disable-dev-shm-usage", + "--use-angle=swiftshader", + "--enable-unsafe-swiftshader", + "--virtual-time-budget=10000", + "--dump-dom", + page.as_uri(), + ] + if no_sandbox: + command.insert(2, "--no-sandbox") + return subprocess.run(command, capture_output=True, text=True, timeout=120) + + +def _write_evidence(path: Path, evidence: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("chromium", nargs="?", default=None) + parser.add_argument( + "--no-sandbox", + action="store_true", + help="explicitly disable Chromium's process sandbox for this CI fixture", + ) + parser.add_argument("--evidence", type=Path, default=None) + args = parser.parse_args(argv) + + evidence: dict[str, Any] = { + "status": "failed", + "launch_sandbox": "disabled-explicitly" if args.no_sandbox else "enabled", + "timed_out": False, + "chromium_returncode": None, + "network_requests": [], + } + try: + executable = find_chromium(args.chromium) + evidence["chromium"] = executable + with network_sentinel() as (sentinel_url, requests): + document, expectations = build_runtime_fixture(sentinel_url) + evidence["expected_surfaces"] = [item["name"] for item in expectations] + with tempfile.TemporaryDirectory(prefix="xy-runtime-security-") as temp_dir: + page = Path(temp_dir) / "runtime-security.html" + page.write_text(document, encoding="utf-8") + try: + completed = _run_browser(executable, page, no_sandbox=args.no_sandbox) + except subprocess.TimeoutExpired as exc: + stderr = exc.stderr or "" + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + evidence.update( + title="XY_RUNTIME_SECURITY_FAIL browser timeout after 120s", + timed_out=True, + stderr_tail=stderr[-4000:], + ) + else: + title, report = _parse_browser_output(completed.stdout) + evidence.update( + title=title, + chromium_returncode=completed.returncode, + stderr_tail=completed.stderr[-4000:], + browser_report=report, + ) + evidence["network_requests"] = list(requests) + except (OSError, RuntimeError, ValueError) as exc: + evidence["title"] = f"XY_RUNTIME_SECURITY_FAIL {exc}" + + report = evidence.get("browser_report") + passed = ( + evidence.get("chromium_returncode") == 0 + and evidence.get("title") == "XY_RUNTIME_SECURITY_OK" + and isinstance(report, dict) + and report.get("failures") == [] + and evidence.get("network_requests") == [] + ) + evidence["status"] = "ok" if passed else "failed" + if evidence.get("network_requests"): + evidence.setdefault("failures", []).append( + "standalone page reached the loopback network sentinel" + ) + if args.evidence is not None: + _write_evidence(args.evidence, evidence) + if passed: + print( + "runtime security smoke OK: " + f"{len(evidence['expected_surfaces'])} hostile text surfaces, " + "CSP-blocked CSS, zero dialogs/requests" + ) + return 0 + print(f"runtime security smoke FAILED: {evidence.get('title', '(no title)')}") + if isinstance(report, dict): + for failure in report.get("failures", []): + print(f"- {failure}") + for failure in evidence.get("failures", []): + print(f"- {failure}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_benchmark_report.py b/scripts/verify_benchmark_report.py index 528d90c8..de6dd7cd 100644 --- a/scripts/verify_benchmark_report.py +++ b/scripts/verify_benchmark_report.py @@ -32,6 +32,7 @@ "install-footprint", "transport-loopback", ) +VALIDATION_PROFILES = ("baseline", "strict") ROW_STATUSES = ("ok", "unavailable", "skipped", "failed") COMPARISON_VERDICTS = {"pass", "watch", "fail", "no-plotly"} INTERACTION_BUDGET_KEYS = ( @@ -1502,7 +1503,9 @@ def _validate_interaction_visual_budget_block( return valid -def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> None: +def _validate_dashboard_browser( + report: dict[str, Any], errors: list[str], *, profile: str = "baseline" +) -> None: _require_keys( report, { @@ -1678,6 +1681,81 @@ def _validate_dashboard_browser(report: dict[str, Any], errors: list[str]) -> No f"dashboard {DASHBOARD_MIN_LOSS_FREE_CHARTS}-chart {metric} " f"{value:.3g} ms exceeds hard smoke budget {limit:.3g} ms" ) + if profile == "strict": + _validate_dashboard_strict(rows, report.get("attempted_chart_counts"), errors) + + +def _validate_dashboard_strict( + rows: list[Any], attempted_chart_counts: Any, errors: list[str] +) -> None: + """Enforce the release-health contract, not just report coherence. + + Baseline validation intentionally preserves measurements from partial and + failed benchmark rows. The strict profile is the hard-CI policy: every + 10/20/50 row must be present exactly once and must be either loss-free or + governed/recoverable, with every chart proven nonblank when visited. + """ + rows_by_count: dict[int, list[dict[str, Any]]] = {} + for row in rows: + if not isinstance(row, dict): + continue + count = row.get("chart_count") + if isinstance(count, int) and not isinstance(count, bool): + rows_by_count.setdefault(count, []).append(row) + + requested_counts = set(DASHBOARD_REQUIRED_COUNTS) + if isinstance(attempted_chart_counts, list): + requested_counts.update( + count + for count in attempted_chart_counts + if isinstance(count, int) and not isinstance(count, bool) and count > 0 + ) + + for count in sorted(requested_counts): + matching = rows_by_count.get(count, []) + if len(matching) != 1: + errors.append( + f"strict dashboard profile requires exactly one {count}-chart row; " + f"found {len(matching)}" + ) + continue + row = matching[0] + path = f"strict dashboard {count}-chart row" + status = _status_kind(row.get("status")) + if status != "ok": + errors.append(f"{path} must have status 'ok'; got {row.get('status')!r}") + continue + render_status = row.get("render_status") + if render_status not in {"complete", "governed"}: + errors.append(f"{path} must be complete or governed; got {render_status!r}") + if row.get("created_charts") != count or row.get("creation_failed_charts") != 0: + errors.append(f"{path} must create all {count} charts without failures") + if row.get("scroll_nonblank_charts") != count or row.get("scroll_blank_chart_ids") != []: + errors.append(f"{path} must prove every chart nonblank when visited") + + # A governed row may release off-screen contexts, but browser evictions, + # ungoverned loss events, and still-lost contexts without a release + # marker are unexplained losses and therefore release-blocking. + if render_status == "governed": + if row.get("evicted_chart_ids") != []: + errors.append(f"{path} contains unexplained browser-evicted charts") + events = row.get("context_events") + if isinstance(events, list) and any( + isinstance(event, dict) + and event.get("type") == "lost" + and event.get("governed") is not True + for event in events + ): + errors.append(f"{path} contains an ungoverned context-loss event") + currently_lost = row.get("currently_lost_chart_ids") + released = row.get("released_chart_ids") + if isinstance(currently_lost, list) and isinstance(released, list): + unexplained = sorted(set(currently_lost) - set(released)) + if unexplained: + errors.append( + f"{path} contains currently lost charts without governed release: " + f"{unexplained}" + ) def _dashboard_id_list( @@ -2111,8 +2189,10 @@ def summarize_report(report: dict[str, Any], *, kind: str) -> list[str]: return summary -def validate_report(path: Path, *, kind: str = "auto") -> list[str]: +def validate_report(path: Path, *, kind: str = "auto", profile: str = "baseline") -> list[str]: errors: list[str] = [] + if profile not in VALIDATION_PROFILES: + return [f"unknown validation profile: {profile!r}"] try: report = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: @@ -2142,7 +2222,7 @@ def validate_report(path: Path, *, kind: str = "auto") -> list[str]: elif selected == "interaction-browser": _validate_interaction_browser(report, errors) elif selected == "dashboard-browser": - _validate_dashboard_browser(report, errors) + _validate_dashboard_browser(report, errors, profile=profile) elif selected == "workflow-native": _validate_workflow_native(report, errors) elif selected == "transport-loopback": @@ -2158,9 +2238,15 @@ def main(argv: Optional[list[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("report", type=Path) parser.add_argument("--kind", choices=KNOWN_KINDS, default="auto") + parser.add_argument( + "--profile", + choices=VALIDATION_PROFILES, + default="baseline", + help="strict makes dashboard outcome health release-blocking; baseline validates artifact coherence", + ) args = parser.parse_args(argv) - errors = validate_report(args.report, kind=args.kind) + errors = validate_report(args.report, kind=args.kind, profile=args.profile) if errors: print(f"benchmark report verification failed for {args.report}:", file=sys.stderr) for error in errors: diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index 2ee5ee39..b1f90723 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -19,21 +19,41 @@ DEFAULT_CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" DEFAULT_CODSPEED_WORKFLOW = ROOT / ".github" / "workflows" / "codspeed.yml" DEFAULT_RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +DEFAULT_DEPLOY_DEV_WORKFLOW = ROOT / ".github" / "workflows" / "deploy-docs-dev.yml" +DEFAULT_DEPLOY_STG_WORKFLOW = ROOT / ".github" / "workflows" / "deploy-docs-stg.yml" +DEFAULT_BUILD_DOCS_WORKFLOW = ROOT / ".github" / "workflows" / "_build-docs-images.yml" +DEFAULT_HELM_DOCS_WORKFLOW = ROOT / ".github" / "workflows" / "_helm-docs-pr.yml" DEFAULT_WORKFLOW = DEFAULT_CI_WORKFLOW REQUIRED_CI_JOBS = { "browser_conformance", + "dependency_audit", + "host_integration", + "javascript_semantics", + "reflex_adapter", "matplotlib_reference", + "native_parity", + "python_coverage", "test", "python_floor", + "rust_release", "benchmark_vs", "benchmark_methodology", "benchmark", "sdist", "wheels", "install_without_rust", + "required_ci", } REQUIRED_CODSPEED_JOBS = {"benchmarks"} -REQUIRED_RELEASE_JOBS = {"wheels", "sdist", "publish", "publish-pyodide", "wasm"} +REQUIRED_RELEASE_JOBS = { + "qualify", + "wheels", + "sdist", + "provenance", + "publish", + "publish-pyodide", + "wasm", +} def _job_blocks(text: str) -> dict[str, str]: @@ -169,8 +189,8 @@ def _require_workflow_contains( errors.append(f"{workflow_label} workflow missing {description}: {missing}") -def _require_docs_spec_pr_paths_ignored(errors: list[str], text: str, workflow_label: str) -> None: - """Require PR-only docs/spec changes to skip an expensive workflow.""" +def _require_pr_all_paths(errors: list[str], text: str, workflow_label: str) -> None: + """Require a PR trigger that cannot omit the workflow's stable result.""" lines = text.splitlines() try: start = next(i for i, line in enumerate(lines) if line == " pull_request:") @@ -178,8 +198,6 @@ def _require_docs_spec_pr_paths_ignored(errors: list[str], text: str, workflow_l errors.append(f"{workflow_label} workflow missing pull_request trigger") return - paths_ignore: list[str] | None = None - collecting_paths_ignore = False for line in lines[start + 1 :]: indent = len(line) - len(line.lstrip()) if line.strip() and indent <= 2: @@ -187,27 +205,62 @@ def _require_docs_spec_pr_paths_ignored(errors: list[str], text: str, workflow_l stripped = line.strip() if not stripped or stripped.startswith("#"): continue - if indent == 4: - collecting_paths_ignore = bool(re.fullmatch(r"paths-ignore:\s*(?:#.*)?", stripped)) - if collecting_paths_ignore: - paths_ignore = [] - continue - if not collecting_paths_ignore or indent <= 4: - continue - item = re.fullmatch(r"-\s+(.+?)\s*", stripped) - if item is None or paths_ignore is None: + if indent == 4 and re.match(r"paths(?:-ignore)?:", stripped): + errors.append( + f"{workflow_label} pull_request trigger must run on every path; found {stripped!r}" + ) + + +def _listed_needs(job_text: str) -> set[str]: + """Return names from a block-form top-level ``needs`` list.""" + needs: set[str] = set() + collecting = False + for line in job_text.splitlines(): + if line == " needs:": + collecting = True continue - value = re.sub(r"\s+#.*$", "", item.group(1)).strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: - value = value[1:-1] - paths_ignore.append(value) + if collecting: + match = re.fullmatch(r" - ([A-Za-z0-9_-]+)", line) + if match: + needs.add(match.group(1)) + continue + if line.strip(): + break + return needs - required = {"docs/**", "spec/**"} - missing = sorted(required - set(paths_ignore or [])) - if missing: - errors.append( - f"{workflow_label} pull_request trigger must skip docs/spec-only changes: {missing}" - ) + +def _python_heredoc_syntax_errors(text: str) -> list[str]: + """Compile Python heredocs after applying YAML block-scalar indentation.""" + lines = text.splitlines() + errors: list[str] = [] + for index, line in enumerate(lines): + if not re.search(r"<<-?['\"]?PY['\"]?\s*$", line): + continue + indent = len(line) - len(line.lstrip()) + prefix = " " * indent + body: list[str] = [] + for end in range(index + 1, len(lines)): + if lines[end] == f"{prefix}PY": + source = "\n".join( + candidate[indent:] if candidate.startswith(prefix) else candidate + for candidate in body + ) + try: + compile(source, f"workflow-heredoc:{index + 1}", "exec") + except (IndentationError, SyntaxError) as exc: + errors.append(f"invalid Python heredoc at line {index + 1}: {exc.msg}") + for offset, candidate in enumerate(source.splitlines(), 1): + leading = len(candidate) - len(candidate.lstrip()) + if candidate.strip() and leading % 4: + errors.append( + "invalid Python heredoc indentation at line " + f"{index + offset + 1}: expected a multiple of four spaces" + ) + break + body.append(lines[end]) + else: + errors.append(f"unterminated Python heredoc at line {index + 1}") + return errors def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: @@ -217,8 +270,8 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: return [f"cannot read CI workflow {path}: {exc}"] jobs = _job_blocks(text) - errors: list[str] = [] - _require_docs_spec_pr_paths_ignored(errors, text, "CI") + errors = _python_heredoc_syntax_errors(text) + _require_pr_all_paths(errors, text, "CI") missing_jobs = sorted(REQUIRED_CI_JOBS - set(jobs)) if missing_jobs: errors.append(f"CI workflow missing required jobs: {missing_jobs}") @@ -271,19 +324,70 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "hard production gates", "scripts/verify_ci_workflow.py", "scripts/check_public_api.py", + "scripts/check_pyplot_options.py", + "--report pyplot-option-contract.json", + "Upload pyplot option contract evidence", + "pyplot-option-contract", "scripts/check_claim_guardrails.py", "ruff check .", "scripts/smoke_render.py", "Browser lifecycle smoke", - "Browser visual regression smoke", + "Browser visual health smoke", + "Reviewed visual baseline", + "Runtime standalone security smoke", + "Animation smoke", + "Pick boundary smoke", "Browser interaction stress smoke", "Browser dashboard reliability smoke", "scripts/reflex_lifecycle_smoke.py", - "scripts/visual_regression_smoke.py", + "scripts/visual_health_smoke.py", + "scripts/visual_baseline.py", + "spec/visual-baselines/v1.json", + "--artifacts visual-baseline-artifacts", + "--evidence visual-baseline-evidence.json", + "Upload visual baseline evidence", + "visual-baseline-evidence", + "Rendered label and formatter oracle (Chromium)", + "npm run test:labels", + "XY_LABEL_EVIDENCE: rendered-label-evidence.json", + "Upload rendered label evidence", + "rendered-label-evidence", + "Every chart-kind render matrix", + "scripts/chart_kind_matrix.py", + "--evidence chart-kind-matrix-evidence.json", + "Upload chart-kind matrix evidence", + "chart-kind-matrix-evidence", + "scripts/runtime_security_smoke.py", + "--evidence runtime-security-evidence.json", + "Upload runtime security evidence", + "runtime-security-evidence", + "scripts/animation_smoke.py", + "--evidence animation-browser-evidence.json", + "Upload animation browser evidence", + "animation-browser-evidence", + "scripts/pick_boundary_smoke.py", + "--evidence pick-boundary-evidence.json", + "Upload pick boundary evidence", + "pick-boundary-evidence", + "if-no-files-found: error", "scripts/interaction_stress_smoke.py", + "--json interaction-worker-evidence.json", + "Upload interaction worker evidence", + "interaction-worker-evidence", + "Pan/zoom acceptance matrix (Chromium)", + "scripts/pan_zoom_matrix.mjs", + "--profile full", + "--browsers chromium", + "--evidence pan-zoom-matrix-evidence.json", + "Upload pan/zoom matrix evidence", + "pan-zoom-matrix-evidence", "benchmarks/bench_dashboard.py", "--chart-counts 10,20,50", "dashboard-smoke.json --kind dashboard-browser", + "--profile strict", + "Upload dashboard health evidence", + "dashboard-health-evidence", + "if-no-files-found: error", "--sizes 1e5,1e6,1e7 --production --json scatter.json", "scripts/bench_native.py --sizes 1e6,1e7 --json kernel.json", "scripts/verify_benchmark_report.py scatter.json --kind scatter-native", @@ -301,6 +405,514 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "spec/benchmarks/metrics.md", "transport.json", ) + hard_test = jobs.get("test", "") + _require_step_contains( + errors, + hard_test, + "Rendered label and formatter oracle (Chromium)", + "hard rendered-label DOM and formatter oracle", + "npm run test:labels", + "XY_LABEL_EVIDENCE: rendered-label-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload rendered label evidence", + "failure-retaining rendered-label artifact policy", + "if: always()", + "actions/upload-artifact@", + "rendered-label-evidence", + "if-no-files-found: error", + "rendered-label-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Pyplot option contract", + "fail-closed structured no-op audit", + "scripts/check_pyplot_options.py", + "--report pyplot-option-contract.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload pyplot option contract evidence", + "failure-retaining option-contract evidence", + "if: always()", + "actions/upload-artifact@", + "pyplot-option-contract", + "if-no-files-found: error", + "pyplot-option-contract.json", + ) + _require_step_contains( + errors, + hard_test, + "Browser visual health smoke (Chromium)", + "honestly named broad visual-health command", + "scripts/visual_health_smoke.py", + ) + _require_step_contains( + errors, + hard_test, + "Reviewed visual baseline (Chromium)", + "pinned baseline command and artifact paths", + "scripts/visual_baseline.py", + "spec/visual-baselines/v1.json", + "--artifacts visual-baseline-artifacts", + "--evidence visual-baseline-evidence.json", + "if: ${{ !cancelled() }}", + ) + baseline_step = _named_step_blocks(hard_test).get("Reviewed visual baseline (Chromium)", "") + if "--update-baselines" in baseline_step: + errors.append("hard visual baseline CI step must never update reviewed baselines") + _require_step_contains( + errors, + hard_test, + "Upload visual baseline evidence", + "failure-retaining expected/actual/diff artifact policy", + "if: always()", + "actions/upload-artifact@", + "visual-baseline-evidence", + "if-no-files-found: error", + "visual-baseline-evidence.json", + "visual-baseline-artifacts/", + ) + _require_step_contains( + errors, + hard_test, + "Every chart-kind render matrix (Chromium)", + "registry-complete browser render command", + "scripts/chart_kind_matrix.py", + "--no-sandbox", + "--evidence chart-kind-matrix-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload chart-kind matrix evidence", + "failure-retaining chart-kind evidence artifact", + "if: always()", + "actions/upload-artifact@", + "chart-kind-matrix-evidence", + "if-no-files-found: error", + "chart-kind-matrix-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Runtime standalone security smoke (Chromium)", + "hard runtime DOM/CSP command and diagnostic path", + "scripts/runtime_security_smoke.py", + "--no-sandbox", + "--evidence runtime-security-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload runtime security evidence", + "failure-retaining runtime security artifact policy", + "if: always()", + "actions/upload-artifact@", + "runtime-security-evidence", + "if-no-files-found: error", + "runtime-security-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Animation smoke (Chromium)", + "hard animation command and diagnostic path", + "scripts/animation_smoke.py", + "--evidence animation-browser-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload animation browser evidence", + "failure-retaining animation artifact policy", + "if: always()", + "actions/upload-artifact@", + "animation-browser-evidence", + "if-no-files-found: error", + "animation-browser-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Pick boundary smoke (Chromium)", + "hard pick-boundary command and diagnostic path", + "scripts/pick_boundary_smoke.py", + "--evidence pick-boundary-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload pick boundary evidence", + "failure-retaining pick-boundary artifact policy", + "if: always()", + "actions/upload-artifact@", + "pick-boundary-evidence", + "if-no-files-found: error", + "pick-boundary-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Browser interaction stress smoke (Chromium)", + "required worker command and diagnostic path", + "scripts/interaction_stress_smoke.py", + "--json interaction-worker-evidence.json", + ) + interaction_step = _named_step_blocks(hard_test).get( + "Browser interaction stress smoke (Chromium)", "" + ) + if "--allow-worker-skip" in interaction_step: + errors.append("hard interaction worker CI step must not allow worker skips") + _require_step_contains( + errors, + hard_test, + "Upload interaction worker evidence", + "failure-retaining worker artifact policy", + "if: always()", + "actions/upload-artifact@", + "interaction-worker-evidence", + "if-no-files-found: error", + "interaction-worker-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Pan/zoom acceptance matrix (Chromium)", + "complete hard Chromium pan/zoom matrix and diagnostic path", + "scripts/pan_zoom_matrix.mjs", + "--profile full", + "--browsers chromium", + '--executable-path "$CHROME"', + "--evidence pan-zoom-matrix-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload pan/zoom matrix evidence", + "failure-retaining pan/zoom matrix artifact policy", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "pan-zoom-matrix-evidence", + "if-no-files-found: error", + "pan-zoom-matrix-evidence.json", + ) + _require_step_contains( + errors, + hard_test, + "Upload regression benchmark report", + "failure-retaining regression artifact policy", + "if: always()", + "regression-benchmark-report", + "if-no-files-found: warn", + ) + _require_step_contains( + errors, + hard_test, + "Upload dashboard health evidence", + "failure-retaining dashboard artifact policy", + "if: always()", + "dashboard-health-evidence", + "if-no-files-found: error", + "dashboard-smoke.json", + ) + _require_step_contains( + errors, + hard_test, + "Protocol catalog and byte-mutation conformance", + "catalog, golden-frame, malformed-frame, and required property suite", + "mkdir -p test-results/protocol", + "tests/test_protocol_catalog.py", + "tests/test_framing.py", + "tests/test_framing_property.py", + "--junitxml=test-results/protocol/junit.xml", + ) + _require_step_contains( + errors, + hard_test, + "Upload protocol conformance evidence", + "failure-retaining protocol JUnit artifact policy", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "protocol-conformance-evidence", + "if-no-files-found: error", + "retention-days: 30", + "test-results/protocol/junit.xml", + ) + javascript_semantics = jobs.get("javascript_semantics", "") + if "continue-on-error:" in javascript_semantics: + errors.append("CI javascript_semantics job must be a hard gate without continue-on-error") + if "npm ci" in javascript_semantics or "npm install" in javascript_semantics: + errors.append( + "CI javascript_semantics job must use only pinned Node built-ins without npm installs" + ) + _require_job_contains( + errors, + jobs, + "javascript_semantics", + "CI", + "pinned dependency-free semantic units with retained test and coverage evidence", + "name: JavaScript semantic unit suite", + "timeout-minutes: 10", + "actions/setup-node@", + 'node-version: "22"', + "node --test --experimental-test-coverage", + "--test-coverage-include=python/xy/static/index.js", + "--test-coverage-lines=15", + "--test-coverage-branches=60", + "--test-coverage-functions=10", + "--test-reporter=junit", + "test-results/javascript/junit.xml", + "NODE_V8_COVERAGE=coverage/javascript", + "js/test/*.test.mjs", + "Upload JavaScript semantic evidence", + "javascript-semantic-evidence", + ) + _require_step_contains( + errors, + javascript_semantics, + "Run JavaScript semantic unit suite", + "hard semantic command, coverage thresholds, and JUnit evidence", + "set -o pipefail", + "NODE_V8_COVERAGE=coverage/javascript", + "node --test --experimental-test-coverage", + "--test-coverage-lines=15", + "--test-coverage-branches=60", + "--test-coverage-functions=10", + "--test-reporter=junit", + "test-results/javascript/junit.xml", + "tee test-results/javascript/coverage.txt", + ) + _require_step_contains( + errors, + javascript_semantics, + "Upload JavaScript semantic evidence", + "failure-retaining JavaScript test and coverage artifact policy", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "javascript-semantic-evidence", + "if-no-files-found: error", + "test-results/javascript/", + "coverage/javascript/", + ) + python_coverage = jobs.get("python_coverage", "") + if "continue-on-error:" in python_coverage: + errors.append("CI python_coverage job must be a hard gate without continue-on-error") + _require_job_contains( + errors, + jobs, + "python_coverage", + "CI", + "branch-aware package/module and changed-line ratchet with retained evidence", + "name: Python branch and diff coverage", + "timeout-minutes: 15", + "fetch-depth: 0", + 'python-version: "3.13"', + "uv sync --frozen --project python/reflex-xy --extra dev", + "UV_PROJECT_ENVIRONMENT=python/reflex-xy/.venv", + "uv sync --frozen --extra dev --inexact", + "coverage run --branch --source=python/xy", + "coverage run --branch --append", + "--source=python/reflex-xy/reflex_xy", + "scripts/run_pytest_no_skips.py -q", + "python/reflex-xy/tests", + "scripts/coverage_ratchet.py", + "spec/testing/coverage-policy.json", + "coverage/python/coverage.json", + "coverage/python/coverage.xml", + "coverage/python/ratchet.json", + "github.event.pull_request.base.sha", + "github.event.pull_request.head.sha", + "Upload Python coverage evidence", + "python-coverage-evidence", + "retention-days: 30", + ) + _require_step_contains( + errors, + python_coverage, + "Enforce reviewed package, module, and changed-line floors", + "branch-aware reports and exact Git comparison", + "coverage json -o coverage/python/coverage.json", + "coverage xml -o coverage/python/coverage.xml", + "scripts/coverage_ratchet.py", + '--base "$BASE_SHA" --head "$HEAD_SHA"', + "--report coverage/python/ratchet.json", + ) + _require_step_contains( + errors, + python_coverage, + "Upload Python coverage evidence", + "failure-retaining raw, JSON, XML, and ratchet evidence", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "python-coverage-evidence", + "if-no-files-found: error", + "retention-days: 30", + ".coverage", + "coverage/python/", + ) + reflex_adapter = jobs.get("reflex_adapter", "") + # This lane is intentionally hard. Check the forbidden form explicitly + # rather than relying on absence of a positive token. + if "continue-on-error:" in reflex_adapter: + errors.append("CI reflex_adapter job must be a hard gate without continue-on-error") + if '-e ".[dev]"' in reflex_adapter: + errors.append( + "CI reflex_adapter job must provision tests from python/reflex-xy[dev], " + "not the root xy dev extra" + ) + expected_reflex_matrix = [ + {"name": "floor", "reflex": "reflex==0.9.6"}, + {"name": "latest", "reflex": "reflex>=0.9.6,<1"}, + ] + if reflex_adapter and _matrix_include_entries(reflex_adapter) != expected_reflex_matrix: + errors.append( + "CI reflex_adapter matrix must exactly declare the reviewed floor/latest range" + ) + _require_job_contains( + errors, + jobs, + "reflex_adapter", + "CI", + "Reflex host matrix, zero-skip suite, compile, and real browser E2E", + "reflex==0.9.6", + "reflex>=0.9.6,<1", + 'python-version: "3.13"', + "npm ci", + 'XY_REQUIRE_CARGO: "1"', + "uv pip install -p .venv/bin/python -e .", + "python/reflex-xy[dev]", + "from importlib.metadata import version", + "version('reflex')", + "scripts/host_integration_policy.py validate", + "--profile ${{ matrix.name }} --hosts reflex", + "reflex-host-versions.json", + "scripts/run_pytest_no_skips.py -q python/reflex-xy/tests", + "reflex-adapter-junit.xml", + "reflex compile --dry", + "reflex run --env prod --single-port", + "scripts/reflex_ws_smoke.py", + "--screenshot reflex-e2e.png", + "scripts/pan_zoom_matrix.mjs --profile reflex --browsers chromium", + "--evidence pan-zoom-reflex-evidence.json", + "npx playwright install --with-deps chromium", + "Upload Reflex E2E evidence", + "reflex-e2e-${{ matrix.name }}", + "examples/reflex/reflex-e2e.log", + "examples/reflex/reflex-e2e.png", + "Upload Reflex pan/zoom evidence", + "reflex-pan-zoom-${{ matrix.name }}", + "examples/reflex/pan-zoom-reflex-evidence.json", + ) + _require_step_contains( + errors, + reflex_adapter, + "Run real Reflex browser E2E", + "browser evidence capture", + "scripts/reflex_ws_smoke.py", + "--screenshot reflex-e2e.png", + "scripts/pan_zoom_matrix.mjs", + "--profile reflex", + "--browsers chromium", + "--url http://localhost:3100", + "--evidence pan-zoom-reflex-evidence.json", + ) + _require_step_contains( + errors, + reflex_adapter, + "Upload Reflex E2E evidence", + "failure-safe browser evidence upload", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "reflex-e2e-${{ matrix.name }}", + "if-no-files-found: error", + "retention-days: 30", + "examples/reflex/reflex-host-versions.json", + "examples/reflex/reflex-adapter-junit.xml", + "examples/reflex/reflex-e2e.log", + "examples/reflex/reflex-e2e.png", + ) + _require_step_contains( + errors, + reflex_adapter, + "Upload Reflex pan/zoom evidence", + "failure-retaining live/static Reflex pan/zoom evidence", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "reflex-pan-zoom-${{ matrix.name }}", + "if-no-files-found: error", + "examples/reflex/pan-zoom-reflex-evidence.json", + ) + host_integration = jobs.get("host_integration", "") + if "continue-on-error:" in host_integration: + errors.append("CI host_integration job must be a hard gate without continue-on-error") + expected_host_matrix = [ + { + "name": "floor", + "anywidget": "anywidget==0.9.0", + "traitlets": "traitlets==5.14.0", + "fastapi": "fastapi==0.110.0", + "starlette": "starlette==0.36.3", + "uvicorn": "uvicorn==0.29.0", + "httpx": "httpx==0.27.0", + }, + { + "name": "latest", + "anywidget": "anywidget>=0.9,<1", + "traitlets": "traitlets>=5.14,<6", + "fastapi": "fastapi>=0.110,<1", + "starlette": "starlette>=0.36.3,<1", + "uvicorn": "uvicorn>=0.29,<1", + "httpx": "httpx>=0.27,<1", + }, + ] + if host_integration and _matrix_include_entries(host_integration) != expected_host_matrix: + errors.append( + "CI host_integration matrix must exactly declare the reviewed floor/latest " + "anywidget and FastAPI host stacks" + ) + _require_job_contains( + errors, + jobs, + "host_integration", + "CI", + "bounded package-owned host matrix with zero-skip compile, mount, transport, " + "browser, and retained evidence gates", + 'python-version: "3.13"', + "cargo build --locked --release", + "npm ci", + "node js/build.mjs --check", + '-e ".[dev]"', + "scripts/host_integration_policy.py validate", + "--profile ${{ matrix.name }} --hosts anywidget fastapi", + "host-integration/versions.json", + "scripts/run_pytest_no_skips.py -q", + "tests/host_integration", + "host-integration/junit.xml", + "npx playwright install --with-deps chromium", + "scripts/fastapi_host_smoke.py", + "--evidence host-integration/fastapi-browser.json", + "--screenshot host-integration/fastapi-browser.png", + "--server-log host-integration/fastapi-server.log", + "Upload host integration evidence", + "host-integration-${{ matrix.name }}", + ) + _require_step_contains( + errors, + host_integration, + "Upload host integration evidence", + "failure-retaining host version, JUnit, screenshot, and browser evidence policy", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "host-integration-${{ matrix.name }}", + "if-no-files-found: error", + "retention-days: 30", + "path: host-integration/", + ) _require_job_contains( errors, jobs, @@ -315,6 +927,66 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "npx playwright install --with-deps chromium firefox webkit", "node js/build.mjs --check", "node scripts/browser_conformance.mjs", + "--evidence browser-conformance-evidence.json", + "Upload browser conformance evidence", + "browser-conformance-evidence", + "if-no-files-found: error", + "Focused pan/zoom matrix (three engines)", + "scripts/pan_zoom_matrix.mjs", + "--profile focused", + "--browsers chromium,firefox,webkit", + "--evidence pan-zoom-cross-engine-evidence.json", + "Upload cross-engine pan/zoom evidence", + "pan-zoom-cross-engine-evidence", + ) + browser_conformance = jobs.get("browser_conformance", "") + _require_step_contains( + errors, + browser_conformance, + "Run accessibility and cross-browser conformance", + "bounded three-engine matrix and retained evidence command", + 'server-args="-screen 0 1920x1200x24"', + "node scripts/browser_conformance.mjs", + "--evidence browser-conformance-evidence.json", + ) + conformance_step = _named_step_blocks(browser_conformance).get( + "Run accessibility and cross-browser conformance", "" + ) + if "--browsers" in conformance_step: + errors.append("hard browser_conformance step must run all three engines") + _require_step_contains( + errors, + browser_conformance, + "Upload browser conformance evidence", + "failure-retaining conformance evidence policy", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "browser-conformance-evidence", + "if-no-files-found: error", + "browser-conformance-evidence.json", + ) + _require_step_contains( + errors, + browser_conformance, + "Focused pan/zoom matrix (three engines)", + "hard focused three-engine pan/zoom subset", + 'XY_PAN_ZOOM_HEADFUL: "1"', + "xvfb-run", + "scripts/pan_zoom_matrix.mjs", + "--profile focused", + "--browsers chromium,firefox,webkit", + "--evidence pan-zoom-cross-engine-evidence.json", + ) + _require_step_contains( + errors, + browser_conformance, + "Upload cross-engine pan/zoom evidence", + "failure-retaining cross-engine pan/zoom artifact policy", + "if: always()", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "pan-zoom-cross-engine-evidence", + "if-no-files-found: error", + "pan-zoom-cross-engine-evidence.json", ) _require_job_contains( errors, @@ -532,6 +1204,110 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "docs/benchmark_ci.md", "if-no-files-found: warn", ) + _require_job_contains( + errors, + jobs, + "dependency_audit", + "CI", + "pinned multi-ecosystem audit and retained machine-readable evidence", + "name: Multi-ecosystem dependency audit", + "timeout-minutes: 15", + "Validate dependency audit policy", + "python3 scripts/dependency_audit.py validate", + "Install pinned OSV-Scanner", + "osv-scanner/releases/download/v2.4.0/osv-scanner_linux_amd64", + "15314940c10d26af9c6649f150b8a47c1262e8fc7e17b1d1029b0e479e8ed8a0", + "sha256sum --check --strict", + "Audit every active dependency environment", + "python3 scripts/dependency_audit.py scan", + "Retain dependency audit evidence", + "if: always()", + "actions/upload-artifact@", + "if-no-files-found: error", + "retention-days: 30", + "dependency-audit/*.json", + "dependency-audit/*.txt", + ) + dependency_audit = jobs.get("dependency_audit", "") + if "continue-on-error:" in dependency_audit: + errors.append("CI dependency_audit must be a hard gate without continue-on-error") + if re.search(r"^ if:", dependency_audit, flags=re.MULTILINE): + errors.append("CI dependency_audit job must be unconditional") + for step_name in ( + "Validate dependency audit policy", + "Install pinned OSV-Scanner", + "Audit every active dependency environment", + ): + if _step_is_conditioned(dependency_audit, step_name): + errors.append(f"CI dependency_audit step {step_name!r} must be unconditional") + + _require_job_contains( + errors, + jobs, + "rust_release", + "CI", + "locked release-profile suite and release-only regression inventory", + "name: Rust release-profile tests", + "timeout-minutes: 15", + "dtolnay/rust-toolchain@", + "Inventory release-only regression coverage", + "cargo test --locked --release -- --list", + "release-tests.txt", + "grep -Fqx", + "tiles::tests::compose_window_astronomically_past_domain_is_empty_not_panic: test", + "Run locked release-profile suite", + "run: cargo test --locked --release", + "Upload release-profile test inventory", + "if: always()", + "actions/upload-artifact@", + "name: rust-release-test-inventory", + "if-no-files-found: error", + "path: release-tests.txt", + ) + release_tests = jobs.get("rust_release", "") + if "continue-on-error:" in release_tests: + errors.append("CI rust_release must be a hard gate without continue-on-error") + for step_name in ( + "Inventory release-only regression coverage", + "Run locked release-profile suite", + ): + if _step_is_conditioned(release_tests, step_name): + errors.append(f"CI rust_release step {step_name!r} must be unconditional") + _require_job_contains( + errors, + jobs, + "native_parity", + "CI", + "hard native architecture matrix and retained common parity probe", + "timeout-minutes: 15", + "fail-fast: false", + "ubuntu-latest", + "ubuntu-24.04-arm", + "windows-latest", + "macos-14", + "cargo build --locked --release", + "scripts/native_parity.py", + "--expect-arch ${{ matrix.arch }}", + "--expect-default ${{ matrix.default }}", + "if: always()", + "actions/upload-artifact@", + "name: native-parity-${{ matrix.os }}", + "if-no-files-found: error", + ) + native_parity = jobs.get("native_parity", "") + expected_native_matrix = [ + {"os": "ubuntu-latest", "arch": "x86_64", "default": "avx2"}, + {"os": "ubuntu-24.04-arm", "arch": "aarch64", "default": "aarch64"}, + {"os": "windows-latest", "arch": "x86_64", "default": "avx2"}, + {"os": "macos-14", "arch": "aarch64", "default": "aarch64"}, + ] + if _matrix_include_entries(native_parity) != expected_native_matrix: + errors.append( + "CI native_parity matrix must exactly cover native Linux x64/ARM64, " + "Windows x64, and macOS ARM64 with explicit dispatch expectations" + ) + if "continue-on-error:" in native_parity: + errors.append("CI native_parity must be a hard gate without continue-on-error") _require_job_contains( errors, jobs, @@ -565,6 +1341,42 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: "--expect-pure", "native Rust core", ) + hard_jobs = { + "browser_conformance", + "dependency_audit", + "host_integration", + "install_without_rust", + "javascript_semantics", + "matplotlib_reference", + "native_parity", + "python_coverage", + "python_floor", + "reflex_adapter", + "rust_release", + "sdist", + "test", + "wheels", + } + aggregate = jobs.get("required_ci", "") + _require_job_contains( + errors, + jobs, + "required_ci", + "CI", + "stable hard-gate aggregation", + "name: Required CI", + "if: always()", + "timeout-minutes: 5", + "NEEDS_JSON: ${{ toJSON(needs) }}", + "scripts/check_required_jobs.py", + ) + if aggregate and _listed_needs(aggregate) != hard_jobs: + errors.append( + "CI required_ci needs must exactly match hard jobs; " + f"got {sorted(_listed_needs(aggregate))}, expected {sorted(hard_jobs)}" + ) + if "continue-on-error:" in aggregate: + errors.append("CI required_ci must not use continue-on-error") return errors @@ -580,8 +1392,9 @@ def validate_codspeed_workflow(path: Path = DEFAULT_CODSPEED_WORKFLOW) -> list[s return [f"cannot read CodSpeed workflow {path}: {exc}"] jobs = _job_blocks(text) - errors: list[str] = [] - _require_docs_spec_pr_paths_ignored(errors, text, "CodSpeed") + errors = _python_heredoc_syntax_errors(text) + # CodSpeed is advisory and may remain path-filtered; only the stable hard + # aggregate in ci.yml is required on every pull request. missing_jobs = sorted(REQUIRED_CODSPEED_JOBS - set(jobs)) if missing_jobs: errors.append(f"CodSpeed workflow missing required jobs: {missing_jobs}") @@ -626,7 +1439,7 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str return [f"cannot read release workflow {path}: {exc}"] jobs = _job_blocks(text) - errors: list[str] = [] + errors = _python_heredoc_syntax_errors(text) missing_jobs = sorted(REQUIRED_RELEASE_JOBS - set(jobs)) if missing_jobs: errors.append(f"release workflow missing required jobs: {missing_jobs}") @@ -635,9 +1448,27 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str errors, text, "release", - "tag and manual triggers", + "tag and manual triggers plus exact-SHA Actions access", 'tags: ["v*"]', "workflow_dispatch:", + "actions: read", + ) + _require_job_contains( + errors, + jobs, + "qualify", + "release", + "unconditional exact-SHA, main-ancestry, hard-CI, tag, version, and changelog preflight", + "fetch-depth: 0", + "main:refs/remotes/origin/main", + "refs/tags/*:refs/tags/*", + "source_sha=$(git rev-parse", + "scripts/verify_source_qualification.py", + '--sha "$SOURCE_SHA"', + '--repository "$GITHUB_REPOSITORY"', + '--tag "$RELEASE_TAG"', + "--release-metadata", + "--wait-seconds 3600", ) _require_job_contains( errors, @@ -662,6 +1493,8 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "assert k.BACKEND=='native'", "actions/upload-artifact@", "dist/*.whl", + "needs: qualify", + "ref: ${{ needs.qualify.outputs.source_sha }}", ) wheels_job = jobs.get("wheels", "") if "continue-on-error:" in wheels_job: @@ -669,6 +1502,38 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "release wheels job must block publishing when any native wheel build or " "verification fails" ) + _require_step_contains( + errors, + wheels_job, + "Run manylinux wheel parity in glibc 2.17", + "pinned manylinux runtime and common artifact parity probe", + "if: matrix.runtime == 'manylinux'", + "manylinux2014_x86_64@sha256:", + "scripts/native_parity.py", + "--wheel", + "--expect-default avx2", + ) + _require_step_contains( + errors, + wheels_job, + "Run musllinux wheel parity in musl 1.2", + "pinned musllinux runtime and common artifact parity probe", + "if: matrix.runtime == 'musllinux'", + "python:3.13-alpine3.22@sha256:", + "scripts/native_parity.py", + "--wheel", + "--expect-default avx2", + ) + _require_step_contains( + errors, + wheels_job, + "Retain native artifact runtime report", + "manylinux/musllinux runtime evidence retention", + "always() && (matrix.runtime == 'manylinux' || matrix.runtime == 'musllinux')", + "actions/upload-artifact@", + "if-no-files-found: error", + "native-parity-${{ matrix.plat }}.json", + ) _require_job_contains( errors, jobs, @@ -686,6 +1551,8 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "scripts/verify_wheel.py", "--expect-native", "name: pyodide-wheel", + "needs: qualify", + "ref: ${{ needs.qualify.outputs.source_sha }}", ) wasm_job = jobs.get("wasm", "") if "continue-on-error:" in wasm_job: @@ -706,7 +1573,39 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "native Rust core", "actions/upload-artifact@", "dist/*.tar.gz", + "needs: qualify", + "ref: ${{ needs.qualify.outputs.source_sha }}", ) + _require_job_contains( + errors, + jobs, + "provenance", + "release", + "one immutable hash manifest for the exact artifacts built in this run", + "needs: [qualify, wheels, sdist, wasm]", + "pattern: dist-*", + "name: pyodide-wheel", + "scripts/release_provenance.py create", + "scripts/release_provenance.py verify", + '--source-sha "$SOURCE_SHA"', + '--repository "$GITHUB_REPOSITORY"', + '--workflow-run-id "$GITHUB_RUN_ID"', + '--tag "$RELEASE_TAG"', + "name: release-provenance", + "if-no-files-found: error", + ) + provenance_job = jobs.get("provenance", "") + for identity_argument in ( + '--source-sha "$SOURCE_SHA"', + '--repository "$GITHUB_REPOSITORY"', + '--workflow-run-id "$GITHUB_RUN_ID"', + '--tag "$RELEASE_TAG"', + ): + if provenance_job.count(identity_argument) != 2: + errors.append( + "release provenance job must bind " + f"{identity_argument} during both creation and self-verification" + ) _require_job_contains( errors, jobs, @@ -714,10 +1613,16 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "release", "trusted PyPI publishing from downloaded artifacts, gated by a dry-run switch " "and a tag/version/CHANGELOG agreement gate", - "needs: [wheels, sdist, wasm]", + "needs: [qualify, wheels, sdist, wasm, provenance]", "environment: pypi", "id-token: write", "scripts/check_release_version.py", + "scripts/release_provenance.py verify", + '--repository "$GITHUB_REPOSITORY"', + '--workflow-run-id "$GITHUB_RUN_ID"', + '--tag "$RELEASE_TAG"', + "name: release-provenance", + "ref: ${{ needs.qualify.outputs.source_sha }}", "actions/download-artifact@", "pattern: dist-*", "merge-multiple: true", @@ -743,7 +1648,7 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "publish-pyodide", "release", "GitHub Release publication of the runtime-verified Pyodide wheel", - "needs: [wheels, sdist, wasm]", + "needs: [qualify, wheels, sdist, wasm, provenance]", "contents: write", "actions/setup-node@", 'node-version: "22"', @@ -751,8 +1656,15 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "name: pyodide-wheel", "dry_run", "scripts/check_release_version.py", + "scripts/release_provenance.py verify", + '--repository "$GITHUB_REPOSITORY"', + '--workflow-run-id "$GITHUB_RUN_ID"', + '--tag "$RELEASE_TAG"', + "name: release-provenance", + "ref: ${{ needs.qualify.outputs.source_sha }}", "GH_TOKEN", "gh release upload", + "provenance/release-provenance.json", "--clobber", "gh release create", "--verify-tag", @@ -762,6 +1674,9 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str ) publish = jobs.get("publish", "") + qualify = jobs.get("qualify", "") + if re.search(r"^ if:", qualify, flags=re.MULTILINE): + errors.append("release qualify job must be unconditional; publication cannot bypass it") if "password:" in publish or "api-token" in publish: errors.append("release publish job should use trusted publishing, not a PyPI token") if "pypa/gh-action-pypi-publish@" in publish and not _step_is_conditioned( @@ -775,15 +1690,222 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str return errors +def validate_deploy_workflows( + dev_path: Path = DEFAULT_DEPLOY_DEV_WORKFLOW, + stg_path: Path = DEFAULT_DEPLOY_STG_WORKFLOW, + build_path: Path = DEFAULT_BUILD_DOCS_WORKFLOW, + helm_path: Path = DEFAULT_HELM_DOCS_WORKFLOW, +) -> list[str]: + """Verify exact-source qualification and immutable docs promotion.""" + texts: dict[str, str] = {} + errors: list[str] = [] + for label, path in ( + ("docs dev deploy", dev_path), + ("docs stage/prod deploy", stg_path), + ("docs image build", build_path), + ("docs Helm promotion", helm_path), + ): + try: + texts[label] = path.read_text(encoding="utf-8") + except OSError as exc: + errors.append(f"cannot read {label} workflow {path}: {exc}") + texts[label] = "" + + dev_text = texts["docs dev deploy"] + dev_jobs = _job_blocks(dev_text) + _require_workflow_contains( + errors, + dev_text, + "docs dev deploy", + "exact-SHA Actions access", + "actions: read", + ) + _require_job_contains( + errors, + dev_jobs, + "qualify", + "docs dev deploy", + "exact-SHA main-ancestry and hard-CI preflight", + "needs: prepare", + "fetch-depth: 0", + "main:refs/remotes/origin/main", + "scripts/verify_source_qualification.py", + '--sha "$SOURCE_SHA"', + '--repository "$GITHUB_REPOSITORY"', + "--wait-seconds 3600", + ) + _require_job_contains( + errors, + dev_jobs, + "build", + "docs dev deploy", + "qualified exact-source build dependency", + "needs: [prepare, qualify]", + "source_ref: ${{ needs.prepare.outputs.source_sha }}", + ) + _require_job_contains( + errors, + dev_jobs, + "helm-pr", + "docs dev deploy", + "immutable build digest promotion", + "frontend_digest: ${{ needs.build.outputs.frontend_digest }}", + "backend_digest: ${{ needs.build.outputs.backend_digest }}", + ) + + stg_text = texts["docs stage/prod deploy"] + stg_jobs = _job_blocks(stg_text) + _require_workflow_contains( + errors, + stg_text, + "docs stage/prod deploy", + "exact-SHA Actions access", + "actions: read", + ) + _require_job_contains( + errors, + stg_jobs, + "qualify", + "docs stage/prod deploy", + "exact tagged source, main-ancestry, and hard-CI preflight", + "needs: prepare", + "fetch-depth: 0", + "main:refs/remotes/origin/main", + "refs/tags/*:refs/tags/*", + "scripts/verify_source_qualification.py", + '--sha "$SOURCE_SHA"', + '--tag "$VERSION"', + "--wait-seconds 3600", + ) + _require_job_contains( + errors, + stg_jobs, + "build", + "docs stage/prod deploy", + "qualified exact-source build dependency", + "needs: [prepare, qualify]", + "source_ref: ${{ needs.prepare.outputs.source_sha }}", + ) + _require_job_contains( + errors, + stg_jobs, + "helm-pr-stg", + "docs stage/prod deploy", + "immutable staging digest promotion", + "frontend_digest: ${{ needs.build.outputs.frontend_digest }}", + "backend_digest: ${{ needs.build.outputs.backend_digest }}", + ) + _require_job_contains( + errors, + stg_jobs, + "verify-prod-artifacts", + "docs stage/prod deploy", + "post-approval ECR digest verification against the original build", + "needs: [prepare, build, await-prod-approval]", + "EXPECTED_FRONTEND: ${{ needs.build.outputs.frontend_digest }}", + "EXPECTED_BACKEND: ${{ needs.build.outputs.backend_digest }}", + '[[ "$ACTUAL_FRONTEND" != "$EXPECTED_FRONTEND" ]]', + '[[ "$ACTUAL_BACKEND" != "$EXPECTED_BACKEND" ]]', + ) + _require_job_contains( + errors, + stg_jobs, + "release", + "docs stage/prod deploy", + "verified digest provenance in the deployment release", + "needs: [prepare, build, verify-prod-artifacts]", + "FRONTEND_DIGEST: ${{ needs.build.outputs.frontend_digest }}", + "BACKEND_DIGEST: ${{ needs.build.outputs.backend_digest }}", + ) + _require_job_contains( + errors, + stg_jobs, + "helm-pr-prod", + "docs stage/prod deploy", + "verified immutable production promotion", + "needs: [prepare, build, verify-prod-artifacts, release]", + "frontend_digest: ${{ needs.build.outputs.frontend_digest }}", + "backend_digest: ${{ needs.build.outputs.backend_digest }}", + ) + + build_text = texts["docs image build"] + build_jobs = _job_blocks(build_text) + _require_workflow_contains( + errors, + build_text, + "docs image build", + "reusable immutable digest outputs", + "frontend_digest:", + "backend_digest:", + "value: ${{ jobs.build-and-push.outputs.frontend_digest }}", + "value: ${{ jobs.build-and-push.outputs.backend_digest }}", + ) + _require_job_contains( + errors, + build_jobs, + "build-and-push", + "docs image build", + "ECR digest resolution and provenance attestations", + "id: digests", + "--repository-name xy/frontend", + "--repository-name xy/backend", + "frontend_digest=$FRONTEND_DIGEST", + "backend_digest=$BACKEND_DIGEST", + "^sha256:[0-9a-f]{64}$", + ) + if build_text.count("--provenance=mode=max") != 2: + errors.append("docs image build must enable max provenance on both image builds") + if "--provenance=false" in build_text: + errors.append("docs image build must not disable image provenance") + + helm_text = texts["docs Helm promotion"] + helm_jobs = _job_blocks(helm_text) + _require_workflow_contains( + errors, + helm_text, + "docs Helm promotion", + "required immutable digest inputs", + "frontend_digest:", + "backend_digest:", + 'description: "Immutable sha256 digest for the frontend image"', + 'description: "Immutable sha256 digest for the backend image"', + ) + _require_job_contains( + errors, + helm_jobs, + "open-helm-pr", + "docs Helm promotion", + "digest validation and tag-at-digest chart pins", + "^sha256:[0-9a-f]{64}$", + 'FRONTEND_REF="${IMAGE_TAG}@${FRONTEND_DIGEST}"', + 'BACKEND_REF="${IMAGE_TAG}@${BACKEND_DIGEST}"', + ".apps.xy.frontend.image.tag = strenv(FRONTEND_REF)", + ".apps.xy.backend.image.tag = strenv(BACKEND_REF)", + "FRONTEND_IMAGE}:${IMAGE_TAG}@${FRONTEND_DIGEST}", + "BACKEND_IMAGE}:${IMAGE_TAG}@${BACKEND_DIGEST}", + ) + return errors + + def validate_all_workflows( ci_path: Path = DEFAULT_CI_WORKFLOW, codspeed_path: Path = DEFAULT_CODSPEED_WORKFLOW, release_path: Path = DEFAULT_RELEASE_WORKFLOW, + deploy_dev_path: Path = DEFAULT_DEPLOY_DEV_WORKFLOW, + deploy_stg_path: Path = DEFAULT_DEPLOY_STG_WORKFLOW, + build_docs_path: Path = DEFAULT_BUILD_DOCS_WORKFLOW, + helm_docs_path: Path = DEFAULT_HELM_DOCS_WORKFLOW, ) -> list[str]: return [ *validate_ci_workflow(ci_path), *validate_codspeed_workflow(codspeed_path), *validate_release_workflow(release_path), + *validate_deploy_workflows( + deploy_dev_path, + deploy_stg_path, + build_docs_path, + helm_docs_path, + ), ] @@ -798,6 +1920,10 @@ def main(argv: Optional[list[str]] = None) -> int: parser.add_argument("--ci-workflow", type=Path, default=DEFAULT_CI_WORKFLOW) parser.add_argument("--codspeed-workflow", type=Path, default=DEFAULT_CODSPEED_WORKFLOW) parser.add_argument("--release-workflow", type=Path, default=DEFAULT_RELEASE_WORKFLOW) + parser.add_argument("--deploy-dev-workflow", type=Path, default=DEFAULT_DEPLOY_DEV_WORKFLOW) + parser.add_argument("--deploy-stg-workflow", type=Path, default=DEFAULT_DEPLOY_STG_WORKFLOW) + parser.add_argument("--build-docs-workflow", type=Path, default=DEFAULT_BUILD_DOCS_WORKFLOW) + parser.add_argument("--helm-docs-workflow", type=Path, default=DEFAULT_HELM_DOCS_WORKFLOW) parser.add_argument("--ci-only", action="store_true") parser.add_argument("--codspeed-only", action="store_true") parser.add_argument("--release-only", action="store_true") @@ -821,9 +1947,23 @@ def main(argv: Optional[list[str]] = None) -> int: checked = [args.release_workflow] else: errors = validate_all_workflows( - args.ci_workflow, args.codspeed_workflow, args.release_workflow + args.ci_workflow, + args.codspeed_workflow, + args.release_workflow, + args.deploy_dev_workflow, + args.deploy_stg_workflow, + args.build_docs_workflow, + args.helm_docs_workflow, ) - checked = [args.ci_workflow, args.codspeed_workflow, args.release_workflow] + checked = [ + args.ci_workflow, + args.codspeed_workflow, + args.release_workflow, + args.deploy_dev_workflow, + args.deploy_stg_workflow, + args.build_docs_workflow, + args.helm_docs_workflow, + ] if errors: print( diff --git a/scripts/verify_local.py b/scripts/verify_local.py index f921309d..d38a6da0 100644 --- a/scripts/verify_local.py +++ b/scripts/verify_local.py @@ -80,6 +80,11 @@ def _base_checks( "public performance-claim guardrails", (py, "scripts/check_claim_guardrails.py"), ), + Check( + "testing_spec", + "whole-spec links, evidence, gap IDs, commands, symbols, and workflow jobs", + (py, "scripts/check_testing_spec.py"), + ), Check( "benchmark_harness", "benchmark metadata, report, regression, and claim guardrail tests", @@ -146,7 +151,12 @@ def _base_checks( "tests/test_scatter.py::test_to_html_escapes_user_strings", "tests/test_components.py::test_component_to_html_escapes_user_strings_across_public_surface", "tests/test_components.py::test_component_to_html_path_keeps_existing_file_on_atomic_replace_failure", - "tests/test_docs_examples.py::test_readme_documents_standalone_html_security_contract", + "tests/test_figure.py::test_html_to_png_uses_browser_sandbox_by_default", + "tests/test_figure.py::test_html_to_png_never_silently_downgrades_after_sandbox_failure", + "tests/test_figure.py::test_html_to_png_reports_explicit_unsandboxed_launch_failure", + "tests/test_batch_export.py::test_browser_session_never_silently_downgrades", + "tests/test_batch_export.py::test_browser_session_honors_explicit_unsandboxed_opt_in", + "tests/test_docs_examples.py::test_security_policy_documents_standalone_html_contract", "tests/test_docs_examples.py::test_production_docs_capture_html_export_dom_text_contract", ), requires_modules=("pytest",), @@ -162,8 +172,9 @@ def _base_checks( "tests/test_figure.py", "tests/test_components.py", "tests/test_lod.py", + "tests/test_property_figure.py", ), - requires_modules=("pytest",), + requires_modules=("pytest", "hypothesis"), ), Check( "ruff_check", "ruff lint", (py, "-m", "ruff", "check", "."), requires_modules=("ruff",) @@ -227,6 +238,13 @@ def _base_checks( requires_paths=chromium_paths, requires_chromium=True, ), + Check( + "runtime_security_smoke", + "standalone DOM-XSS, CSP, hostile-CSS, and network-isolation smoke in Chromium", + (py, "scripts/runtime_security_smoke.py", chromium_arg), + requires_paths=chromium_paths, + requires_chromium=True, + ), Check( "reflex_lifecycle_smoke", "Reflex example iframe lifecycle smoke in Chromium", @@ -235,13 +253,36 @@ def _base_checks( requires_chromium=True, ), Check( - "visual_regression_smoke", - "representative chart screenshot smoke in Chromium", - (py, "scripts/visual_regression_smoke.py", chromium_arg), + "visual_health_smoke", + "gallery visual health/occupancy smoke in Chromium", + (py, "scripts/visual_health_smoke.py", chromium_arg), requires_modules=("numpy",), requires_paths=chromium_paths, requires_chromium=True, ), + Check( + "visual_baseline", + "reviewed semantic/perceptual visual baselines in pinned Chromium", + ( + py, + "scripts/visual_baseline.py", + chromium_arg, + "--baseline", + "spec/visual-baselines/v1.json", + ), + requires_modules=("numpy",), + requires_paths=chromium_paths, + requires_chromium=True, + ), + Check( + "chart_kind_matrix", + "registry-complete payload, GPU-geometry, and pixel matrix in Chromium", + (py, "scripts/chart_kind_matrix.py", chromium_arg), + requires_modules=("numpy",), + requires_executables=("node",), + requires_paths=chromium_paths, + requires_chromium=True, + ), Check( "step_tier_smoke", "step traces stay step-shaped through kernel tier updates", @@ -250,6 +291,20 @@ def _base_checks( requires_paths=chromium_paths, requires_chromium=True, ), + Check( + "animation_smoke", + "keyed/replacement/frozen animation smoke in Chromium", + (py, "scripts/animation_smoke.py", chromium_arg), + requires_paths=chromium_paths, + requires_chromium=True, + ), + Check( + "pick_boundary_smoke", + "256-slot and large-index pick boundary/cache smoke in Chromium", + (py, "scripts/pick_boundary_smoke.py", chromium_arg), + requires_paths=chromium_paths, + requires_chromium=True, + ), Check( "interaction_stress_smoke", "browser interaction stress smoke with latency/visual budgets", @@ -258,6 +313,26 @@ def _base_checks( requires_paths=chromium_paths, requires_chromium=True, ), + Check( + "pan_zoom_matrix", + "complete bounded pan/zoom acceptance matrix in Chromium", + ( + "node", + "scripts/pan_zoom_matrix.mjs", + "--profile", + "full", + "--browsers", + "chromium", + "--executable-path", + chromium_arg, + "--evidence", + "/tmp/xy-pan-zoom-matrix-evidence.json", + ), + requires_executables=("node",), + requires_paths=chromium_paths, + requires_chromium=True, + requires_node_major=18, + ), Check( "sdist_artifact", "source distribution artifact contents", @@ -280,6 +355,7 @@ def _base_checks( "python_floor", "public_api", "claim_guardrails", + "testing_spec", "ci_workflow", "ruff_check", "ruff_format", @@ -296,10 +372,16 @@ def _base_checks( BROWSER_CHECKS = ( "render_smoke_nonumpy", "smoke_render", + "runtime_security_smoke", "reflex_lifecycle_smoke", - "visual_regression_smoke", + "visual_health_smoke", + "visual_baseline", + "chart_kind_matrix", "step_tier_smoke", + "animation_smoke", + "pick_boundary_smoke", "interaction_stress_smoke", + "pan_zoom_matrix", ) PACKAGING_CHECKS = ("sdist_artifact", "wheel_artifact") diff --git a/scripts/verify_sdist.py b/scripts/verify_sdist.py index 8e9f301b..7a363262 100644 --- a/scripts/verify_sdist.py +++ b/scripts/verify_sdist.py @@ -58,6 +58,8 @@ "spec/api/chart-roadmap.md", "spec/benchmarks/results.md", "spec/design/renderer-architecture.md", + "spec/design/host-compatibility.md", + "spec/testing/host-integration-policy.json", "spec/matplotlib/compat.md", "spec/process/contributing.md", "spec/process/production-readiness.md", @@ -108,29 +110,41 @@ "scripts/check_public_api.py", "scripts/check_claim_guardrails.py", "scripts/check_python_floor.py", + "scripts/check_release_version.py", "scripts/check_regressions.py", + "scripts/fastapi_host_smoke.py", + "scripts/host_integration_policy.py", "scripts/bench_dashboard.py", "scripts/bench_interaction.py", "scripts/bench_pyplot_vs_matplotlib.py", + "scripts/native_parity.py", "scripts/verify_ci_workflow.py", "scripts/verify_benchmark_report.py", "scripts/verify_local.py", + "scripts/release_provenance.py", "scripts/verify_sdist.py", + "scripts/verify_source_qualification.py", "scripts/verify_wheel.py", "src/kernels.rs", "src/lib.rs", "tests/test_public_api.py", "tests/test_claim_guardrails.py", + "tests/test_check_release_version.py", "tests/test_benchmark_environment.py", "tests/test_bench_pyplot_vs_matplotlib.py", "tests/test_check_regressions.py", "tests/test_docs_examples.py", "tests/test_example_apps.py", + "tests/host_integration/test_host_matrix.py", + "tests/test_fastapi_host_smoke.py", + "tests/test_host_integration_policy.py", "tests/test_type_surface.py", "tests/test_verify_benchmark_report.py", "tests/test_verify_ci_workflow.py", "tests/test_verify_local.py", + "tests/test_release_provenance.py", "tests/test_verify_sdist.py", + "tests/test_verify_source_qualification.py", "tests/test_verify_wheel.py", } @@ -189,11 +203,17 @@ def _normalized_files(path: str) -> tuple[str, set[str]]: def _dependency_satisfies_floor(requirement: str, package: str, minimum: str) -> bool: + if _dependency_name(requirement) != package.lower().replace("_", "-"): + return False + specifiers = re.sub( + r"^\s*[A-Za-z0-9_.-]+\s*(?:\[[^\]]+\])?\s*", + "", + requirement.split(";", 1)[0], + ) return bool( - re.match( - rf"^\s*{re.escape(package)}\s*(?:\[[^\]]+\])?\s*>=\s*" - rf"{re.escape(minimum)}(?:\b|[,;\s])", - requirement, + re.search( + rf"(?:^|,)\s*>=\s*{re.escape(minimum)}(?:\b|,|\s|$)", + specifiers, flags=re.IGNORECASE, ) ) @@ -226,7 +246,11 @@ def _require_pkg_info(path: str, root: str) -> None: if metadata.get("Requires-Python", "").strip() != ">=3.11": missing.append("Requires-Python: >=3.11") requirements = metadata.get_all("Requires-Dist") or [] - for package, minimum in (("anywidget", "0.9"), ("numpy", "1.24")): + for package, minimum in ( + ("anywidget", "0.9"), + ("traitlets", "5.14"), + ("numpy", "1.24"), + ): if not any( _dependency_satisfies_floor(requirement, package, minimum) for requirement in requirements @@ -450,6 +474,8 @@ def verify_sdist(path: str) -> None: "pypa/gh-action-pypi-publish@", "scripts/verify_wheel.py", "scripts/verify_sdist.py", + "scripts/verify_source_qualification.py", + "scripts/release_provenance.py", "id-token: write", }, ) diff --git a/scripts/verify_source_qualification.py b/scripts/verify_source_qualification.py new file mode 100644 index 00000000..57ceed7a --- /dev/null +++ b/scripts/verify_source_qualification.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Qualify an exact source commit before release or deployment. + +The gate proves that the source is a real commit on ``main`` and that a +successful ``Required CI`` job ran for that exact SHA. Release publication can +add ``--release-metadata`` to require tag, project-version, and dated changelog +agreement as part of the same preflight. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +SHA_RE = re.compile(r"[0-9a-f]{40}") +REPOSITORY_RE = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") +REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,255}") +JsonGetter = Callable[[str], dict[str, Any]] + + +@dataclass(frozen=True) +class RequiredCIInspection: + run_id: int | None + active: bool + errors: tuple[str, ...] + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ("git", *args), + cwd=repo, + text=True, + capture_output=True, + check=False, + ) + + +def check_git_source( + repo: Path, + sha: str, + *, + main_ref: str = "origin/main", + tag: str | None = None, +) -> list[str]: + """Validate commit identity, main ancestry, and an optional exact tag.""" + errors: list[str] = [] + if SHA_RE.fullmatch(sha) is None: + return [f"source SHA must be 40 lowercase hexadecimal characters, got {sha!r}"] + if REF_RE.fullmatch(main_ref) is None or main_ref.startswith("-"): + return [f"invalid main ref {main_ref!r}"] + if tag is not None and TAG_RE.fullmatch(tag) is None: + return [f"invalid tag {tag!r}"] + + resolved = _git(repo, "rev-parse", "--verify", f"{sha}^{{commit}}") + if resolved.returncode != 0: + errors.append(f"source SHA {sha} is not a commit in the checkout") + return errors + resolved_sha = resolved.stdout.strip() + if resolved_sha != sha: + errors.append(f"source SHA resolved to {resolved_sha}, expected exact commit {sha}") + + main = _git(repo, "rev-parse", "--verify", f"{main_ref}^{{commit}}") + if main.returncode != 0: + errors.append(f"main ancestry ref {main_ref!r} is unavailable") + else: + ancestor = _git(repo, "merge-base", "--is-ancestor", sha, main_ref) + if ancestor.returncode != 0: + errors.append(f"source SHA {sha} is not an ancestor of {main_ref}") + + if tag is not None: + tagged = _git(repo, "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}") + if tagged.returncode != 0: + errors.append(f"tag {tag!r} does not exist in the checkout") + elif tagged.stdout.strip() != sha: + errors.append( + f"tag {tag!r} resolves to {tagged.stdout.strip()}, expected exact SHA {sha}" + ) + return errors + + +def check_release_metadata(tag: str, pyproject: Path, changelog: Path) -> list[str]: + """Reuse the release-version checker without weakening either CLI.""" + module_path = ROOT / "scripts" / "check_release_version.py" + spec = importlib.util.spec_from_file_location("_source_qualification_release", module_path) + if spec is None or spec.loader is None: + return [f"cannot load release metadata checker from {module_path}"] + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.check_release(tag, pyproject, changelog) + + +def inspect_required_ci( + get_json: JsonGetter, + repository: str, + sha: str, + *, + workflow: str = "ci.yml", + required_job: str = "Required CI", +) -> RequiredCIInspection: + """Inspect Actions runs and require the aggregate job on the exact SHA.""" + workflow_id = urllib.parse.quote(workflow, safe="") + query = urllib.parse.urlencode({"head_sha": sha, "per_page": 100}) + payload = get_json(f"/repos/{repository}/actions/workflows/{workflow_id}/runs?{query}") + runs = payload.get("workflow_runs") + if not isinstance(runs, list): + return RequiredCIInspection(None, False, ("Actions response has no workflow_runs list",)) + + exact = [run for run in runs if isinstance(run, dict) and run.get("head_sha") == sha] + if not exact: + return RequiredCIInspection( + None, + False, + (f"no {workflow} run exists for exact SHA {sha}",), + ) + + # Never fall back to an older success. A new run (or rerun attempt) may be + # correcting a failure; accepting an older run while it is active or after + # it fails would make the preflight raceable. + newest = max( + exact, + key=lambda run: (int(run.get("id") or 0), int(run.get("run_attempt") or 0)), + ) + run_id = newest.get("id") + if not isinstance(run_id, int): + return RequiredCIInspection(None, False, ("newest Actions run has no integer id",)) + status = newest.get("status") + if status != "completed": + return RequiredCIInspection( + None, + True, + (f"newest Actions run {run_id} for exact SHA {sha} is {status!r}",), + ) + + jobs_payload = get_json( + f"/repos/{repository}/actions/runs/{run_id}/jobs?filter=latest&per_page=100" + ) + jobs = jobs_payload.get("jobs") + if not isinstance(jobs, list): + return RequiredCIInspection(None, False, (f"Actions run {run_id} has no jobs list",)) + matches = [job for job in jobs if isinstance(job, dict) and job.get("name") == required_job] + if not matches: + return RequiredCIInspection( + None, + False, + (f"Actions run {run_id} has no {required_job!r} job",), + ) + if any( + job.get("status") == "completed" and job.get("conclusion") == "success" for job in matches + ): + # Advisory jobs are intentionally non-blocking. Required CI is the + # stable aggregate of the hard suite, so it alone controls promotion. + return RequiredCIInspection(run_id, False, ()) + conclusions = sorted({str(job.get("conclusion")) for job in matches}) + return RequiredCIInspection( + None, + False, + ( + f"newest Actions run {run_id} {required_job!r} conclusions are " + f"{conclusions}, expected success", + ), + ) + + +def _github_getter(api_url: str, token: str) -> JsonGetter: + base = api_url.rstrip("/") + + def get_json(path: str) -> dict[str, Any]: + request = urllib.request.Request( + base + path, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 + payload = json.load(response) + except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as exc: + raise RuntimeError(f"GitHub API request failed for {path}: {exc}") from exc + if not isinstance(payload, dict): + raise RuntimeError(f"GitHub API returned a non-object for {path}") + return payload + + return get_json + + +def wait_for_required_ci( + get_json: JsonGetter, + repository: str, + sha: str, + *, + workflow: str, + required_job: str, + wait_seconds: float, + poll_seconds: float, +) -> int: + deadline = time.monotonic() + wait_seconds + while True: + inspection = inspect_required_ci( + get_json, + repository, + sha, + workflow=workflow, + required_job=required_job, + ) + if inspection.run_id is not None: + return inspection.run_id + remaining = deadline - time.monotonic() + if remaining <= 0 or (inspection.errors and not inspection.active and wait_seconds == 0): + raise RuntimeError("; ".join(inspection.errors)) + no_run_yet = bool( + inspection.errors + and inspection.errors[0].startswith("no ") + and " run exists for exact SHA " in inspection.errors[0] + ) + if inspection.errors and not inspection.active and not no_run_yet: + raise RuntimeError("; ".join(inspection.errors)) + time.sleep(min(poll_seconds, remaining)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sha", required=True) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "")) + parser.add_argument("--repo", type=Path, default=ROOT) + parser.add_argument("--main-ref", default="origin/main") + parser.add_argument("--tag") + parser.add_argument("--release-metadata", action="store_true") + parser.add_argument("--pyproject", type=Path, default=ROOT / "pyproject.toml") + parser.add_argument("--changelog", type=Path, default=ROOT / "CHANGELOG.md") + parser.add_argument("--workflow", default="ci.yml") + parser.add_argument("--required-job", default="Required CI") + parser.add_argument("--wait-seconds", type=float, default=0.0) + parser.add_argument("--poll-seconds", type=float, default=15.0) + parser.add_argument( + "--api-url", default=os.environ.get("GITHUB_API_URL", "https://api.github.com") + ) + parser.add_argument( + "--token", default=os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + ) + args = parser.parse_args(argv) + + errors = check_git_source(args.repo, args.sha, main_ref=args.main_ref, tag=args.tag) + if args.release_metadata: + if args.tag is None: + errors.append("--release-metadata requires --tag") + else: + errors.extend(check_release_metadata(args.tag, args.pyproject, args.changelog)) + if REPOSITORY_RE.fullmatch(args.repository) is None: + errors.append(f"repository must be owner/name, got {args.repository!r}") + if not args.token: + errors.append("GitHub token is required for exact-SHA Actions verification") + if args.wait_seconds < 0 or args.poll_seconds <= 0: + errors.append("wait seconds must be non-negative and poll seconds must be positive") + if errors: + print("source qualification failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + try: + run_id = wait_for_required_ci( + _github_getter(args.api_url, args.token), + args.repository, + args.sha, + workflow=args.workflow, + required_job=args.required_job, + wait_seconds=args.wait_seconds, + poll_seconds=args.poll_seconds, + ) + except RuntimeError as exc: + print(f"source qualification failed: {exc}", file=sys.stderr) + return 1 + print( + f"source qualification OK: {args.sha} is on {args.main_ref}; " + f"Actions run {run_id} passed {args.required_job!r}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_wheel.py b/scripts/verify_wheel.py index 8b5cd633..04b30ec3 100644 --- a/scripts/verify_wheel.py +++ b/scripts/verify_wheel.py @@ -129,11 +129,17 @@ def _require_filename_tag(path: Path, tags: list[str]) -> None: def _dependency_satisfies_floor(requirement: str, package: str, minimum: str) -> bool: + if _dependency_name(requirement) != package.lower().replace("_", "-"): + return False + specifiers = re.sub( + r"^\s*[A-Za-z0-9_.-]+\s*(?:\[[^\]]+\])?\s*", + "", + requirement.split(";", 1)[0], + ) return bool( - re.match( - rf"^\s*{re.escape(package)}\s*(?:\[[^\]]+\])?\s*>=\s*" - rf"{re.escape(minimum)}(?:\b|[,;\s])", - requirement, + re.search( + rf"(?:^|,)\s*>=\s*{re.escape(minimum)}(?:\b|,|\s|$)", + specifiers, flags=re.IGNORECASE, ) ) @@ -162,7 +168,11 @@ def _require_metadata(names: set[str], data: bytes) -> None: if metadata.get("Requires-Python", "").strip() != ">=3.11": missing.append("Requires-Python: >=3.11") requirements = metadata.get_all("Requires-Dist") or [] - for package, minimum in (("anywidget", "0.9"), ("numpy", "1.24")): + for package, minimum in ( + ("anywidget", "0.9"), + ("traitlets", "5.14"), + ("numpy", "1.24"), + ): if not any( _dependency_satisfies_floor(requirement, package, minimum) for requirement in requirements diff --git a/scripts/visual_baseline.py b/scripts/visual_baseline.py new file mode 100644 index 00000000..2a15ec00 --- /dev/null +++ b/scripts/visual_baseline.py @@ -0,0 +1,786 @@ +#!/usr/bin/env python3 +"""Reviewed visual-identity baselines for a small deterministic gallery set. + +Unlike ``visual_health_smoke.py`` (broad nonblank/occupancy coverage), this +gate compares three representative charts with a versioned semantic + +perceptual oracle. Chromium, the bundled font, viewport, DPR, downsample, and +tolerances are pinned in the manifest. Every run also executes real-browser +data, color, label, and geometry negative controls so a weakened oracle fails. +""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import contextlib +import hashlib +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import zlib +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "examples" / "fastapi")) + +import charts # noqa: E402 + +from _app_smoke import ChromiumSession, Probe, find_chromium # noqa: E402 +from visual_health_smoke import ( # noqa: E402 + VIEW_H, + VIEW_W, + _assert_visual, + _png_rgba, +) + +SCHEMA_VERSION = 1 +BASELINE_PATH = ROOT / "spec" / "visual-baselines" / "v1.json" +FONT_PATH = ROOT / "docs" / "app" / "xy_docs" / "assets" / "InstrumentSans-wdth-wght.ttf" +FONT_FAMILY = "XY Visual Baseline" +BASELINE_ROUTES = ("grouped-bars", "heatmap", "composed-layers") +VIEW_DPR = 1.0 +DOWNSAMPLE = 10 +SAMPLE_W, SAMPLE_H = VIEW_W // DOWNSAMPLE, VIEW_H // DOWNSAMPLE +MAX_MEAN_ABS_ERROR = 10.0 +MAX_CHANGED_CELL_FRACTION = 0.12 +CHANGED_CELL_DELTA = 30 +MAX_GEOMETRY_DELTA_PX = 2.0 + +_CAPTURE_INIT_SCRIPT = r""" +(() => { + let runtime; + Object.defineProperty(window, "xy", { + configurable: true, + get: () => runtime, + set: (value) => { + if (value && typeof value.renderStandalone === "function") { + const original = value.renderStandalone; + value.renderStandalone = function(el, spec, buffer) { + const view = original.call(this, el, spec, buffer); + window.__xyVisualBaseline = {view, spec, buffer}; + return view; + }; + } + runtime = value; + }, + }); +})() +""" + +_CORRUPT_DATA_MUTATION = r""" +const state = window.__xyVisualBaseline; +if (!state || !state.view || !state.spec || !state.buffer) { + throw new Error("captured chart payload is unavailable"); +} +const source = state.buffer instanceof ArrayBuffer + ? new Uint8Array(state.buffer) + : new Uint8Array(state.buffer.buffer, state.buffer.byteOffset, state.buffer.byteLength); +const bytes = new Uint8Array(source); +const spec = structuredClone(state.spec); +let changed = 0; +for (const trace of spec.traces || []) { + const columnIndex = trace.bar && trace.bar.value1; + const column = Number.isInteger(columnIndex) && spec.columns[columnIndex]; + if (!column || column.kind !== "float") continue; + const values = new Float32Array(bytes.buffer, column.byte_offset, column.len); + const encoded = (5 - (column.offset || 0)) / (column.scale || 1); + for (let i = 0; i < values.length; i++) { + if (values[i] !== encoded) changed++; + values[i] = encoded; + } +} +if (!changed) throw new Error("corrupted-data control did not alter any values"); +if (!state.view.updatePayload(spec, bytes)) { + throw new Error("corrupted-data payload update was rejected by the renderer"); +} +state.view.draw(); +document.querySelector("[data-xy-slot='root']").dataset.xyBaselineCorruptedValues = String(changed); +""" + +NEGATIVE_CONTROLS = { + "corrupted-data": ("grouped-bars", _CORRUPT_DATA_MUTATION), + "wrong-color": ( + "heatmap", + "document.querySelector('[data-xy-slot=canvas]').style.filter=" + "'hue-rotate(125deg) saturate(1.8)'", + ), + "wrong-label": ( + "grouped-bars", + "document.querySelector('[data-xy-slot=title]').textContent=" + "'CORRUPTED VISUAL BASELINE LABEL'", + ), + "wrong-geometry": ( + "composed-layers", + "document.querySelector('[data-xy-slot=canvas]').style.transform=" + "'translateX(52px) scaleX(.72)';" + "document.querySelector('[data-xy-slot=canvas]').style.transformOrigin='left top'", + ), +} + +_SEMANTIC_EXPR = r""" +(() => { + const text = (selector) => Array.from(document.querySelectorAll(selector)).map((el) => ({ + axis: el.dataset.xyAxis || "", + side: el.dataset.xyAxisSide || "", + text: (el.textContent || "").trim(), + })).filter((item) => item.text); + const slots = {}; + for (const slot of ["root", "canvas", "title", "legend", "colorbar"]) { + slots[slot] = Array.from(document.querySelectorAll(`[data-xy-slot='${slot}']`)).map((el) => { + const r = el.getBoundingClientRect(); + return {x: +r.x.toFixed(1), y: +r.y.toFixed(1), + w: +r.width.toFixed(1), h: +r.height.toFixed(1)}; + }); + } + const root = document.querySelector("[data-xy-slot='root']"); + return { + title: text("[data-xy-slot='title']"), + axis_titles: text("[data-xy-label-kind='axis-title']"), + tick_labels: text("[data-xy-label-kind='tick']"), + legend: text("[data-xy-slot='legend_item']"), + colorbar: text("[data-xy-slot='colorbar_tick'],[data-xy-slot='colorbar_title']"), + slots, + font_family: root ? getComputedStyle(root).fontFamily : "", + dpr: window.devicePixelRatio, + }; +})() +""" + + +class BaselineError(RuntimeError): + pass + + +@contextlib.contextmanager +def _baseline_pages() -> Iterator[dict[str, str]]: + """Build deterministic standalone gallery pages without a network hop.""" + with tempfile.TemporaryDirectory(prefix="xy-visual-baseline-pages-") as directory: + root = Path(directory) + urls: dict[str, str] = {} + for route in BASELINE_ROUTES: + path = root / f"{route}.html" + path.write_text(charts.BY_ID[route].builder().to_html(), encoding="utf-8") + urls[route] = path.as_uri() + yield urls + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _font_bytes() -> bytes: + try: + return FONT_PATH.read_bytes() + except OSError as exc: + raise BaselineError(f"cannot read pinned font {FONT_PATH}: {exc}") from exc + + +def _playwright_version() -> str: + package = json.loads((ROOT / "package.json").read_text(encoding="utf-8")) + version = package.get("devDependencies", {}).get("playwright") + if not isinstance(version, str) or not re.fullmatch(r"\d+\.\d+\.\d+", version): + raise BaselineError("package.json must pin an exact Playwright version") + return version + + +def _browser_version(executable: str) -> str: + try: + completed = subprocess.run( + [executable, "--version"], capture_output=True, text=True, timeout=15 + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise BaselineError(f"cannot execute configured Chromium {executable}: {exc}") from exc + match = re.search(r"(\d+\.\d+\.\d+\.\d+)", completed.stdout + completed.stderr) + if completed.returncode != 0 or match is None: + raise BaselineError( + f"configured Chromium did not report an exact version: " + f"exit={completed.returncode} output={(completed.stdout + completed.stderr)[-300:]!r}" + ) + return match.group(1) + + +def _resolve_chromium(explicit: str | None) -> str: + if explicit: + candidate = explicit if Path(explicit).is_file() else shutil.which(explicit) + if not candidate: + raise BaselineError(f"configured chromium not found: {explicit}") + return str(candidate) + return find_chromium(None) + + +def _font_expr(font: bytes) -> str: + encoded = base64.b64encode(font).decode("ascii") + return f""" +(async () => {{ + const family = {json.dumps(FONT_FAMILY)}; + const bin = atob({json.dumps(encoded)}); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + const face = new FontFace(family, bytes.buffer, {{weight: "100 900", style: "normal"}}); + await face.load(); + document.fonts.add(face); + const root = document.querySelector("[data-xy-slot='root']"); + if (!root) throw new Error("visual baseline root is missing"); + root.style.fontFamily = `"${{family}}"`; + await document.fonts.ready; + window.dispatchEvent(new Event("resize")); + await new Promise((resolve) => requestAnimationFrame(() => + requestAnimationFrame(() => requestAnimationFrame(resolve)))); + return {{loaded: document.fonts.check(`12px "${{family}}"`), + family: getComputedStyle(root).fontFamily, + dpr: window.devicePixelRatio}}; +}})() +""" + + +def _capture_route( + session: ChromiumSession, + url: str, + name: str, + font: bytes, + *, + mutation: str | None = None, +) -> tuple[bytes, dict, dict]: + probe = None + stage = "create page" + try: + probe = Probe( + session, + url, + init_script=_CAPTURE_INIT_SCRIPT, + emulate=(VIEW_W, VIEW_H, VIEW_DPR), + ) + stage = "wait for canvas" + probe.wait_for( + "!!document.querySelector('[data-xy-slot=canvas]')", + timeout_s=60.0, + label=f"{name}: canvas mounted", + ) + stage = "load pinned font" + font_info = probe.eval(_font_expr(font), timeout_s=60.0) + if not font_info or font_info.get("loaded") is not True: + raise BaselineError(f"{name}: pinned browser font did not load: {font_info!r}") + if mutation: + stage = "apply negative control" + probe.eval(f"(() => {{{mutation}; return true;}})()") + stage = "settle negative control" + probe.eval("new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))") + stage = "capture screenshot" + shot = probe._call( + "Page.captureScreenshot", + { + "format": "png", + "clip": {"x": 0, "y": 0, "width": VIEW_W, "height": VIEW_H, "scale": 1}, + }, + ) + png = base64.b64decode(shot["data"], validate=True) + stage = "capture semantics" + semantic = probe.eval(_SEMANTIC_EXPR) + if not isinstance(semantic, dict): + raise BaselineError(f"{name}: semantic probe returned {semantic!r}") + return png, semantic, font_info + except BaselineError: + raise + except SystemExit as exc: + raise BaselineError(f"{name}: browser capture failed during {stage}: {exc}") from exc + except Exception as exc: + raise BaselineError( + f"{name}: browser capture failed during {stage} ({type(exc).__name__}): {exc}" + ) from exc + finally: + if probe is not None: + probe.close() + + +def _downsample(png: bytes) -> bytes: + width, height, rgba = _png_rgba(png) + if (width, height) != (VIEW_W, VIEW_H): + raise BaselineError(f"expected {VIEW_W}x{VIEW_H} PNG, got {width}x{height}") + out = bytearray(SAMPLE_W * SAMPLE_H * 3) + for sy in range(SAMPLE_H): + for sx in range(SAMPLE_W): + sums = [0, 0, 0] + count = 0 + for y in range(sy * DOWNSAMPLE, (sy + 1) * DOWNSAMPLE): + for x in range(sx * DOWNSAMPLE, (sx + 1) * DOWNSAMPLE): + offset = (y * width + x) * 4 + r, g, b, a = rgba[offset : offset + 4] + alpha = a / 255.0 + sums[0] += round(r * alpha + 255 * (1 - alpha)) + sums[1] += round(g * alpha + 255 * (1 - alpha)) + sums[2] += round(b * alpha + 255 * (1 - alpha)) + count += 1 + target = (sy * SAMPLE_W + sx) * 3 + out[target : target + 3] = bytes(round(value / count) for value in sums) + return bytes(out) + + +def _decode_expected(case: dict, name: str) -> bytes: + raster = case.get("raster") + if not isinstance(raster, str): + raise BaselineError(f"{name}: baseline raster must be base64 text") + try: + raw = base64.b64decode(raster, validate=True) + except (binascii.Error, ValueError) as exc: + raise BaselineError(f"{name}: corrupt baseline raster: {exc}") from exc + expected_len = SAMPLE_W * SAMPLE_H * 3 + if len(raw) != expected_len: + raise BaselineError( + f"{name}: baseline raster has {len(raw)} bytes, expected {expected_len}" + ) + if case.get("raster_sha256") != _sha256(raw): + raise BaselineError(f"{name}: baseline raster checksum mismatch") + return raw + + +def _semantic_errors(expected: dict, actual: dict) -> list[str]: + errors: list[str] = [] + for key in ("title", "axis_titles", "tick_labels", "legend", "colorbar", "font_family", "dpr"): + if expected.get(key) != actual.get(key): + errors.append(f"semantic {key} differs") + expected_slots = expected.get("slots") + actual_slots = actual.get("slots") + if not isinstance(expected_slots, dict) or not isinstance(actual_slots, dict): + return [*errors, "semantic slot geometry is missing"] + if set(expected_slots) != set(actual_slots): + errors.append("semantic slot set differs") + return errors + for slot, expected_rects in expected_slots.items(): + actual_rects = actual_slots.get(slot) + if not isinstance(expected_rects, list) or not isinstance(actual_rects, list): + errors.append(f"semantic {slot} geometry is malformed") + continue + if len(expected_rects) != len(actual_rects): + errors.append(f"semantic {slot} geometry count differs") + continue + for index, (expected_rect, actual_rect) in enumerate( + zip(expected_rects, actual_rects, strict=True) + ): + for dimension in ("x", "y", "w", "h"): + before = expected_rect.get(dimension) + after = actual_rect.get(dimension) + if not isinstance(before, int | float) or not isinstance(after, int | float): + errors.append(f"semantic {slot}[{index}].{dimension} is malformed") + elif abs(before - after) > MAX_GEOMETRY_DELTA_PX: + errors.append( + f"semantic {slot}[{index}].{dimension} delta " + f"{abs(before - after):.1f}px exceeds {MAX_GEOMETRY_DELTA_PX:.1f}px" + ) + return errors + + +def compare_case( + name: str, png: bytes, semantic: dict, case: dict +) -> tuple[list[str], dict, bytes, bytes]: + expected = _decode_expected(case, name) + actual = _downsample(png) + deltas = [abs(before - after) for before, after in zip(expected, actual, strict=True)] + mae = sum(deltas) / len(deltas) + changed = 0 + cells = SAMPLE_W * SAMPLE_H + for cell in range(cells): + start = cell * 3 + if max(deltas[start : start + 3]) > CHANGED_CELL_DELTA: + changed += 1 + changed_fraction = changed / cells + errors = _semantic_errors(case.get("semantic", {}), semantic) + if mae > MAX_MEAN_ABS_ERROR: + errors.append(f"perceptual MAE {mae:.3f} exceeds {MAX_MEAN_ABS_ERROR:.3f}") + if changed_fraction > MAX_CHANGED_CELL_FRACTION: + errors.append( + f"changed-cell fraction {changed_fraction:.4f} exceeds {MAX_CHANGED_CELL_FRACTION:.4f}" + ) + metrics = { + "mean_abs_error": round(mae, 4), + "changed_cell_fraction": round(changed_fraction, 6), + "semantic_error_count": len(_semantic_errors(case.get("semantic", {}), semantic)), + } + return errors, metrics, expected, actual + + +def _png_chunk(kind: bytes, payload: bytes) -> bytes: + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + +def _rgb_png(width: int, height: int, rgb: bytes) -> bytes: + if len(rgb) != width * height * 3: + raise BaselineError("RGB artifact byte length does not match geometry") + raw = b"".join(b"\x00" + rgb[y * width * 3 : (y + 1) * width * 3] for y in range(height)) + return ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + _png_chunk(b"IDAT", zlib.compress(raw, 9)) + + _png_chunk(b"IEND", b"") + ) + + +def _diff_rgb(expected: bytes, actual: bytes) -> bytes: + out = bytearray(len(expected)) + for offset in range(0, len(expected), 3): + delta = max(abs(expected[offset + i] - actual[offset + i]) for i in range(3)) + if delta <= 2: + out[offset : offset + 3] = b"\xff\xff\xff" + else: + out[offset : offset + 3] = bytes( + (min(255, 40 + delta * 5), max(0, 220 - delta * 3), 32) + ) + return bytes(out) + + +def write_artifacts( + root: Path, + name: str, + *, + full_png: bytes, + expected: bytes, + actual: bytes, + expected_semantic: dict, + actual_semantic: dict, +) -> None: + destination = root / name + destination.mkdir(parents=True, exist_ok=True) + (destination / "expected.png").write_bytes(_rgb_png(SAMPLE_W, SAMPLE_H, expected)) + (destination / "actual.png").write_bytes(_rgb_png(SAMPLE_W, SAMPLE_H, actual)) + (destination / "actual-full.png").write_bytes(full_png) + (destination / "diff.png").write_bytes( + _rgb_png(SAMPLE_W, SAMPLE_H, _diff_rgb(expected, actual)) + ) + (destination / "expected-semantic.json").write_text( + json.dumps(expected_semantic, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (destination / "actual-semantic.json").write_text( + json.dumps(actual_semantic, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def _load_manifest(path: Path) -> dict: + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise BaselineError(f"cannot read visual baseline manifest {path}: {exc}") from exc + if manifest.get("schema_version") != SCHEMA_VERSION: + raise BaselineError( + f"unsupported visual baseline schema {manifest.get('schema_version')!r}" + ) + if manifest.get("suite") != "xy-reviewed-visual-baselines": + raise BaselineError("visual baseline suite identity is missing or invalid") + if set(manifest.get("cases", {})) != set(BASELINE_ROUTES): + raise BaselineError("visual baseline route set does not match the required bounded set") + pinned = manifest.get("pinned", {}) + if pinned.get("browser_engine") != "chromium": + raise BaselineError("visual baselines require the Chromium browser engine") + if not isinstance(pinned.get("browser_version"), str) or not re.fullmatch( + r"\d+\.\d+\.\d+\.\d+", pinned["browser_version"] + ): + raise BaselineError("visual baseline browser_version must be an exact four-part version") + expected_pin = { + "playwright": _playwright_version(), + "viewport": [VIEW_W, VIEW_H], + "dpr": VIEW_DPR, + "font_path": str(FONT_PATH.relative_to(ROOT)), + "font_sha256": _sha256(_font_bytes()), + "downsample": DOWNSAMPLE, + } + for key, value in expected_pin.items(): + if pinned.get(key) != value: + raise BaselineError( + f"visual baseline pin {key!r} is stale: {pinned.get(key)!r} != {value!r}" + ) + expected_tolerances = { + "max_mean_abs_error": MAX_MEAN_ABS_ERROR, + "max_changed_cell_fraction": MAX_CHANGED_CELL_FRACTION, + "changed_cell_delta": CHANGED_CELL_DELTA, + "max_geometry_delta_px": MAX_GEOMETRY_DELTA_PX, + } + if manifest.get("tolerances") != expected_tolerances: + raise BaselineError("visual baseline tolerances differ from the reviewed code constants") + update = manifest.get("update", {}) + if ( + not isinstance(update.get("prepared_by"), str) + or not update["prepared_by"].strip() + or not isinstance(update.get("reason"), str) + or not update["reason"].strip() + or update.get("review_required") is not True + ): + raise BaselineError("visual baseline update provenance is missing or invalid") + for name, case in manifest["cases"].items(): + _decode_expected(case, name) + return manifest + + +def _load_review_source(path: Path) -> dict | None: + """Load the previous reviewed pixels for proposal artifacts. + + Existing environment pins are deliberately not checked here: updating a + browser, font, or Playwright pin is one reason a proposal may be needed. + The old raster checksums and bounded route set must still be intact. + """ + if not path.exists(): + return None + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise BaselineError(f"cannot read prior visual baseline {path}: {exc}") from exc + if manifest.get("schema_version") != SCHEMA_VERSION: + raise BaselineError("prior visual baseline schema cannot be used for review artifacts") + if set(manifest.get("cases", {})) != set(BASELINE_ROUTES): + raise BaselineError("prior visual baseline route set is incomplete") + for name, case in manifest["cases"].items(): + _decode_expected(case, name) + return manifest + + +def _new_manifest( + *, + browser_version: str, + font: bytes, + prepared_by: str, + reason: str, + captures: dict[str, tuple[bytes, dict]], +) -> dict: + return { + "schema_version": SCHEMA_VERSION, + "suite": "xy-reviewed-visual-baselines", + "pinned": { + "browser_engine": "chromium", + "browser_version": browser_version, + "playwright": _playwright_version(), + "viewport": [VIEW_W, VIEW_H], + "dpr": VIEW_DPR, + "font_path": str(FONT_PATH.relative_to(ROOT)), + "font_sha256": _sha256(font), + "downsample": DOWNSAMPLE, + }, + "tolerances": { + "max_mean_abs_error": MAX_MEAN_ABS_ERROR, + "max_changed_cell_fraction": MAX_CHANGED_CELL_FRACTION, + "changed_cell_delta": CHANGED_CELL_DELTA, + "max_geometry_delta_px": MAX_GEOMETRY_DELTA_PX, + }, + "update": { + "prepared_by": prepared_by, + "reason": reason, + "prepared_at_utc": datetime.now(UTC).replace(microsecond=0).isoformat(), + "review_required": True, + }, + "cases": { + name: { + "semantic": semantic, + "raster": base64.b64encode(sample).decode("ascii"), + "raster_sha256": _sha256(sample), + } + for name, (sample, semantic) in captures.items() + }, + } + + +def _write_evidence(path: Path | None, payload: dict) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _run(args: argparse.Namespace, artifacts: Path) -> tuple[int, dict]: + executable = _resolve_chromium(args.chromium) + browser_version = _browser_version(executable) + font = _font_bytes() + prepared_by = args.prepared_by.strip() if isinstance(args.prepared_by, str) else "" + reason = args.reason.strip() if isinstance(args.reason, str) else "" + review_source = None + if args.update_baselines: + if os.environ.get("CI"): + raise BaselineError("baseline updates are forbidden in CI") + if not prepared_by or not reason: + raise BaselineError("--update-baselines requires --prepared-by and --reason") + review_source = _load_review_source(args.baseline) + manifest = None if args.update_baselines else _load_manifest(args.baseline) + if manifest is not None and manifest["pinned"].get("browser_version") != browser_version: + raise BaselineError( + f"visual baselines require Chromium {manifest['pinned'].get('browser_version')}, " + f"got {browser_version}; use the pinned Playwright executable" + ) + + captures: dict[str, tuple[bytes, dict]] = {} + full_captures: dict[str, bytes] = {} + results: dict[str, dict] = {} + failures: list[str] = [] + with ( + _baseline_pages() as urls, + ChromiumSession(executable, gl="software", sandbox=False) as session, + ): + for route in BASELINE_ROUTES: + png, semantic, font_info = _capture_route(session, urls[route], route, font) + health_error = None + try: + _assert_visual(route, png) + except SystemExit as exc: + health_error = f"visual health failure: {exc}" + sample = _downsample(png) + captures[route] = (sample, semantic) + full_captures[route] = png + if manifest is not None: + errors, metrics, expected, actual = compare_case( + route, png, semantic, manifest["cases"][route] + ) + if health_error: + errors.insert(0, health_error) + results[route] = {"errors": errors, "metrics": metrics, "font": font_info} + if errors: + failures.extend(f"{route}: {error}" for error in errors) + write_artifacts( + artifacts, + route, + full_png=png, + expected=expected, + actual=actual, + expected_semantic=manifest["cases"][route]["semantic"], + actual_semantic=semantic, + ) + elif health_error: + failures.append(f"{route}: {health_error}") + results[route] = {"errors": [health_error], "font": font_info} + + if args.update_baselines: + manifest = _new_manifest( + browser_version=browser_version, + font=font, + prepared_by=prepared_by, + reason=reason, + captures=captures, + ) + for route, (actual, actual_semantic) in captures.items(): + if review_source is None: + expected = actual + expected_semantic = actual_semantic + else: + expected_case = review_source["cases"][route] + expected = _decode_expected(expected_case, route) + expected_semantic = expected_case["semantic"] + write_artifacts( + artifacts, + route, + full_png=full_captures[route], + expected=expected, + actual=actual, + expected_semantic=expected_semantic, + actual_semantic=actual_semantic, + ) + + negative_results: dict[str, dict] = {} + for control, (route, mutation) in NEGATIVE_CONTROLS.items(): + png, semantic, _ = _capture_route( + session, + urls[route], + f"negative-{control}", + font, + mutation=mutation, + ) + errors, metrics, expected, actual = compare_case( + f"negative-{control}", png, semantic, manifest["cases"][route] + ) + rejected = bool(errors) + negative_results[control] = { + "route": route, + "rejected": rejected, + "errors": errors, + "metrics": metrics, + } + write_artifacts( + artifacts, + f"negative-{control}", + full_png=png, + expected=expected, + actual=actual, + expected_semantic=manifest["cases"][route]["semantic"], + actual_semantic=semantic, + ) + if not rejected: + failures.append(f"negative control {control!r} escaped the visual oracle") + + if args.update_baselines and not failures: + args.baseline.parent.mkdir(parents=True, exist_ok=True) + args.baseline.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"visual baseline proposal written: {args.baseline}") + print("attach the generated expected/actual/diff artifacts and request independent review") + + evidence = { + "status": "failed" if failures else "ok", + "browser_version": browser_version, + "playwright_version": _playwright_version(), + "font_sha256": _sha256(font), + "viewport": [VIEW_W, VIEW_H], + "dpr": VIEW_DPR, + "baseline": str(args.baseline), + "updated": bool(args.update_baselines), + "cases": results, + "negative_controls": negative_results, + "failures": failures, + } + if failures: + print("reviewed visual baseline FAILED:", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + return 1, evidence + print( + f"reviewed visual baseline OK: {len(BASELINE_ROUTES)} cases, " + f"{len(NEGATIVE_CONTROLS)} real-browser negative controls" + ) + return 0, evidence + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("chromium", nargs="?", default=None) + parser.add_argument("--baseline", type=Path, default=BASELINE_PATH) + parser.add_argument("--artifacts", type=Path, default=None) + parser.add_argument("--evidence", type=Path, default=None) + parser.add_argument("--update-baselines", action="store_true") + parser.add_argument("--prepared-by", default=None) + parser.add_argument("--reason", default=None) + args = parser.parse_args(argv) + + temporary = tempfile.TemporaryDirectory() if args.artifacts is None else None + artifacts = args.artifacts or Path(temporary.name) + evidence = { + "status": "failed", + "error": "visual baseline exited before producing evidence", + "baseline": str(args.baseline), + "updated": bool(args.update_baselines), + } + try: + rc, evidence = _run(args, artifacts) + except (Exception, SystemExit) as exc: + evidence = { + "status": "failed", + "error": str(exc), + "baseline": str(args.baseline), + "updated": bool(args.update_baselines), + } + print(f"reviewed visual baseline FAILED: {exc}", file=sys.stderr) + rc = 1 + finally: + _write_evidence(args.evidence, evidence) + if temporary is not None: + temporary.cleanup() + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/visual_regression_smoke.py b/scripts/visual_health_smoke.py similarity index 96% rename from scripts/visual_regression_smoke.py rename to scripts/visual_health_smoke.py index ef9f71c6..a3f91876 100644 --- a/scripts/visual_regression_smoke.py +++ b/scripts/visual_health_smoke.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 -"""Screenshot smoke for the example gallery. +"""Visual-health smoke for the example gallery. Runs the FastAPI example and, for each chart route plus the drilldown, screenshots it in headless Chromium and asserts the render is not blank, flat, or collapsed, and that axis tick labels do not overlap. -Usage: python scripts/visual_regression_smoke.py [/path/to/chrome] +This is deliberately not an image-identity regression oracle. Reviewed visual +identity lives in ``scripts/visual_baseline.py``; this gate retains the broader +gallery health, occupancy, and tick-overlap coverage. + +Usage: python scripts/visual_health_smoke.py [/path/to/chrome] """ from __future__ import annotations @@ -251,7 +255,7 @@ def main() -> int: png, _ = _screenshot_route(session, f"{base_url}{DRILLDOWN_PATH}", "drilldown") _assert_visual("drilldown", png) print(" drilldown: rendered") - print(f"visual regression smoke OK: {len(GALLERY_IDS)} charts + drilldown") + print(f"visual health smoke OK: {len(GALLERY_IDS)} charts + drilldown") return 0 diff --git a/spec/README.md b/spec/README.md index 19509017..f649886a 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,8 +1,8 @@ # Specification The root-level `spec/` directory is XY's engineering source of truth: intended -behavior, architecture, compatibility, benchmarks, release readiness, and -contributor contracts. The public documentation lives directly under `docs/`, +behavior, architecture, compatibility, benchmarks, testing, release readiness, +and contributor contracts. The public documentation lives directly under `docs/`, while the Reflex application that renders it lives in `docs/app/`. Keep this tree current with every relevant code, configuration, build, and @@ -39,6 +39,9 @@ Internal architecture: how the engine is built and why. - [`chart-grammar.md`](design/chart-grammar.md) — the declarative composition model (`Chart` + `Mark` + `Axis` + `Legend`), fixed before the catalog grows. +- [`host-compatibility.md`](design/host-compatibility.md) — bounded supported + Anywidget, Reflex, and FastAPI host ranges plus their executable floor/latest + matrix. - [`animation.md`](design/animation.md) — declarative entrance/update/exit motion, stable identity, interruption, reduced motion, and export determinism. - [`lod-architecture.md`](design/lod-architecture.md) — the Tier 0/1/2/3 @@ -86,6 +89,18 @@ Measured numbers and the rules that make them defensible. - [`methodology.md`](benchmarks/methodology.md) — how numbers are produced: mode-scoped, reproducible, oracle-checked, and publishing the cases we lose. +## testing/ + +The living test contract: what is enforced today and what remains to be built. + +- [`README.md`](testing/README.md) — status vocabulary, evidence layers, gate + tiers, and rules for promoting planned coverage. +- [`current.md`](testing/current.md) — current test surfaces, commands, + enforcement, and the boundary of each existing protection. +- [`gaps.md`](testing/gaps.md) — prioritized additions with explicit completion + criteria. Every entry remains `NOT IMPLEMENTED` until its automated evidence + and intended gate exist. + ## process/ Release bar, contribution rules, and audit trail. diff --git a/spec/api/export.md b/spec/api/export.md index eba531b6..d02e8344 100644 --- a/spec/api/export.md +++ b/spec/api/export.md @@ -172,25 +172,19 @@ end-to-end parity: the two download paths can disagree on the same chart. Closing this requires a theme snapshot on the comm channel and a bump of the message contract; nothing in the current protocol carries it. -## 7. The `--no-sandbox` auto-fallback - -Chromium launches sandboxed by default. Both browser paths silently downgrade on -failure: - -- `html_to_png` (`export.py:509-526`): if the sandboxed run produces no - screenshot, it rebuilds the argv with `--no-sandbox` inserted and re-runs - before raising. The final error reports both attempts. -- `_browser_session` (`export.py:926-931`): retries - `ChromiumSession(..., sandbox=False)` on `ChromiumError`. - -So `--no-sandbox` can appear without the caller requesting it, on input that -`html_to_png` accepts as arbitrary HTML. This is a known, accepted residual risk -taken to keep CI and container rasterization working where the sandbox cannot -initialize — see [XY-SEC-2026-03 and its 2026-07-20 status -note](../process/security-audit-2026-07-06.md#status-as-of-2026-07-20-xy-sec-2026-03). The -pending follow-up is to make the fallback opt-in, or at minimum warn, so a -sandbox loss is observable. `sandbox=False` remains the explicit escape hatch -for trusted HTML. +## 7. Chromium sandbox policy + +Chromium launches sandboxed by default, and that choice is a guarantee rather +than a preference. Neither the one-shot `html_to_png` path nor the persistent +`_browser_session` path retries with `--no-sandbox` after a launch failure. +Sandboxed failures say that no downgrade was attempted and identify the only +escape hatch: an explicit `sandbox=False` from a caller that has established the +HTML is trusted and the surrounding CI/container isolation is sufficient. + +The repository's PNG browser smoke owns its generated fixture and makes that +trusted-runner exception explicit. Public export calls never acquire +`--no-sandbox` merely because the sandbox failed to initialize. See the resolved +[XY-SEC-2026-03 audit note](../process/security-audit-2026-07-06.md#resolution-as-of-2026-07-21-xy-sec-2026-03). ## 8. Batch export diff --git a/spec/api/styling.md b/spec/api/styling.md index 6e9faa55..0c994fa8 100644 --- a/spec/api/styling.md +++ b/spec/api/styling.md @@ -163,27 +163,22 @@ output at 200 ticks. `xy.x_axis(format=...)` and `xy.y_axis(format=...)` take a format string whose grammar depends on the axis kind. Both are deliberately small subsets, not full -d3-format or strftime, and neither raises on a spec it does not understand — -but they fail differently, and only the numeric grammar falls back. - -- **Numeric axes** accept `.Nf` (fixed decimals), `,.Nf` (fixed decimals with - locale group separators, via the runtime's default locale), and either form - with a trailing `%`, which multiplies the value by 100 and appends the sign — - for example `.2f`, `,.0f`, `.1%`. The trailing `f` is optional. Any other - string **falls back**: `fmtNumberSpec` returns `null` - (`js/src/30_ticks.ts:168`) and `fmtAxis` takes its `|| fmtLinear(...)` branch - (`:209`), so the axis silently reverts to the automatic formatter. On a log - axis, a value in `(0, 1)` that the spec would render as `"0"` falls back the - same way. -- **Time axes** accept a strftime subset of exactly `%Y %m %d %H %M %S %b %B`. - All fields are **UTC**; `%b`/`%B` are English month names. A time spec - **never** falls back: `fmtTimeSpec` (`js/src/30_ticks.ts:180-200`) - substitutes the tokens it knows and copies every other character through - verbatim, so it always returns a string and the `|| fmtTime(...)` branch at - `:204` is unreachable. An unrecognized `%` token such as `%y` therefore - renders literally as `%y`. The automatic time formatter is reached only when - `format` is absent or not a string. -- **Category axes** ignore `format=` and render the category label. +d3-format or strftime. Unsupported and kind-mismatched formats raise at Python +construction/build time and throw when a raw browser spec bypasses Python. + +- **Numeric and log axes** accept fixed precision with optional locale grouping + (`.2f`, `,.0f`), one literal currency prefix (`$`, `€`, `£`, or `¥`), a + literal unit after `f` (`.3f GiB`, `,.0fK`), or percent scaling (`.1%`, + `.0f%`). Precision is 0 through 20. A unit and percent cannot be combined. +- **Time axes** accept exactly `%Y %m %d %H %M %S %b %B` plus literal text. + All fields are UTC and month names are English, so output is invariant across + host locales and time zones. Unknown or incomplete `%` tokens are rejected. +- **Category axes** reject `format=` and render the category label. Use matching + `tick_values` / `tick_labels` for explicit replacements. + +The complete grammar, tooltip behavior, colorbar rule, locale contract, and + executable evidence are normative in + [`rendered-label-policy.md`](../testing/rendered-label-policy.md). ## Slot reference diff --git a/spec/benchmarks/methodology.md b/spec/benchmarks/methodology.md index 3f173baf..9f167a00 100644 --- a/spec/benchmarks/methodology.md +++ b/spec/benchmarks/methodology.md @@ -133,10 +133,13 @@ reported). 7. `dashboard_scale`: `benchmarks/bench_dashboard.py` attempts 10/20/50 mixed charts, checks every canvas initially and while scrolling, and records payload prep, navigation readiness, JS heap, redraw-submission p95, per-chart context - loss/restoration events, and the stable loss-free chart-count ceiling. Partial - dashboards remain successful measurement rows rather than losing their metrics. - CI hard-gates the 10-chart row as loss-free/nonblank and applies deliberately - loose catastrophic budgets to its render, scroll, and redraw timings. + loss/restoration events, and the stable loss-free chart-count ceiling. Baseline + report validation retains coherent partial dashboards rather than losing their + metrics. Hard CI selects the strict outcome profile: 10 must be loss-free, and + 20/50 must be complete or governed with every chart created and nonblank when + visited; failed, partial, browser-evicted, and unexplained-loss rows fail. The + 10-chart row also has deliberately loose catastrophic render, scroll, and + redraw budgets. 8. `install_import`: lower-bound distribution size plus opt-in fresh-venv total site-packages, transitive distribution count, install time, and cold import. 9. `public_workflows`: `benchmarks/bench_workflows.py` tracks ingestion shapes, @@ -295,8 +298,9 @@ performance data. Two `make` targets bound the `xy.pyplot` shim from opposite sides. Both appear in the focused-gate table of `spec/process/production-readiness.md`. -- **`make check-pyplot`** → `pytest tests/pyplot -q`. The correctness side: shim - behavior, matplotlib interoperability, and the reference corpus. It also +- **`make check-pyplot`** → the structured pyplot accepted-option audit followed + by `pytest tests/pyplot -q`. The correctness side: shim behavior, matplotlib + interoperability, and the reference corpus. It also carries the *relative* speed gate, `tests/pyplot/test_perf_guardrail.py`, which asserts the pyplot build stays within 1.6x the declarative build at 10k points and 1.5x at 100k (best-of-N, plus a 100us absolute allowance for CI diff --git a/spec/benchmarks/results.md b/spec/benchmarks/results.md index 1feaa224..ed2ef4ea 100644 --- a/spec/benchmarks/results.md +++ b/spec/benchmarks/results.md @@ -179,6 +179,12 @@ count. Partial rows retain their timing and memory metrics. `bench_workflows.py` updates, and separate HTML/SVG/native-PNG/Chromium-PNG export rows. All three emit schema-versioned JSON with environment metadata and benchmark category IDs. +Use `--profile strict` when verifying the dashboard artifact for CI or release +health. That profile requires healthy 10/20/50 rows and rejects missing, failed, +partial, visited-blank, browser-evicted, and unexplained context-loss outcomes; +the default profile continues to validate and retain non-healthy measurement +rows without presenting them as release evidence. + ## Copyable claim taxonomy Use these shapes when turning benchmark rows into README text, release notes, or diff --git a/spec/design-dossier.md b/spec/design-dossier.md index 9df9427e..bfe632e3 100644 --- a/spec/design-dossier.md +++ b/spec/design-dossier.md @@ -1,4 +1,4 @@ -# Building a Faster Charting Engine — Complete Design Dossier +# XY Charting Engine — Complete Design Dossier *A single compiled record of the design, the competitive research that validates it, the performance estimates, and the full audit trail. Python-only binding.* @@ -28,7 +28,7 @@ Every claim in this dossier is **mode-scoped and testable** — no universal num 1. **Part 1 — The Design** (§1–§37): the full specification. §1–§14 are the core; §15–§31 fold in two prior audit rounds; §32–§37 add the Python-only architecture, distribution, filtering, theming, and the transfer protocol. -2. **Part 2 — Competitive Research**: how the fastest libraries in the field actually +2. **Part 2 — Competitive Research**: how high-throughput libraries in the field actually work, and where each of the six core bets is validated, corrected, or extended. All sourced. 3. **Part 3 — Performance Estimates**: projected standing vs standard Python **and** React @@ -1271,7 +1271,7 @@ on either side is always safe; worst case is a re-send or re-bin, never wrongnes # Part 2 — Competitive Research -# How the fastest graphing libraries work — research findings +# Graphing-library architecture — research findings Companion to the charting-engine design plan. Every load-bearing design decision was checked against production libraries and the academic literature. **Headline: all six @@ -1436,10 +1436,12 @@ frames. (It's made bearable by Numba + multicore + Dask/CUDA: ~1B points in seco in <3 min on 32 cores.) A *separate* offline module, `render_tiles`, does build a static power-of-two pyramid — but not in the interactive path. -**Verdict:** for **navigation**, our live data-space pyramid genuinely beats datashader's -interactive path — **O(visible tiles), no per-frame round-trip, client-side -compositing, re-bin only below the floor** = strictly lower latency and jitter. It does -**not** beat datashader's own `render_tiles` in concept (same pyramid idea) and it +**Research comparison, not a release claim:** for **navigation**, our live data-space +pyramid has lower algorithmic cost than datashader's interactive path — **O(visible +tiles), no per-frame round-trip, client-side +compositing, re-bin only below the floor** = strictly lower latency and jitter. This is +not a release claim that we beat datashader's own `render_tiles` in concept (same +pyramid idea), and it inherits the **static-pyramid-is-stale-under-filtering** tradeoff. Defensible framing: we **unify** them — a *live* pyramid with client compositing — and layer a Falcon-style active-dimension index when dynamic filtering is needed. @@ -1535,7 +1537,7 @@ Two structural notes for React specifically: ## The verdict that matters -The honest headline is not "N× faster than everything." It is: **no existing Python or +**Research hypothesis, not a release claim:** no existing Python or React library gives you all four of {100M+ points, fully interactive pan/zoom/hover, low/bounded memory, one simple API in a notebook or React app} at once.** diff --git a/spec/design/animation.md b/spec/design/animation.md index 53e95138..9b4a9340 100644 --- a/spec/design/animation.md +++ b/spec/design/animation.md @@ -153,10 +153,13 @@ an exact deterministic progress without starting a frame loop. - `tests/test_animation.py` owns validation, serialization, identity, wire, sorting, errorbar expansion, and deterministic-export contracts. -- `scripts/animation_smoke.py` exercises pixel-checked, ghost-free keyed interpolation, - explicit partial-match fallback, GPU scratch buffers, rapid replacement, - bounded lifetime, lifecycle balance (including destroy), and reduced motion - in headless Chrome. +- `scripts/animation_smoke.py` is a hard CI and `make check-browser` gate. Its + fixtures derive the shared wire protocol and real Chromium must pass + pixel-checked, ghost-free keyed interpolation, explicit partial-match + fallback, GPU scratch buffers, rapid replacement, bounded lifetime, + lifecycle balance (including destroy), representative marks, reduced motion, + and deterministic frozen capture. Browser startup, timeout, assertion, and + nonzero-exit failures are blocking and retain JSON evidence. - `benchmarks/test_codspeed_animation.py` attributes key encoding and animated payload build overhead separately from the plain payload path. - Browser frame/allocation measurements belong to the real-Chrome benchmark diff --git a/spec/design/host-compatibility.md b/spec/design/host-compatibility.md new file mode 100644 index 00000000..b312df1f --- /dev/null +++ b/spec/design/host-compatibility.md @@ -0,0 +1,67 @@ +# Host Compatibility + +XY supports a bounded set of host-library versions. The executable source of +truth is [`../testing/host-integration-policy.json`](../testing/host-integration-policy.json); +package metadata must declare the same ranges, and +`scripts/host_integration_policy.py validate` fails when they diverge. A new +major version is outside the supported range until a reviewed change widens the +policy and runs the expanded matrix. + +## Supported versions + +| Host surface | Package | Supported range | CI floor | +|---|---|---|---| +| Anywidget | `anywidget` | `>=0.9,<1` | `0.9.0` | +| Anywidget trait transport | `traitlets` | `>=5.14,<6` | `5.14.0` | +| Reflex adapter | `reflex` | `>=0.9.6,<1` | `0.9.6` | +| FastAPI example | `fastapi` | `>=0.110,<1` | `0.110.0` | +| FastAPI ASGI layer | `starlette` | `>=0.36.3,<1` | `0.36.3` | +| FastAPI production server | `uvicorn` | `>=0.29,<1` | `0.29.0` | +| FastAPI in-process transport | `httpx` | `>=0.27,<1` | `0.27.0` | + +`pyproject.toml` owns the Anywidget runtime declarations and the FastAPI test +stack. `python/reflex-xy/pyproject.toml` owns the Reflex declaration. +`examples/fastapi/pyproject.toml` owns every library that its application or +focused host tests import directly. Transitive installation alone does not +count as support metadata. + +## Executable matrix + +Every pull request runs two hard profiles on Python 3.13: + +- `floor` installs every exact floor above; and +- `latest` resolves the newest mutually compatible release inside every + supported range. + +The `host_integration` job builds the locked native core, verifies the committed +client is fresh, then runs `tests/host_integration/` through +`scripts/run_pytest_no_skips.py`. It +compiles the FastAPI example, mounts its routes through `TestClient`, exercises +HTTP drilldown transport, instantiates the real `AnyWidget`, and drives split +binary comm messages. `scripts/fastapi_host_smoke.py` starts Uvicorn with the +matrix interpreter, requires a nonblank WebGL2 canvas, and sends the drilldown +request from Chromium. The job retains installed-version JSON, JUnit, browser +JSON, a screenshot, and the server log for both profiles. + +The package-owned `reflex_adapter` job independently installs the Reflex floor +or latest supported release. It records the installed version, runs the adapter +suite with zero skips, compiles and starts the production example, and proves +the browser/socket transport. It retains the version report, JUnit, server log, +and screenshot for each profile. + +| Host | Compile | Mount | Transport | Browser | +|---|---|---|---|---| +| Anywidget | Widget construction and bundled ESM contract | Real Python `AnyWidget` trait model | Pick reply and binary append comm buffers | Not claimed here; a real notebook frontend remains TST-NI-018 | +| Reflex | Production example compile | Production Reflex application | Shared socket, state, stream, and close behavior | Real Chromium paint and interaction | +| FastAPI/Starlette | Every example module | In-process routes and current-environment Uvicorn | Health, chart, and drilldown requests | Real Chromium WebGL2 paint and browser-originated drilldown | + +The matrix has no optional-import or browser skip path. A missing dependency, +unsupported installed version, skipped test, blank canvas, failed transport, or +missing evidence artifact fails the hard `required_ci` aggregate. + +## Boundary + +This contract proves the declared Python host seams at both supported edges. It +does not claim that a JupyterLab or classic-notebook frontend mounts the widget; +that separate real-frontend lifecycle and bidirectional E2E remains tracked by +TST-NI-018 in [`../testing/gaps.md`](../testing/gaps.md). diff --git a/spec/design/reflex-integration.md b/spec/design/reflex-integration.md index 95211424..a6a8a040 100644 --- a/spec/design/reflex-integration.md +++ b/spec/design/reflex-integration.md @@ -1,7 +1,7 @@ # Reflex integration — design Status: **prototype landed** (`python/reflex-xy`, tests under -`tests/reflex_adapter/`). This document is the authoritative design; the +`python/reflex-xy/tests/`). This document is the authoritative design; the prototype implements it end to end over Reflex 0.9.6. The deliverable is an external adapter package (`reflex-xy`) that makes a xy figure a first-class Reflex component with the same performance contract as the @@ -69,7 +69,7 @@ from `xy.channel`) for HTTP/export hosts; the namespace does not use it. - **Version coupling.** The wrapper mirrors Reflex's socket options (`transports`, ws subprotocol, `?token=` query) so the manager cache merges the connections. Those names are pinned by - `tests/reflex_adapter/test_assets.py` — a Reflex upgrade that renames + `python/reflex-xy/tests/test_assets.py` — a Reflex upgrade that renames them fails loudly in CI, not silently in prod. - **One engine.io connection per tab** stays the invariant. If a chart page somehow loads without state enabled there is no socket at all — but @@ -550,7 +550,7 @@ examples/reflex/ (repo root) reflex-xy showcase: figure-var drilldown with fixed-data tiers (direct Chart + inline() token) examples/fastapi/ (repo root) the same charts + a live 100M drilldown served from a plain FastAPI app (no committed HTML) -tests/reflex_adapter/ 69 tests: token/registry/var/bridge/payload-asset +python/reflex-xy/tests/ token/registry/var/bridge/payload-asset units, component compile, and a real-websocket integration suite (uvicorn + socketio client) covering payload/pick/select/affinity/rebuild/ diff --git a/spec/design/renderer-architecture.md b/spec/design/renderer-architecture.md index ce92e9b1..76b27b41 100644 --- a/spec/design/renderer-architecture.md +++ b/spec/design/renderer-architecture.md @@ -9,7 +9,7 @@ module per former concat part; explicit imports replaced filename-order concatenation), bundled and minified by vite via `js/build.mjs`. The audit findings in §2 cover the render path — `40_gl.ts`, `45_lod.ts`, `50_chartview.ts`, `55_marks.ts`, `60_entries.ts`. -The module inventory below covers all 15. Ticks and axis label formatting +The module inventory below covers all 17. Ticks and axis label formatting (`30_ticks.ts`) are specified in §6; the gesture contract in `53_interaction.ts` is specified in [interaction.md](../api/interaction.md) §5, and the client half of the kernel channel (`54_kernel.ts`) in [wire-protocol.md](wire-protocol.md). @@ -23,21 +23,23 @@ relative mass, not as a budget (see §3 on why a line count failed as a metric). | Module | Lines | Responsibility | | --- | ---: | --- | -| `00_header.ts` | 172 | Bundle preamble: the client-wide design comment, the `PROTOCOL` version constant, and the binary frame codec (`XY_FRAME_MAGIC` `"XYBF"`, `decodeFrame`, 8-byte alignment). Every inbound frame is validated here — magic/version, u64 fields, zero-padding, and per-field size limits — before any other module sees it. | +| `00_header.ts` | 179 | Bundle preamble: the client-wide design comment, the `PROTOCOL` version constant, and the binary frame codec (`XY_FRAME_MAGIC` `"XYBF"`, `decodeFrame`, 8-byte alignment). Every inbound frame is validated here — magic/version, u64 fields, zero-padding, and per-field size limits — before any other module sees it. | | `10_colormaps.ts` | 51 | The `COLORMAP_STOPS` table (§36 CVD-safe defaults) as compact RGB stop lists, and `buildLutData`, which linearly interpolates a stop list into the 256-texel RGBA LUT uploaded once per colormap as a texture. | | `20_theme.ts` | 163 | Resolves chrome and mark colors: arbitrary CSS color expressions and `--chart-*` custom properties are resolved against a live probe element into f32 RGBA for GL, with a fallback on unparseable input. Also owns `XY_CHROME_CSS` and its one-time stylesheet injection. | -| `30_ticks.ts` | 224 | CPU-side tick generation in f64 for linear, log, category and time axes, plus every axis/colorbar label formatter (automatic and `format=`-driven). Specified in §6. | -| `40_gl.ts` | 829 | WebGL2 primitives: shader compile/link, `makeProgram` with its per-program uniform-location memo (R1), the fixed `ATTR_SLOTS` attribute-slot table bound at link time, and the shader inventory itself. The only module that is GPU-API-specific by design (§4). | -| `45_lod.ts` | 567 | View-dependent level-of-detail orchestration, deliberately chart-agnostic: tier selection, drill enter/exit hysteresis (`LOD_DRILL_EXIT_FACTOR`), cross-tier fades, and the retained tier caches. Calls back into `view._draw*` rather than drawing itself, which is the seam tests intercept. | +| `30_ticks.ts` | 285 | CPU-side tick generation in f64 for linear, log, category and time axes, plus strict axis/colorbar/tooltip format validation and every rendered label formatter. Specified in §6 and the normative rendered-label policy. | +| `40_gl.ts` | 938 | WebGL2 primitives: shader compile/link, `makeProgram` with its per-program uniform-location memo (R1), the fixed `ATTR_SLOTS` attribute-slot table bound at link time, and the shader inventory itself. The only module that is GPU-API-specific by design (§4). | +| `45_lod.ts` | 664 | View-dependent level-of-detail orchestration, deliberately chart-agnostic: tier selection, drill enter/exit hysteresis (`LOD_DRILL_EXIT_FACTOR`), cross-tier fades, and the retained tier caches. Calls back into `view._draw*` rather than drawing itself, which is the seam tests intercept. | | `46_worker.ts` | 59 | The standalone density re-bin worker: a worker source string carried inside the bundle and booted from a Blob URL. Re-bins the retained sample off the main thread so kernel-less (`to_html`) density charts refine on zoom instead of stretching the overview texture; absence of workers falls back to stretching. | -| `50_chartview.ts` | 4175 | The `ChartView` class: the four drawing surfaces, scale/view state, chrome (background, grid, axes, legend, colorbar), GL buffer and VAO management (R2), and pick orchestration. Modules 51–54 extend this same class. | -| `51_annotations.ts` | 591 | The 2D overlay canvas above the marks canvas: annotation markers, arrows, shape fills, and collision-nudged labels. Separates canvas shape style keys from label CSS so annotation styling never leaks into the DOM label. | -| `52_tooltip.ts` | 321 | Hit → source row → tooltip DOM. Anchors the tooltip at the picked point's data coordinates and reprojects it every draw ([interaction.md](../api/interaction.md) §7). Renders the local f32-decoded row immediately, then replaces it with the kernel's exact f64 row when that reply arrives (sequence- and `drill_seq`-guarded); composes text nodes, never HTML. | -| `53_interaction.ts` | 1820 | The entire user-facing interaction surface: pointer/drag/wheel wiring, crosshair, box select, box zoom, lasso, the modebar and its export menu, and the animated pan/zoom view state machine. The gesture→action mapping, the modebar tool inventory and the `interaction_config` switches are specified in [interaction.md](../api/interaction.md) §2 and §5. | -| `54_kernel.ts` | 437 | The client half of the kernel channel: debounced density/decimated view-requests, streaming `append` handling, the inbound message dispatcher, and the deep-zoom drill lifecycle (§16). Degrades to `46_worker.ts` when there is no comm. The message catalog it consumes is specified in [wire-protocol.md](wire-protocol.md). | -| `55_marks.ts` | 220 | The `MARK_KINDS` registry — `build`/`draw` per kind plus the `pointPick`/`retainCpu`/`refreshColor` capability flags — mirroring the kernel's `_emit_` dispatch so adding a 2D chart is an entry here, not a branch in `ChartView`. | -| `56_animation.ts` | — | Declarative entrance/update/exit state machine, easing/spring evaluation, bounded identity matching, full-payload replacement, interruption, reduced-motion resolution, and lifecycle events. Its normative behavior is in [animation.md](animation.md). | -| `60_entries.ts` | 76 | Mount/unmount entry points for both hosts (`render` for anywidget, `renderStandalone` for exported HTML) and `payloadBuffers`, which materializes first-paint columns in whichever layout the spec declares. Keeps aligned views zero-copy; a spec/transport disagreement throws. | +| `50_chartview.ts` | 4941 | The `ChartView` class: the four drawing surfaces, scale/view state, chrome (background, grid, axes, legend, colorbar), GL buffer and VAO management (R2), and pick orchestration. Modules 51–54 and 57 extend this same class. | +| `51_annotations.ts` | 594 | The 2D overlay canvas above the marks canvas: annotation markers, arrows, shape fills, and collision-nudged labels. Separates canvas shape style keys from label CSS so annotation styling never leaks into the DOM label. | +| `52_tooltip.ts` | 332 | Hit → source row → tooltip DOM. Anchors the tooltip at the picked point's data coordinates and reprojects it every draw ([interaction.md](../api/interaction.md) §7). Renders the local f32-decoded row immediately, then replaces it with the kernel's exact f64 row when that reply arrives (sequence- and `drill_seq`-guarded); composes text nodes, never HTML. | +| `53_interaction.ts` | 2255 | The entire user-facing interaction surface: pointer/drag/wheel wiring, crosshair, box select, box zoom, lasso, the modebar and its export menu, and the animated pan/zoom view state machine. The gesture→action mapping, the modebar tool inventory and the `interaction_config` switches are specified in [interaction.md](../api/interaction.md) §2 and §5. | +| `54_kernel.ts` | 496 | The client half of the kernel channel: debounced density/decimated view-requests, streaming `append` handling, the inbound message dispatcher, and the deep-zoom drill lifecycle (§16). Degrades to `46_worker.ts` when there is no comm. The message catalog it consumes is specified in [wire-protocol.md](wire-protocol.md). | +| `55_marks.ts` | 222 | The `MARK_KINDS` registry — `build`/`draw` per kind plus the `pointPick`/`retainCpu`/`refreshColor` capability flags — mirroring the kernel's `_emit_` dispatch so adding a 2D chart is an entry here, not a branch in `ChartView`. | +| `56_animation.ts` | 504 | Declarative entrance/update/exit state machine, easing/spring evaluation, bounded identity matching, full-payload replacement, interruption, reduced-motion resolution, and lifecycle events. Its normative behavior is in [animation.md](animation.md). | +| `57_viewstate.ts` | 574 | The canonical durable view-state document and state-patch application path, including local history, linked-chart synchronization, axis-band gesture scoping, and the structured hover payload. Its normative behavior is in [view-state.md](view-state.md). | +| `60_entries.ts` | 137 | Mount/unmount entry points for both hosts (`render` for anywidget, `renderStandalone` for exported HTML) and `payloadBuffers`, which materializes first-paint columns in whichever layout the spec declares. Keeps aligned views zero-copy; a spec/transport disagreement throws. Its named, frozen `__testing` export gives the dependency-free semantic suite access to production seams in the exact committed ESM bundle without changing the default widget entry point. | +| `61_standalone.ts` | 11 | Public-only IIFE entry wrapper. It re-exports the runtime namespace used by `window.xy` while deliberately excluding the ESM-only `__testing` seam. | ## 1. What's structurally right (keep) @@ -196,8 +198,8 @@ Ordered by how much each compounds as kinds multiply. document states *normatively* is the DOM that implements them: the `role="region"` root with its `aria-label`/`aria-describedby` summary, the `role="status"` `aria-live="polite"` readout, the `role="img"` focusable - canvas (`50_chartview.ts:1110-1147`, `:1212`), and the modebar's - toolbar/menu roles and roving `tabIndex` (`53_interaction.ts:718-1000`). + canvas (`50_chartview.ts:1538-1575`), and the modebar's toolbar/menu roles + and roving `tabIndex` (`53_interaction.ts:864-1172`). Those citations are accurate but descriptive — dossier §20 sketches the intent, production-readiness.md tracks the status, and this paragraph inventories the attributes. None of the three binds an implementer, so a @@ -292,8 +294,8 @@ subset: ## 6. Axis ticks and label formatting (`30_ticks.ts`) Ticks are computed on the CPU in f64 and never round-trip through the f32 -render path (§16). `ChartView._ticksFor` dispatches on the axis -(`50_chartview.ts:395–398`), checking in this order: `kind === "time"` → +render path (§16). `ChartView._axisTicks` dispatches on the axis +(`50_chartview.ts:681-692`), checking in this order: `kind === "time"` → `timeTicks`, `kind === "category"` → `categoryTicks`, `scale === "log"` → `logTicks`, otherwise `linearTicks`. Every generator takes `(lo, hi, target)` with `target = 6` by default and returns `{ ticks, step }`; `logTicks` adds @@ -358,54 +360,48 @@ With no `format=` on the axis, labels come from the step: - `fmtGeneral` reproduces Python's `:g` (default 6 significant digits) and is used for *explicit* colorbar ticks, whose precision is authored and must not be inferred from an unrelated automatic step. The colorbar itself ticks with - `linearTicks(lo, hi, 8)` (`50_chartview.ts:1467`). + `linearTicks(lo, hi, 8)` (`50_chartview.ts:1895`). ### 6.3 The `format=` mini-language -`fmtAxis` consults the axis's `format` string before falling back to the -automatic formatter. The two accepted grammars are narrow. +`fmtAxis` consults the axis's `format` string only after the Python builder and +the raw-spec browser boundary validate it. The accepted grammars are narrow; +an invalid or kind-mismatched string throws and never selects an automatic +fallback. -**Numeric axes** (`fmtNumberSpec`). One optional trailing `%` is stripped -first; the remainder must match exactly +**Numeric and log axes** (`fmtNumberSpec`) match ``` -/^(,)?\.([0-9]+)f?$/ +/^([$€£¥]?)(,)?\.([0-9]{1,2})(?:f([ A-Za-z0-9/_-]*))?(%)?$/ ``` -That is: an optional thousands-separator comma, a literal `.`, one or more -digits of precision, and an optional trailing `f`. The comma selects -`toLocaleString` with fixed fraction digits; without it the value goes through -`toFixed`. A `%` suffix multiplies by 100 and re-appends `%`. So `.2f`, -`,.0f`, `.1%` and `,.2f%` are the entire accepted surface. There is no -currency prefix, no sign flag, no `e`/`g`/`s` type, no explicit width or fill. +with a semantic precision check from 0 through 20 and a check that a literal +unit suffix and percent are not combined. The first group is a literal currency +prefix, the second selects locale grouping/decimal separators, the third is +fixed precision, the fourth is a literal unit suffix following `f`, and the +last multiplies by 100 and appends `%`. `.2f`, `,.0f`, `$,.2f`, `.3f GiB`, +`$,.0fK`, `.1%`, and `.0f%` are accepted. Width, alignment, sign policy, +scientific notation, arbitrary prefixes, and broader d3/Python syntax are not. -**Time axes** (`fmtTimeSpec`). A strftime *subset* substituted by -`/%[YmdHMSbB]/g`: `%Y`, `%m`, `%d`, `%H`, `%M`, `%S`, `%b` (short month name), -`%B` (long month name). All fields are UTC — there is no timezone support and -no `%j`, `%p`, `%I`, `%Z` or literal-`%%` escape. Any other text in the string -passes through verbatim, which means an unrecognized token such as `%y` renders -literally as `%y` rather than falling back. +**Time axes** (`fmtTimeSpec`) require at least one token from `%Y`, `%m`, `%d`, +`%H`, `%M`, `%S`, `%b`, or `%B`; other text is literal. All fields are UTC and +month names are English. Unknown/incomplete `%` tokens, local-time tokens, and +`%%` are rejected, so output is invariant across host locale and time zone. + +**Category axes** reject `format=`. Their category strings remain the labels; +explicit tick labels are a separate authored replacement surface. **Log-axis carve-out.** On a log axis, a value in `(0, 1)` that a numeric format renders as the string `"0"` is re-rendered with `fmtLinear` instead, so a low decade is not labelled as a row of zeros. -### 6.4 Sharp edge: silent fallback on unmatched formats - -`fmtNumberSpec` returns `null` on anything the regex rejects, and `fmtAxis` -treats that as "no format" — it silently uses the automatic formatter. No -warning is raised anywhere on the path: the Python side stores `format` as free -text with no validation (`python/xy/_figure.py:204` via `_optional_text`), so -an unsupported spec survives the whole pipeline and simply does nothing. - -The in-tree example is `tests/test_components.py:555`, which sets -`format="$,.0f"` on a y-axis. The leading `$` cannot match the regex, so the -whole format is discarded and the axis renders with `fmtLinear` — the `$` never -appears. The test does not catch this because it asserts on the emitted spec, -not on rendered labels. - -Two consequences to keep in mind when extending this: the failure mode for a -typo'd numeric format is a *plausible-looking wrong label*, not an error; and -the two grammars fail differently, since an unrecognized `%`-token on a time -axis is echoed literally instead of triggering the fallback. Making either loud -is a behavior change, not a doc fix, and is not proposed here. +### 6.4 Validation and rendered oracle + +`python/xy/_validate.py` is the public-builder/build boundary and +`validateAxisFormat` / `validateValueFormat` are the raw browser-spec boundary. +Tooltip fields are checked again against their resolved numeric or `time_ms` +kind. The `$,.0f` example therefore renders its currency prefix instead of +silently losing it, while a typo such as `.2q` fails loudly. + +The normative grammar and two-locale/non-UTC DOM evidence are in +[`rendered-label-policy.md`](../testing/rendered-label-policy.md). diff --git a/spec/design/wire-protocol.md b/spec/design/wire-protocol.md index 60a7df51..6c990c77 100644 --- a/spec/design/wire-protocol.md +++ b/spec/design/wire-protocol.md @@ -13,6 +13,13 @@ has the shape specified here, byte for byte. What varies is *which* messages a host sends — the Reflex wrapper resolves `view_change` in the browser and never emits it (§2). +The machine-readable test companion is +[`../testing/protocol-catalog.json`](../testing/protocol-catalog.json). Its +request inventory is checked against the dispatcher AST, and its committed +`XYBF` reply frames are decoded by both Python and the shipped JavaScript client +under the policy in +[`../testing/protocol-conformance.md`](../testing/protocol-conformance.md). + ## 1. Dispatch contract `handle_message(fig, content, buffers=None, callbacks=ChannelCallbacks())` diff --git a/spec/matplotlib/shim-todo.md b/spec/matplotlib/shim-todo.md index dcdf2c03..3cf8f955 100644 --- a/spec/matplotlib/shim-todo.md +++ b/spec/matplotlib/shim-todo.md @@ -61,11 +61,13 @@ evidence is not oversold: - The PNG comparisons in the same file are coarse structural smoke checks (aspect-preserving normalized-mask IoU, a 2x ink-area band, and a 0.20 luma band), plus negative controls proving blank and wrong geometry fail. -- `test_silent_drop_regressions.py` mechanically scans every public adapter: - bare option pops and deleted signature parameters fail unless an explicit - `compat-noop:` rationale is attached. Corpus coverage only credits calls on - proven pyplot/Axes receivers, so unrelated `anything.fill()` calls cannot - satisfy the inventory. +- `scripts/check_pyplot_options.py` performs the accepted-option audit: unread + named parameters and discarded literal option pops fail unless the exact + function/option appears in `spec/testing/pyplot-noops.json` with a substantive + rationale and a live behavioral contract in `test_compat_noops.py`. Reachable + local closures are followed; unused nested helpers cannot hide a dead option. + Corpus coverage separately credits calls only on proven pyplot/Axes receivers, + so unrelated `anything.fill()` calls cannot satisfy the inventory. - `test_p3_option_contracts.py`, `test_silent_drop_regressions.py`, `test_artist_transform_contracts.py`, `test_rc_chrome_contracts.py`, and `test_rc_color_export_contracts.py` cover implemented-or-rejected option @@ -233,7 +235,7 @@ appear frequently in ordinary scripts and notebooks. ### Stateful pyplot and figure management -- [x] `plt.clf()` and `Figure.clear()`/`Figure.clf()`. Evidence: `tests/pyplot/test_pyplot_state_management.py::test_pyplot_cla_and_clf_clear_current_scope` and `tests/pyplot/test_figure_state.py::test_figure_clear_and_clf_reset_axes`. +- [x] `plt.clf()` and `Figure.clear()`/`Figure.clf()`. Evidence: `tests/pyplot/test_pyplot_state_management.py::test_pyplot_cla_and_clf_clear_current_scope` and `tests/pyplot/test_figure_state.py::test_figure_clear_clf_removes_axes_and_chrome`. - [x] `plt.cla()` and `Axes.clear()`/`Axes.cla()`. Evidence: `tests/pyplot/test_pyplot_state_management.py::test_pyplot_cla_and_clf_clear_current_scope` clears only the current axes entries. - [x] `plt.axes()` and `plt.delaxes()`/`Figure.delaxes()`. Evidence: `tests/pyplot/test_pyplot_state_management.py::test_pyplot_axes_delaxes_figtext_and_figlegend` covers absolute axes creation and deletion. - [x] `plt.fignum_exists()`, `get_fignums()`, and `get_figlabels()`. Evidence: `tests/pyplot/test_pyplot_state_management.py::test_pyplot_figure_registry_and_labels` covers numeric and labeled figures. @@ -596,10 +598,13 @@ streamplot margins are thin (a correct image scores ~0.567 vs the 0.55 floor). The negative control is tied to the live `MINIMUM_IOU` values, so loosening them below a wrong-geometry score fails the control. -- Discard detector: mechanically scans public shim adapters for bare - `kwargs.pop`/`del` statements without a `compat-noop:` marker. It does NOT - catch a named parameter that is accepted and never read, or an - assigned-then-unused pop; the `compat-noop:` escape hatch is free text. +- Accepted-option detector: AST/dataflow checks unread public signature + parameters, bare literal option pops, and assigned-then-unused pops; it + follows reachable local closures without crediting dead nested helpers. + The exact 34-option compatibility boundary is structured in + `spec/testing/pyplot-noops.json`, and every entry names a behavioral invariant + test. Dynamic option keys are not credited as declarations; public adapters + still reject remaining unknown keys through `check_unsupported`. - Corpus coverage credits calls only on receivers traced to `subplots`/`gca`/... — but the names `ax`/`axes` are trusted unconditionally, so it is a naming heuristic, not type resolution. diff --git a/spec/process/contributing.md b/spec/process/contributing.md index c1f88f1f..46288e18 100644 --- a/spec/process/contributing.md +++ b/spec/process/contributing.md @@ -81,6 +81,13 @@ numbers into docs or posts: make check-benchmark-report BENCHMARK_JSON=benchmark.json BENCHMARK_KIND=scatter-vs ``` +For the dashboard release contract, select outcome enforcement explicitly: + +```bash +make check-benchmark-report BENCHMARK_JSON=dashboard.json \ + BENCHMARK_KIND=dashboard-browser BENCHMARK_PROFILE=strict +``` + `BENCHMARK_KIND` accepts `auto`, `scatter-vs`, `core-2d`, `pyplot-vs-matplotlib`, `scatter-native`, `heatmap-native`, `kernel-native`, `interaction-browser`, `dashboard-browser`, `workflow-native`, @@ -116,6 +123,21 @@ When you touch `python/xy/pyplot/` or the matplotlib compatibility corpus, run: make check-pyplot ``` +This first rejects newly accepted-but-unused pyplot options against the +reviewed `spec/testing/pyplot-noops.json` contract, then runs the shim suite. + +When reviewing coverage evidence or changing a shipped Python branch, validate +the retained report against the exact comparison range: + +```bash +make check-coverage COVERAGE_JSON=coverage/python/coverage.json \ + COVERAGE_BASE=origin/main COVERAGE_HEAD=HEAD +``` + +Package/module floors and exclusions live in +`spec/testing/coverage-policy.json`; they are reviewed policy, not generated +output. + When you change shim rendering performance, run `make check-pyplot-speed`, which enforces the per-family 10x static-PNG target via `benchmarks/bench_pyplot_vs_matplotlib.py` and requires the `.[bench]` extra. @@ -153,14 +175,17 @@ For browser render smoke checks, pass a local Chrome/Chromium executable: make check-browser CHROMIUM=/path/to/chrome ``` -This runs six split browser checks, which CI runs as separate steps: +This runs split browser checks, which CI runs as separate steps: | Check | CI step name | | --- | --- | | `render_smoke_nonumpy` | `Headless render smoke (stdlib + Chromium)` | | `smoke_render` | `Real-Figure render smoke (numpy + Chromium)` | +| `runtime_security_smoke` | `Runtime standalone security smoke (Chromium)` | | `reflex_lifecycle_smoke` | `Browser lifecycle smoke (Chromium)` | -| `visual_regression_smoke` | `Browser visual regression smoke (Chromium)` | +| `visual_health_smoke` | `Browser visual health smoke (Chromium)` | +| `visual_baseline` | `Reviewed visual baseline (Chromium)` | +| `chart_kind_matrix` | `Every chart-kind render matrix (Chromium)` | | `step_tier_smoke` | `Step tier-update smoke (Chromium)` | | `interaction_stress_smoke` | `Browser interaction stress smoke (Chromium)` | @@ -193,12 +218,36 @@ nonblank. A final pass loads the index page and confirms its embedded iframes paint. A blank, destroyed, shortened lifecycle, failed context restore, or missing DOM slot is a failing browser gate. -The visual gate runs `scripts/visual_regression_smoke.py`. It boots the same +The visual-health gate runs `scripts/visual_health_smoke.py`. It boots the same app and screenshots every gallery chart route plus `/drilldown`, checking global nonblank/color/unique-color invariants, rejecting collapsed plot -occupancy, and running tick-label overlap probes. This catches charts that -render pixels in the wrong place or collapse to nothing while avoiding a fragile -pixel-perfect golden file. +occupancy, and running tick-label overlap probes. It remains broad health +coverage and does not claim image identity. + +The reviewed identity gate runs `scripts/visual_baseline.py` against the small +versioned set in `spec/visual-baselines/v1.json`. It pins Playwright Chromium, +the repository Instrument Sans font and its checksum, viewport, DPR, +downsample, and explicit semantic/geometry/perceptual tolerances. Every hard +run also proves that real-browser corrupted-data, wrong-color, wrong-label, and +wrong-geometry controls are rejected, and CI retains expected, actual, and diff +PNGs plus semantic JSON. + +Baseline changes are proposals, never self-approval. Generate one only with the +pinned Playwright executable, a named preparer, and a concrete reason: + +```bash +CHROMIUM="$(node -e "const {chromium}=require('playwright'); process.stdout.write(chromium.executablePath())")" +python scripts/visual_baseline.py "$CHROMIUM" --update-baselines \ + --prepared-by "Your Name" --reason "intentional renderer change" \ + --artifacts visual-baseline-review +``` + +Attach `visual-baseline-review/` to the pull request. An independent reviewer +must inspect the expected/actual/diff images, semantic changes, fixture intent, +browser/font pins, and tolerances before approving the manifest. In proposal +artifacts, `expected` is the prior reviewed baseline and `actual` is the new +candidate. CI refuses update mode; do not refresh a baseline merely to make an +unexplained failure green. The interaction gate runs `scripts/interaction_stress_smoke.py`, a smoke-sized interaction benchmark that validates p95 budgets and visual invariants for @@ -210,6 +259,13 @@ repeated hover sample, missing crosshair chrome, missing view changes, box zoom that does not narrow/restore the viewport, brush selection that does not select/clear eligible marks, undersized lit-pixel readbacks, and oversized frame color jumps. +The same gate launches the standalone density re-bin path with real wall-clock +Playwright, and requires proof that a worker was created, returned a re-binned +nonblank view, changed the requested range, and was terminated and cleared by +`ChartView.destroy()`. Missing Node/Playwright, skipped workers, incomplete +evidence, and failed teardown are blocking by default. Only a direct local +diagnostic run may opt out with `--allow-worker-skip`; `make check-browser` and +CI never pass that flag. On macOS, pass the executable inside the app bundle, for example `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`, not the @@ -232,7 +288,8 @@ the `` placeholder, name it explicitly, for example - `import xy` stays lazy and under budget in fresh interpreters; no NumPy/native-core import on package import. - Standalone HTML handles hostile user strings in every text surface touched by - the patch; run `make check-security` for export/client text-sink changes. + the patch; run `make check-security` for export/client text-sink changes and + `make check-browser CHROMIUM=/path/to/chrome` for runtime DOM/CSP changes. - Benchmarks label mode truthfully: `direct`, `decimated`, `density`, `sampled`, or `adaptive`. - README/docs examples still match the current public API. diff --git a/spec/process/production-readiness.md b/spec/process/production-readiness.md index 0ba59a43..ee337b21 100644 --- a/spec/process/production-readiness.md +++ b/spec/process/production-readiness.md @@ -4,6 +4,11 @@ This is the release bar for xy while the core renderer is still moving. It separates hard gates from advisory measurements so performance claims, packaging promises, and API stability do not depend on memory or vibes. +The canonical [testing specification](../testing/README.md) inventories the +evidence behind this release bar. It distinguishes checks enforced today from +partial coverage and planned work; a planned test is not a release protection +until that specification marks it `IMPLEMENTED` with automated evidence. + ## Current Contract xy is early alpha. The goal is Plotly-class chart breadth with a @@ -67,7 +72,7 @@ These must pass before publishing or making a broad performance claim. | Import-time budget | `xy.__init__`, `dir(xy)`, export helpers, chart construction, and `.widget()` keep their lazy import boundaries | `make check-import` | | Claim guardrails | Public docs and package metadata avoid broad, unqualified performance claims | `make check-claims` | | CI/release workflows | Hard gates, non-blocking benchmarks, best-effort benchmark artifact upload/download, trusted publishing, and no-Rust clear-error jobs stay wired | `make check-ci` | -| HTML export safety | Inline JSON/script escaping, atomic path writes, hostile user strings, and browser client text-node insertion stay protected | `make check-security` | +| HTML export safety | Inline JSON/script escaping, atomic path writes, hostile user strings, and browser client text-node insertion stay protected; standalone CSP and network isolation are also enforced | `make check-security` and `make check-browser CHROMIUM=/path/to/chrome` | | Python tests | Native backend passes | `pytest -q` | | Python style | Library, tests, scripts, and benchmarks lint clean | `ruff check .` and `ruff format --check .` | | Matplotlib reference | The reviewed compatibility snapshot matches the pinned released matplotlib reference, and the `xy.pyplot` shim passes its interoperability and dual-engine corpus suites | `python scripts/sync_matplotlib_compat.py --check` and `pytest tests/pyplot` | @@ -75,16 +80,29 @@ These must pass before publishing or making a broad performance claim. | Native ABI | C ABI can be loaded from the built core | `python scripts/abi_smoke.py` | | JavaScript | Committed bundles match source | `node js/build.mjs --check` | | Browser render | WebGL smoke reaches real pixels | `python scripts/render_smoke_nonumpy.py ` | +| Host compatibility | Bounded Anywidget/FastAPI and Reflex floor/latest stacks pass compile, mount, transport, and browser evidence with zero skips | Hard `host_integration` and `reflex_adapter` jobs; see `spec/design/host-compatibility.md` | | Accessibility / cross-browser | Semantic interaction checks plus tolerant WebGL/layout comparison pass in Chromium, Firefox, and WebKit | `make check-conformance` | | Real chart render | A real composed chart exports and paints in Chromium | `python scripts/smoke_render.py ` | | Step tier update | A decimated `step` chart keeps its risers after a synthetic kernel `tier_update` replaces the vertex buffers | `python scripts/step_tier_smoke.py ` | -| Dashboard reliability | 10/20/50-chart dashboards stay nonblank under the render client's context governor | `python benchmarks/bench_dashboard.py --chart-counts 10,20,50 --chromium --json dashboard-smoke.json` then `python scripts/verify_benchmark_report.py dashboard-smoke.json --kind dashboard-browser` | +| Pick boundaries | All 256 trace slots (including 255), large/global-to-local point IDs, and pick-cache invalidation/reuse remain exact | `python scripts/pick_boundary_smoke.py --evidence pick-boundary-evidence.json` | +| Dashboard reliability | 10/20/50-chart dashboards stay nonblank under the render client's context governor | `python benchmarks/bench_dashboard.py --chart-counts 10,20,50 --chromium --json dashboard-smoke.json` then `python scripts/verify_benchmark_report.py dashboard-smoke.json --kind dashboard-browser --profile strict` | | sdist | Source archive contains required source/bundles, benchmark regression harness/baseline, release docs/tests/scripts, the example apps' source, `PKG-INFO` version/dependencies matching `pyproject.toml`, no duplicate members, and no generated junk | `python scripts/verify_sdist.py dist/*.tar.gz` | | Native wheel | Platform wheel contains package-only files, exactly one native library, `METADATA` version/dependencies matching `pyproject.toml`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, and is tagged non-pure | `python scripts/verify_wheel.py dist/*.whl --expect-native` | | Fallback wheel | No-toolchain wheel contains package-only files, `METADATA` version/dependencies matching `pyproject.toml`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, is pure, and contains no native library | `python scripts/verify_wheel.py dist/*.whl --expect-pure` | | Wheel size | Platform wheel remains small enough for notebook installs | CI budget: 15 MB | | Benchmark artifact | JSON benchmark reports carry schema, environment, categories, row status, and finite non-negative metrics; native reports must declare the native backend | `python scripts/verify_benchmark_report.py benchmark.json --kind scatter-vs`; repeat for line, install, core-2D, pyplot-vs-matplotlib, native, interaction, dashboard, and workflow artifacts | +Dashboard reports have two validation modes. The default `baseline` profile +keeps coherent partial/failed measurement rows publishable. Hard CI selects +`--profile strict`: exactly one 10-, 20-, and 50-chart row must be present; each +must be complete or governed, every chart must be created and prove nonblank +when visited, and failed, partial, browser-evicted, or otherwise unexplained +context-loss rows fail the step. The probe's virtual-time budget scales with +chart count so the 50-chart visit contract cannot outlive the harness itself; +the independent wall-clock timeout remains blocking. CI uploads the generated +`dashboard-smoke.json` with `if: always()` so failed/partial row telemetry is +retained even when strict verification stops the job. + Type checking is **advisory, not release-blocking**. CI runs `ty check python` and reports findings without failing the build, and `scripts/verify_local.py` registers the same check with `advisory=True`, so `make check-full` prints @@ -129,8 +147,10 @@ reports, and sharing a single file, but it has a clear security contract: ## Local Verification Shortcut -Use the focused gates below while iterating, then run the full gate before a -production-facing push: +Use the focused gates below while iterating. `make check-full` is the full +non-browser local gate; it is not equivalent to the browser, host-integration, +packaging, cross-platform, or exact-SHA release evidence cataloged in the +[testing specification](../testing/current.md). | Changed surface | Focused gate | |---|---| @@ -142,16 +162,16 @@ production-facing push: | `xy.pyplot` shim behavior, matplotlib interoperability, reference corpus | `make check-pyplot` | | Reviewed matplotlib compatibility snapshot (`spec/matplotlib/compat-matrix.md`) | `python scripts/sync_matplotlib_compat.py --check` | | `xy.pyplot` speed margin against matplotlib | `make check-pyplot-speed` | -| Standalone HTML export, path writes, user text, tooltips, legends, browser DOM insertion | `make check-security` | +| Standalone HTML export, path writes, user text, tooltips, legends, browser DOM insertion, CSP, and network isolation | `make check-security` plus `make check-browser CHROMIUM=/path/to/chrome` | | Benchmark harness code, environment metadata, report schema, regressions | `make check-benchmark-harness` | | Generated benchmark JSON artifacts | `make check-benchmark-report BENCHMARK_JSON=benchmark.json BENCHMARK_KIND=scatter-vs` | | CI/release workflows, artifact upload/download, no-Rust clear-error jobs | `make check-ci` | | Source distributions and wheels | `make check-sdist` and `make check-wheel` | | Existing release artifacts | `make check-artifacts SDIST=/path/to/xy.tar.gz WHEEL=/path/to/xy.whl` | | Browser render/lifecycle/interaction smoke | `make check-browser CHROMIUM=/path/to/chrome` | -| Production-facing PR | `make check-full` | +| Production-facing non-browser change | `make check-full` | -Use this before pushing production-facing changes: +Use this before pushing production-facing non-browser changes: ```bash make check-full @@ -165,9 +185,12 @@ make check-docs ``` The browser gates are split into app-facing checks that match the CI step -names: `Browser lifecycle smoke (Chromium)`, `Browser visual regression smoke -(Chromium)`, `Step tier-update smoke (Chromium)`, `Browser interaction stress -smoke (Chromium)`, and `Browser dashboard reliability smoke (Chromium)`. +names: `Browser lifecycle smoke (Chromium)`, `Browser visual health smoke +(Chromium)`, `Reviewed visual baseline (Chromium)`, `Every chart-kind render +matrix (Chromium)`, `Step tier-update smoke +(Chromium)`, `Animation smoke (Chromium)`, `Pick boundary smoke (Chromium)`, +`Browser interaction stress smoke (Chromium)`, and `Browser dashboard +reliability smoke (Chromium)`. `make check-browser` runs all of these except the dashboard reliability smoke, which runs in CI only. The lifecycle and visual smokes both boot the `examples/fastapi` app under uvicorn and drive Chromium at its live routes (no @@ -175,21 +198,34 @@ committed HTML): the lifecycle smoke loads every gallery chart and the live drilldown and requires each to report nonblank pixels through `initial`, `narrow-resize`, `wide-resize`, `visibility-change`, `context-restore`, and `restore` (and to keep its runtime DOM slots), then confirms the index page's -embedded iframes paint; the visual regression smoke screenshots every gallery +embedded iframes paint; the visual-health smoke screenshots every gallery route and checks nonblank/colored/occupancy plus tick-label overlap. The `context-restore` phase forces `WEBGL_lose_context` loss/restoration and -requires the rebuilt chart to remain nonblank. The interaction stress smoke +requires the rebuilt chart to remain nonblank. The animation smoke requires +keyed and fallback matching, ghost-free interpolation pixels, rapid +latest-wins replacement, the previous+next allocation bound, balanced +lifecycle events, representative errorbar/vertical-bar/horizontal-bar marks, +reduced motion, and deterministic frozen capture in real Chromium. A missing +or hung browser is a hard failure, and CI retains its JSON title/assertion +diagnostic with `if: always()`. The interaction stress smoke validates the real `ChartView` wheel zoom, pan, hover, crosshair, box zoom, and brush-select paths with p95 budgets plus visual invariants for blank frames, tick-label overlap, tooltip stability, crosshair visibility, view changes, box zoom narrow/restore behavior, brush select count/clear behavior, lit-pixel -readback floors, and frame-to-frame color jumps. The visual regression smoke +readback floors, and frame-to-frame color jumps. The visual-health smoke also validates title, plot, x-axis, and y-axis regions plus plot-region occupancy, and it screenshots static Reflex-style chrome shells for the custom legend/tooltip and annotated heatmap examples. A chart cannot collapse into a corner, lose axis/custom chrome, or pass merely because some pixels exist somewhere. +The pick-boundary smoke is a hard local/CI browser gate. It covers trace slots +0, 127, 253, 254, and 255 in a 256-trace fixture, an exact point index of +69,999, a second trace whose global ID follows that range, steady-hover pick-FBO +reuse, and view-change invalidation. CI retains its compact JSON title/stderr +diagnostic with `if: always()` so a GPU/ID regression is inspectable after the +step fails. + Use this after packaging, workflow, or source-distribution changes: ```bash @@ -276,8 +312,10 @@ make check-claims Browser smoke and package artifact verification need a built bundle, Chromium, and wheel/sdist outputs. The interaction gate's real-wall-clock worker probe also uses the pinned development-only Playwright driver; install it once with -`make setup-browser` (or `npm install`). These gates are required in CI and -release workflows even if they are skipped locally. +`make setup-browser` (or `npm install`). The worker proof is required by +`make check-browser` and CI. A direct local diagnostic may explicitly pass +`--allow-worker-skip` only when the Node/Playwright harness is unavailable; +the hard suites never pass that option. For browser checks, pass the local Chromium/Chrome binary explicitly: @@ -296,18 +334,31 @@ nonblank. A final pass loads the index page and confirms its embedded iframes paint. Empty canvases, destroyed views, shortened lifecycle reports, failed context restores, or missing DOM slots fail the gate. -The visual gate runs `scripts/visual_regression_smoke.py`. It boots the same +The visual-health gate runs `scripts/visual_health_smoke.py`. It boots the same app and screenshots every gallery chart route plus `/drilldown`, checking nonblank, colored, unique-color, plot-occupancy, and tick-label-overlap invariants so a blank, flat, or collapsed chart fails the gate. +The visual-identity gate runs `scripts/visual_baseline.py` against +`spec/visual-baselines/v1.json` with the Playwright Chromium version, bundled +font checksum, 900×470 viewport, DPR 1, downsample, semantic geometry, and +perceptual tolerances pinned. Its required real-browser negative controls alter +numeric payload values, color, a rendered label, and canvas geometry; every +mutation must be rejected. CI always retains the evidence plus expected, +actual, and diff PNGs. Baseline update mode is forbidden in CI and follows the +independent review procedure in [contributing.md](contributing.md). + The interaction gate runs `scripts/interaction_stress_smoke.py`, which is a smaller gated version of `benchmarks/bench_interaction.py`. The smoke validates interaction budgets for direct scatter, density scatter, line, histogram, bar, and heatmap rows so performance regressions are not scatter-only and not direct-scatter-only. For pickable rows, tooltip stability means every declared repeated hover sample must remain visible, so a tooltip that appears and -immediately disappears fails the gate. +immediately disappears fails the gate. Its real-wall-clock standalone density +probe must also prove worker creation, a returned re-bin with a changed range +and nonblank pixels, and teardown through worker termination, cleared worker +state, and a removed chart root. CI retains this evidence even when the gate +fails. Use `make list-checks` to see the individual check names, or `python scripts/verify_local.py --dry-run --full` to print commands without @@ -321,8 +372,11 @@ Before tagging a release: - Refresh benchmark reports or explicitly document why the previous report still applies. -- Run `make check-full` locally or confirm the equivalent - CI gates passed on the release commit. +- Run `make check-full` for the non-browser layer, then confirm each applicable + browser, conformance, dashboard, packaging, and host-integration gate in the + [current testing inventory](../testing/current.md). Exact-SHA automated + qualification remains [TST-NI-003](../testing/gaps.md#tst-ni-003--exact-sha-release-deployment-and-provenance-preflight), + so this confirmation is manual until that gap is implemented. - Run `make check-ci` to confirm CI and release workflow gates still include artifact verification, upload/download, and trusted PyPI publishing. @@ -357,7 +411,8 @@ Before tagging a release: - Confirm every wheel passes `scripts/verify_wheel.py --expect-native` and the install smoke loads `xy.kernels.BACKEND == "native"`. Wheel `METADATA` must keep `Name: xy`, `Requires-Python: >=3.11`, - `anywidget>=0.9`, and `numpy>=1.24`; wheel `RECORD` must list every archive + `anywidget>=0.9,<1`, `traitlets>=5.14,<6`, and `numpy>=1.24`; wheel `RECORD` + must list every archive file exactly once with matching `sha256` and size fields. Wheels must stay package-only: docs, tests, benchmarks, scripts, and the `examples/` apps are sdist-only. @@ -396,9 +451,9 @@ Not yet safe: - Cross-browser/perceptual rendering conformance. - Exact-marker interaction for every possible 100M-point zoom path. - Production Reflex state integration as a first-class API. -- More than ~12 charts *simultaneously in view* holding live WebGL contexts. +- More than ~10 charts *simultaneously in view* holding live WebGL contexts. Browsers cap live contexts per page (~16 in Chrome); the render client's - context governor keeps xy inside a budget (default 12) by having + context governor keeps xy inside a budget (default 10) by having the least-recently-visible off-screen chart release its context and re-acquire on scroll-into-view. Measured (`benchmarks/bench_dashboard.py`, 2026-07-09, Chrome/macOS): 10/20/50-chart dashboards are all fully usable — diff --git a/spec/process/security-audit-2026-07-06.md b/spec/process/security-audit-2026-07-06.md index f3a8ed06..48a82c87 100644 --- a/spec/process/security-audit-2026-07-06.md +++ b/spec/process/security-audit-2026-07-06.md @@ -93,6 +93,38 @@ residual risk taken to keep CI and container rasterization working where the sandbox cannot initialize. Follow-up pending: make the fallback opt-in (or at minimum warn on the downgrade) so a silent sandbox loss is observable. +#### Resolution as of 2026-07-21 (XY-SEC-2026-03) + +The automatic fallback has been removed from both public browser-export paths. +A sandboxed launch now fails once with an actionable diagnostic and never adds +`--no-sandbox`. Trusted repository CI that needs the downgrade passes +`sandbox=False` at the call site, making the exception reviewable. Unit tests +assert the exact launch count, sandbox arguments, and failure guidance for the +one-shot and persistent paths. This closes TST-NI-025. + +### XY-SEC-2026-05: Standalone sink and CSP controls lacked runtime proof + +Severity: medium regression risk. + +The export tests decoded escaped JSON and the client-security tests constrained +source sinks, but neither observed all public hostile-text surfaces or hostile +CSS in a browser. A source-only contract could stay green while a later DOM +composition change parsed text, opened a dialog, or bypassed the standalone +network boundary. + +#### Resolution as of 2026-07-21 (XY-SEC-2026-05) + +`scripts/runtime_security_smoke.py` now loads a production standalone export in +Chromium with hostile strings across 16 title/axis/tick/trace/category/ +annotation/legend/colorbar/tooltip surfaces. It asserts literal text, no +executable user DOM/script/dialogs, applies hostile custom CSS, observes the +expected CSP `img-src` violation, and requires that no HTTP request reaches a +loopback sentinel. CI retains `runtime-security-evidence.json` on every outcome, +and the only remaining `innerHTML` assignments are structurally allowlisted +calls to the fixed internal icon factory. This closes TST-NI-024. Chromium's +process sandbox and any explicit trusted-fixture opt-out remain the separate +XY-SEC-2026-03 / TST-NI-025 policy. + ### XY-SEC-2026-04: Pyramid native-boundary validation was weaker than other kernels Severity: low/medium robustness. @@ -156,9 +188,17 @@ path — which pulls in eight transitive crates: `bitflags`, `crc32fast`, `Cargo.lock` therefore holds nine third-party packages plus `xy-core`. The tree is still shallow and single-rooted, but "no third-party Rust crates" -is no longer an accurate standing control. Follow-up pending: add `cargo audit` -(or `cargo deny`) to CI now that a third-party tree exists, matching the -`pip-audit` coverage already run on the Python side. +is no longer an accurate standing control. + +#### Status as of 2026-07-21 (Dependency Audit Follow-up Closed) + +The pending dependency-audit follow-up is now closed by the hard +`dependency_audit` CI job. Pinned OSV-Scanner coverage includes `Cargo.lock`, +all active Python locks, the root npm lock, and the documentation app's +committed Bun lock under the reviewed policy in +`spec/testing/dependency-audit-policy.json`. This dated audit remains historical +evidence; [`dependency-auditing.md`](../testing/dependency-auditing.md) is the +durable current contract. ## Tooling Evidence @@ -198,10 +238,9 @@ is no longer an accurate standing control. Follow-up pending: add `cargo audit` - The Reflex demo live-drilldown endpoint is local/demo infrastructure and is not authenticated or rate-limited as a production API. Add auth, quotas, and request-size policy before exposing an equivalent service publicly. -- Bun was not installed locally and the Reflex-generated frontend uses - `bun.lock`, not an npm lockfile, so the JS dependency advisory audit for the - demo app could not be run faithfully here. Run `bun audit` in an environment - with Bun installed. +- Bun was not installed during this dated manual audit, so its generated lock + was not checked at that time. The current hard dependency lane now scans the + committed `docs/app/reflex.lock/bun.lock` directly with OSV-Scanner. - This audit did not perform browser fuzzing, GPU-driver fuzzing, native memory sanitizer runs, or a hosted-app penetration test. @@ -226,3 +265,6 @@ sandbox cannot initialize. Container/worker isolation is therefore the load- bearing control, not the sandbox flag. Follow-up pending (same item as XY-SEC-2026-03): make the fallback opt-in, or at minimum warn on the downgrade, so a sandbox loss is observable. + +The 2026-07-21 resolution under XY-SEC-2026-03 supersedes this residual-risk +status: `sandbox=True` is now enforced end to end and never downgrades itself. diff --git a/spec/testing/README.md b/spec/testing/README.md new file mode 100644 index 00000000..40f9d6d6 --- /dev/null +++ b/spec/testing/README.md @@ -0,0 +1,136 @@ +# Testing Specification + +This directory is the canonical inventory of XY test evidence. It records what +the repository tests today, how that evidence is enforced, and what additional +coverage is still required. It complements the release bar in +[`production-readiness.md`](../process/production-readiness.md); it does not +silently strengthen or weaken that bar. + +The testing specification has three responsibilities: + +1. distinguish an existing test from an enforced gate; +2. connect every material product claim to executable evidence; and +3. label missing coverage honestly until it is implemented. + +## Documents + +- [`current.md`](current.md) inventories the current unit, integration, + browser, benchmark, packaging, workflow, and release evidence. +- [`gaps.md`](gaps.md) is the stable, prioritized implementation register for + coverage and enforcement. IDs remain in place after completion; incomplete + entries are explicitly `NOT IMPLEMENTED`, while closed entries cite their + evidence and are marked `IMPLEMENTED`. +- [`dependency-auditing.md`](dependency-auditing.md) defines the hard + multi-ecosystem lock inventory, severity and exception policy, and retained + vulnerability evidence. +- [`protocol-conformance.md`](protocol-conformance.md) defines the executable + request/reply catalog, shared Python/JavaScript golden frames, and structural + byte-mutation parity policy. +- [`host-integration-policy.json`](host-integration-policy.json) is the + executable version inventory behind the bounded host contract in + [`../design/host-compatibility.md`](../design/host-compatibility.md). + +## Status vocabulary + +Use only these status values in this directory: + +| Status | Meaning | +|---|---| +| `IMPLEMENTED` | The stated scope has executable evidence, runs in its documented environment, and enforces the documented outcome. | +| `PARTIALLY IMPLEMENTED` | Useful evidence exists, but the stated scope is incomplete, can skip unexpectedly, is not wired into the intended gate, or has an oracle that can accept a known failure. | +| `NOT IMPLEMENTED` | The desired protection does not yet exist as reliable automated evidence. A supporting script or dormant test does not change this status. | +| `OUT OF SCOPE` | The product does not claim the behavior. Use this only when the corresponding product specification also excludes it. | + +Status applies to the exact capability in a row. For example, the focused +three-engine browser fixture is `IMPLEMENTED`, while a full cross-engine chart +catalog is `NOT IMPLEMENTED`. The stronger planned capability must not inherit +the status of the narrower existing one. + +`PARTIALLY IMPLEMENTED` and `NOT IMPLEMENTED` do not provide release evidence. +Another specification may define either as an intended hard gate only when it +also says that current automation does not yet satisfy the contract and links +the corresponding gap. + +## Promotion rule + +A gap may move from `NOT IMPLEMENTED` only when the same change provides all of +the following: + +- executable positive and negative evidence for the stated requirement; +- a stable local command or workflow job; +- the dependencies, platforms, browsers, and skip policy needed by that job; +- failure semantics that reject the defect the test is intended to catch; +- durable failure output, such as JUnit, structured JSON, screenshots, traces, + or package artifacts where appropriate; +- an update to [`current.md`](current.md) and the relevant product spec; and +- required-check or release wiring when the item is meant to block a merge or + release. + +Merely adding a test file, smoke script, workflow step, or report row is not +enough if the intended environment can omit it or the verifier accepts failure. + +## Evidence layers + +XY uses complementary layers. A product claim should use the lowest-cost layer +that can independently observe the behavior, with a higher layer for seams that +cannot be proved in isolation. + +| Layer | Purpose | Typical evidence | +|---|---|---| +| Static contract | API inventory, generated-file freshness, forbidden constructs, workflow shape | Ruff, public API checker, bundle check, workflow verifier | +| Unit | Pure validation, transforms, kernels, encoders, state transitions | Python, Rust, and planned JavaScript unit tests | +| Property and mutation | Broad input domains, rollback, malformed bytes, oracle strength | Hypothesis, deterministic randomized tests, planned mutation tests | +| Integration | Python/native, Python/JavaScript, host adapters, artifact install | ABI smoke, golden frames, adapter tests, wheel/sdist probes | +| Browser | Pixels, DOM semantics, interaction, lifecycle, accessibility | Chromium smokes and Playwright conformance | +| End to end | Real host, socket, package, release, and deployment paths | FastAPI/Reflex app smokes, package installs, exact-SHA qualification | +| Measurement | Performance, memory, payload, dashboard pressure | Benchmark reports with separate integrity and timing policy | + +Source-substring assertions are appropriate only for narrow static contracts, +such as forbidden parser sinks or generated bundle freshness. They are not a +substitute for runtime semantics, rendered output, or workflow dependency +behavior. + +## Gate tiers + +| Tier | Required outcome | +|---|---| +| PR required | Runs for every relevant pull request and contributes to the stable required aggregate. Unexpected failure, cancellation, or skip is blocking. | +| Release required | Qualifies the exact artifact SHA before publication or deployment. | +| Advisory | The job may remain non-blocking, but its inputs and report integrity must still be valid. Ordinary shared-runner timing belongs here. | +| Scheduled | Longer cross-platform, fuzz, sanitizer, visual, and soak evidence that is too costly for every pull request. | +| Local | Developer feedback that is useful but is not evidence that automation ran. | + +The target tier and the current enforcement are separate facts. A row is not a +PR or release gate merely because a product spec says it should be one. + +## Audits + +Point-in-time reconciliations of this catalog against the repository live in +[`audits/`](audits/). They record observed numbers on a date and are expected to +go stale; this document and [`current.md`](current.md) hold the durable rules. + +- [`audits/2026-07-21.md`](audits/2026-07-21.md) — the reconciliation this + catalog was first written against, including the failed 50-chart dashboard row + that a green verifier accepted. + +## Maintenance rules + +- Update this section in the same pull request as a material test, workflow, + platform-support, release, or product-contract change. +- Run `make check-testing-spec` after editing this directory. It validates links + and anchors, the status vocabulary, gap-ID uniqueness and reachability, and + every referenced `make` target, repository path, and workflow job. The root + test suite runs the same checker, so a renamed script or retired target fails + a normal pull request rather than rotting here unnoticed. +- Prefer file, symbol, command, and workflow-job references over volatile line + numbers or collected-test counts. +- A skip counts as evidence only when the job declares and enforces that skip as + allowed. A missing required dependency or configured browser is a failure. +- A retry may handle a classified infrastructure failure; it must not turn a + semantic assertion failure green. +- Benchmark schema validity, expected scenario availability, and deterministic + correctness are integrity checks. Ordinary comparative timing may remain + advisory. +- Keep good existing foundations while closing gaps. Do not replace malformed + framing tests, API parity, native parity, focused cross-engine conformance, + packaging verification, or visual-health smokes with a narrower new lane. diff --git a/spec/testing/audits/2026-07-21.md b/spec/testing/audits/2026-07-21.md new file mode 100644 index 00000000..343d2e2c --- /dev/null +++ b/spec/testing/audits/2026-07-21.md @@ -0,0 +1,24 @@ +# Testing audit — 2026-07-21 + +Point-in-time reconciliation of [`../current.md`](../current.md) against the +repository. These numbers are an evidence snapshot, not permanent thresholds: +they record what was observed on this date, and they are expected to go stale. +Stable requirements and commands live in [`../current.md`](../current.md); new +required work lives in [`../gaps.md`](../gaps.md). + +- Commit: [`47b484c2fdcba8359b1a9214aab34a30596952a5`](https://github.com/reflex-dev/xy/commit/47b484c2fdcba8359b1a9214aab34a30596952a5) +- CI run: [29878728616](https://github.com/reflex-dev/xy/actions/runs/29878728616) + +| Evidence | Observed result | How observed | +|---|---|---| +| Main Python lane | 2,070 passed, 69 skipped, 3 warnings | CI run `test` job | +| Local repository-wide lane | 2,070 passed, 69 skipped, 3 warnings | `uv run pytest -q` | +| Local Python branch coverage, at prior commit [`eddb2c14`](https://github.com/reflex-dev/xy/commit/eddb2c14) | 82% over 20,538 statements and 7,800 branches | Local branch-aware run; not measured at the audited commit | +| Rust debug / release | 102 / 103 passed | `cargo test` and `cargo test --locked --release -q` | +| Reflex adapter with dependencies installed | 83 passed | The corresponding package-owned suite is now `python/reflex-xy/tests/` | +| Dashboard 10/20/50 | 10 healthy, 20 governed and recoverable, 50 failed; verifier still passed | CI dashboard smoke step and its report | +| Repository merge rule | No required status check; zero required approvals | GitHub repository ruleset UI, observed by a maintainer with admin access on this date | + +The dashboard row is the observation that motivates +[TST-NI-002](../gaps.md#tst-ni-002--strict-dashboard-102050-health-gate): a +failed 50-chart row did not turn the job red. diff --git a/spec/testing/browser-conformance-policy.md b/spec/testing/browser-conformance-policy.md new file mode 100644 index 00000000..d8faaed9 --- /dev/null +++ b/spec/testing/browser-conformance-policy.md @@ -0,0 +1,67 @@ +# Browser Conformance Matrix Policy + +This is the normative TST-NI-015 contract for the shared shipped-renderer +fixture. It is deliberately bounded rather than a Cartesian product: six +reviewed cases cover every required tier, representative mark family, DPR, +motion preference, and axis class, and every case runs unchanged in Chromium, +Firefox, and WebKit. + +## Reviewed matrix + +| Case | Tier / mark | DPR | Motion | Axis contract | +| --- | --- | ---: | --- | --- | +| `direct-linear-scatter-dpr1-reduced` | direct scatter | 1 | reduced | linear x/y; full keyboard/live-region anchor | +| `decimated-log-line-dpr2-motion` | decimated line | 2 | no preference | logarithmic x | +| `direct-category-bar-dpr1-motion` | direct bar | 1 | no preference | category x with exact category labels | +| `direct-linear-heatmap-dpr2-reduced` | direct heatmap | 2 | reduced | linear x/y | +| `direct-named-mesh-dpr1-motion` | direct triangle mesh | 1 | no preference | trace bound to named `x2` / `y2` axes | +| `density-linear-scatter-dpr2-reduced` | density scatter | 2 | reduced | linear x/y | + +The catalog is exact. `scripts/browser_conformance.mjs --list-cases` exposes it, +and validation fails on a missing, extra, duplicate, or metadata-mismatched +case. This keeps the required coverage explicit while preventing accidental +growth into hundreds of redundant browser contexts. + +## Required assertions + +Every case must prove a nonzero root/canvas layout, accessible chart region and +plot image, named toolbar controls and one active toggle, expected summary and +axis-title text, the declared GPU kind/tier/axis binding, nonblank WebGL pixels, +the requested media preference, and a backing store matching DPR 1 or 2. +Reduced-motion cases must bypass animated view state; no-preference cases must +enter it. The scatter anchor additionally preserves the existing keyboard, +live-region, exact-reply, transition-suppression, Escape, and stale-reply +assertions. + +The same case is compared to Chromium in each other selected engine. DOM layout +uses CSS pixels and may differ by at most 4 px. The 32 × 20 RGBA signature has a +mean absolute per-channel tolerance of 12 on the 0–255 scale. Lit-pixel count +must stay within 0.8–1.2 of Chromium, and every render must contain at least 80 +lit WebGL pixels. These reviewed tolerances accommodate engine rasterization +without turning the gate into a blank-chart health check; changes require +review of this policy and retained evidence. + +## Environment, evidence, and failure policy + +The hard `browser_conformance` CI job uses Node 22, package-pinned Playwright +`1.61.1`, its bundled Chromium/Firefox/WebKit revisions, Ubuntu's Xvfb display, +a 760 × 480 CSS viewport, and a 1920 × 1200 virtual screen for DPR 2. It runs: + +```bash +node scripts/browser_conformance.mjs --evidence browser-conformance-evidence.json +``` + +`make check-conformance` is the local entry point after all three Playwright +engines are installed. Default and CI execution require all three engines with +no skip path; an explicit `--browsers` subset is only a local diagnostic. A +missing selected engine, launch failure, missing WebGL2, page error, absent +matrix case, semantic failure, or comparison failure exits nonzero. CI retains +`browser-conformance-evidence.json` as the `browser-conformance-evidence` +artifact even on failure. The JSON records the exact matrix, environment, +browser versions, semantic/layout/DPR/motion observations, raster metrics, +thresholds, and signature digests. + +Independent negative controls remove a required catalog case, corrupt a pixel +signature, and shift layout beyond tolerance. All three mutations must be +rejected before the real browser matrix can pass; `--self-test` exposes those +controls without launching a browser. diff --git a/spec/testing/coverage-policy.json b/spec/testing/coverage-policy.json new file mode 100644 index 00000000..be35dcd8 --- /dev/null +++ b/spec/testing/coverage-policy.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "coverage_format": "coverage.py-json-v3", + "source_roots": [ + "python/xy", + "python/reflex-xy/reflex_xy" + ], + "packages": [ + { + "name": "core", + "path_prefix": "python/xy/", + "exclude_prefixes": ["python/xy/pyplot/"], + "minimum_line_percent": 87.5, + "minimum_branch_percent": 78.5 + }, + { + "name": "pyplot", + "path_prefix": "python/xy/pyplot/", + "exclude_prefixes": [], + "minimum_line_percent": 80.5, + "minimum_branch_percent": 68.0 + }, + { + "name": "reflex_adapter", + "path_prefix": "python/reflex-xy/reflex_xy/", + "exclude_prefixes": [], + "minimum_line_percent": 91.5, + "minimum_branch_percent": 80.5 + } + ], + "modules": [ + { + "path": "python/xy/channel.py", + "minimum_line_percent": 96.0, + "minimum_branch_percent": 87.5 + }, + { + "path": "python/xy/widget.py", + "minimum_line_percent": 97.5, + "minimum_branch_percent": 91.0 + }, + { + "path": "python/xy/pyplot/_axes.py", + "minimum_line_percent": 80.5, + "minimum_branch_percent": 70.5 + }, + { + "path": "python/reflex-xy/reflex_xy/component.py", + "minimum_line_percent": 97.5, + "minimum_branch_percent": 82.0 + } + ], + "diff": { + "minimum_line_percent": 90.0, + "missing_coverage_file": "fail" + }, + "exclusions": [ + { + "pattern": "python/xy/static/**", + "rationale": "Generated JavaScript bundles are governed by the separate Node semantic coverage gate and bundle-freshness check." + }, + { + "pattern": "tests/**", + "rationale": "Test implementation is evidence, while this ratchet governs shipped Python package behavior." + }, + { + "pattern": "python/reflex-xy/tests/**", + "rationale": "Adapter tests drive the shipped reflex_xy package but are not themselves production code." + }, + { + "pattern": "scripts/**", + "rationale": "Repository automation has dedicated verifier and mutation gates rather than the shipped-package baseline." + }, + { + "pattern": "benchmarks/**", + "rationale": "Benchmark harnesses are governed by their schema and methodology gates, not production-package coverage." + }, + { + "pattern": "examples/**", + "rationale": "Examples are exercised by example and browser checks but are not distributed as the core Python package." + } + ] +} diff --git a/spec/testing/coverage.md b/spec/testing/coverage.md new file mode 100644 index 00000000..9aa4a04d --- /dev/null +++ b/spec/testing/coverage.md @@ -0,0 +1,47 @@ +# Coverage ratchet + +`python_coverage` is a hard pull-request job and a dependency of the stable +`Required CI` result. It installs the package-owned, locked +`python/reflex-xy[dev]` environment, measures the full root suite plus the +zero-skip Reflex adapter suite with branch tracing, and retains raw coverage, +Coverage.py JSON/XML, and `coverage/python/ratchet.json` for 30 days. + +The reviewed policy is `coverage-policy.json`; `scripts/coverage_ratchet.py` +owns its schema and enforcement. At the policy's 2026-07-21 review point, the +measured package values and blocking floors were: + +| Package | Measured line | Floor | Measured branch | Floor | +|---|---:|---:|---:|---:| +| Core `python/xy` excluding pyplot | 88.27% | 87.5% | 79.09% | 78.5% | +| `python/xy/pyplot` | 81.07% | 80.5% | 68.98% | 68.0% | +| `python/reflex-xy/reflex_xy` | 92.22% | 91.5% | 81.41% | 80.5% | + +Critical module floors separately protect `channel.py`, `widget.py`, pyplot's +`_axes.py`, and the Reflex component boundary. Every shipped Python file must +appear in the branch-aware report and map to exactly one package group, so a +new unmeasured module fails rather than diluting or evading the aggregate. + +Changed-line policy is 90%. The verifier compares the immutable pull-request +base/head SHAs with a zero-context Git diff, intersects added production lines +with Coverage.py's executable-line inventory, and reports every covered and +missing line. A production file absent from coverage fails closed. A change +with no executable Python lines is recorded as not requiring diff coverage; +package and module floors still run. + +Exclusions are exact patterns with substantive rationales in the policy. They +cover generated JavaScript, tests, automation, benchmarks, and examples—not +shipped Python modules. Lowering a floor or adding an exclusion is a policy +change requiring normal review; generated evidence never rewrites the policy. + +JavaScript is measured separately by the hard `javascript_semantics` job, +which retains raw V8/text/JUnit evidence and enforces line, branch, and function +floors. Rust has no coverage ratchet yet and must first gain a retained, +toolchain-pinned coverage report before any Rust percentage can become policy; +the Python ratchet deliberately cannot manufacture or imply one. + +To re-check an existing report locally: + +```bash +make check-coverage COVERAGE_JSON=coverage/python/coverage.json \ + COVERAGE_BASE=origin/main COVERAGE_HEAD=HEAD +``` diff --git a/spec/testing/current.md b/spec/testing/current.md new file mode 100644 index 00000000..38764eb8 --- /dev/null +++ b/spec/testing/current.md @@ -0,0 +1,243 @@ +# Current Testing Inventory + +This document describes evidence that exists in the repository today. Status +uses the vocabulary in [`README.md`](README.md). The scope column is important: +`IMPLEMENTED` applies only to the exact behavior stated, not to a broader +product area with a similar name. + +The inventory groups tests by risk surface rather than listing every test +function. Test locations, stable commands, and workflow jobs are the executable +index. Missing additions are tracked by ID in [`gaps.md`](gaps.md). + +## Repository contracts and static quality + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| Python lint and formatting | `ruff check .`; `ruff format --check .`; repository pre-commit hooks | Hard CI and required before push by contributor policy | `IMPLEMENTED` | Covers the configured Python source set. | +| Public API and annotations | `scripts/check_public_api.py`; `tests/test_public_api.py`; `tests/test_type_surface.py`; `tests/test_api_parity.py` | Hard CI / `make check-api` | `IMPLEMENTED` | Dynamic inventories cover exports, lazy mappings, builder/applier identity, defaults, and the `py.typed` surface. | +| Python floor syntax and imports | `scripts/check_python_floor.py`; `tests/test_python_floor.py`; `tests/test_import.py`; `tests/test_dependencies.py` | Hard CI / `make python-floor` / `make check-import` | `IMPLEMENTED` | The syntax/import contract is distinct from a full supported-version and minimum-dependency matrix. See TST-NI-036. | +| Documentation examples and public claims | `tests/test_docs_examples.py`; `tests/test_claim_guardrails.py`; `scripts/check_claim_guardrails.py` scans every specification and public document | Hard all-path CI / `make check-docs` | `IMPLEMENTED` | Public examples and broad benchmark wording are checked, including specification-only changes. See TST-NI-005. | +| Generated JavaScript bundle freshness | `node js/build.mjs --check`; main-CI ESM/IIFE parse; release freshness check and rebuild | Hard CI and release | `IMPLEMENTED` | This proves source-to-bundle freshness and parseability; the dedicated semantic suite below tests the same committed ESM artifact. | +| Default CodeQL analysis | GitHub default setup for Actions, JavaScript/TypeScript, Python, and Rust | Separate GitHub code-scanning workflow; not required by the main ruleset | `IMPLEMENTED` | This status applies to code scanning, not dependency-vulnerability auditing. | +| Generated native font freshness | `scripts/gen_font.py`; committed `src/font.rs` | No check mode or comparison gate | `NOT IMPLEMENTED` | The generator relationship is documented in source but cannot fail CI when stale. See TST-NI-043. | +| Matplotlib compatibility snapshot freshness | `scripts/sync_matplotlib_compat.py --check` | Hard Matplotlib reference job | `IMPLEMENTED` | The generated method inventory is current for the pinned reference. Semantic gaps are separate. | +| Workflow contract checking | `scripts/verify_ci_workflow.py`; `tests/test_verify_ci_workflow.py` | Hard CI / `make check-ci` | `PARTIALLY IMPLEMENTED` | Checks CI, CodSpeed, release, docs deployment, and reusable image/Helm text and wiring with negative controls. It remains a string checker rather than a semantic dependency validator and does not cover every workflow. See TST-NI-038. | +| Specification contract checking | Whole-tree link/anchor, command, path/symbol/job, status/evidence-row, gap-ID, and public-claim validation in `scripts/check_testing_spec.py` and `scripts/check_claim_guardrails.py` | Hard all-path CI / `make check-testing-spec` and `make check-docs` | `IMPLEMENTED` | Stable completed gap IDs remain checkable and require explicit evidence; generated artifact paths require a workflow producer. See TST-NI-005. | +| Type checking | `ty check python` | Advisory; current CI permits diagnostics | `PARTIALLY IMPLEMENTED` | The latest audit observed 25 accepted diagnostics and an unprovisioned adapter surface. No baseline ratchet exists. See TST-NI-044. | + +## Python API, data, and protocol + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| Core Python behavior | Repository-wide `pytest -q`, recursively including root tests and `tests/pyplot/`; adapter tests are package-owned below | Hard main and Python 3.11 jobs | `PARTIALLY IMPLEMENTED` | Broad native-backed coverage exists, but optional root dependencies can still skip without a per-job allowlist. See TST-NI-031 and TST-NI-036. | +| Figure grammar and builder parity | `tests/test_figure.py`; `tests/test_components.py`; `tests/test_api_parity.py`; `tests/test_property_figure.py`; plot-family tests | Hard root suite | `IMPLEMENTED` | Public composition methods, appliers, signatures, defaults, representative payloads, and valid/invalid strategies for all 20 public builders are exercised. | +| Validation and transactional rollback | `tests/test_components.py`; `tests/test_figure.py`; `tests/test_property_figure.py`; focused error/LOD/cache tests; `make check-errors` | Hard root suite | `IMPLEMENTED` | Every public builder is injected with a failure after trace insertion on nonempty seeded state and must restore exact traces, columns/dedup keys, axes/categories, annotations, caches, pyramids, and drill state. See TST-NI-006. | +| Property-based figure and frame tests | `tests/test_property_figure.py`; `tests/test_framing_property.py` | Hard root suite and named protocol CI step; Hypothesis is a required development dependency | `IMPLEMENTED` | All 20 figure builders have must-succeed valid and classified-invalid strategies; frame properties cover valid zero-copy round trips plus batched structural mutations with Python/shipped-JavaScript parity. See TST-NI-006 and TST-NI-028. | +| Column ingestion and geometry | `tests/test_arrow_ingest.py`; `tests/test_arrowgeom.py`; scatter, matrix, facets, and plot-family tests | Hard root suite when optional dependencies are present | `PARTIALLY IMPLEMENTED` | Lists, NumPy, Arrow, geometry, and selected null/copy paths have evidence. A catalog for dtype/shape/stride/endian/null/pandas/Arrow and cross-renderer semantics is not enforced. See TST-NI-010 and TST-NI-047. | +| LOD, precision, streaming, and cache behavior | `tests/test_lod.py`; `tests/test_streaming.py`; density, zoom-precision, bounds, tier-update, and pan/zoom no-op tests | Hard root suite plus browser matrices | `PARTIALLY IMPLEMENTED` | Core direct/decimated/density and mutation paths are covered, and TST-NI-011 now proves that clamped navigation schedules no LOD work; the broader drill/stream/resource matrix and soak bounds are not. See TST-NI-034. | +| Wire framing | `spec/testing/protocol-catalog.json`; `tests/test_protocol_catalog.py`; `tests/test_framing.py`; `tests/test_framing_property.py` | Hard named protocol CI step / `make check-protocol`; retained JUnit | `IMPLEMENTED` | Exact shared golden frames, truncation/corrupt-header/padding/metadata/zero-copy evidence, valid properties, and header/length/count/padding/metadata mutation parity cover the Python and shipped JavaScript decoders. TST-NI-028 is complete. | +| Widget/channel dispatch | `spec/testing/protocol-catalog.json`; `tests/test_protocol_catalog.py`; `tests/test_channel.py`; `tests/test_widget.py`; `tests/test_html_transport.py` | Hard named protocol and root suites | `IMPLEMENTED` | The catalog exactly matches all dispatcher branches and generates valid/missing/wrong-type/boundary/callback cases plus reply schemas; the real notebook frontend mount remains TST-NI-018. | +| Governed branch and diff coverage | `scripts/coverage_ratchet.py`; `spec/testing/coverage-policy.json`; core, pyplot, and zero-skip adapter branch runs | Hard `python_coverage` job with retained raw/JSON/XML/ratchet evidence | `IMPLEMENTED` | Reviewed package and critical-module line/branch floors, exact shipped-file inventory, and a 90% changed-executable-line threshold fail closed. JavaScript retains its independent V8 report; Rust is explicitly not ratcheted before a pinned report. See TST-NI-029. | +| Mutation score | No mutation lane | None | `NOT IMPLEMENTED` | Oracle strength is inferred from tests and selected negative cases. See TST-NI-030. | + +## Native Rust and JavaScript client + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| Rust debug correctness | `cargo test` | Hard main CI / `make rust-check` | `IMPLEMENTED` | Includes deterministic randomized kernel and scalar/native parity cases. | +| Hard Rust release-test gate | `cargo test --locked --release`; release-test inventory requires `compose_window_astronomically_past_domain_is_empty_not_panic` | Hard `rust_release` job and `required_ci` dependency | `IMPLEMENTED` | The optimized suite cannot pass by silently compiling out the known release-only regression. See TST-NI-020. | +| Rust lint | `cargo clippy --all-targets -- -D warnings` | Hard main CI | `IMPLEMENTED` | Covers configured targets on the main Linux host. | +| Native C ABI | `scripts/abi_smoke.py` | Hard main CI / `make abi-smoke` | `IMPLEMENTED` | Loads the built core and checks the exported ABI surface. | +| SIMD and architecture parity | `xy_runtime_capabilities`; `scripts/native_parity.py`; fixed-seed Rust parity tests | Hard native Linux x64/ARM64, Windows x64, and macOS ARM64 matrix plus selected release-artifact runtimes | `IMPLEMENTED` | Default and forced-scalar subprocesses compare an exact kernel, invalid-pointer FFI, and framebuffer oracle; x64 requires AVX2 and ARM64 reports its baseline. See TST-NI-023. | +| Native sanitizers, Miri, and coverage-guided fuzzing | Deterministic randomized tests only | None | `NOT IMPLEMENTED` | There is no sanitizer, Miri, or cargo-fuzz lane. See TST-NI-021. | +| Native dependency audit | Point-in-time security review | None in regular automation | `NOT IMPLEMENTED` | No `cargo audit` or `cargo deny` policy is enforced. See TST-NI-022. | +| JavaScript build and syntax | `node js/build.mjs`; main-CI ESM import and IIFE parse checks; release rebuild | Hard main CI plus release build | `IMPLEMENTED` | Main CI proves parseability; release rebuilds the exact artifact after the freshness check. | +| JavaScript semantic units | `js/test/frame.test.mjs`; `js/test/semantics.test.mjs`; `js/test/worker.test.mjs`; `make js-test` | Hard Node 22 `javascript_semantics` job and `required_ci` dependency | `IMPLEMENTED` | The dependency-free suite exercises the exact fresh ESM bundle across frame decode, ticks/formatters, transforms/bounds, theme/style normalization, the 18-kind registry, LOD selection, ChartView state, and the standalone worker protocol. Malformed cases and real mutation controls fail closed; line/branch/function floors gate retained JUnit, coverage text, and raw V8 coverage. | +| Protocol-version implementation coherence | Python and JavaScript runtime constants currently agree | Indirect tests | `PARTIALLY IMPLEMENTED` | Some specification and smoke fixtures can drift; the animation smoke used protocol 3 while runtime uses 4. See TST-NI-046. | + +## Rendering, interaction, and host integration + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| Dependency-light WebGL render | `scripts/render_smoke_nonumpy.py ` | Hard main CI | `IMPLEMENTED` | Exercises representative marks, interaction, context loss, and nonblank pixels without NumPy. It is not an every-kind registry. | +| Real Figure browser render | `scripts/smoke_render.py ` | Hard main CI | `IMPLEMENTED` | A composed Figure reaches nonblank Chromium pixels. | +| Native and browser export | PNG/JPEG/WebP/SVG/PDF unit tests; `scripts/png_export_smoke.py`; image and batch-export tests | Hard root/browser lanes, with optional external PDF tooling | `PARTIALLY IMPLEMENTED` | Broad byte/pixel validation exists. Public persistent browser batch reuse, mixed-format cleanup, and a guaranteed independent PDF oracle are incomplete. See TST-NI-045. | +| Standalone HTML and text security | `tests/test_static_client_security.py`; `scripts/runtime_security_smoke.py`; HTML transport/CSS and sandbox-policy tests; `make check-security`; `make check-browser` | Hard root and Chromium CI/browser lanes with retained runtime JSON | `IMPLEMENTED` | Static escaping/sink constraints and real-browser literal text across 16 public surfaces, hostile CSS, CSP blocking, zero executable user nodes/dialogs, and zero loopback requests are enforced (TST-NI-024). Browser launch never silently downgrades its process sandbox (the distinct TST-NI-025 policy). | +| FastAPI gallery lifecycle | `scripts/reflex_lifecycle_smoke.py` | Hard Chromium CI/browser lane | `IMPLEMENTED` | Exercises gallery routes, resize, visibility, context loss/restore, drilldown, and embedded iframes. The filename is misleading: it serves `examples/fastapi`, not Reflex. | +| Visual health | `scripts/visual_health_smoke.py` | Hard Chromium CI/browser lane | `IMPLEMENTED` | Honestly scoped broad-gallery protection for blank/collapsed output, occupancy, regions, and label overlap; it does not claim identity comparison. | +| Reviewed visual identity | `scripts/visual_baseline.py`; `spec/visual-baselines/v1.json` | Hard pinned-Chromium CI/browser lane with retained artifacts | `IMPLEMENTED` | Bounded TST-NI-014 evidence pins browser/font/viewport/DPR, compares semantics plus tolerant pixels, and rejects real data/color/label/geometry mutations. Baseline proposals require independent review. | +| Rendered labels and value formats | `js/tests/rendered_labels.test.mjs`; `tests/test_rendered_label_formats.py`; normative `spec/testing/rendered-label-policy.md` | Hard main-CI `test` job / `npm run test:labels` / `make check-labels`; retained JSON | `IMPLEMENTED` | Exact numeric, grouping, literal currency, percent, UTC time, log, category, named-axis, tooltip, and colorbar DOM labels run under two non-UTC locale/time-zone contexts; malformed raw formats and a corrupted DOM label are negative controls. TST-NI-008 is complete. | +| Step tier replacement | `scripts/step_tier_smoke.py` | Hard Chromium CI/browser lane | `IMPLEMENTED` | Protects step risers across a synthetic tier-buffer replacement. | +| Interaction stress | `scripts/interaction_stress_smoke.py`; `scripts/pan_zoom_matrix.mjs`; focused wheel/pan/zoom/pick tests | Hard Chromium CI/browser lane with retained worker and matrix JSON | `PARTIALLY IMPLEMENTED` | TST-NI-011, TST-NI-012, and TST-NI-016 jointly hard-gate the bounded action/axis/host matrix, core interaction budgets, pick boundaries, and standalone worker re-bin/paint/teardown; long-duration and compound-gesture stress remain outside this bounded evidence. See TST-NI-034. | +| Bounded cross-browser conformance | `scripts/browser_conformance.mjs`; normative `spec/testing/browser-conformance-policy.md` | Hard `browser_conformance` CI / `make check-conformance` / retained JSON | `IMPLEMENTED` | Six reviewed cases run unchanged in Chromium, Firefox, and WebKit, covering direct/decimated/density, scatter/line/bar/heatmap/mesh, DPR 1/2, reduced/no-preference motion, and linear/log/category/named axes with semantics, layout, DPR, motion, and tolerant raster assertions. TST-NI-015 is complete. | +| Browser version support policy | Playwright-pinned current Chromium, Firefox, and WebKit | Hard CI for the pinned versions only | `NOT IMPLEMENTED` | No normative engine/version floor, WebGL2 prerequisite statement, or oldest-claimed-version lane exists, and recorded renderer versions can drift from the docs. See TST-NI-054. | +| GPU and driver realism | Headless CI lanes running software rendering | Hard Chromium CI on software rasterization | `NOT IMPLEMENTED` | No lane exercises representative hardware WebGL2 drivers or rejects software fallback, so driver-specific defects are invisible. See TST-NI-052. | +| Renderer failure-mode behavior | Successful context loss/restore paths plus source-marker assertions | Mixed | `PARTIALLY IMPLEMENTED` | Restoration is proved in a real browser, but WebGL2 acquisition, shader/program, and permanent-restore failures are not forced and their user-visible contract is unasserted. See TST-NI-053. | +| Chart-kind render contract | `scripts/chart_kind_matrix.py`; `tests/test_chart_kind_matrix.py` | Hard Chromium CI/browser lane with retained JSON | `IMPLEMENTED` | Sixteen public-builder fixtures exactly cover all 18 shipped registry kinds with payload tier/count, live GPU geometry, and independently measured nonblank pixels. Adding a registry kind without a fixture fails (TST-NI-009); cross-renderer parity remains TST-NI-010. | +| Axes, layout, styling, chrome, and facets | Axis/viewport, facets, CSS mark styles, export, legend-resize, rendered-label oracle, pan/zoom matrix, and gallery browser tests | Mixed hard evidence | `PARTIALLY IMPLEMENTED` | Formatter semantics and linear/log/reversed/category/dual/named-axis pan/zoom are hard-gated; the broader collision, renderer-parity, and general axis-catalog contracts remain open. See TST-NI-010 and TST-NI-014. | +| Pan/zoom contract | `scripts/pan_zoom_matrix.mjs`; `tests/test_pan_zoom_matrix.py`; workflow negative controls | Hard full Chromium, focused Chromium/Firefox/WebKit, and Reflex floor/latest CI with retained JSON; `make check-pan-zoom` | `IMPLEMENTED` | Five bounded standalone cases drive drag, wheel, box, toolbar zoom, and reset over linear/log/reversed/category/dual/named axes; assert bounds, default/finite/partial limits, actual changed axes, exact nonparticipants, link/no-echo, reduced motion, clamped no-op event/LOD silence, and semantic/layout health. Two real-host cases prove JSON-safe live Reflex view/LOD transport and kernel-less static Reflex navigation. | +| Animation Python contract | `tests/test_animation.py`; Python-only CodSpeed microbenchmarks | Hard root suite / advisory CodSpeed | `IMPLEMENTED` | Validation, payload, and deterministic static behavior have evidence. | +| Required animation browser gate | `scripts/animation_smoke.py`; `tests/test_animation_smoke.py` | Hard Chromium CI/browser lane with retained JSON evidence | `IMPLEMENTED` | Protocol-derived fixtures prove keyed/fallback interpolation, representative marks, ghost-free pixels, allocation, replacement, lifecycle, reduced motion, frozen capture, and teardown. TST-NI-013 records the completed gate; real-browser performance measurement remains TST-NI-048. | +| Dashboard context governance | `benchmarks/bench_dashboard.py` at 10/20/50 plus strict outcome validation and mutation negatives in `tests/test_verify_benchmark_report.py` | Hard CI selects `--profile strict` and retains `dashboard-health-evidence`; timing reports remain advisory | `IMPLEMENTED` | Every requested row must be complete or governed, create and visit-paint every chart, and contain no unexplained context loss. See TST-NI-002. | +| Accessibility | Source contracts plus the focused three-engine semantic fixture | Mixed hard evidence | `PARTIALLY IMPLEMENTED` | Roles and selected keyboard/live behavior have evidence, but there is no normative rendered matrix for direct/aggregate charts, focus, forced colors, or a table alternative. See TST-NI-017 and TST-NI-039. | +| Anywidget/Jupyter | `tests/host_integration/`; Python widget and transport tests | Hard Anywidget floor/latest host matrix and root suite | `PARTIALLY IMPLEMENTED` | Both supported edges instantiate the real trait model and exercise split-buffer comm transport without skips. No supported notebook frontend mounts the shipped widget and drives its full browser lifecycle; see TST-NI-018. | +| Reflex adapter | `python/reflex-xy/tests/`; `scripts/reflex_ws_smoke.py`; `scripts/host_integration_policy.py` | Hard dedicated Reflex floor/latest job with zero skips and retained version/JUnit/log/screenshot evidence | `IMPLEMENTED` | Package-owned bounded dependencies make missing or unsupported test requirements fail; the production example proves one shared socket, binary paint, drill/hover state, streaming, renderer teardown, and host transport close. This supplies TST-NI-004 and the Reflex portion of TST-NI-019. | +| FastAPI and host versions | `tests/host_integration/`; `scripts/fastapi_host_smoke.py`; `spec/testing/host-integration-policy.json` | Hard bounded floor/latest `host_integration` job and stable `required_ci` aggregate | `IMPLEMENTED` | Package-owned Anywidget, Traitlets, FastAPI, Starlette, Uvicorn, and HTTPX ranges are code-grounded; compile, mount, HTTP/comm transport, real-browser paint, and browser-originated drilldown run with zero skips and retained evidence. This supplies TST-NI-019; the real notebook frontend remains TST-NI-018. | + +## Matplotlib compatibility and documentation application + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| `xy.pyplot` API and state | `tests/pyplot/`; compatibility snapshot; public annotations and state/artist/options tests | Hard dedicated Matplotlib 3.11 and root suites | `IMPLEMENTED` | The declared compatibility inventory and broad behavioral corpus are substantial. | +| Matplotlib semantic/perceptual parity | Pinned reference subsets and a 54-case corpus | Hard dedicated lane | `PARTIALLY IMPLEMENTED` | The tolerant corpus can accept wrong or empty data for some cases; contour, vector magnitude, transforms, and tick semantics need stronger independent oracles. See TST-NI-026. | +| Pyplot accepted-option use | `scripts/check_pyplot_options.py`; `spec/testing/pyplot-noops.json`; behavioral no-op contracts and detector mutation tests | Hard root CI with retained JSON | `IMPLEMENTED` | Unread named options and discarded literal option pops fail unless the exact function/option is reviewed with a substantive rationale and executable invariant test. TST-NI-027 records the completed gate. | +| Required pandas interoperability lane | Tests guarded by `pytest.importorskip("pandas")` are supporting evidence | Dependency absent from normal CI | `NOT IMPLEMENTED` | No supported pandas floor/latest lane enforces Period/Series interoperability. See TST-NI-047. | +| Documentation app | `docs/app/tests`; production app on Python 3.11/3.12; sitemap, Markdown asset, route, and preview checks | Docs workflow | `IMPLEMENTED` | Applies to selected docs paths, not specification-only changes. | +| Published quickstart | Installs the configured released `xy` wheel and executes public quickstarts | Docs workflow | `IMPLEMENTED` | The published version is currently hard-coded and must be maintained deliberately. | + +## Performance, packaging, platforms, and delivery + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| Benchmark harness and report schema | Benchmark-specific tests; `scripts/verify_benchmark_report.py`; environment/category/status metadata | Hard harness tests; benchmark workflows mostly advisory | `PARTIALLY IMPLEMENTED` | Schema is strong, but unexpected failures, skips, and unavailable rivals can still produce green integrity results. See TST-NI-042. | +| Deterministic hard regressions | Native scatter/kernel/transport metrics and `scripts/check_regressions.py` | Hard main CI | `IMPLEMENTED` | Catastrophic configured regressions block; ordinary shared-runner timing remains advisory. | +| Regular-CI cross-library measurements | Isolated scatter, line, install, workflow, interaction, and dashboard benchmarks | Advisory workflow artifacts | `PARTIALLY IMPLEMENTED` | Expected competitor availability and skip ceilings are not enforced; plotly-resampler was unavailable in the audited run. See TST-NI-042. | +| Manual benchmark refresh | Scatter and core-2D browser measurements in `benchmark-refresh.yml` | Manual workflow | `PARTIALLY IMPLEMENTED` | It produces reviewable measurements but is not part of regular pull-request evidence. See TST-NI-042. | +| Full pyplot-vs-Matplotlib speed comparison | `make check-pyplot-speed` | Local only | `PARTIALLY IMPLEMENTED` | The executable comparison is not automated in CI or benchmark refresh. See TST-NI-042. | +| CodSpeed microbenchmarks | `benchmarks/test_codspeed_*.py --codspeed` | Advisory CodSpeed workflow | `IMPLEMENTED` | Covers kernel, transport, pyplot, and Python animation microbenchmarks. It is not the real-browser animation benchmark. | +| Real-browser animation benchmark | `benchmarks/bench_animation.py` exists | No CI, Makefile, refresh, or report-verifier lane | `NOT IMPLEMENTED` | See TST-NI-048. | +| Source distribution structure | `uv build --sdist`; `scripts/verify_sdist.py`; expected no-Rust error path | Hard CI/release | `IMPLEMENTED` | A clean Rust-enabled install from the built sdist is missing. See TST-NI-040. | +| Native and pure wheel structure | `scripts/verify_wheel.py`; metadata/tags/RECORD/content/native-presence checks; 15 MB budget | Hard CI/release | `IMPLEMENTED` | Structural coverage is strong. Runtime coverage is host-selective. | +| No-toolchain pure-wheel behavior | CI removes Cargo, builds and verifies `py3-none-any`, installs it, imports the top-level package, and requires an actionable compute-layer error | Hard CI | `IMPLEMENTED` | This is the supported negative runtime path; it does not replace positive native sdist installation. | +| CI wheel runtime | Linux x86, Linux ARM64, and Windows x64 build/install/native import jobs; common native parity matrix also covers macOS ARM64 | Hard CI | `IMPLEMENTED` | The focused native contract runs on all four host classes; broader installed-wheel public behavior remains separate. See TST-NI-023 and TST-NI-049. | +| Release wheel matrix | Eleven native platform wheels with structural verification; selected host-native imports; x64 manylinux 2.17 and musllinux 1.2 run the common native parity oracle in pinned containers | Release workflow | `PARTIALLY IMPLEMENTED` | Selected glibc/musl artifacts execute before publish, while the remaining cross-built artifacts are structurally checked. Broader artifact behavior is tracked by TST-NI-049. | +| Pyodide artifact | Pinned Rust/Emscripten/Pyodide build and runtime load probe | Release workflow | `IMPLEMENTED` | This status applies only to the exact release job; no regular PR/scheduled regression lane builds and runs the WASM artifact. See TST-NI-041. | +| Supported Python matrix | Explicit Python 3.11 plus the main runner's incidental Python | Hard CI | `PARTIALLY IMPLEMENTED` | The project declares Python 3.11+ without explicitly testing every currently claimed version or newest supported interpreter. See TST-NI-036. | +| Cross-platform behavioral parity | Focused kernel/FFI/raster parity runs on native Linux x64/ARM64, Windows x64, and macOS ARM64; wheel jobs install/import selected hosts | Hard CI for the focused native contract | `NOT IMPLEMENTED` | Public Python, framing, and export behavior still run primarily on Ubuntu; the focused native parity closure does not substitute for the broader matrix. See TST-NI-049. | +| Dependency reproducibility | Benchmark lock and docs lock; floating root `.[dev]` hard job | Mixed | `PARTIALLY IMPLEMENTED` | The hard root gate is not frozen; there is no separate latest-dependency canary or true minimum-dependency lane. See TST-NI-036. | +| Required merge gate | `required_ci` evaluates the complete hard-job result map with negative controls for failure, cancellation, skip, omission, and accidental advisory inclusion | Stable all-path result exists; repository ruleset does not yet require it | `PARTIALLY IMPLEMENTED` | The aggregate is executable, but `main` is not protected until the repository ruleset requires `Required CI`. See TST-NI-001. | +| Exact-SHA release qualification | `scripts/verify_source_qualification.py`; exact-tag/main ancestry; newest exact-SHA `Required CI`; package tag/version/dated-changelog agreement | Unconditional release `qualify` dependency before every build or publish job | `IMPLEMENTED` | A dry run may defer nonexistent tag metadata because it cannot publish; a tag push or manual real publication cannot bypass it. See TST-NI-003. | +| Exact-SHA deployment qualification | The same source qualifier in dev and stage/production; exact deploy-tag check for tagged promotion | Required `qualify` dependency before docs image builds | `IMPLEMENTED` | Dev is intentionally SHA-addressed rather than tagged; tagged stage/production promotion verifies the CalVer tag and reuses the qualified SHA. See TST-NI-003. | +| Release provenance | `scripts/release_provenance.py`; exact-set SHA-256 manifest; BuildKit max-provenance image builds; ECR digest comparison and `tag@sha256` Helm pins | Required release `provenance` job plus staging/production digest gates | `IMPLEMENTED` | Package publishers verify the complete downloaded artifact set; docs promotion reuses the same image digests. This is artifact provenance, not a general dependency SBOM. See TST-NI-003. | +| Continuous dependency-vulnerability auditing | A dated point-in-time security audit is historical evidence | No recurring dependency-audit lane | `NOT IMPLEMENTED` | Python, Rust, and JavaScript dependency audits are not enforced as one policy. See TST-NI-022. | + +## Test reliability and evidence + +| Surface | Current executable evidence | Enforcement | Status | Current boundary / gap | +|---|---|---|---|---| +| Suite markers and skip policy | Ad hoc markers, `importorskip`, and environment-dependent skips | No common per-job allowlist or budget | `NOT IMPLEMENTED` | Required dependencies/browsers can disappear into a skip. See TST-NI-031. | +| Retry and flake provenance | Browser launch retries and focused retry tests | Broad retry behavior; incomplete attempt reporting | `PARTIALLY IMPLEMENTED` | Semantic and infrastructure failures are not uniformly classified, and flaky passes are not governed. See TST-NI-032. | +| Async synchronization | Bounded waits plus several fixed sleeps in host-adapter tests | Mixed | `PARTIALLY IMPLEMENTED` | Fixed sleeps remain load-bearing in selected socket/registry cases. See TST-NI-033. | +| Resource-leak and soak testing | Lifecycle teardown assertions and short context/worker paths | No quantitative repeated-lifecycle gate | `NOT IMPLEMENTED` | Listener, GL, worker, socket, file-descriptor, temporary-file, and heap growth are not bounded over repetitions. See TST-NI-034. | +| Failure artifacts | Benchmark/package artifacts and normal job logs | Inconsistent for hard tests | `PARTIALLY IMPLEMENTED` | JUnit, coverage, screenshots/diffs, console/CDP traces, and retry history are not retained consistently. See TST-NI-035. | +| Execution timeouts | Some workflows and scripts have local bounds | No uniform hard/release policy | `PARTIALLY IMPLEMENTED` | Several jobs and hang-prone paths depend on runner ceilings. See TST-NI-051. | +| Fixture/golden provenance | Individual benchmark baselines, protocol vectors, and compatibility corpora | Per-surface conventions | `PARTIALLY IMPLEMENTED` | There is no shared rule for source, seed/version/license, update command, review, checksum, tolerance, and negative controls. See TST-NI-055. | +| Claim-to-test traceability | This risk-surface inventory and product-specific evidence links | Not machine checked claim by claim | `PARTIALLY IMPLEMENTED` | Stable normative claim IDs and orphan/dead-reference validation do not exist. See TST-NI-050. | + +## Stable local commands + +These commands are useful entry points. Their names do not broaden the scopes +described above. + +| Command | Current scope | +|---|---| +| `make check` | Fast local verifier selection | +| `make check-full` | Non-browser Python, JavaScript freshness, Rust debug/clippy, Rust release build, and ABI checks; not a release-equivalent suite | +| `make check-browser CHROMIUM=...` | Configured Chromium smokes including runtime page-content security, animation, and pick-boundary but except dashboard; does not include Reflex or cross-engine conformance | +| `make check-labels` | Strict authored-source formatter units plus shipped-bundle DOM labels in two locale/time-zone contexts | +| `make check-pan-zoom CHROMIUM=...` | Complete five-case standalone pan/zoom matrix in the configured Chromium; writes validated JSON evidence under `/tmp` | +| `make check-conformance` | Exact six-case semantic/layout/DPR/motion/raster matrix across Chromium, Firefox, and WebKit | +| `make check-protocol` | Catalog-generated request/reply cases, exact shared golden frames, handwritten corruption cases, and Hypothesis structural-mutation parity | +| `make check-docs` | Executable examples and claim guardrails | +| `make check-examples` | README/API examples and Reflex asset-registry checks | +| `make check-security` | Standalone HTML safety and client text-sink checks | +| `make check-errors` | Public validation, rollback, LOD, drill, and cache focus | +| `make check-api` | Public API and type-surface contracts | +| `make check-import` | Import budget and dependency boundaries | +| `make check-ci` | Current workflow text/wiring invariants | +| `make check-claims` | Public performance-claim guardrails | +| `make check-testing-spec` | Validate this catalog's links, gap IDs, commands, paths, and workflow jobs | +| `make check-benchmark-harness` | Benchmark metadata, schema, and regression tests | +| `make check-coverage` | Validate a branch-aware report against reviewed package/module floors and the configured Git diff | +| `make check-pyplot` | Structured accepted-option audit plus the full `tests/pyplot` suite in the active environment | +| `make check-pyplot-speed` | Local full pyplot-vs-Matplotlib static-PNG target; requires benchmark dependencies | +| `make check-sdist` | Build and structurally verify a source archive | +| `make check-wheel` | Build and structurally verify a wheel; native expectation is caller-selected | +| `make check-artifacts` | Verify supplied prebuilt sdist and wheel paths | +| `make check-benchmark-report` | Validate one supplied benchmark JSON report and kind | +| `make list-checks` | Print the checks known to `scripts/verify_local.py` | +| `make test` | Repository-wide `pytest -q` in the active environment | +| `make lint` | `ruff check .` | +| `make format` | `ruff format --check .` | +| `make typecheck` | Advisory `ty check python` command; the command itself exits with the type checker result | +| `make public-api` | Direct public API checker | +| `make python-floor` | Direct Python-floor checker | +| `make js-check` | Committed JavaScript bundle freshness | +| `make js-test` | Dependency-free JavaScript semantic units with coverage floors | +| `make rust-check` | Rust debug tests and clippy | +| `make abi-smoke` | Direct C ABI smoke | + +The target replacement for the ambiguous `make check-full` contract is tracked +as TST-NI-037. + +## Test suite and executable-helper registry + +This registry makes dormant or optional evidence visible. `Wired` describes +automation, not whether the helper is valuable. + +| Suite or helper | Scope | Wiring / status | +|---|---|---| +| Repository `pytest -q` | Root tests plus recursively collected pyplot tests | Hard main/Python-floor jobs; `PARTIALLY IMPLEMENTED` because optional dependencies can skip | +| `tests/pyplot/` | Matplotlib API/state/artist/options/corpus/reference behavior | Root collection plus dedicated hard Matplotlib reference job; `IMPLEMENTED` for its documented scope | +| `python/reflex-xy/tests/` | Adapter assets, components, vars, socket plane, state bridge, and tokens | Package-owned dev environment plus zero-skip Reflex floor/latest job; `IMPLEMENTED` | +| `docs/app/tests` | Documentation application unit/route/content behavior | Docs quality job; `IMPLEMENTED` | +| `benchmarks/test_codspeed_*.py` | Kernel, transport, pyplot, and Python animation microbenchmarks | Advisory CodSpeed job; `IMPLEMENTED` | +| Rust tests in `src/` | Native kernels, encoding, raster, tiles, SIMD, module invariants, and a release-only extreme-window regression | Debug and locked release-profile hard CI | +| `js/test/*.test.mjs` | Exact-bundle frame, tick/formatter, transform/bounds, theme/style, mark-registry, LOD, ChartView-state, worker-protocol, malformed-input, and mutation-control semantics | Hard Node 22 `javascript_semantics` job with retained JUnit and coverage; `IMPLEMENTED` for TST-NI-007 | +| `spec/testing/protocol-catalog.json` / `tests/test_protocol_catalog.py` / `tests/test_framing_property.py` | Exact dispatcher/reply inventory, shared Python/JavaScript `XYBF` frames, generated case classes, and structural byte-mutation parity | Hard main-CI protocol step with retained JUnit; `IMPLEMENTED` for TST-NI-028 | +| `scripts/coverage_ratchet.py` | Exact shipped-file inventory, reviewed core/pyplot/adapter line and branch floors, critical modules, and 90% changed executable lines | Hard `python_coverage` job with retained raw/JSON/XML/ratchet evidence; `IMPLEMENTED` for TST-NI-029 | +| `scripts/abi_smoke.py` | Exported native C ABI | Hard main CI | +| `scripts/render_smoke_nonumpy.py` | Dependency-light WebGL marks, pixels, interaction, and context recovery | Hard Chromium CI | +| `scripts/png_export_smoke.py` | Native and Chromium PNG health | Hard Chromium CI | +| `scripts/smoke_render.py` | Real composed Figure to Chromium pixels | Hard Chromium CI | +| `scripts/runtime_security_smoke.py` | Production standalone DOM-XSS, CSP, hostile-CSS, dialog, and wire-level network-isolation behavior | Hard Chromium CI/browser lane with retained JSON evidence; `IMPLEMENTED` for TST-NI-024 | +| `scripts/reflex_lifecycle_smoke.py` | FastAPI gallery lifecycle, drilldown, resize, visibility, context recovery, and iframes | Hard Chromium CI; misleading historical filename | +| `scripts/visual_health_smoke.py` | Broad gallery visual-health invariants | Hard Chromium CI | +| `scripts/visual_baseline.py` | Reviewed semantic/perceptual identity plus real negative controls | Hard pinned-Chromium CI; expected/actual/diff artifacts retained | +| `scripts/step_tier_smoke.py` | Step geometry after tier-buffer replacement | Hard Chromium CI | +| `scripts/interaction_stress_smoke.py` | Wheel/pan/hover/crosshair/box/brush behavior, budgets, and standalone density worker re-bin/paint/teardown | Hard Chromium CI with retained JSON; worker skips are blocking; `IMPLEMENTED` for TST-NI-016 | +| `scripts/pan_zoom_matrix.mjs` / `tests/test_pan_zoom_matrix.py` | Catalog-validated action/axis/host matrix; semantic/layout, link, bounds/limits, no-op LOD/event, cross-engine, and live/static Reflex assertions | Hard full Chromium, focused three-engine, and Reflex floor/latest CI with failure-retaining JSON; `IMPLEMENTED` for TST-NI-011 | +| `js/tests/rendered_labels.test.mjs` | Formatter units plus exact rendered numeric/time/category/named-axis/tooltip/colorbar DOM labels and independent negatives | Hard Chromium main CI with retained JSON; `IMPLEMENTED` for TST-NI-008 | +| `scripts/browser_conformance.mjs` | Six-case tier/mark/DPR/motion/axis catalog with semantic, layout, and tolerant raster comparison | Hard Chromium/Firefox/WebKit CI with always-retained JSON; `IMPLEMENTED` for TST-NI-015 | +| `scripts/pick_boundary_smoke.py` | Large trace/index picking limits | Hard Chromium CI/browser lane with retained JSON evidence; `IMPLEMENTED` | +| `scripts/animation_smoke.py` | Real-browser animation lifecycle/pixels/allocation | Hard Chromium CI/browser lane with retained JSON evidence; `IMPLEMENTED` | +| `scripts/reflex_ws_smoke.py` | Real Reflex websocket/browser path: shared socket, binary paint, drill/pick state, streaming, renderer teardown, and transport close | Hard dedicated Reflex floor/latest CI with retained log/screenshot evidence; `IMPLEMENTED` for TST-NI-004 | +| `scripts/host_integration_policy.py` and `tests/test_host_integration_policy.py` | Exact supported/floor host inventory, package-metadata grounding, and installed-version reports | Hard Anywidget/FastAPI and Reflex floor/latest jobs; `IMPLEMENTED` for TST-NI-019 | +| `tests/host_integration/` | Anywidget trait/comm mount plus FastAPI compile, route mount, and HTTP transport at both supported edges | Hard zero-skip `host_integration` job; `IMPLEMENTED` for TST-NI-019 | +| `scripts/fastapi_host_smoke.py` | Current-matrix Uvicorn, nonblank Chromium WebGL2 mount, and browser-originated drilldown | Hard floor/latest `host_integration` job with JSON/screenshot/log evidence; `IMPLEMENTED` for TST-NI-019 | +| `scripts/pyodide_load_smoke.py` | Built WASM wheel runtime load | Release-only exact-artifact evidence | +| `scripts/native_parity.py` | Scalar/AVX2/aarch64 capability plus exact kernel/FFI/raster parity | Hard native host matrix and selected manylinux/musllinux artifact runtimes; `IMPLEMENTED` | +| `scripts/verify_released_docs_quickstart.py` | Public quickstarts against the published wheel | Docs released-quickstart job | +| `scripts/check_release_version.py` and its tests | Tag, project version, and changelog coherence | Reused by exact-SHA release qualification for tag and manual real publication | +| `scripts/verify_source_qualification.py` and its tests | Exact commit, `main` ancestry, exact tag, newest exact-SHA `Required CI`, and release metadata | Required release/dev/stage-production preflight; `IMPLEMENTED` | +| `scripts/release_provenance.py` and its tests | Exact artifact-set names, sizes, SHA-256 hashes, source SHA, and duplicate-record rejection | Required before package publication; `IMPLEMENTED` | +| `scripts/verify_sdist.py` / `scripts/verify_wheel.py` and tests | Package contents, metadata, tags, hashes, and native/pure expectations | Hard CI/release structural gates | +| `scripts/verify_benchmark_report.py`, merge/regression scripts, and tests | Benchmark schema, coherence, merge, and deterministic regression policy | Hard harness tests; outcome-integrity mode `NOT IMPLEMENTED` | +| `scripts/verify_local.py` and tests | Named local-suite composition and execution | Current local command router; release-equivalent aggregation `NOT IMPLEMENTED` | +| `scripts/check_testing_spec.py` and its tests | Whole-spec links, anchors, commands, repository paths/symbols/jobs, evidence rows, status vocabulary, and stable gap-ID integrity | Hard all-path suite via `make check-testing-spec` and `make check-docs`; `IMPLEMENTED` | +| Hatch build-hook tests | Native build/no-toolchain build behavior | Repository pytest and packaging jobs | + +## Workflow and job registry + +| Workflow | Current jobs | Testing role | +|---|---|---| +| `ci.yml` | `dependency_audit`, `rust_release`, `native_parity`, `javascript_semantics`, `python_coverage`, `matplotlib_reference`, `test`, `host_integration`, `reflex_adapter`, `browser_conformance`, `python_floor`, `benchmark_vs`, `benchmark_methodology`, `benchmark`, `sdist`, `wheels`, `install_without_rust`, `required_ci` | Main hard and advisory code/package evidence; the stable hard aggregate includes dependency auditing, optimized Rust, four-host native parity, Python and JavaScript semantic/coverage gates, the bounded cross-browser and pan/zoom matrices, and Anywidget, FastAPI, and live/static Reflex floor/latest evidence on every pull-request path. | +| `codspeed.yml` | `benchmarks` | Advisory microbenchmark evidence | +| `docs.yml` | `released-quickstart`, `quality`, `production` | Published-wheel quickstart, docs tests/lint, and production-route matrix | +| `benchmark-refresh.yml` | `cross-library` | Manual scatter and core-2D refresh evidence | +| `release.yml` | `qualify`, `wheels`, `wasm`, `sdist`, `provenance`, `publish`, `publish-pyodide` | Exact-source qualification, artifact build/verification/provenance, and publication | +| `_build-docs-images.yml` | `build-and-push` | Reusable provenance-attested docs-image build with immutable digest outputs | +| `_helm-docs-pr.yml` | `open-helm-pr` | Reusable deployment PR automation that validates and pins tag-at-digest image references | +| `deploy-docs-dev.yml` | `prepare`, `qualify`, `build`, `helm-pr`, `update-last-sha` | Development deployment waits for exact-SHA hard CI and promotes built digests | +| `deploy-docs-stg.yml` | `prepare`, `qualify`, `build`, `helm-pr-stg`, `await-prod-approval`, `verify-prod-artifacts`, `release`, `helm-pr-prod` | Stage/production promotion qualifies the tag and verifies the same digests after approval | +| GitHub default CodeQL | Dynamic Actions, JavaScript/TypeScript, Python, and Rust analyses | Implemented code scanning; not a required merge check or dependency audit | diff --git a/spec/testing/dependency-audit-policy.json b/spec/testing/dependency-audit-policy.json new file mode 100644 index 00000000..050cacaf --- /dev/null +++ b/spec/testing/dependency-audit-policy.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "scanner": { + "name": "osv-scanner", + "version": "2.4.0", + "artifacts": { + "darwin-arm64": { + "url": "https://github.com/google/osv-scanner/releases/download/v2.4.0/osv-scanner_darwin_arm64", + "sha256": "9ca3185ad63e9ab54f7cb90f46a7362be02d80e37f0123d095a54355ea202f5d" + }, + "linux-x86_64": { + "url": "https://github.com/google/osv-scanner/releases/download/v2.4.0/osv-scanner_linux_amd64", + "sha256": "15314940c10d26af9c6649f150b8a47c1262e8fc7e17b1d1029b0e479e8ed8a0" + } + } + }, + "fail_severities": [ + "CRITICAL", + "HIGH", + "MODERATE", + "LOW", + "UNKNOWN" + ], + "environments": [ + { + "subsystem": "root-python", + "path": "uv.lock", + "format": "uv.lock" + }, + { + "subsystem": "docs-python", + "path": "docs/app/uv.lock", + "format": "uv.lock" + }, + { + "subsystem": "docs-javascript", + "path": "docs/app/reflex.lock/bun.lock", + "format": "bun.lock" + }, + { + "subsystem": "reflex-adapter-python", + "path": "python/reflex-xy/uv.lock", + "format": "uv.lock" + }, + { + "subsystem": "benchmark-python", + "path": "benchmarks/requirements-ci.lock", + "format": "requirements.txt" + }, + { + "subsystem": "native-rust", + "path": "Cargo.lock", + "format": "Cargo.lock" + }, + { + "subsystem": "browser-javascript", + "path": "package-lock.json", + "format": "package-lock.json" + } + ], + "excluded_locks": [ + { + "path": "benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/uv.lock", + "owner": "@reflex-dev/xy", + "reason": "Immutable historical launch-measurement evidence; it is never installed or executed by a current subsystem." + }, + { + "path": "examples/reflex/.web/bun.lock", + "owner": "@reflex-dev/xy", + "reason": "Example fixture lock is not an installed production subsystem." + }, + { + "path": "examples/reflex/reflex.lock/bun.lock", + "owner": "@reflex-dev/xy", + "reason": "Example fixture lock is not an installed production subsystem." + } + ], + "exceptions": [] +} diff --git a/spec/testing/dependency-auditing.md b/spec/testing/dependency-auditing.md new file mode 100644 index 00000000..e88e7792 --- /dev/null +++ b/spec/testing/dependency-auditing.md @@ -0,0 +1,88 @@ +# Dependency Vulnerability Auditing + +The repository has one hard vulnerability policy for every active committed +dependency environment. The `dependency_audit` CI job runs on every pull +request, is a dependency of `required_ci`, and cannot be made advisory without +failing the workflow contract tests. + +## Audited environments + +[`dependency-audit-policy.json`](dependency-audit-policy.json) is the +machine-readable inventory. [`scripts/dependency_audit.py`](../../scripts/dependency_audit.py) +requires this exact set and also discovers supported lockfile names so a newly +committed lock cannot remain unaudited silently. + +| Subsystem | Lock | OSV-Scanner format | +|---|---|---| +| Root Python package and development environment | `uv.lock` | `uv.lock` | +| Documentation Python application | `docs/app/uv.lock` | `uv.lock` | +| Documentation generated frontend | `docs/app/reflex.lock/bun.lock` | `bun.lock` | +| Reflex adapter | `python/reflex-xy/uv.lock` | `uv.lock` | +| CI benchmark environment | `benchmarks/requirements-ci.lock` | `requirements.txt` | +| Native Rust core | `Cargo.lock` | `Cargo.lock` | +| Root browser tooling | `package-lock.json` | `package-lock.json` | + +The sole inventory exclusion is +`benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/uv.lock`. It is an +immutable historical measurement fixture and is not installed or executed by a +current subsystem. Its exact path, owner, and reason are fixed in the policy; +adding another exclusion requires changing and reviewing the validator. + +## Hard policy + +CI downloads OSV-Scanner 2.4.0 from its versioned release URL and verifies the +reviewed Linux x86-64 SHA-256 before execution. The runner verifies the binary +again, validates its reported version, commit, and build timestamp, downloads +fresh offline advisory archives, and submits every lock explicitly. A scan is +invalid if any inventoried source is missing, repeated, unrequested, or yields +zero packages. + +Every severity class is blocking: `CRITICAL`, `HIGH`, `MODERATE`, `LOW`, and +`UNKNOWN`. This deliberately makes a vulnerability without a published CVSS +score fail closed. The job succeeds only when every finding has been removed or +has one valid exact exception. + +An exception has this shape: + +```json +{ + "id": "GHSA-1234-5678-9abc", + "package": {"ecosystem": "PyPI", "name": "example"}, + "environment": "root-python", + "owner": "@reflex-dev/xy", + "reason": "Why this exact affected path is temporarily acceptable.", + "expires": "2026-08-20" +} +``` + +IDs, ecosystems, package names, and audited subsystem names match exactly; +wildcards are forbidden. Owners must be a GitHub user/team or email address, +reasons must be substantive, and expiration must be in the future and no more +than 180 days from validation. Expired, duplicate, malformed, and no-longer-used +exceptions fail the job. + +## Retained evidence + +The `dependency-audit` artifact is retained for 30 days and contains: + +- `osv-results.json`, the scanner's full machine-readable findings; +- `dependency-audit.json`, the normalized policy outcome, package counts, + exception decisions, scanner identity, and per-ecosystem database retrieval + timestamp, latest archive-entry timestamp, entry count, size, and SHA-256; +- `scanner-version.txt`, the scanner, engine, commit, and build timestamp; and +- `scanner-output.txt`, the execution log. + +The advisory database archives are used during the job but are not uploaded; +their recorded hashes and timestamps identify the exact snapshots used. This +gate detects vulnerabilities represented in the OSV databases. It does not +replace source code scanning, artifact provenance, or review of malicious or +compromised packages that have no advisory. + +Run policy and lock-inventory validation locally with: + +```console +python3 scripts/dependency_audit.py validate +``` + +The full scan additionally requires a reviewed OSV-Scanner binary and is the +command executed by the hard CI job. diff --git a/spec/testing/gaps.md b/spec/testing/gaps.md new file mode 100644 index 00000000..29429181 --- /dev/null +++ b/spec/testing/gaps.md @@ -0,0 +1,717 @@ +# Testing Gap Register + +This is the stable, prioritized implementation register for XY testing and +enforcement. IDs are never removed or renumbered when work closes. Incomplete +entries are `NOT IMPLEMENTED`; an entry becomes `IMPLEMENTED` only after all of +its completion criteria are automated, explicit evidence is recorded here, and +[`current.md`](current.md) is updated. Supporting scripts alone do not qualify. + +Priority means: + +- P0: required before a green check can be treated as a release certificate; +- P1: material product, platform, oracle, or integration coverage; and +- P2: reliability, maintainability, observability, and longer-horizon depth. + +Implementation pull requests should add an owner or issue, exact environment, +independent oracle and negative control, command/job, gate tier, skip policy, +and retained artifact to the relevant entry before changing its status. + +## P0 — Release confidence + +### TST-NI-001 — Required hard-CI aggregate and repository rule + +- Status: `NOT IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Current gap: `required_ci` now evaluates every declared hard dependency on + every pull-request path and rejects failed, cancelled, skipped, missing, or + unexpected jobs. `main` still has no rule requiring the stable `Required CI` + result, so the protection can be bypassed until the repository ruleset is + updated. +- Implemented when: one `required-ci` job uses `if: always()`, runs on every pull + request path, fails for failed/cancelled/unexpectedly skipped hard jobs, excludes + advisory timing, and is required by the repository ruleset. + +### TST-NI-002 — Strict dashboard 10/20/50 health gate + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `benchmarks/bench_dashboard.py`, `scripts/verify_benchmark_report.py`, `tests/test_verify_benchmark_report.py`, and the hard-CI `test` job's `dashboard-health-evidence` artifact. +- Current gap: closed; hard CI selects the strict profile, and the context + governor now completes the 10/20/50 visit matrix without unexplained loss. +- Implemented when: a strict profile requires one healthy row for every requested + count; governed charts must all be created and nonblank when visited; missing, + failed, partial, or unexplained-loss rows exit nonzero; mutation tests prove + rejection; and hard CI explicitly selects that profile. + +### TST-NI-003 — Exact-SHA release, deployment, and provenance preflight + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `scripts/verify_source_qualification.py`, `scripts/release_provenance.py`, `tests/test_verify_source_qualification.py`, `tests/test_release_provenance.py`, `tests/test_verify_ci_workflow.py`, and the `qualify` / `provenance` release and deployment jobs. +- Current gap: closed; package publication qualifies the exact tagged source + and full artifact set, while docs deployments qualify the exact source and + pin the same provenance-attested images by ECR digest across environments. +- Implemented when: every publish or promotion path verifies tag/version/ + changelog, tag SHA, `main` ancestry, exact-SHA hard-test success, and artifact + hashes/provenance; manual dispatch cannot bypass it; dev/stage/production wait + for it; and the same immutable artifacts are promoted. + +### TST-NI-004 — Required Reflex adapter and browser E2E lane + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `python/reflex-xy/tests/`, `scripts/run_pytest_no_skips.py`, + `scripts/reflex_ws_smoke.py`, and the required `reflex_adapter` CI job's + floor/latest `reflex-e2e-*` artifacts. +- Current gap: closed; package-owned dependencies fail collection when missing, + and the real production example is a zero-skip required browser lane. +- Implemented when: a dedicated job installs root XY and + `python/reflex-xy[dev]`, requires zero dependency skips, runs all adapter cases, + compiles/starts the real example, and proves socket, binary paint, drill, + state, streaming, and teardown behavior at the supported Reflex floor/latest. + +### TST-NI-005 — Specification and testing-catalog validation + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `scripts/check_testing_spec.py::main`, `scripts/check_claim_guardrails.py`, + `make check-testing-spec`, and the all-path `required_ci` job. +- Current gap: closed; the whole specification tree and its public claims are + checked, and specification-only pull requests receive the stable hard result. +- Implemented when: spec-only pull requests receive the stable required result + and validation extends across `spec/` — Markdown links, commands, referenced + files/symbols/jobs, status vocabulary, evidence rows, claim guardrails, and + dead or duplicate gap IDs. + +## P1 — Product and integration coverage + +### TST-NI-006 — All-public-builder property and transaction matrix + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `tests/test_property_figure.py`, the required Hypothesis development + dependency, and `make check-errors` / the hard root test suite. +- Current gap: closed; all 20 public builders have must-succeed valid strategies, + classified-invalid strategies, and append-then-raise rollback proofs against + a seeded figure with exact state snapshots. +- Implemented when: all builders have valid strategies that must succeed, + classified invalid strategies, injected late failures on nonempty state, and + exact pre/post snapshots of traces, columns/dedup keys, axes/categories, + annotations, caches, pyramids, and drill state. + +### TST-NI-007 — First-class JavaScript semantic unit suite + +- Status: `IMPLEMENTED` +- Evidence: `js/test/frame.test.mjs`, `js/test/semantics.test.mjs`, + `js/test/worker.test.mjs`, `make js-test`, and the required + `javascript_semantics` job's `javascript-semantic-evidence` artifact. +- Current gap: closed; Node 22 runs the exact fresh ESM bundle with no test + dependencies, rejects malformed and deliberately mutated behavior, enforces + coverage floors, and retains JUnit, textual coverage, and raw V8 coverage. +- Implemented when: a pinned `node --test` or equivalent hard lane covers frame + decode, ticks/formatters, transforms/bounds, theme/style normalization, mark + registry, LOD choice, ChartView state, and worker protocol with malformed + cases, negative controls, and coverage output. + +### TST-NI-008 — Rendered-label and formatter oracle + +- Status: `IMPLEMENTED` +- Owner: renderer maintainers +- Evidence: `spec/testing/rendered-label-policy.md`, `js/tests/rendered_labels.test.mjs`, `tests/test_rendered_label_formats.py`, and the hard `Rendered label and formatter oracle (Chromium)` CI step with retained `rendered-label-evidence` JSON. +- Current gap: closed; Python and raw-spec validation reject unsupported or + kind-mismatched formats, while unit and real-DOM oracles cover the declared + label surface in two non-UTC locale/time-zone contexts. +- Environment: Node 22 in CI, package-pinned Playwright `1.61.1` Chromium, + `en-US` / `America/Los_Angeles`, and `de-DE` / `Asia/Tokyo`. +- Oracle and negative control: exact DOM text is independent of serialized + specs; malformed raw axis/tooltip formats must throw, and a corrupted DOM + label must make the comparator fail. +- Command / gate / artifact: `npm run test:labels` and `make check-labels`; hard + main-CI `test` job; always-retained `rendered-label-evidence.json`. +- Skip policy: package-owned dependencies are required; missing Playwright or + Chromium fails, with no `importorskip`, optional-import, or browser skip path. +- Implemented when: policy explicitly supports or rejects each format and unit/ + DOM tests assert numeric, grouping, currency-or-error, percent, time, log, + category, named-axis, tooltip, and colorbar labels under at least two locales + and non-UTC host time zones, with UTC output invariant and a broken-formatter + negative control. + +### TST-NI-009 — Registry-driven every-chart-kind render matrix + +- Status: `IMPLEMENTED` +- Evidence: `scripts/chart_kind_matrix.py`, `tests/test_chart_kind_matrix.py`, + the hard `Every chart-kind render matrix (Chromium)` CI step, and retained + `chart-kind-matrix-evidence` JSON. +- Current gap: closed; 16 public-builder fixtures exactly cover all 18 shipped + registry kinds, validate payload kind/tier/counts and live GPU geometry, and + require independently measured colored pixels. Missing-kind, wrong-kind, and + blank-render controls are rejected. +- Implemented when: a fixture catalog keyed to the public mark registry requires + payload/tier and nonblank semantic geometry/pixel evidence for every primitive, + including error, step/stem/segment, box/violin, contour/hexbin, and triangle + mesh families; adding a kind without evidence fails. + +### TST-NI-010 — Cross-renderer semantic parity catalog + +- Status: `NOT IMPLEMENTED` +- Current gap: strong spot checks do not guarantee equivalent geometry/style + across WebGL, SVG, native raster, and PDF for every claimed feature. +- Implemented when: each mark/feature declares applicable renderers and an + independent coordinate/count/label oracle; exact buffers match where identity + is claimed, pixels use reviewed tolerances, and each primitive has a negative + control. + +### TST-NI-011 — Pan/zoom acceptance matrix + +- Status: `IMPLEMENTED` +- Evidence: `scripts/pan_zoom_matrix.mjs`, `tests/test_pan_zoom_matrix.py`, + `make check-pan-zoom`, the hard `Pan/zoom acceptance matrix (Chromium)` and + `Focused pan/zoom matrix (three engines)` CI steps, the Reflex floor/latest + host matrix, and retained `pan-zoom-matrix-evidence`, + `pan-zoom-cross-engine-evidence`, and `reflex-pan-zoom-*` JSON artifacts. +- Current gap: closed; a bounded catalog drives real pointer, wheel, and + modebar input, verifies semantic/layout invariants and failure-safe evidence, + and its catalog, evidence schema, local wiring, and workflow gates have + negative controls. +- Implemented when: data-driven browser cases cover drag, wheel, box, toolbar + zoom, and reset across linear/log/reversed/category/dual/named axes, bounds, + limits, linking, reduced motion, no-op event/LOD semantics, and live/static + Reflex, with Chromium hard and a focused three-engine subset. + +### TST-NI-012 — Pick-boundary browser gate + +- Status: `IMPLEMENTED` +- Evidence: `scripts/pick_boundary_smoke.py`, `tests/test_pick_boundary_smoke.py`, + `make check-browser`, and the hard-CI `pick-boundary-evidence` artifact. +- Current gap: closed; the configured Chromium executable is authoritative, + boundary and cache semantics are hard-gated, and failures retain JSON evidence. +- Implemented when: it is a named hard check for 256 trace slots including 255, + large indices, global/local mapping, and pick-cache reuse/invalidation, with + catalog wiring and retained failure evidence. + +### TST-NI-013 — Animation browser gate + +- Status: `IMPLEMENTED` +- Evidence: `scripts/animation_smoke.py`, `tests/test_animation_smoke.py`, + `make check-browser`, and the hard-CI `animation-browser-evidence` artifact. +- Current gap: closed; fixtures derive the shared protocol and configured Chrome + hard-gates animation semantics, pixels, allocation, lifecycle, and teardown. +- Implemented when: fixtures derive the shared protocol, the smoke runs in local + browser checks and hard CI, and it asserts keyed interpolation, fallback, + ghost-free pixels, allocation bounds, replacement, balanced lifecycle, + representative marks, reduced motion, and frozen export. Browser startup + failure must fail rather than skip. + +### TST-NI-014 — Reviewed visual regression baselines + +- Status: `IMPLEMENTED` +- Evidence: `scripts/visual_health_smoke.py`, `scripts/visual_baseline.py`, + `tests/test_visual_baseline.py`, `spec/visual-baselines/v1.json`, the hard + `Reviewed visual baseline (Chromium)` CI step, and its retained expected/ + actual/diff artifacts. +- Current gap: closed; the health probe remains separate while the reviewed, + pinned baseline gate checks semantic identity and tolerant pixels and rejects + corrupted data, color, label, and geometry controls. +- Implemented when: the health smoke is retained and honestly named, while a + small versioned baseline set uses pinned browser/font/viewport/DPR, semantic + plus perceptual diff, explicit tolerances, corrupted-data/color/label/geometry + negative controls, reviewed update policy, and expected/actual/diff artifacts. + +### TST-NI-015 — Broader cross-browser, DPR, and motion conformance + +- Status: `IMPLEMENTED` +- Owner: renderer maintainers +- Evidence: `spec/testing/browser-conformance-policy.md`, `scripts/browser_conformance.mjs`, `tests/test_browser_conformance.py`, and the hard `browser_conformance` CI job's retained `browser-conformance-evidence` JSON artifact. +- Current gap: closed; the exact six-case catalog runs 18 browser contexts and + spans direct/decimated/density tiers, scatter/line/bar/heatmap/mesh families, + DPR 1/2, both motion preferences, and linear/log/category/named axes. +- Environment: Node 22, package-pinned Playwright `1.61.1`, bundled Chromium/ + Firefox/WebKit, Ubuntu Xvfb at 1920×1200, and a 760×480 CSS viewport. +- Oracle and negative control: each engine must satisfy semantic, layout, DPR, + motion, axis-binding, and nonblank-pixel assertions before live cross-engine + tolerant comparison; missing-catalog, corrupted-signature, and shifted-layout + mutations must fail. +- Command / gate / artifact: `make check-conformance`; required + `browser_conformance` job; always-retained `browser-conformance-evidence.json`. +- Skip policy: none; missing engines, launch/WebGL2 failures, page errors, or + incomplete cases are blocking. +- Implemented when: a bounded matrix spans direct/decimated/density and + representative line/bar/heatmap/mesh, DPR 1/2, reduced/no-preference motion, + and linear/log/category/named axes in Chromium, Firefox, and WebKit, while + preserving semantic/layout checks and tolerant pixel comparison. + +### TST-NI-016 — Required standalone-worker evidence + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `benchmarks/bench_interaction.py::run_worker_probe`, + `scripts/interaction_stress_smoke.py`, + `tests/test_interaction_stress_smoke.py`, and the hard-CI `test` job's + `interaction-worker-evidence` artifact. +- Current gap: closed; the hard browser lane rejects unavailable, skipped, + failed, incomplete, blank, or unterminated worker evidence. Only a direct + local diagnostic can explicitly opt out with `--allow-worker-skip`. +- Implemented when: required CI proves worker creation, message/re-bin result, + nonblank paint, and teardown; unavailable/skipped/failed is blocking in that + environment, while explicitly optional local use may skip. + +### TST-NI-017 — Normative accessibility behavior matrix + +- Status: `NOT IMPLEMENTED` +- Current gap: source assertions and one scatter fixture do not define the full + accessibility contract. +- Implemented when: a normative API spec covers roles/names, summaries/live + regions, focus order, keyboard behavior, aggregate-bin semantics, toolbar + state, reduced motion, forced colors, and table-alternative policy; browser + tests inspect DOM/accessibility behavior for direct and aggregate charts and + explicitly state the supported engine/OS/assistive-technology scope. + +### TST-NI-018 — Real anywidget/Jupyter frontend E2E + +- Status: `NOT IMPLEMENTED` +- Current gap: Python widget behavior is tested, but no supported notebook + frontend mounts the shipped ESM widget. +- Implemented when: a pinned frontend/anywidget environment mounts the widget, + receives split binary buffers, paints, sends pick/view/select, applies append/ + tier updates, and unmounts without listener, worker, GL, or socket leaks. + +### TST-NI-019 — Host-integration version matrix + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `spec/testing/host-integration-policy.json`, + `spec/design/host-compatibility.md`, `scripts/host_integration_policy.py`, + `tests/host_integration/`, `scripts/fastapi_host_smoke.py`, + `tests/test_host_integration_policy.py`, `tests/test_fastapi_host_smoke.py`, + `tests/test_verify_ci_workflow.py`, and the hard `host_integration` and + `reflex_adapter` floor/latest CI jobs with retained version, JUnit, browser, + log, and screenshot artifacts. +- Current gap: closed; bounded package-owned ranges are checked against project + metadata and both supported edges run compile, mount, transport, and browser + evidence without hidden dependency skips. A real notebook frontend remains + separately tracked by TST-NI-018. +- Implemented when: the supported ranges are documented and focused compile, + mount, transport, and browser tests run at floor/latest versions with zero + hidden dependency skips. + +### TST-NI-020 — Rust release tests + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: the hard `rust_release` job and its retained + `rust-release-test-inventory` artifact, `scripts/verify_ci_workflow.py`, + `tests/test_verify_ci_workflow.py`, and + `src/tiles.rs::compose_window_astronomically_past_domain_is_empty_not_panic`. +- Current gap: closed; locked release-profile tests are a required aggregate + dependency, and CI inventories the named release-only regression before + running the full optimized suite. +- Implemented when: `cargo test --locked --release` is a hard job, workflow + validation requires it, and release-only test coverage cannot silently drop. + +### TST-NI-021 — Native fuzzing, sanitizers, and Miri + +- Status: `NOT IMPLEMENTED` +- Current gap: fixed-seed randomized loops are useful regressions, not + coverage-guided fuzzing; unsafe FFI and raster/parser boundaries lack dynamic + hardening lanes. +- Implemented when: committed cargo-fuzz targets/corpora cover pointer/length and + command-buffer inputs, scheduled ASan/UBSan/Miri-compatible jobs run with + bounded budgets, and crashes/reproducers are retained. Deterministic randomized + tests remain in the normal suite and are named accurately. + +### TST-NI-022 — Multi-ecosystem dependency vulnerability policy + +- Status: `IMPLEMENTED` +- Evidence: [`dependency-auditing.md`](dependency-auditing.md), + `spec/testing/dependency-audit-policy.json`, + `scripts/dependency_audit.py`, the hard `dependency_audit` job in `ci.yml`, + the patched `docs/app/reflex.lock/package.json` and + `docs/app/reflex.lock/bun.lock`, + `scripts/verify_ci_workflow.py::validate_ci_workflow`, + `tests/test_dependency_audit.py`, and + `tests/test_verify_ci_workflow.py`. +- Current gap: closed; every active committed root, docs, adapter, benchmark, + Rust, Bun, and npm lock is scanned under one fail-closed policy, while the + only historical-lock exclusion is exact and validator-owned. +- Implemented when: supported scanners audit the relevant Python locks/ + environments, `Cargo.lock`, and `package-lock.json`; reviewed severities fail; + scanner/database timestamps and machine-readable findings are retained; and + exceptions have owners, reasons, and expirations. + +### TST-NI-023 — SIMD and native artifact runtime parity + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `src/lib.rs::xy_runtime_capabilities`, the stdlib-only + `scripts/native_parity.py` oracle, the hard `native_parity` matrix in + `ci.yml` with retained `native-parity-*` reports, and the pinned manylinux + 2.17/musllinux 1.2 runtime steps and artifacts in `release.yml`. +- Current gap: closed for the focused native contract. The wider public Python + behavior matrix remains tracked by TST-NI-049. +- Implemented when: scalar/AVX2/aarch64 capability is reported and exercised; + focused kernel/FFI/raster parity runs on native Linux x64/ARM64, Windows x64, + and macOS ARM64; and selected manylinux/musllinux release artifacts run under + appropriate native or emulated environments. + +### TST-NI-024 — Runtime DOM-XSS, CSP, and network-isolation tests + +- Status: `IMPLEMENTED` +- Evidence: `scripts/runtime_security_smoke.py`, + `tests/test_runtime_security_smoke.py`, the structural icon-sink allowlist in + `tests/test_static_client_security.py`, `make check-browser`, and the hard-CI + `runtime-security-evidence` artifact. +- Current gap: closed for the standalone page-content boundary. A real + production export sends hostile strings through 16 title/axis/tick/trace/ + category/annotation/legend/colorbar/tooltip surfaces, exercises hostile CSS, + and requires literal DOM text, no executable user nodes/script/dialogs, a + browser-observed CSP block, and zero loopback HTTP requests. Chromium process + sandbox policy remains the separate TST-NI-025 scope. +- Implemented when: hostile strings flow through every public text surface; + tests assert literal text, no executable nodes/dialogs, and no network request + under standalone CSP; hostile CSS is exercised; and fixed internal icon sinks + are structurally allowlisted. + +### TST-NI-025 — Observable or opt-in Chromium sandbox downgrade + +- Status: `IMPLEMENTED` +- Owner: unassigned — file a tracking issue before implementation starts +- Evidence: `python/xy/export.py::html_to_png`, + `python/xy/export.py::_browser_session`, sandbox-policy tests in + `tests/test_figure.py` and `tests/test_batch_export.py`, and the explicit + trusted-CI opt-in in `scripts/png_export_smoke.py`. +- Current gap: closed; sandboxed launches fail once with actionable guidance, + never retry unsandboxed, and `--no-sandbox` requires explicit + `sandbox=False` at the call site. +- Implemented when: policy chooses explicit opt-in or an observable warning, + launch-failure tests assert exact arguments and diagnostics, no silent downgrade + occurs, and trusted CI requests `sandbox=False` explicitly when necessary. + +### TST-NI-026 — Stronger Matplotlib semantic and perceptual oracles + +- Status: `NOT IMPLEMENTED` +- Current gap: the broad corpus can accept incorrect or empty data in some + cases, and contour/vector/transform/tick semantics have known weak spots. +- Implemented when: high-risk scripts gain independent contour geometry/levels, + quiver magnitude/direction, transform, tick, category, and axis-content + comparisons plus reviewed perceptual fixtures and wrong-data negative controls. + +### TST-NI-027 — Sound pyplot unused-option detector + +- Status: `IMPLEMENTED` +- Evidence: `scripts/check_pyplot_options.py`, + `spec/testing/pyplot-noops.json`, `tests/test_check_pyplot_options.py`, + `tests/pyplot/test_compat_noops.py`, `make check-pyplot`, and the hard-CI + `pyplot-option-contract` artifact. +- Current gap: closed; the AST/dataflow audit detects unread named options and + discarded literal `kwargs`/`options` pops, follows only reachable local + closures, and requires an exact structured registry with substantive + rationale and executable behavioral evidence for all 34 deliberate no-ops. +- Implemented when: AST/dataflow or runtime instrumentation proves each supported + option affects state/geometry or maps to a structured, reviewed no-op with a + rationale and test; adding an accepted unused option fails. + +### TST-NI-028 — Catalog-driven protocol conformance and byte mutation + +- Status: `IMPLEMENTED` +- Evidence: [`protocol-conformance.md`](protocol-conformance.md), + `spec/testing/protocol-catalog.json`, `tests/test_protocol_catalog.py`, + `tests/test_framing.py`, `tests/test_framing_property.py`, + `make check-protocol`, and the hard-CI `protocol-conformance-evidence` + JUnit artifact. +- Current gap: closed; the catalog is checked against the dispatcher's AST and + generates all five case classes for every request, all reply schemas carry + committed Python/JavaScript golden frames, and required Hypothesis batches + mutate every structural frame class while enforcing decoder parity. +- Implemented when: one protocol catalog generates valid, missing, wrong-type, + bounds, and callback cases for every request/reply; shared golden frames run in + Python and JavaScript; and property mutations of headers, lengths, counts, + padding, and metadata always reject safely or preserve parity. + +### TST-NI-029 — Branch and diff-coverage ratchet + +- Status: `IMPLEMENTED` +- Evidence: [`coverage.md`](coverage.md), + `spec/testing/coverage-policy.json`, `scripts/coverage_ratchet.py`, + `tests/test_coverage_ratchet.py`, the hard `python_coverage` job, and its + retained `python-coverage-evidence` raw/JSON/XML/ratchet artifact. +- Current gap: closed for shipped Python and the report-before-ratchet rule. + Reviewed package and critical-module line/branch floors cannot regress, every + shipped module must be measured, and changed executable lines require 90% + coverage. JavaScript already retains its independent V8 report and floors; + Rust remains explicitly unratcheted until a pinned Rust coverage report exists. +- Implemented when: branch-aware artifacts are uploaded, reviewed package/module + baselines cannot regress, executable changed lines meet a concrete threshold, + and exclusions are explicit. Rust/JavaScript reports precede ratchets for those + languages. + +### TST-NI-030 — Targeted mutation testing + +- Status: `NOT IMPLEMENTED` +- Current gap: there is no mutation score or survivor review for high-risk + validators, protocols, or report verifiers. +- Implemented when: scheduled or changed-path mutation lanes cover Python + validators/protocol/report policy, JavaScript frame/format/bounds logic, and + safe Rust kernel/raster parsers; a baseline and justified survivors are + recorded; and seeded known mutants prove the lane detects weak oracles. + +## P2 — Test-system reliability and depth + +### TST-NI-031 — Strict marker, skip, and xfail policy + +- Status: `NOT IMPLEMENTED` +- Current gap: a single dependency skip can hide a suite and configured browser + failures can become skips without an enforced per-job budget. +- Implemented when: registered markers describe all layers; each job declares its + selection and allowed skips; skip reasons/budgets are enforced; required + dependency/browser skips fail; and strict xfails carry an issue and expiry. + +### TST-NI-032 — Classified retry and flake provenance + +- Status: `NOT IMPLEMENTED` +- Current gap: browser helpers retry broad failures and do not retain complete + attempt history. +- Implemented when: only enumerated launch/GPU infrastructure failures retry, + semantic assertions never retry, every attempt/status/duration is retained in + JSON/JUnit and job summaries, and flaky-pass thresholds require quarantine or + repair with an issue. + +### TST-NI-033 — Event-driven asynchronous adapter tests + +- Status: `NOT IMPLEMENTED` +- Current gap: several adapter/socket assertions depend on fixed sleeps. +- Implemented when: positive paths use events, queues, acknowledgements, and + bounded waits; unsubscribe is acknowledged before a documented no-delivery + window; and arbitrary sleep is not load-bearing. + +### TST-NI-034 — Resource-leak and soak evidence + +- Status: `NOT IMPLEMENTED` +- Current gap: lifecycle smokes check successful teardown but do not quantify + repeated listener/context/worker/socket/temp-file/FD/heap growth. +- Implemented when: short hard loops and longer scheduled soaks repeatedly create, + update, and destroy ChartView, worker, GL context, adapter subscription, + streaming, and persistent browser export resources with published bounds. + +### TST-NI-035 — Failure evidence artifacts + +- Status: `NOT IMPLEMENTED` +- Current gap: benchmarks and packages retain artifacts, but hard tests do not + consistently retain JUnit, coverage, browser screenshots, console output, + traces, or retry history. +- Implemented when: every hard job emits a structured result; browser failures + upload expected/actual/diff images plus console/page/CDP evidence; coverage, + skip, and retry summaries are visible; and uploads use `if: always()`. + +### TST-NI-036 — Reproducible hard environment and support matrix + +- Status: `NOT IMPLEMENTED` +- Current gap: root hard CI floats `.[dev]` and tool versions, Python versions are + partly incidental, and the floor lane omits relevant optional test dependencies. +- Implemented when: the hard baseline uses a frozen lock and pinned uv/Python/ + Node/Rust/browser versions; full floor and newest claimed Python lanes plus + focused middle versions run; a genuine minimum-dependency lane exists; and a + separate latest-dependency canary exposes ecosystem drift. + +### TST-NI-037 — Honest local and release suite aggregation + +- Status: `NOT IMPLEMENTED` +- Current gap: `make check-full` excludes browser conformance, dashboard, + packaging, animation, pick-boundary, and Reflex evidence. +- Implemented when: it is renamed to `check-full-nonbrowser` everywhere or a + manifest-driven `check-release` runs all locally possible hard gates, prints + the exact CI-only remainder, and is kept coherent with Makefile, workflows, + docs, and this catalog. + +### TST-NI-038 — Semantic workflow and gate-manifest validation + +- Status: `NOT IMPLEMENTED` +- Current gap: the bespoke workflow checker relies on strings/regexes and does + not validate dependency dominance across all workflows. +- Implemented when: actionlint and real YAML parsing cover CI, CodSpeed, docs, + deploy, reusable, benchmark-refresh, and release workflows; a declarative + manifest defines job tier/dependencies/outcome/skips/artifacts; and redundant + exact-step tests are retired after semantic parity exists. + +### TST-NI-039 — Source-substring test reduction + +- Status: `NOT IMPLEMENTED` +- Current gap: many tests inspect exact source snippets for behavior, creating + false failures and false confidence; the stale Reflex assertion is one example. +- Implemented when: source checks are limited to freshness, generated boundaries, + and forbidden constructs; runtime/DOM units own semantics; shipped bundles are + exercised; and duplicate CI/local/docs command strings derive from the gate + manifest. + +### TST-NI-040 — Positive Rust-enabled sdist installation + +- Status: `NOT IMPLEMENTED` +- Current gap: sdist validation proves structure and the expected no-Rust failure, + not a fresh successful native build from the produced archive. +- Implemented when: the exact sdist installs in a clean Rust-enabled environment, + loads the native core, runs ABI plus representative kernel/export smoke, and is + retained as an artifact. The clear no-Rust failure lane remains. + +### TST-NI-041 — Regular Pyodide regression lane + +- Status: `NOT IMPLEMENTED` +- Current gap: the real WASM build/runtime probe runs only in the release workflow. +- Implemented when: relevant Rust/build/package changes trigger a pull-request or + scheduled build and runtime kernel probe using the pinned toolchain tuple, while + release continues to test the exact artifact it publishes. + +### TST-NI-042 — Benchmark availability and integrity mode + +- Status: `NOT IMPLEMENTED` +- Current gap: expected rivals or rows can be unavailable, skipped, timed out, or + failed while schema verification and the advisory job remain green. +- Implemented when: expected competitors have import smokes with retained error + details and version metadata; required scenarios/sizes have an allowlisted + status policy; integrity failures are internally nonzero; report summaries + separate integrity from advisory speed; and ordinary timing stays non-blocking. + +### TST-NI-043 — Generated native-font freshness + +- Status: `NOT IMPLEMENTED` +- Current gap: `src/font.rs` says it is generated by `scripts/gen_font.py`, but no + deterministic check mode compares regenerated output. +- Implemented when: pinned font/tool inputs regenerate to a temporary location, + byte-for-byte comparison is a relevant hard gate, and a stale committed font + fails without rewriting the tree. + +### TST-NI-044 — Type and static-analysis ratchets + +- Status: `NOT IMPLEMENTED` +- Current gap: `ty` accepts an unbounded diagnostic count, JavaScript has syntax + checks but no lint/static semantic policy, and workflow syntax uses a bespoke + checker. +- Implemented when: core and adapter type checks run in correctly provisioned + environments against reviewed baselines and reject new findings; JavaScript + lint/static sink rules are hard; actionlint covers workflows; and existing + Ruff/format/clippy gates remain hard. + +### TST-NI-045 — Independent PDF and browser-export oracle policy + +- Status: `NOT IMPLEMENTED` +- Current gap: local PDF checks can omit external tools, and public persistent + Chromium batch behavior is not exercised through all formats in CI. +- Implemented when: at least one required job guarantees an independent PDF + parser/renderer and asserts pages, dimensions, content, and nonblank semantics; + PNG/JPEG/WebP/PDF browser fixtures use independent decode where applicable; + and session reuse, mixed formats, failure cleanup, and sandbox policy run E2E. + +### TST-NI-046 — Protocol/version documentation coherence + +- Status: `NOT IMPLEMENTED` +- Current gap: Python and JavaScript runtime protocol constants are 4, while + documentation and an unwired animation fixture retained protocol 3. +- Implemented when: a machine-readable source or checker keeps Python, JavaScript + source/bundles, wire-protocol docs, examples, browser fixtures, and mismatch + tests aligned; a protocol bump updates compatibility/changelog evidence in the + same pull request; and hard-coded fixture versions are rejected. + +### TST-NI-047 — Required pandas interoperability lane + +- Status: `NOT IMPLEMENTED` +- Current gap: pandas Series/Period compatibility tests use + `pytest.importorskip("pandas")`, while normal CI does not install pandas. +- Implemented when: a focused optional-integration job runs at the documented + pandas floor/latest with zero pandas skips and verifies Period/datetime + converter semantics plus rendered output. If pandas is unsupported, the tests + and public boundary must say so explicitly instead. + +### TST-NI-048 — Real-browser animation measurement lane + +- Status: `NOT IMPLEMENTED` +- Current gap: `benchmarks/bench_animation.py` measures Chrome frame pacing, + memory, and scratch allocation but is absent from CI, Makefile, benchmark + refresh, and report verification. +- Implemented when: an advisory browser/scheduled lane emits the standard + environment/category/status schema, validates lifecycle and bounded scratch/ + heap as integrity separate from timing, uploads JSON, and has verifier negative + tests. Existing Python-only CodSpeed animation microbenchmarks remain distinct. + +### TST-NI-049 — Cross-platform behavioral test matrix + +- Status: `NOT IMPLEMENTED` +- Current gap: Python/Rust behavior suites run primarily on Ubuntu; Windows, + Linux ARM64, and macOS lanes mostly prove wheel build/install/import. +- Implemented when: supported host classes install the actual built wheel and run + one documented public API, kernel, encoding/framing, and export parity subset + with identical fixture digests and zero unapproved skips. Keep this distinct + from artifact structure and import smoke. + +### TST-NI-050 — Machine-checkable claim-to-test traceability + +- Status: `NOT IMPLEMENTED` +- Current gap: broad risk-surface rows do not yet map every material normative + claim to a stable ID and exact evidence record. +- Implemented when: material requirements have stable IDs and a checked registry + records authoritative spec, current/target gate, environment, evidence, status, + oracle, skips, artifacts, and owner; validation rejects orphan claims, dead + references, duplicate IDs, and `IMPLEMENTED` evidence that can silently skip. + +### TST-NI-051 — Bounded execution and timeout policy + +- Status: `NOT IMPLEMENTED` +- Current gap: several hard jobs and hang-prone subprocess/browser paths rely on + runner ceilings rather than reviewed bounds. +- Implemented when: every hard/release job has `timeout-minutes`, subprocess and + browser waits are explicit and clean up, targeted per-test timeout enforcement + covers hang risks, timeout failures retain diagnostics, and workflow tests + reject removing the bounds. + +### TST-NI-052 — Hardware GPU and driver validation + +- Status: `NOT IMPLEMENTED` +- Current gap: deterministic headless lanes primarily exercise CI software + rendering, not representative hardware WebGL2 drivers. +- Implemented when: a scheduled or release lane runs a high-risk render, + interaction, and context-loss subset on declared hardware-backed macOS, + Windows, and/or Linux environments; records renderer/driver metadata; rejects + software fallback for that lane; and retains semantic/golden evidence. + +### TST-NI-053 — Browser renderer failure-mode E2E + +- Status: `NOT IMPLEMENTED` +- Current gap: real-browser tests prove successful context restoration, while + WebGL2 acquisition, shader/program, and permanent-restore failures are mostly + protected by source markers. +- Implemented when: browser fixtures force each failure and assert one documented + user-visible error/lifecycle event, host callback propagation, no uncaught + exception/retry loop/resource leak, and retained logs/traces. + +### TST-NI-054 — Browser compatibility policy and floor/latest matrix + +- Status: `NOT IMPLEMENTED` +- Current gap: Playwright-pinned current engines run, but no browser-version + support floor or latest-only policy is normative. +- Implemented when: product specs state engine/version ranges and WebGL2 + prerequisites; CI tests current plus oldest claimed versions where feasible; + actual browser/renderer versions are recorded; and docs, dependency locks, and + the test catalog cannot drift independently. + +### TST-NI-055 — Fixture and golden provenance policy + +- Status: `NOT IMPLEMENTED` +- Current gap: protocol bytes, benchmark baselines, reference corpora, generated + snapshots, and future visual goldens do not share one provenance/review policy. +- Implemented when: every durable fixture declares source/oracle, seed/version/ + license, update command, reviewer rule, checksum/freshness method, and tolerance + rationale; deterministic artifacts regenerate in check mode; and a corrupted + fixture negative control proves the comparator fails. + +## Recommended implementation order + +1. Complete TST-NI-001 through TST-NI-005 so automation can be trusted to report + the state of the existing suite. +2. Wire strict dashboard, Reflex, animation, pick-boundary, skip policy, and + release/deploy qualification before expanding broad matrices. +3. Add the JavaScript unit layer, all-builder properties, protocol catalog, + rendered-label/accessibility behavior, and coverage artifacts as fast oracle + foundations. +4. Expand chart/axis/browser/platform matrices and packaging runtime evidence. +5. Add scheduled mutation, fuzz/sanitizer, GPU, visual, security, and soak depth. + +No entry is complete merely because work started. Remove it from this file only +after its exact protection appears as `IMPLEMENTED` in [`current.md`](current.md) +with executable evidence and the intended gate wiring. diff --git a/spec/testing/host-integration-policy.json b/spec/testing/host-integration-policy.json new file mode 100644 index 00000000..87c40b26 --- /dev/null +++ b/spec/testing/host-integration-policy.json @@ -0,0 +1,55 @@ +{ + "schema_version": 1, + "packages": { + "anywidget": { + "distribution": "anywidget", + "supported": ">=0.9,<1", + "floor": "==0.9.0" + }, + "traitlets": { + "distribution": "traitlets", + "supported": ">=5.14,<6", + "floor": "==5.14.0" + }, + "reflex": { + "distribution": "reflex", + "supported": ">=0.9.6,<1", + "floor": "==0.9.6" + }, + "fastapi": { + "distribution": "fastapi", + "supported": ">=0.110,<1", + "floor": "==0.110.0" + }, + "starlette": { + "distribution": "starlette", + "supported": ">=0.36.3,<1", + "floor": "==0.36.3" + }, + "uvicorn": { + "distribution": "uvicorn", + "supported": ">=0.29,<1", + "floor": "==0.29.0" + }, + "httpx": { + "distribution": "httpx", + "supported": ">=0.27,<1", + "floor": "==0.27.0" + } + }, + "hosts": { + "anywidget": [ + "anywidget", + "traitlets" + ], + "reflex": [ + "reflex" + ], + "fastapi": [ + "fastapi", + "starlette", + "uvicorn", + "httpx" + ] + } +} diff --git a/spec/testing/protocol-catalog.json b/spec/testing/protocol-catalog.json new file mode 100644 index 00000000..0aa5d182 --- /dev/null +++ b/spec/testing/protocol-catalog.json @@ -0,0 +1,144 @@ +{ + "schema_version": 1, + "request_case_kinds": ["valid", "missing", "wrong_type", "bounds", "callback"], + "requests": [ + { + "type": "view", + "cases": { + "valid": {"message": {"type": "view", "x0": 10, "x1": 100, "px": 320, "seq": 1}, "reply": "tier_update"}, + "missing": {"message": {"type": "view", "x1": 100}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "view", "x0": "left", "x1": 100}, "outcome": "drop"}, + "bounds": {"message": {"type": "view", "x0": 100, "x1": 10}, "outcome": "drop"}, + "callback": {"message": {"type": "view", "x0": 10, "x1": 100, "px": 320}, "reply": "tier_update", "callbacks": []} + } + }, + { + "type": "density_view", + "cases": { + "valid": {"message": {"type": "density_view", "trace": 2, "x0": 0, "x1": 50, "y0": 0, "y1": 50, "w": 32, "h": 32, "seq": 2}, "reply": "density_update"}, + "missing": {"message": {"type": "density_view", "trace": 2, "x0": 0, "x1": 50, "y0": 0}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "density_view", "trace": 2, "x0": 0, "x1": 50, "y0": 0, "y1": 50, "w": "wide"}, "outcome": "drop"}, + "bounds": {"message": {"type": "density_view", "trace": 999, "x0": 0, "x1": 50, "y0": 0, "y1": 50}, "outcome": "drop"}, + "callback": {"message": {"type": "density_view", "trace": 2, "x0": 0, "x1": 50, "y0": 0, "y1": 50, "w": 32, "h": 32}, "reply": "density_update", "callbacks": []} + } + }, + { + "type": "pick", + "cases": { + "valid": {"message": {"type": "pick", "trace": 1, "index": 0, "seq": 3}, "reply": "pick_result"}, + "missing": {"message": {"type": "pick", "trace": 1}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "pick", "trace": [], "index": 0}, "outcome": "drop"}, + "bounds": {"message": {"type": "pick", "trace": 1, "index": 999, "seq": 4}, "reply": "pick_result"}, + "callback": {"message": {"type": "pick", "trace": 1, "index": 0}, "reply": "pick_result", "callbacks": ["on_hover"]} + } + }, + { + "type": "click", + "cases": { + "valid": {"message": {"type": "click", "trace": 1, "index": 0}, "outcome": "none"}, + "missing": {"message": {"type": "click", "trace": 1}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "click", "trace": 1, "index": {}}, "outcome": "drop"}, + "bounds": {"message": {"type": "click", "trace": 1, "index": 999}, "outcome": "none"}, + "callback": {"message": {"type": "click", "trace": 1, "index": 0}, "outcome": "none", "callbacks": ["on_click"]} + } + }, + { + "type": "view_change", + "cases": { + "valid": {"message": {"type": "view_change", "ranges": {"x": [0, 10], "y": [0, 10]}, "source": "catalog"}, "outcome": "none"}, + "missing": {"message": {"type": "view_change"}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "view_change", "ranges": []}, "outcome": "drop"}, + "bounds": {"message": {"type": "view_change", "ranges": {"x": [1, 1]}}, "outcome": "drop"}, + "callback": {"message": {"type": "view_change", "ranges": {"x": [0, 10], "y": [0, 10]}}, "outcome": "none", "callbacks": ["on_view_change"]} + } + }, + { + "type": "select", + "cases": { + "valid": {"message": {"type": "select", "x0": 0, "x1": 10, "y0": 0, "y1": 10}, "reply": "selection"}, + "missing": {"message": {"type": "select", "x0": 0, "x1": 10, "y0": 0}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "select", "x0": [], "x1": 10, "y0": 0, "y1": 10}, "outcome": "drop"}, + "bounds": {"message": {"type": "select", "x0": 10, "x1": 0, "y0": 10, "y1": 0}, "reply": "selection"}, + "callback": {"message": {"type": "select", "x0": 0, "x1": 10, "y0": 0, "y1": 10}, "reply": "selection", "callbacks": ["on_brush", "on_select"]} + } + }, + { + "type": "select_polygon", + "cases": { + "valid": {"message": {"type": "select_polygon", "points": [[0, 0], [10, 0], [5, 10]]}, "reply": "selection"}, + "missing": {"message": {"type": "select_polygon"}, "outcome": "drop"}, + "wrong_type": {"message": {"type": "select_polygon", "points": "triangle"}, "outcome": "drop"}, + "bounds": {"message": {"type": "select_polygon", "points": [[0, 0], [1, 1]]}, "outcome": "drop"}, + "callback": {"message": {"type": "select_polygon", "points": [[0, 0], [10, 0], [5, 10]]}, "reply": "selection", "callbacks": ["on_brush", "on_select"]} + } + }, + { + "type": "select_clear", + "cases": { + "valid": {"message": {"type": "select_clear"}, "reply": "selection"}, + "missing": {"message": {}, "outcome": "drop"}, + "wrong_type": {"message": {"type": []}, "outcome": "drop"}, + "bounds": {"message": {"type": "select_clear", "seq": 9007199254740991}, "reply": "selection"}, + "callback": {"message": {"type": "select_clear"}, "reply": "selection", "callbacks": ["on_select"]} + } + }, + { + "type": "animation_start", + "cases": { + "valid": {"message": {"type": "animation_start", "phase": "enter"}, "outcome": "none"}, + "missing": {"message": {}, "outcome": "drop"}, + "wrong_type": {"message": {"type": 1, "phase": "enter"}, "outcome": "drop"}, + "bounds": {"message": {"type": "animation_start", "phase": ""}, "outcome": "none"}, + "callback": {"message": {"type": "animation_start", "phase": "enter"}, "outcome": "none", "callbacks": ["on_animation_start"]} + } + }, + { + "type": "animation_end", + "cases": { + "valid": {"message": {"type": "animation_end", "phase": "update", "cancelled": true}, "outcome": "none"}, + "missing": {"message": {}, "outcome": "drop"}, + "wrong_type": {"message": {"type": {}, "phase": "update"}, "outcome": "drop"}, + "bounds": {"message": {"type": "animation_end", "phase": "", "cancelled": false}, "outcome": "none"}, + "callback": {"message": {"type": "animation_end", "phase": "update", "cancelled": true}, "outcome": "none", "callbacks": ["on_animation_end"]} + } + } + ], + "replies": { + "tier_update": { + "required": ["type", "seq", "traces"], + "buffers": "nonempty", + "golden": { + "message": {"type": "tier_update", "seq": 7, "traces": [{"id": 0, "x": {"buf": 0, "len": 2, "offset": 0.0, "scale": 1.0}, "y": {"buf": 1, "len": 2, "offset": 0.0, "scale": 1.0}}]}, + "buffers_base64": ["AACAPwAAAEA=", "AABAQAAAgEA="], + "frame_base64": "WFlCRgEAGACQAAAAAgAAAMgAAAAAAAAAeyJ0eXBlIjoidGllcl91cGRhdGUiLCJzZXEiOjcsInRyYWNlcyI6W3siaWQiOjAsIngiOnsiYnVmIjowLCJsZW4iOjIsIm9mZnNldCI6MC4wLCJzY2FsZSI6MS4wfSwieSI6eyJidWYiOjEsImxlbiI6Miwib2Zmc2V0IjowLjAsInNjYWxlIjoxLjB9fV19CAAAAAAAAAAAAIA/AAAAQAgAAAAAAAAAAABAQAAAgEA=" + } + }, + "density_update": { + "required": ["type", "seq", "traces"], + "buffers": "nonempty", + "golden": { + "message": {"type": "density_update", "seq": 8, "traces": [{"id": 1, "mode": "density", "tier": "density", "visible": 3, "reduction": "bin", "binning": "exact", "density": {"buf": 0, "w": 2, "h": 2, "max": 3, "enc": "log-u8", "x_range": [0.0, 1.0], "y_range": [0.0, 1.0]}}]}, + "buffers_base64": ["AAECAw=="], + "frame_base64": "WFlCRgEAGADmAAAAAQAAABABAAAAAAAAeyJ0eXBlIjoiZGVuc2l0eV91cGRhdGUiLCJzZXEiOjgsInRyYWNlcyI6W3siaWQiOjEsIm1vZGUiOiJkZW5zaXR5IiwidGllciI6ImRlbnNpdHkiLCJ2aXNpYmxlIjozLCJyZWR1Y3Rpb24iOiJiaW4iLCJiaW5uaW5nIjoiZXhhY3QiLCJkZW5zaXR5Ijp7ImJ1ZiI6MCwidyI6MiwiaCI6MiwibWF4IjozLCJlbmMiOiJsb2ctdTgiLCJ4X3JhbmdlIjpbMC4wLDEuMF0sInlfcmFuZ2UiOlswLjAsMS4wXX19XX0AAAQAAAAAAAAAAAECAwAAAAA=" + } + }, + "pick_result": { + "required": ["type", "seq", "row"], + "buffers": "none", + "golden": { + "message": {"type": "pick_result", "seq": 9, "row": {"trace": 1, "index": 2, "x": 2.0, "y": 4.0, "x_kind": "number", "y_kind": "number"}}, + "buffers_base64": [], + "frame_base64": "WFlCRgEAGABuAAAAAAAAAIgAAAAAAAAAeyJ0eXBlIjoicGlja19yZXN1bHQiLCJzZXEiOjksInJvdyI6eyJ0cmFjZSI6MSwiaW5kZXgiOjIsIngiOjIuMCwieSI6NC4wLCJ4X2tpbmQiOiJudW1iZXIiLCJ5X2tpbmQiOiJudW1iZXIifX0AAA==" + } + }, + "selection": { + "required": ["type", "traces", "total"], + "buffers": "trace_count", + "golden": { + "message": {"type": "selection", "seq": "selection:4", "traces": [{"id": 1, "count": 2, "buf": 0, "drill_seq": 3}], "total": 2}, + "buffers_base64": ["AQAAAAMAAAA="], + "frame_base64": "WFlCRgEAGABmAAAAAQAAAJAAAAAAAAAAeyJ0eXBlIjoic2VsZWN0aW9uIiwic2VxIjoic2VsZWN0aW9uOjQiLCJ0cmFjZXMiOlt7ImlkIjoxLCJjb3VudCI6MiwiYnVmIjowLCJkcmlsbF9zZXEiOjN9XSwidG90YWwiOjJ9AAAIAAAAAAAAAAEAAAADAAAA" + } + } + } +} diff --git a/spec/testing/protocol-conformance.md b/spec/testing/protocol-conformance.md new file mode 100644 index 00000000..2c7c7ab4 --- /dev/null +++ b/spec/testing/protocol-conformance.md @@ -0,0 +1,30 @@ +# Protocol conformance and mutation policy + +Status: **implemented** for TST-NI-028. + +[`protocol-catalog.json`](protocol-catalog.json) is the executable catalog for +every request dispatched by `xy.channel.handle_message` and every reply it can +return. The catalog is deliberately data, not a second dispatcher: tests derive +the implementation's request branches from the `handle_message` AST and require +an exact match. Adding or removing a branch without updating the catalog fails. + +Each request has five generated cases: a valid message, a missing-field or +missing-discriminator message, a wrong-type message, a boundary message, and a +callback case. The callback case records exact hook order. Reply entries declare +required fields and buffer cardinality and carry committed `XYBF` frames. Python +must reproduce and decode those bytes exactly; the shipped JavaScript +`decodeFrame` consumes the same base64 frames without copies. + +The required Hypothesis suite mutates every structural class in batches: +magic/version/flags/header size, metadata and total lengths, buffer count, +metadata bytes, metadata padding, buffer lengths, and buffer padding. For every +mutation Python and the shipped JavaScript decoder must either both reject it or +produce the same metadata and buffers. Rejection is the normal result for +structural corruption; an accidental mutation that remains valid is acceptable +only when the decoded results preserve cross-language parity. + +`make check-protocol` is the stable local entry point. Main CI runs the same +catalog, golden-frame, handwritten framing, and property-mutation suites as a +hard step and retains JUnit evidence even on failure. Node and Hypothesis are +required lane dependencies; missing either is a collection or command failure, +never a skip. diff --git a/spec/testing/pyplot-noops.json b/spec/testing/pyplot-noops.json new file mode 100644 index 00000000..cc511dac --- /dev/null +++ b/spec/testing/pyplot-noops.json @@ -0,0 +1,173 @@ +{ + "schema_version": 1, + "noops": [ + { + "path": "python/xy/pyplot/_artists.py", + "function": "PathCollection.set_sizes", + "options": ["dpi"], + "rationale": "Marker sizes are converted with the point scale owned by the axes; an unrelated renderer DPI cannot change that stored geometry.", + "test": "tests/pyplot/test_compat_noops.py::test_artist_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_artists.py", + "function": "Text.get_window_extent", + "options": ["renderer"], + "rationale": "The shim has one deterministic text metric approximation and no renderer-specific font-metric object to consult.", + "test": "tests/pyplot/test_compat_noops.py::test_artist_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axes.py", + "function": "Axes.axis", + "options": ["emit"], + "rationale": "Axes limit callbacks are not part of the supported shim API, so changing callback emission cannot affect the resulting view.", + "test": "tests/pyplot/test_compat_noops.py::test_axes_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axes.py", + "function": "Axes.get_figure", + "options": ["root"], + "rationale": "The shim does not implement nested subfigures; the immediate and root figure are therefore the same supported object.", + "test": "tests/pyplot/test_compat_noops.py::test_axes_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axes.py", + "function": "Axes.get_position", + "options": ["original"], + "rationale": "There is one stored figure-fraction axes rectangle and no active-versus-original transform graph in the shim.", + "test": "tests/pyplot/test_compat_noops.py::test_axes_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axes.py", + "function": "Axes.margins", + "options": ["tight"], + "rationale": "The engine always derives auto limits directly from entries plus the requested margin, with no separate sticky-edge tightness pass.", + "test": "tests/pyplot/test_compat_noops.py::test_axes_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axes.py", + "function": "Axes.relim", + "options": ["visible_only"], + "rationale": "Visibility changes presentation opacity but deliberately retain every entry in the data-extent model used by relim.", + "test": "tests/pyplot/test_compat_noops.py::test_axes_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axes.py", + "function": "Axes.set_aspect", + "options": ["anchor", "share"], + "rationale": "The supported aspect model has no independent layout-anchor graph, and shared axes already resolve through shared axis state.", + "test": "tests/pyplot/test_compat_noops.py::test_axes_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_axisgrid.py", + "function": "FacetGrid.__init__", + "options": ["legend_out"], + "rationale": "External facet legends are observable only with hue mapping, which this bounded FacetGrid surface rejects explicitly.", + "test": "tests/pyplot/test_compat_noops.py::test_facetgrid_legend_out_noop_is_bounded_by_hue_rejection" + }, + { + "path": "python/xy/pyplot/_mplfig.py", + "function": "Figure.clear", + "options": ["keep_observers"], + "rationale": "The shim exposes no figure observer registry, so both values perform the same complete supported-state reset.", + "test": "tests/pyplot/test_compat_noops.py::test_figure_clear_observer_noop_is_invariant" + }, + { + "path": "python/xy/pyplot/_plot_types.py", + "function": "PlotTypeMixin.clabel", + "options": ["fontsize", "inline", "inline_spacing", "use_clabeltext", "rightside_up", "zorder"], + "rationale": "Contour labels use deterministic shim-owned placement and styling; unsupported manual layout controls cannot alter that bounded output.", + "test": "tests/pyplot/test_compat_noops.py::test_clabel_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "FixedFormatter.__call__", + "options": ["value"], + "rationale": "FixedFormatter selects its predeclared label exclusively by tick position, matching its supported sequence contract.", + "test": "tests/pyplot/test_compat_noops.py::test_formatter_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "FixedLocator.tick_values", + "options": ["vmin", "vmax"], + "rationale": "FixedLocator returns its configured locations regardless of the current view bounds; clipping is performed by the axis renderer.", + "test": "tests/pyplot/test_compat_noops.py::test_locator_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "FormatStrFormatter.__call__", + "options": ["pos"], + "rationale": "Percent-style formatting depends only on the numeric value and has no position placeholder in its supported contract.", + "test": "tests/pyplot/test_compat_noops.py::test_formatter_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "LogLocator.__init__", + "options": ["numticks"], + "rationale": "The bounded log locator emits every in-view requested-decade tick and leaves density clipping to the axis layout.", + "test": "tests/pyplot/test_compat_noops.py::test_locator_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "MaxNLocator.__init__", + "options": ["prune"], + "rationale": "Edge ticks remain part of the locator result and are clipped at draw time, so constructor pruning does not alter locations.", + "test": "tests/pyplot/test_compat_noops.py::test_locator_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "NullFormatter.__call__", + "options": ["value", "pos"], + "rationale": "NullFormatter intentionally returns an empty label for every numeric value and every tick position.", + "test": "tests/pyplot/test_compat_noops.py::test_formatter_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "NullLocator.tick_values", + "options": ["vmin", "vmax"], + "rationale": "NullLocator intentionally returns no locations for every possible pair of view bounds.", + "test": "tests/pyplot/test_compat_noops.py::test_locator_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/_ticker.py", + "function": "ScalarFormatter.__call__", + "options": ["pos"], + "rationale": "Scalar formatting is a pure numeric conversion and does not vary according to the tick's sequence position.", + "test": "tests/pyplot/test_compat_noops.py::test_formatter_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/dates.py", + "function": "DateFormatter.__call__", + "options": ["pos"], + "rationale": "Date label text is derived from epoch milliseconds and the stored format, independent of tick sequence position.", + "test": "tests/pyplot/test_compat_noops.py::test_date_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/dates.py", + "function": "DateFormatter.__init__", + "options": ["tz", "usetex"], + "rationale": "The supported date contract is naive UTC plain text, without timezone conversion or a TeX rendering backend.", + "test": "tests/pyplot/test_compat_noops.py::test_date_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/dates.py", + "function": "DayLocator.__init__", + "options": ["tz"], + "rationale": "Calendar locators operate in the shim's documented naive UTC millisecond coordinate space.", + "test": "tests/pyplot/test_compat_noops.py::test_date_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/dates.py", + "function": "MonthLocator.__init__", + "options": ["tz"], + "rationale": "Calendar locators operate in the shim's documented naive UTC millisecond coordinate space.", + "test": "tests/pyplot/test_compat_noops.py::test_date_compat_noops_are_invariant" + }, + { + "path": "python/xy/pyplot/dates.py", + "function": "YearLocator.__init__", + "options": ["tz"], + "rationale": "Calendar locators operate in the shim's documented naive UTC millisecond coordinate space.", + "test": "tests/pyplot/test_compat_noops.py::test_date_compat_noops_are_invariant" + } + ] +} diff --git a/spec/testing/rendered-label-policy.md b/spec/testing/rendered-label-policy.md new file mode 100644 index 00000000..e4fb1b87 --- /dev/null +++ b/spec/testing/rendered-label-policy.md @@ -0,0 +1,77 @@ +# Rendered Label and Value-Format Policy + +This is the normative contract for browser-visible axis and tooltip values. +Accepted formats must render as specified; unsupported or kind-mismatched +formats raise `ValueError` at a Python boundary and throw for a raw browser +spec. Silent fallback is not permitted. + +## Numeric grammar + +A numeric format has this shape: + +```text +[currency][,].N[f[unit]][%] +``` + +- `currency` is one optional literal prefix from `$`, `€`, `£`, or `¥`. +- `,` enables grouping and locale decimal/group separators through the host's + `Intl.NumberFormat`. Without it, fixed output uses an ASCII decimal point. +- `N` is an integer precision from 0 through 20. +- `f` is optional for an unadorned fixed value. A literal unit requires `f`; + the unit may contain ASCII letters, digits, spaces, `/`, `_`, or `-`. +- `%` multiplies by 100 and appends `%`. It may follow the precision directly + or an otherwise suffix-free `f`; a unit and percent cannot be combined. + +Examples include `.2f`, `,.0f`, `$,.2f`, `.1%`, `.0f%`, `.4f s`, +`.3f GiB`, `,.0fK`, and `$,.0fK`. Currency symbols are literal prefixes, +not ISO-code or exchange-rate semantics. Width, alignment, signs, scientific +notation, arbitrary prefixes, and d3/Python format features outside this +grammar are rejected. + +Linear and log axes use the same numeric grammar. On a log axis, a positive +subunit value that `.0f` would collapse to `0` retains the automatic nonzero +label. Category labels are the category strings themselves and reject +`format=`; explicit `tick_labels` remain the way to author replacements. + +## Time grammar + +Time formats contain at least one token from `%Y`, `%m`, `%d`, `%H`, `%M`, +`%S`, `%b`, or `%B`; other text is literal. Unknown or incomplete `%` tokens, +including `%y`, `%Z`, and `%%`, are rejected. + +Every token is evaluated in UTC. `%b` and `%B` use English month names. Output +therefore remains identical across host locales and host time zones; local-time +formatting is intentionally not part of this API. + +Tooltip formats use the numeric grammar for numeric fields and the time grammar +for `time_ms` fields. Applying a format to a string, a non-finite value, or the +wrong field kind throws instead of falling back. + +## Automatic and colorbar labels + +Absent `format=`, linear/log, category, and time axes keep their automatic tick +formatters. Automatic time labels are also UTC. Colorbars do not expose a +custom format: automatic ticks use the domain-derived fixed formatter, while +explicit ticks use the six-significant-digit general formatter. The colorbar +title is literal text. + +## Executable oracle + +`npm run test:labels` (or `make check-labels`) runs package-owned Vite, +TypeScript source, Playwright `1.61.1`, and Chromium with no optional imports or +skip path. It combines formatter units with real shipped-bundle DOM assertions +for numeric, grouping, currency, percent, UTC time, log, category, named-axis, +tooltip, and colorbar labels in: + +- `en-US` / `America/Los_Angeles`; and +- `de-DE` / `Asia/Tokyo`. + +Both time zones are non-UTC, and the test requires the UTC labels to match +exactly. Independent negative controls inject malformed raw axis and tooltip +formats and corrupt a captured DOM label to prove both runtime rejection and +oracle sensitivity. The hard `test` CI job uploads +`rendered-label-evidence.json` as `rendered-label-evidence` even on failure. + +The executable sources are `js/tests/rendered_labels.test.mjs`, +`tests/test_rendered_label_formats.py`, `js/src/30_ticks.ts`, and +`python/xy/_validate.py`. diff --git a/spec/visual-baselines/README.md b/spec/visual-baselines/README.md new file mode 100644 index 00000000..c8b10a7d --- /dev/null +++ b/spec/visual-baselines/README.md @@ -0,0 +1,42 @@ +# Reviewed Visual Baselines + +Status: hard CI evidence for TST-NI-014 is implemented by +`scripts/visual_baseline.py`, while broad gallery health remains independently +enforced by `scripts/visual_health_smoke.py`. + +`v1.json` is a bounded, versioned identity oracle for grouped bars, heatmap, +and composed layers. It stores exact rendered semantics and a checksummed +10×-downsampled RGB raster. The manifest pins the exact Playwright Chromium +version, repository font and SHA-256, 900×470 viewport, DPR 1, and explicit +geometry and perceptual tolerances. This is intentionally separate from the +all-gallery health smoke. + +Every gate run must reject four real-browser negative controls: + +- corrupted numeric payload values; +- wrong mark colors; +- changed rendered labels; and +- shifted/scaled geometry. + +Failures retain expected, actual, full-resolution actual, diff, and semantic +artifacts. CI cannot update the manifest. + +## Update policy + +Install the pinned development browser tooling with `make setup-browser`, then +prepare a proposal with the exact Playwright executable: + +```bash +CHROMIUM="$(node -e "const {chromium}=require('playwright'); process.stdout.write(chromium.executablePath())")" +python scripts/visual_baseline.py "$CHROMIUM" --update-baselines \ + --prepared-by "Your Name" --reason "intentional renderer change" \ + --artifacts visual-baseline-review +``` + +An update is not approval. Commit the manifest only alongside the intentional +renderer/spec change, attach `visual-baseline-review/` to the pull request, and +request an independent reviewer. The reviewer must inspect expected/actual/diff +PNGs and semantic JSON (`expected` is the prior reviewed baseline and `actual` +is the proposal), confirm the fixture and visual change are intended, and +reject unexplained browser/font pin or tolerance changes. Never regenerate the +file solely to turn an unexplained failure green. diff --git a/spec/visual-baselines/v1.json b/spec/visual-baselines/v1.json new file mode 100644 index 00000000..86ed17fa --- /dev/null +++ b/spec/visual-baselines/v1.json @@ -0,0 +1,434 @@ +{ + "cases": { + "composed-layers": { + "raster": "////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8/Pz/////////////////////f39+Pj4////////////////+vr6+vr6////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////+/v7l5eXi4uLlJSUl5eXlJSUioqKhoaGqKioh4eHiIiImZmZkZGRn5+flpaWj4+Pjo6O+Pj4////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8fHx////////////////8fHx////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////9Pf+4en82+X81uH74en82+X81uH7/Pz/////////////////////////////8fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+f87+f87+f87+f87+f87+f87+f87+f87OP87+f87+f87OP82cb57OP87OP87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f88+/6/////////////////////////////////////v7++/v7+/v75ev46u/50t734Oj509732eL42eP40t74+/z/////////////////////////////8fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+f87+f87+f87+f87+f87+f87+f87+f81sH5z7b4073428j5s4z0waL2u5j1xKb27+f87+f87+f87+f87+f87+f87+f87+f87+f88+/6/////////////////////////////vfq/e7W++3U9NGV9btZtrSyvLeut6+iycCxyMCx8+bP9PDq9fX1////////////////////+/v73d3d8fHx8fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f828n57+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f88+/6/////////////////////////////e7W+96s+dyr4MeWzbmE3KmG5Mug6M6i5c2h2sKZ8dep9Oze8/Pz////////////////////9/f3ysrK7u7u7u7u+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v79PH37OT57OT57OT57OT57OT57OT57OT57OT57OT57OT57OT57OT57OT52Mb37OT57OT57OT57OT57OT57OT57OT57OT57OT57OT57OT57OT58Oz3+/v7+/v7+/v7+vT09tfX9NDQ9NDQ88Ov77eP5reRm76eWMGmq6qItaF+vKeDr5x6u6eDsp58u7Wq4ODg/f39////////////////////////////8fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f828n57+f87+f87+f87+f87+f87+f87+f87NLH7NDC68q96Lew57Cq4K+o9sCi9sCi9b6d8cCZ7d676fj34fbz1+XO0der0daqzNCof57Hzc6m5cyh5Myg4sqf5cyg8dep9Oze9fX1////////////////////////////////8fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f87+f828n57+f87Nbr6cLX6cDV6cHW6cHW6cHW5qyh5qqc57Cg6sOu58mzoarC5dqr3dmr1Ner0der0tqy1fLv1fLv1OTN0ter0daqzNCooLO6ysujvaiEybSNvqqFuKSB79Wo9Oze9fX1////////////////////////////////8fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+f87+f87+f87+f87+f87+f87+f87Nfs6cHW6cHW6cHW1rPUv5jX6cHW7NLn7+X67+f87+f87+f86uX63su618ey0MWyycSyxsSyy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0daqzcagz66O0LeUzreQwayHvqmF3cWb8dep9Oze9fX1////////////////////////8/Pz/Pz88fHx////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////9/X77+f87+X668zh6cDV6cDV6cDV6cDV6LvI6b+57dXQ7dbQ7dbQ7dbQ2brV7dbQ59XQ39PQ2NLQ0tfgy9zuydzux8a4x8Syx8Syx8Syx8Syy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0darzdGpzdGpz9Cm6dCj48qf4cid7tSm89iq9u7g9/f3/////////////////////v7+tra25OTk7u7u+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+vPz9dXV9NDQ9dHR8cvR5r3S5r/U6tjt7OT57OT57OT57OT569vj6cix5cex3sax2cWw08Sxv6u7xsKwxcKwxcKwxcKwxs7Ox9nrx9nrxsW2xcKwxcKwxcKwxcKwycqr0NWp0NWp0NWp0NWp0Niw0u/s0u/s0eLL0NWp0NWp0NWp0NWp0tOn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7/f39////////////////////////////8fHx////////////////////////+/v7////////////////////////////////////////////////+/z/09/5/////////Ozs+NPT+NPT+NPT+NPT98i298yz/OnQ/e7W/e7W7eHW7djY7djY7djX5tfX4NbX3ODy1uDzz9beyMSyxsSyx8Syx8Syx8Syuqy8x8Syx8Syx8Syx8SyyNDQytzuytzux8a4x8Syx8Syx8Syx8Syy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0ter0ter0ter1NWp+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx////////////////////////+/v7/////////////////////////fT0++Tk+NTU+NPT+NPT+NPT9dHTz7XN+NPT+NPT++bm/////////////////OjF+d6s8tys7dys6Nusoq/AzMO0zMWyxsSyx8Syx8SyytroytzuydXcx8Syx8Syx8Syx8Syx8Syuqy8x8Syx8Syx8Syx8SyyNDQytzuytzux8a4x8Syx8Syx8Syx8Syy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0ter0ter0ter1NWp+96s+96s+96s+96s+96s/vXm////////////7Ozs3d3d////////////////8fHx////////////////////////+/Pz+NPT+NPT+NPT+NPT+NPT+NPT+t7e/Ovn+92r+96s+96s+96s+96s+Nyq9d2s8Nys69us5tur4uLB4PXz2/Tx1N/A0Ner0ter0ter0terzdCsx8Syx8Syx8Syx8Syx8SyytroytzuydXcx8Syx8Syx8Syx8Syx8Syuqy8x8Syx8Syx8Syx8SyyNDQytzuytzux8a4x8Syx8Syx8Syx8Syy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0ter0ter0ter1NWp+96s+96s+96s+96s+96s/vXm////////////39/f1tbW////////////////8fHx/////vju/fLe/fLe/fLe/fLe7eff9+7g/fLe/fLe/fLe/fLe/fr09vz88fjy6dus5Nqr39mr29ir1dirztWp0der0ter0ter0ter09+/1fLv1fLv09+/0ter0ter0ter0terzdCsx8Syx8Syx8Syx8Syx8SyytroytzuydXcx8Syx8Syx8Syx8Syx8Syuqy8x8Syx8Syx8Syx8SyyNDQytzuytzux8a4x8Syx8Syx8Syx8Syy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0ter0ter0ter1NWp+96s+96s+96s+96s+96s/vXm////////////2dnZzMzM/////Pz8tLS05eXl7u7u+/v7+uzS+dyq+dyq+dyq+dyqzMK1z8yw4Niq29ep1tap0dWpz+fX0u/s0uzl0NWp0NWp0NWp0NWp0NWpztOn0NWp0NWp0NWp0NWp0d290u/s0u/s0d290NWp0NWp0NWp0NWpy86qxcKwxcKwxcKwxcKwxcKwx9fmx9nrx9LaxcKwxcKwxcKwxcKwxcKwuaq7xcKwxcKwxcKwxcKwxs7Ox9nrx9nrxsW2xcKwxcKwxcKwxcKwycqr0NWp0NWp0NWp0NWp0Niw0u/s0u/s0eLL0NWp0NWp0NWp0NWp0tOn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7/f39////39/fxcXF////////+Pj4+/v78fHx/////e7W+96s+96s+96s+96s9duq0ter0ter0ter0ter0ter1Orb1fLv1e/o0ter0ter0ter0ter0ter0NWp0ter0ter0ter0ter09+/1fLv1fLv09+/0ter0ter0ter0terzdCsx8Syx8Syx8Syx8Syx8SyytroytzuydXcx8Syx8Syx8Syx8Syx8Syuqy8x8Syx8Syx8Syx8SyyNDQytzuytzux8a4x8Syx8Syx8Syx8Syy8ut0ter0ter0ter0ter0tqy1fLv1fLv1OTN0ter0ter0ter0ter1NWp+96s+96s+96s+96s+96s/vXm////////////4uLiy8vL////////////////8fHx/////e7W+96s+96s+96s+96s9tuq2tir2tir2tir2tir2tir3Ozd3fXy3fLr2tir2tir2tir2tir2tir2Nap2tir2tir2tir2tir2+HA3fXy3fXy2+HA2tir2tir2tir2tir1dGszsWyzsWyzsWyzsWyzsWy0dzr0d7x0NfezsWyzsWyzsWyzsWyzsWywK28zsWyzsWyzsWyzsWy0NLS0d7x0d7xz8i4zsWyzsWyzsWyzsWy0s2t2tir2tir2tir2tir29uy3fXy3fXy3ObP2tir2tir2tir2tir29ep+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm/////////////////////////v7+z8/P7u7u8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm/////////////////////////v7+2NjY8fHx7u7u+/v7+uzS+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7+/jz+dyq+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+eXC+/v7+/v7+eXC+dyq+dyq+dyq+dyq8tSq6cix6cix6cix6cix6cix7OHy7OT569vj6cix6cix6cix6cix6cix1q+76cix6cix6cix6cix6tbV7OT57OT56cu46cix6cix6cix6cix7s+r+dyq+dyq+dyq+dyq+d+y+/v7+/v7+uzS+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7/f39////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm/////////////////////////v7+4ODg9PT08fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm/////////////////////////v7+ysrK6+vr7u7u+/v7+uzS+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7+/jz+dyq+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+eXC+/v7+/v7+eXC+dyq+dyq+dyq+dyq8tSq6cix6cix6cix6cix6cix7OHy7OT569vj6cix6cix6cix6cix6cix1q+76cix6cix6cix6cix6tbV7OT57OT56cu46cix6cix6cix6cix7s+r+dyq+dyq+dyq+dyq+d+y+/v7+/v7+uzS+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7/f39////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////v7+/4eHh7u7u+/v7+uzS+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7+/jz+dyq+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+eXC+/v7+/v7+eXC+dyq+dyq+dyq+dyq8tSq6cix6cix6cix6cix6cix7OHy7OT569vj6cix6cix6cix6cix6cix1q+76cix6cix6cix6cix6tbV7OT57OT56cu46cix6cix6cix6cix7s+r+dyq+dyq+dyq+dyq+d+y+/v7+/v7+uzS+dyq+dyq+dyq+dyq9tmn+dyq+dyq+dyq+dyq+dyq+/Lj+/v7/f39////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////////////8fHx/////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm//////z3+96s+96s+96s+96s+96s+dyq+96s+96s+96s+96s/OjF/////////OjF+96s+96s+96s+96s9Nas68qz68qz68qz68qz68qz7+T17+f87t7m68qz68qz68qz68qz68qz2LG968qz68qz68qz68qz7djY7+f87+f8682668qz68qz68qz68qz8NGt+96s+96s+96s+96s++G0/////////e7W+96s+96s+96s+96s+dyq+96s+96s+96s+96s+96s/vXm////////////////////////////6Ojo6Ojo6Ojo7+/v7uPR7dey7dey7dey7dey69aw7dey7dey7dey7dey7dey7+jd7+/v7+3p7dey7dey7dey7dey7dey69aw7dey7dey7dey7dey7t/E7+/v7+/v7t/E7dey7dey7dey7dey6NGy4cm34cm34cm34cm34cm349zo5N7t49fd4cm34cm34cm34cm34cm307a/4cm34cm34cm34cm34tPS5N7t5N7t4cu94cm34cm34cm34cm35c6z7dey7dey7dey7dey7dq47+/v7+/v7uPR7dey7dey7dey7dey69aw7dey7dey7dey7dey7dey7+jd7+/v9vb2////////////////////9/f39/f3////////////////////////////7e3t9vb2////////////////////////////////////////////6enp6urq+/v7////////////////////////////////////////6+vr6Ojo9PT0////////////////////////////////////////////6urq8vLy/Pz8////////////////////////////////////////7Ozs5OTk+vr6////////////////////////////////////////////8vLy9PT0/////////////////////////////////////////////////////////////////////////////////v7+ysrK3d3d////////////////////////////////////////////5ubmw8PD7u7u////////////////////////////////////////7u7uzMzM2dnZ////////////////////////////////////////////2dnZwcHB////////////////////////////////////////////3t7exsbG3d3d/////////////////////////////////////////v7+1dXV1dXV////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////7e3t2NjY2NjYy8vL9fX1////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////9fX13t7e3t7e5+fn9fX1////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////", + "raster_sha256": "e4a58aa7d901efd0bb36a4911c94ed05fe9079517206d674edb51d5a43c94b0d", + "semantic": { + "axis_titles": [], + "colorbar": [], + "dpr": 1, + "font_family": "\"XY Visual Baseline\"", + "legend": [ + { + "axis": "", + "side": "", + "text": "bookings" + }, + { + "axis": "", + "side": "", + "text": "forecast band" + }, + { + "axis": "", + "side": "", + "text": "samples" + }, + { + "axis": "", + "side": "", + "text": "target" + } + ], + "slots": { + "canvas": [ + { + "h": 348, + "w": 824, + "x": 62, + "y": 40 + } + ], + "colorbar": [], + "legend": [ + { + "h": 72, + "w": 102.3, + "x": 777.7, + "y": 46 + } + ], + "root": [ + { + "h": 430, + "w": 900, + "x": 0, + "y": 0 + } + ], + "title": [ + { + "h": 18, + "w": 900, + "x": 0, + "y": 6 + } + ] + }, + "tick_labels": [ + { + "axis": "x", + "side": "bottom", + "text": "Jan" + }, + { + "axis": "x", + "side": "bottom", + "text": "Feb" + }, + { + "axis": "x", + "side": "bottom", + "text": "Mar" + }, + { + "axis": "x", + "side": "bottom", + "text": "Apr" + }, + { + "axis": "x", + "side": "bottom", + "text": "May" + }, + { + "axis": "x", + "side": "bottom", + "text": "Jun" + }, + { + "axis": "y", + "side": "left", + "text": "0" + }, + { + "axis": "y", + "side": "left", + "text": "10" + }, + { + "axis": "y", + "side": "left", + "text": "20" + }, + { + "axis": "y", + "side": "left", + "text": "30" + }, + { + "axis": "y", + "side": "left", + "text": "40" + }, + { + "axis": "y", + "side": "left", + "text": "50" + }, + { + "axis": "y", + "side": "left", + "text": "60" + } + ], + "title": [ + { + "axis": "", + "side": "", + "text": "Composed layered chart" + } + ] + } + }, + "grouped-bars": { + "raster": "////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////+fn5+vr6////////////////+vr6////////////////////+vr6////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////wcHBjY2NnJyci4uLjY2NkJCQoaGhi4uLiIiIkpKSh4eHpaWlpaWlkpKSm5ububm5////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8fHx////////////////////6Ojo9/f39fX1+/v7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////f399vb2/Pz8////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////39/ftLS05OTk7u7u+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v79vb29/f39/f39/f39/f39/f39/f3+Pj4/f39////////////////////////////8fHx////orz2RXruRXruRXru7PL9+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////9vb2cZnvtMfyurq6yMjIvb29xsbG6urq////////////////////////////////8fHx////orz2RXruRXruRXru7PL9+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////9vb2s9LZ1eTo39/f5ubm5ubm4ODg9PT0////////////////////////////////8fHx////orz2RXruRXruRXru7PL9+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////9vb2aMGJr9u/srKytLS0tbW13Nzc9fX1////////////////////////////////8fHx////orz2RXruRXruRXru7PL9+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////9vb29cyG9OC+zs7Ozs7O0NDQ9PT09fX1////////////////////6urq5eXl9PT08fHx////orz2RXruRXruRXru7PL9+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////9vb29dqs9ejR29vb1tbW2tra9PT09fX1////////////////////39/f0tLS6+vr7u7u+/v7oLr0RHruRHruRHru6e76+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7/f39////////////////////////////8fHx////orz2RXruRXruRXru7PL9+/v7/////////////////////////////f7/7PL97PL97PL99vj+////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////5eXl1NTU////////////////8fHx////orz2RXruRXruRXru7PL9+/v7////////////////////////////7PL9RXruRXruRXruorz2////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////3t7e0NDQ////////////////8fHx////orz2RXruRXruRXrupdbGrd2/sODByOnU////////////////////7PL9RXruRXruRXruorz2////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////4eHh0tLS////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruorz2////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////39/fzs7O////+/v7wMDA6+vr7u7u+/v7oLr0RHruRHruRHruOqtyOLBkOLBkc8eS+/v7+/v7+/v7+/v7+/v76e76RHruRHruRHruoLr0+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7/f39////5eXl1tbW/////Pz84ODg9PT08fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruorz2////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////39/f1NTU////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////////////////////////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////6+vr4eHh////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////l7X1apXxapXxeZ/z////+/v7////////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruWIfw////+/v7////////////////////////////+fv+7PL97PL97PL9+fv+////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruWIfw////+/v7////////////////////////////x9f6RXruRXruRXrux9f6////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////9PT0qKio4eHh7u7u+/v7oLr0RHruRHruRHruOqtyOLBkOLBkc8eS+/v7+/v7+/v7+/v7+/v76e76RHruRHruRHruPpWpOLBkOLBkOLBk6PTs+/v7+/v7+/v7+/v7+/v7e6DyRHruRHruV4bv+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7xdT3RHruRHruRHruxdT3+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7/f39////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruWIfw////+/v7////////////////////////////x9f6RXruRXruRXrux9f6////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruRoDiTbl0TLh0Tbl0pty6////////////////////x9f6RXruRXruRXrux9f6////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruRIDgObFlOLBkObFlnNiy////////////////////x9f6RXruRXruRXrux9f6////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFldMiT////////////////////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruRIDgObFlOLBkObFlnNiy////////////////////x9f6RXruRXruRXruSqaZTbl0TLh0X8CC////////////////////////////////////////////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////8PDw+Pj48fHx////orz2RXruRXruRXruOqxzOLBkObFldMOH/e7V/e7V/e7V/vrz////7PL9RXruRXruRXruP5aqObFlOLBkObFl6/fw////////////////////faLzRXruRXruRIDgObFlOLBkObFlnNiy////////////////////x9f6RXruRXruRXruPaCOObFlOLBkTbl0////////////////////////q8P3orz2orz2vtD5////+/v7////////////////////////////////////////////////+/v7/////////////////////////////////////////////////Pz8vLy86Ojo7u7u+/v7oLr0RHruRHruRHruOqtyOLBkOLBkca9U9qwu9qwu9qwu+uO++/v76e76RHruRHruRHruPpWpOLBkOLBkOLBk5dah+Nqo+Nqo+eTB+/v7+/v7e6DyRHruRHruQ3/gOLBkOLBkOLBkmtaw+/v7+/v7+/v7+/v7+/v7xdT3RHruRHruRHruPKCOOLBkOLBkTLh0+/v7+/v7+/v7+/v7+/v7+/v7V4bvRHruRHrue6Dy+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+Pj4+/v7+/v7+/v7+/v7+/v7+/v7+/v7/f39////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlnNiy////////////////////x9f6RXruRXruRXruPaCOObFlOLBkTbl0////////////////////////WIfwRXruRXrufaLz////+/v7////////////////////////////////////////////////+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlnNiy////////////////////x9f6RXruRXruRXruPaCOObFlOLBkTbhy/vfq/vfq/vfq//v1////////WIfwRXruRXrufaLz////+/v7////////////////////////////9vj+7PL97PL97PL9/f7/+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlnNiy////////////////////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruX5bcnNiymtawnNiy4fPo////////////////////orz2RXruRXruRXru7PL9+/v7////////////////////////////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlmb90+s2C+s2C+s2C/vrz////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruQYrFObFlOLBkObFlxOjR////////////////////orz2RXruRXruRXru7PL9+/v7/////////////////////////////////////////////////v7+2tra8fHx8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlmK5K9qwv9qwv9qwv/vfq////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruQYrFObFlOLBkObFlxOjR////////////////////orz2RXruRXruRXruk8+4mtawnNiyuuTJ/////////////////////////////////////////v7+0dHR7u7u7u7u+/v7oLr0RHruRHruRHruOqtyOLBkOLBkca9U9qwu9qwu9qwu+uO++/v76e76RHruRHruRHruPpWpOLBkOLBkOLBk46w09qwu9qwu98Ns+/v7+/v7e6DyRHruRHruQ3/gOLBkOLBkOLBkl65K9qwu9qwu9qwu+/Pn+/v7xdT3RHruRHruRHruPKCOOLBkOLBkS7Bf9qwu9qwu9qwu+NOV+/v7+/v7V4bvRHruRHruQYrEOLBkOLBkOLBkvcJ39sp/9sp/98+L+/v7+/v7oLr0RHruRHruRHruOqtyOLBkOLBkc8eS+/v7+/v7+/v7+/v7+/v7/f39////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlmK5K9qwv9qwv9qwv/vfq////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruQYrFObFlOLBkObFlva4/9qwv9qwv97RE////////orz2RXruRXruRXruOqxzOLBkObFldMOH/e7V/e7V/e7V/vrz////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlmK5K9qwv9qwv9qwv/vfq////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruQYrFObFlOLBkObFlva4/9qwv9qwv97RE////////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlmK5K9qwv9qwv9qwv/vfq////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruQYrFObFlOLBkObFlva4/9qwv9qwv97RE////////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////////////////////////////////////8fHx////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////7PL9RXruRXruRXruP5aqObFlOLBkObFl46w09qwv9qwv+cVt////////faLzRXruRXruRIDgObFlOLBkObFlmK5K9qwv9qwv9qwv/vfq////x9f6RXruRXruRXruPaCOObFlOLBkTLBg9qwv9qwv9qwv+taX////////WIfwRXruRXruQYrFObFlOLBkObFlva4/9qwv9qwv97RE////////orz2RXruRXruRXruOqxzOLBkObFlcrBV9qwv9qwv9qwv/ObB////////////////////////////6Ojo6Ojo6Ojo7+/vq7/qZo7kZo7kZo7kXrOIXbZ+Xbd+h7Zy6rNW6rNW6rNW7t3B7+/v4ubuZo7kZo7kZo7kYqKxXbd+XbZ+Xbd+3LNa6rNW6rNW7MWE7+/v7+/vj6vnZo7kZo7kZZLaXbd+XbZ+Xbd+pLVq6rNW6rNW6rNW7+ng7+/vxtLsZo7kZo7kZo7kYKqdXbd+XbZ+a7Z66rNW6rNW6rNW7dGj7+/v7+/vdJflZo7kZo7kZJrFXbd+XbZ+Xbd+wLRi6rNW6rNW67ll7+/v7+/vq7/qZo7kZo7kZo7kXrOIXbZ+Xbd+h7Zy6rNW6rNW6rNW7t3B7+/v9vb2////////////////////9/f39/f3////////////////////////5OTk8/Pz8fHx7u7u////////////////////////////////////////6+vr7Ozs+fn5////////////////////////////////////////6Ojo8PDw8vLy9fX1////////////////////////////////////6urq6enp9PT06+vr////////////////////////////////////6+vr7Ozs6enp8fHx+Pj4////////////////////////////////////5ubm9fX17+/v8/Pz////////////////////////////////////////////////////////////////////////////ysrKx8fH1dXV39/f////////////////////////////////////////29vbzc3N6+vr////////////////////////////////////////29vb0NDQzMzM8vLy////////////////////////////////////5+fn1NTUycnJ4uLi////////////////////////////////////6OjozMzM2NjYz8/P6enp////////////////////////////////////3t7ezs7O09PT39/f////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////3d3d2dnZ3d3d1dXV3d3d////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////4+Pj39/f4eHh3Nzc6urq////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////", + "raster_sha256": "7065989daac239bd926946fd597d820470fa389b16f168c2c14953c2d4072a0e", + "semantic": { + "axis_titles": [], + "colorbar": [], + "dpr": 1, + "font_family": "\"XY Visual Baseline\"", + "legend": [ + { + "axis": "", + "side": "", + "text": "Desktop" + }, + { + "axis": "", + "side": "", + "text": "Mobile" + }, + { + "axis": "", + "side": "", + "text": "Tablet" + } + ], + "slots": { + "canvas": [ + { + "h": 348, + "w": 824, + "x": 62, + "y": 40 + } + ], + "colorbar": [], + "legend": [ + { + "h": 54, + "w": 75.7, + "x": 804.3, + "y": 46 + } + ], + "root": [ + { + "h": 430, + "w": 900, + "x": 0, + "y": 0 + } + ], + "title": [ + { + "h": 18, + "w": 900, + "x": 0, + "y": 6 + } + ] + }, + "tick_labels": [ + { + "axis": "x", + "side": "bottom", + "text": "Search" + }, + { + "axis": "x", + "side": "bottom", + "text": "Ads" + }, + { + "axis": "x", + "side": "bottom", + "text": "Email" + }, + { + "axis": "x", + "side": "bottom", + "text": "Direct" + }, + { + "axis": "x", + "side": "bottom", + "text": "Partner" + }, + { + "axis": "x", + "side": "bottom", + "text": "Social" + }, + { + "axis": "y", + "side": "left", + "text": "0" + }, + { + "axis": "y", + "side": "left", + "text": "20" + }, + { + "axis": "y", + "side": "left", + "text": "40" + }, + { + "axis": "y", + "side": "left", + "text": "60" + }, + { + "axis": "y", + "side": "left", + "text": "80" + }, + { + "axis": "y", + "side": "left", + "text": "100" + }, + { + "axis": "y", + "side": "left", + "text": "120" + } + ], + "title": [ + { + "axis": "", + "side": "", + "text": "Grouped category bars" + } + ] + } + }, + "heatmap": { + "raster": "/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////f399vb2+vr6////9vb2////////////+fn5+fn5////+vr6/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////v7+fn5+fn5+mZmZc3Nzubm5mZmZiYmJm5ubhYWFrKysnZ2dm5ubcXFxmZmZl5eXgYGB9/f3////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8fHx////////////////8fHx////////////////////8fHx////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8fHx////////////////////////////+/v7////////////////////////////////////////+/v7////////////////////////////////////////+/v7////////////////////////////////////////+/v7////////////////////////////////////////+/v7////////////////////////////////////////+/v7////////////////////////////////////////+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7////////////////////////////////8fHx////lOjqKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtXQKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGMeG7Qu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gSvCYaPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3uvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF0OlC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hCkfddgftkgftkgftkgftkgftkd9hwlbRcc9dbasZUbMlVpsCg7e3t////////////////////////////////8fHx////lOjqKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtXQKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGMeG7Qu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gSvCYaPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3uvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF0OlC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hCkfddgftkgftkgftkgftkgftkf+5ohuVkfetie+hhfexixOW89/f3/////////////////////v7+vb295eXl7u7u+/v7k+bpKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtXPKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGMeG6Qu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gSfCYaPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3uvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF0OlC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hCkfddgftkgftkgftkgftkgftkgftkgftkgftkgftkgftk1/vO+/v7/f39/////////////////v7+7e3t+/v78fHx////lOjqKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtXQKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGMeG7Qu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gSvCYaPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3uvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF0OlC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hCkfddgftkgftkgftkgftkgftkgftkgftkgftkgftkgftk2f7Q////////////////////////////////////8fHx////lOjqKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtXQKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGMeG7Qu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gSvCYaPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3aPp3uvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF0OlC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hCkfddgftkgftkgftkgftkgftkgftkgftkgftkgftkgftk2f7Q////////////////////////////////////8fHx////yeDEksCKksCKksCKksCKksCKksCJksCKksCKksCKksCKkr2Ekrh8krh8krh8krh8krh8krh8krh8krh8krh8krh8lbV0mq1hmq1hmq1hmq1hmq1hmq1hmq1hmq1hmq1hmq1hnKpcoKBGoKBGoKBGoKBGoKBGoJ9GoKBGoKBGoKBGoKBGoKBGvJAqvJAqvJAqvJAqvJAqu48qvJAqvJAqvJAqvJAqvJAq2Z0u3J4u3J4u3J4u3J4u3J4u3J4u3J4u3J4u3J4u3J4uxMRIvs1Ovs1Ovs1Ovs1Ovs1Ovs1Ovs1Ovs1Ovs1Ovs1O7PDK////////////////////////////////////8fHx/////die+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+6U5+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+Yku82wi82wi82wi82wi82wi82wi82wi82wi82wi82wi7mQg2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUWvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQ4lEZ5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa+JAy/J84/J84/J84/J84/J84/J84/J84/J84/J84/J84/uLD////////////////////////////5+fn8/Pz8fHx/////die+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+6U5+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+Yku82wi82wi82wi82wi82wi82wi82wi82wi82wi82wi7mQg2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUWvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQ4lEZ5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa+JAy/J84/J84/J84/J84/J84/J84/J84/J84/J84/J84/uLD////////////////////////////19fX6Ojo7u7u+/v7+9ac+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+6U5+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+Igu82wi82wi82wi82wi82wi82wi82wi82wi82wi82wi7WQf2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUWvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQ4lEZ5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa95Ay/J84/J84/J84/J84/J84/J84/J84/J84/J84/J84/ODB+/v7/f39////////////////////////////8fHx/////die+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+7A9+6U5+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+5Uz+Yku82wi82wi82wi82wi82wi82wi82wi82wi82wi82wi7mQg2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUW2EUWvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQvSsQ4lEZ5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa+JAy/J84/J84/J84/J84/J84/J84/J84/J84/J84/J84/uLD////////////////////////////////////8fHx/////dWd+qo7+qo7+qo7+qo7+qo7+qo7+qo7+qo7+qo7+qo7+p83+Y8w+Y8w+Y8w+Y8w+Y8w+Y4w+Y8w+Y8w+Y8w+Y8w9oMs72cg72cg72cg72cg72cg72Yg72cg72cg72cg72cg6l8e00IV00IV00IV00IV00IV00EV00IV00IV00IV00IV00IVtygQtygQtygQtygQtygQtygQtygQtygQtygQtygQtygQ41Ub6Foc6Foc6Foc6Foc6Foc6Foc6Foc6Foc6Foc6Foc95Qz+6I4+6I4+6I4+6I4+6I4+6I4+6I4+6I4+6I4+6I4/uPD////////////////////////////////////8fHx////+rqT9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn72ki5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa3kwYzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSxTMSqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQ7X0r+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku9LQ68789878987898789878987898789878987898789++zF////////////////////////////////////8fHx////+rqT9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn72ki5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa3kwYzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSxTMSqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQ7X0r+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku9LQ68789878987898789878987898789878987898789++zF////////////6enp3t7e////////1dXV5OTk7u7u+/v7+LmR9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn72ki5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa3kwXzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSxTMRqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQ7X0r+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku9LQ68789878987898789878987898789878987898789+enC+/v7/f39////4+Pjzc3N////////9/f39fX18fHx////+rqT9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn72ki5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa3kwYzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSxTMSqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQ7X0r+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku9LQ68789878987898789878987898789878987898789++zF////////////4ODg4uLi////////////////8fHx////+rqT9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn9XYn72ki5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa5lUa3kwYzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSzDcSxTMSqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQqiMQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQgREQ7X0r+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku9LQ68789878987898789878987898789878987898789++zF////////////////////////////////////8fHx////8N2b4bo34bo34bo34bo34bo34Lo34bo34bo34bo34bo35a0065kw65kw65kw65kw65kw65kv65kw65kw65kw65kw65Au6nsr6nsr6nsr6nsr6nsr6nsq6nsr6nsr6nsr6nsr53Up2WAi2WAi2WAi2WAi2WAi2WAi2WAi2WAi2WAi2WAi2WAixEIYxEIYxEIYxEIYxEIYxEIYxEIYxEIYxEIYxEIYxEIY7Kc18bI48bI48bI48bI48LI48bI48bI48bI48bI48bI419ZA0d9C0d9C0d9C0d9C0d9C0d9C0d9C0d9C0d9C0d9C8fXG////////////////////////////////////8fHx////6fSg0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC3ttA78c+78c+78c+78c+78c+78c+78c+78c+78c+78c+9L49/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/aI4+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe7MM8684/684/684/684/684/684/684/684/684/684/xOxEuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF6vzH/////////////////////////f392dnZ7+/v8fHx////6fSg0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC3ttA78c+78c+78c+78c+78c+78c+78c+78c+78c+78c+9L49/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/aI4+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe7MM8684/684/684/684/684/684/684/684/684/684/xOxEuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF6vzH////////////////////////+/v7z8/P5+fn7u7u+/v75/Kf0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC3ttA78c+78c+78c+78c+78c+78c+78c+78c+78c+78c+8749/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/aI4+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe68M8684/684/684/684/684/684/684/684/684/684/xOxEuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF6PnF+/v7/f39////////////////////////////8fHx////6fSg0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC0+hC3ttA78c+78c+78c+78c+78c+78c+78c+78c+78c+78c+9L49/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/qg7/aI4+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku+Yku8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe8WMe7MM8684/684/684/684/684/684/684/684/684/684/xOxEuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRFuvRF6vzH////////////////////////////////////8fHx////4u6oxt1Sxt1Sxt1Sxt1Sxt1Sxd1Sxt1Sxt1Sxt1Sxt1S0NJR3sFQ3sFQ3sFQ3sFQ3sFQ3sFQ3sFQ3sFQ3sFQ3sFQ4rpP6qpM6qpM6qpM6qpM6qpM6qlM6qpM6qpM6qpM6qpM6aRK5JA/5JA/5JA/5JA/5JA/5JA/5JA/5JA/5JA/5JA/5JA/3nAt3nAt3nAt3nAt3nAt3nAt3nAt3nAt3nAt3nAt3nAt4MlB4NJD4NJD4NJD4NJD4NJC4NJD4NJD4NJD4NJD4NJDue5Kr/RMr/RMr/RMr/RMr/RMr/RMr/RMr/RMr/RMr/RM5/zJ////////////////////////////////////8fHx////pr3wTHviTHviTHviTHviTHviTHviTHviTHviTHviTHviS4PoSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xRZvvObjqObjqObjqObjqObjqObjqObjqObjqObjqObjqNr3mKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzevlsgftkgftkgftkgftkgftkgftkgftkgftkgftkgftkWfmCT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKyv3c////////////////////////////////////8fHx////pr3wTHviTHviTHviTHviTHviTHviTHviTHviTHviTHviS4PoSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xRZvvObjqObjqObjqObjqObjqObjqObjqObjqObjqObjqNr3mKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzevlsgftkgftkgftkgftkgftkgftkgftkgftkgftkgftkWfmCT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKyv3c/////////////////////////Pz8tLS02dnZ7u7u+/v7pLvvTHviTHviTHviTHviTHviTHviTHviTHviTHviTHviS4PoSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xRZvvObjqObjqObjqObjqObjqObjqObjqObjqObjqObjqNr3mKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzeflsgftkgftkgftkgftkgftkgftkgftkgftkgftkgftkWfmCT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKyPvZ+/v7/f39////////////////////////////8fHx////pr3wTHviTHviTHviTHviTHviTHviTHviTHviTHviTHviS4PoSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xRZvvObjqObjqObjqObjqObjqObjqObjqObjqObjqObjqNr3mKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzevlsgftkgftkgftkgftkgftkgftkgftkgftkgftkgftkWfmCT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKyv3c////////////////////////////////////8fHx////pr3wTHviTHviTHviTHviTHviTHviTHviTHviTHviTHviS4PoSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xSo/xRZvvObjqObjqObjqObjqObjqObjqObjqObjqObjqObjqNr3mKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWKtDWN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzN+WzevlsgftkgftkgftkgftkgftkgftkgftkgftkgftkgftkWfmCT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKT/mKyv3c////////////////////////////////////8fHx////oaPHREaPREaPREaPREaPREaPQ0aOREaPREaPREaPREaPQkSGQEB5QEB5QEB5QEB5QEB5P0B4QEB5QEB5QEB5QEB5QEyJQmivQmivQmivQmivQmivQmivQmivQmivQmivQmivQm+3Q4nVQ4nVQ4nVQ4nVQ4nVQ4jVQ4nVQ4nVQ4nVQ4nVQ4nVQ7XoQ7XoQ7XoQ7XoQ7XoQ7TnQ7XoQ7XoQ7XoQ7XoQ7XoU+yXVfKOVfKOVfKOVfKOVfKOVfKOVfKOVfKOVfKOVfKOO+esNeW0NeW0NeW0NeW0NeS0NeW0NeW0NeW0NeW0NeW0wvfo////////////////////////////////////8fHx////oJi1QDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrPilcOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FPipdRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWSE2jTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVSKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+Q+apQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gL+C+KtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGv/Tu////////////////////////9/f339/f8fHx8fHx////oJi1QDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrPilcOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FPipdRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWSE2jTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVSKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+Q+apQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gL+C+KtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGv/Tu////////////////////////8/Pz2NjY7u7u7u7u+/v7npazQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrPilcOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FPipdRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWR02iTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVSKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+QuapQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gL9++KtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGvfLr+/v7/f39////////////////////////////8fHx////oJi1QDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrQDBrPilcOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FOx5FPipdRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWRkaWSE2jTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVTmrVSKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+SKD+Q+apQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gQu6gL+C+KtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGKtzGv/Tu////////////////////////////////////8fHx////s6zEZlmJZlmJZlmJZlmJZlmJZVmIZlmJZlmJZlmJZlmJZVR8YktqYktqYktqYktqYktqYUppYktqYktqYktqYktqZVV+a2ura2ura2ura2ura2uramqqa2ura2ura2ura2urbHG1cYjdcYjdcYjdcYjdcYjdcYfdcYjdcYjdcYjdcYjdcYjdbbP+bbP+bbP+bbP+bbP+bLL9bbP+bbP+bbP+bbP+bbP+aOu7aPGzaPGzaPGzaPGzZ/GyaPGzaPGzaPGzaPGzaPGzWObLVePRVePRVePRVePRVOLRVePRVePRVePRVePRVePRzPfx////////////////////////////////////6enp8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx7u7u8fHx8fHx8fHx8fHx8fHx8fHx8fHx9/f3////////////////////////////////////////////////////+Pj439/f8vLy+/v7////////////////////////////////5OTk9vb2/v7+////////////////////////////////4eHh7e3t7e3t////////////////////////////////5+fn6+vr/f39////////////////////////////////8/Pz7u7u+fn5////////////////////////////////8/Pz6+vr8PDw////////////////////////////////8vLy7+/v9vb2////////////////////////////////////////////////////////////////////////////////9fX109PTzs7O8fHx////////////////////////////////5ubmy8vL9PT0////////////////////////////////0tLSzMzM3t7e////////////////////////////////8vLy1NTU6enp////////////////////////////////8PDw5ubm8vLy////////////////////////////////9fX1w8PD7u7u////////////////////////////////8vLyyMjI4eHh////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////4eHh19fX9PT0////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////5ubm1tbW7e3t////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////", + "raster_sha256": "9e722d0bd90f18712ce81722812acadf5afe773ec5617cc35a22469996cce734", + "semantic": { + "axis_titles": [], + "colorbar": [], + "dpr": 1, + "font_family": "\"XY Visual Baseline\"", + "legend": [ + { + "axis": "", + "side": "", + "text": "activity" + } + ], + "slots": { + "canvas": [ + { + "h": 348, + "w": 824, + "x": 62, + "y": 40 + } + ], + "colorbar": [], + "legend": [ + { + "h": 22, + "w": 69.7, + "x": 810.3, + "y": 46 + } + ], + "root": [ + { + "h": 430, + "w": 900, + "x": 0, + "y": 0 + } + ], + "title": [ + { + "h": 18, + "w": 900, + "x": 0, + "y": 6 + } + ] + }, + "tick_labels": [ + { + "axis": "x", + "side": "bottom", + "text": "Mon" + }, + { + "axis": "x", + "side": "bottom", + "text": "Tue" + }, + { + "axis": "x", + "side": "bottom", + "text": "Wed" + }, + { + "axis": "x", + "side": "bottom", + "text": "Thu" + }, + { + "axis": "x", + "side": "bottom", + "text": "Fri" + }, + { + "axis": "x", + "side": "bottom", + "text": "Sat" + }, + { + "axis": "x", + "side": "bottom", + "text": "Sun" + }, + { + "axis": "y", + "side": "left", + "text": "00" + }, + { + "axis": "y", + "side": "left", + "text": "04" + }, + { + "axis": "y", + "side": "left", + "text": "08" + }, + { + "axis": "y", + "side": "left", + "text": "12" + }, + { + "axis": "y", + "side": "left", + "text": "16" + }, + { + "axis": "y", + "side": "left", + "text": "20" + } + ], + "title": [ + { + "axis": "", + "side": "", + "text": "Weekly activity heatmap" + } + ] + } + } + }, + "pinned": { + "browser_engine": "chromium", + "browser_version": "149.0.7827.55", + "downsample": 10, + "dpr": 1.0, + "font_path": "docs/app/xy_docs/assets/InstrumentSans-wdth-wght.ttf", + "font_sha256": "b24f1812584816958afcf22e22d08e44318c5e51651e25d2438efdde389b33b1", + "playwright": "1.61.1", + "viewport": [ + 900, + 470 + ] + }, + "schema_version": 1, + "suite": "xy-reviewed-visual-baselines", + "tolerances": { + "changed_cell_delta": 30, + "max_changed_cell_fraction": 0.12, + "max_geometry_delta_px": 2.0, + "max_mean_abs_error": 10.0 + }, + "update": { + "prepared_at_utc": "2026-07-22T03:10:26+00:00", + "prepared_by": "Alek Petuskey", + "reason": "Establish TST-NI-014 reviewed visual baselines", + "review_required": true + } +} diff --git a/src/lib.rs b/src/lib.rs index fa061108..bfaf04f4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,7 +80,11 @@ unsafe fn borrowed_byte_spans<'a>( /// ABI version — bumped on any signature change. The Python wrapper checks this /// at load time and refuses a mismatched library loudly (§33 comm-versioning /// rule, applied to the in-process boundary). -pub const ABI_VERSION: u32 = 37; +pub const ABI_VERSION: u32 = 38; +pub const NATIVE_CAP_SCALAR: u32 = 1 << 0; +pub const NATIVE_CAP_AVX2_AVAILABLE: u32 = 1 << 1; +pub const NATIVE_CAP_AVX2_SELECTED: u32 = 1 << 2; +pub const NATIVE_CAP_AARCH64: u32 = 1 << 3; const FACTORIZE_CAPACITY_EXCEEDED: usize = usize::MAX - 1; #[no_mangle] @@ -88,6 +92,27 @@ pub extern "C" fn xy_abi_version() -> u32 { ABI_VERSION } +/// Report native dispatch capabilities for executable architecture evidence. +/// Scalar kernels are always available. AVX2 availability describes the host +/// CPU, while AVX2 selection also reflects the `XY_SIMD=0` kill switch. +/// AArch64 reports its mandatory baseline SIMD separately because those scalar +/// Rust loops are autovectorized to NEON rather than runtime-dispatched. +#[no_mangle] +pub extern "C" fn xy_runtime_capabilities() -> u32 { + let mut capabilities = NATIVE_CAP_SCALAR; + if simd::avx2_available() { + capabilities |= NATIVE_CAP_AVX2_AVAILABLE; + } + if simd::use_avx2() { + capabilities |= NATIVE_CAP_AVX2_SELECTED; + } + #[cfg(target_arch = "aarch64")] + { + capabilities |= NATIVE_CAP_AARCH64; + } + capabilities +} + /// Serialize parallel f64 screen coordinates into SVG polyline path data. /// Returns the required byte count, or `usize::MAX` for invalid inputs. When /// `out_cap` is too small no bytes are written, allowing callers to retry. @@ -2839,6 +2864,19 @@ pub unsafe extern "C" fn xy_pyramid_free(handle: u64) -> i32 { mod tests { use super::*; + #[test] + fn runtime_capability_bits_are_coherent() { + let capabilities = xy_runtime_capabilities(); + assert_ne!(capabilities & NATIVE_CAP_SCALAR, 0); + if capabilities & NATIVE_CAP_AVX2_SELECTED != 0 { + assert_ne!(capabilities & NATIVE_CAP_AVX2_AVAILABLE, 0); + } + #[cfg(target_arch = "aarch64")] + assert_ne!(capabilities & NATIVE_CAP_AARCH64, 0); + #[cfg(not(target_arch = "aarch64"))] + assert_eq!(capabilities & NATIVE_CAP_AARCH64, 0); + } + #[test] fn ffi_guard_maps_panic_to_sentinel() { // A panic anywhere behind the C ABI must become the entry point's diff --git a/src/simd.rs b/src/simd.rs index 96c7bb0a..00d339c0 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -44,14 +44,21 @@ const BLOCK: usize = 1024; /// bucket indices are always well below this (grids are screen-sized). const SKIP: u32 = u32::MAX; +#[cfg(target_arch = "x86_64")] +pub(crate) fn avx2_available() -> bool { + std::arch::is_x86_feature_detected!("avx2") +} + +#[cfg(not(target_arch = "x86_64"))] +pub(crate) fn avx2_available() -> bool { + false +} + #[cfg(target_arch = "x86_64")] pub(crate) fn use_avx2() -> bool { use std::sync::OnceLock; static ON: OnceLock = OnceLock::new(); - *ON.get_or_init(|| { - std::env::var_os("XY_SIMD").is_none_or(|v| v != "0") - && std::arch::is_x86_feature_detected!("avx2") - }) + *ON.get_or_init(|| std::env::var_os("XY_SIMD").is_none_or(|v| v != "0") && avx2_available()) } #[cfg(not(target_arch = "x86_64"))] diff --git a/tests/host_integration/test_host_matrix.py b/tests/host_integration/test_host_matrix.py new file mode 100644 index 00000000..7c2bd73f --- /dev/null +++ b/tests/host_integration/test_host_matrix.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import anywidget +from fastapi.testclient import TestClient +from scripts import host_integration_policy + +from xy._figure import Figure +from xy.widget import FigureWidget, bundled_js + +ROOT = Path(__file__).resolve().parents[2] +FASTAPI_DIR = ROOT / "examples" / "fastapi" + + +def _load_fastapi_app(): + sys.path.insert(0, str(FASTAPI_DIR)) + spec = importlib.util.spec_from_file_location( + "xy_host_matrix_fastapi_app", FASTAPI_DIR / "app.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_installed_anywidget_and_fastapi_stack_matches_selected_profile() -> None: + profile = os.environ.get("XY_HOST_PROFILE", "latest") + policy = host_integration_policy.load_policy() + errors, installed = host_integration_policy.validate_installed( + policy, profile, ["anywidget", "fastapi"] + ) + assert errors == [] + assert set(installed) == { + "anywidget", + "traitlets", + "fastapi", + "starlette", + "uvicorn", + "httpx", + } + + +def test_anywidget_mount_and_binary_comm_transport() -> None: + figure = Figure().scatter([0.0, 1.0, 2.0], [0.0, 1.0, 0.0]) + widget = FigureWidget(figure) + sent: list[tuple[dict, object]] = [] + widget.send = lambda content, buffers=None: sent.append((content, buffers)) + try: + assert isinstance(widget, anywidget.AnyWidget) + assert "export" in bundled_js("widget") + assert widget.spec["buffer_layout"] == "split" + assert widget.buffers and all(isinstance(value, memoryview) for value in widget.buffers) + + widget._on_custom_msg( + None, + {"type": "pick", "trace": 0, "index": 1, "seq": 19}, + None, + ) + assert sent[-1][0]["type"] == "pick_result" + assert sent[-1][0]["seq"] == 19 + assert sent[-1][0]["row"]["x"] == 1.0 + + widget.append(figure.traces[0].id, [3.0], [1.0]) + assert sent[-1][0]["type"] == "append" + assert sent[-1][1] and isinstance(sent[-1][1][0], (bytes, memoryview)) + finally: + widget.close() + + +def test_fastapi_compile_mount_and_http_transport(monkeypatch) -> None: + for path in sorted(FASTAPI_DIR.glob("*.py")): + compile(path.read_text(encoding="utf-8"), str(path), "exec") + + monkeypatch.setenv("XY_LIVE_POINTS", "2000") + module = _load_fastapi_app() + with TestClient(module.app) as client: + health = client.get("/healthz") + assert health.status_code == 200 + + chart = client.get("/chart/line-walk") + assert chart.status_code == 200 + assert "renderStandalone" in chart.text + assert " None: + plt.close("all") + plt.rcdefaults() + + +def test_artist_compat_noops_are_invariant() -> None: + _fig, ax = plt.subplots() + collection = ax.scatter([0.0, 1.0], [1.0, 2.0]) + collection.set_sizes([9.0, 25.0], dpi=72) + at_72 = collection.get_sizes().copy() + collection.set_sizes([9.0, 25.0], dpi=288) + np.testing.assert_array_equal(collection.get_sizes(), at_72) + + text = ax.text(0.25, 0.75, "renderer independent") + assert ( + text.get_window_extent(renderer=None).bounds + == text.get_window_extent(renderer=object()).bounds + ) + + +def _new_axes() -> tuple[object, object]: + fig, ax = plt.subplots() + ax.plot([0.0, 1.0, 2.0], [1.0, 3.0, 2.0]) + return fig, ax + + +def _aspect_snapshot(ax: object) -> tuple[object, ...]: + return ( + ax._aspect_equal, + ax._aspect_adjustable, + ax._aspect_bounds, + ax.get_xlim(), + ax.get_ylim(), + ) + + +def test_axes_compat_noops_are_invariant() -> None: + fig, ax = _new_axes() + assert ax.get_figure(root=False) is fig + assert ax.get_figure(root=True) is fig + assert ax.get_position(original=False).bounds == ax.get_position(original=True).bounds + + _fig_a, ax_a = _new_axes() + _fig_b, ax_b = _new_axes() + assert ax_a.axis(xmin=-1.0, ymax=5.0, emit=False) == ax_b.axis(xmin=-1.0, ymax=5.0, emit=True) + + _fig_a, ax_a = _new_axes() + _fig_b, ax_b = _new_axes() + ax_a.margins(x=0.2, y=0.3, tight=False) + ax_b.margins(x=0.2, y=0.3, tight=True) + assert (ax_a.get_xlim(), ax_a.get_ylim()) == (ax_b.get_xlim(), ax_b.get_ylim()) + + _fig_a, ax_a = _new_axes() + _fig_b, ax_b = _new_axes() + ax_a.lines[0].set_visible(False) + ax_b.lines[0].set_visible(False) + ax_a.relim(visible_only=False) + ax_b.relim(visible_only=True) + assert (ax_a.get_xlim(), ax_a.get_ylim()) == (ax_b.get_xlim(), ax_b.get_ylim()) + + _fig_a, ax_a = _new_axes() + _fig_b, ax_b = _new_axes() + ax_a.set_aspect("equal", adjustable="box") + ax_b.set_aspect("equal", adjustable="box", anchor="NE", share=True) + assert _aspect_snapshot(ax_a) == _aspect_snapshot(ax_b) + + +def test_facetgrid_legend_out_noop_is_bounded_by_hue_rejection() -> None: + data = { + "group": np.asarray(["a", "a", "b", "b"]), + "x": np.asarray([0.0, 1.0, 0.0, 1.0]), + "y": np.asarray([1.0, 2.0, 3.0, 4.0]), + } + + def snapshot(legend_out: bool) -> tuple[object, ...]: + grid = FacetGrid(data, row="group", legend_out=legend_out) + grid.map(plt.plot, "x", "y") + + def stable(value: object) -> object: + if isinstance(value, dict): + return {key: stable(item) for key, item in value.items() if key != "link_group"} + if isinstance(value, list): + return [stable(item) for item in value] + return value + + payloads = tuple( + (stable(spec), blob) + for spec, blob in (chart.figure().build_payload() for chart in grid.figure._charts()) + ) + return grid.axes.shape, tuple(grid.row_names), payloads + + assert snapshot(False) == snapshot(True) + with pytest.raises(NotImplementedError): + FacetGrid(data, row="group", hue="group", legend_out=False) + with pytest.raises(NotImplementedError): + FacetGrid(data, row="group", hue="group", legend_out=True) + + +def test_figure_clear_observer_noop_is_invariant() -> None: + def cleared(keep_observers: bool) -> tuple[object, ...]: + fig, ax = _new_axes() + ax.set_title("discard me") + fig.clear(keep_observers=keep_observers) + return ( + tuple(fig.axes), + fig._current_ax, + fig._nrows, + fig._ncols, + fig._suptitle, + fig._gci, + ) + + assert cleared(False) == cleared(True) + + +def test_clabel_compat_noops_are_invariant() -> None: + def labels(**kwargs: object) -> tuple[tuple[object, ...], ...]: + _fig, ax = plt.subplots() + z = np.asarray([[0.0, 1.0, 2.0], [1.0, 3.0, 4.0], [2.0, 4.0, 6.0]]) + contours = ax.contour(z, levels=[2.0, 4.0]) + result = ax.clabel(contours, fmt="L=%.0f", **kwargs) + return tuple( + ( + label.get_text(), + tuple(float(value) for value in label._entry["args"][:2]), + tuple(sorted(label._entry["kwargs"].items())), + ) + for label in result + ) + + baseline = labels() + altered = labels( + fontsize=31, + inline=False, + inline_spacing=40, + use_clabeltext=True, + rightside_up=False, + zorder=99, + ) + assert altered == baseline + + +def test_locator_compat_noops_are_invariant() -> None: + np.testing.assert_array_equal( + plt.NullLocator().tick_values(-10.0, 20.0), + plt.NullLocator().tick_values(100.0, 200.0), + ) + fixed = plt.FixedLocator([-1.0, 0.0, 3.0]) + np.testing.assert_array_equal( + fixed.tick_values(-1_000.0, -900.0), fixed.tick_values(900.0, 1_000.0) + ) + np.testing.assert_array_equal( + plt.MaxNLocator(4, prune=None).tick_values(-2.0, 9.0), + plt.MaxNLocator(4, prune="both").tick_values(-2.0, 9.0), + ) + np.testing.assert_array_equal( + plt.LogLocator(numticks=2).tick_values(0.1, 10_000.0), + plt.LogLocator(numticks=200).tick_values(0.1, 10_000.0), + ) + + +def test_formatter_compat_noops_are_invariant() -> None: + assert plt.ScalarFormatter()(12.5, pos=0) == plt.ScalarFormatter()(12.5, pos=99) + assert plt.NullFormatter()(-1.0, pos=0) == plt.NullFormatter()(999.0, pos=99) == "" + fixed = plt.FixedFormatter(["first", "second"]) + assert fixed(-1_000.0, pos=1) == fixed(1_000.0, pos=1) == "second" + percent = plt.FormatStrFormatter("%.1f") + assert percent(3.25, pos=0) == percent(3.25, pos=99) + + +def _ms(value: dt.datetime) -> float: + return (value - dt.datetime(1970, 1, 1)).total_seconds() * 1000.0 + + +def test_date_compat_noops_are_invariant() -> None: + value = _ms(dt.datetime(2025, 6, 15, 12, 30)) + baseline = plt.dates.DateFormatter("%Y-%m-%d %H:%M") + configured = plt.dates.DateFormatter( + "%Y-%m-%d %H:%M", tz=dt.timezone(dt.timedelta(hours=9)), usetex=True + ) + assert baseline(value, pos=0) == configured(value, pos=99) + + lo = _ms(dt.datetime(2024, 1, 1)) + hi = _ms(dt.datetime(2025, 12, 31)) + factories = ( + lambda tz: plt.dates.DayLocator(bymonthday=[1, 15], interval=2, tz=tz), + lambda tz: plt.dates.MonthLocator(bymonth=[1, 4, 7, 10], interval=2, tz=tz), + lambda tz: plt.dates.YearLocator(base=1, month=6, day=1, tz=tz), + ) + for factory in factories: + np.testing.assert_array_equal( + factory(None).tick_values(lo, hi), + factory(dt.timezone(dt.timedelta(hours=-7))).tick_values(lo, hi), + ) diff --git a/tests/pyplot/test_silent_drop_regressions.py b/tests/pyplot/test_silent_drop_regressions.py index 55e897e3..644d2a96 100644 --- a/tests/pyplot/test_silent_drop_regressions.py +++ b/tests/pyplot/test_silent_drop_regressions.py @@ -1,9 +1,7 @@ """Regressions from the adversarial completion review: values that previously crashed, were silently dropped, or bypassed validation must now behave.""" -import ast import warnings -from pathlib import Path import numpy as np import pytest @@ -12,58 +10,6 @@ from xy.pyplot._colors import Cmap -def test_public_adapters_cannot_discard_parameters_without_an_explicit_marker(): - """Mechanical guard against newly accepted-and-dropped kwargs. - - A deliberate compatibility no-op must carry an inline ``compat-noop:`` - explanation. Bare ``kwargs.pop`` calls and deleting named public method - parameters otherwise fail automatically; no hand-maintained keyword list - is involved. - """ - root = Path(__file__).resolve().parents[2] / "python" / "xy" / "pyplot" - violations = [] - for path in root.glob("*.py"): - source = path.read_text() - lines = source.splitlines() - tree = ast.parse(source, filename=str(path)) - for function in ( - node - for node in ast.walk(tree) - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - and not node.name.startswith("_") - ): - parameters = { - arg.arg - for arg in ( - *function.args.posonlyargs, - *function.args.args, - *function.args.kwonlyargs, - ) - } - for node in ast.walk(function): - if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): - marked = "compat-noop:" in lines[node.lineno - 1] - call = node.value - if ( - isinstance(call.func, ast.Attribute) - and call.func.attr == "pop" - and isinstance(call.func.value, ast.Name) - and call.func.value.id in {"kwargs", "options"} - and not marked - ): - violations.append( - f"{path.name}:{node.lineno} bare {call.func.value.id}.pop" - ) - if isinstance(node, ast.Delete): - marked = "compat-noop:" in lines[node.lineno - 1] - discarded = { - target.id for target in node.targets if isinstance(target, ast.Name) - } & parameters - if discarded and not marked: - violations.append(f"{path.name}:{node.lineno} deletes {sorted(discarded)}") - assert not violations, "unexplained accepted-and-dropped options:\n" + "\n".join(violations) - - def teardown_function(): plt.close("all") plt.rcdefaults() diff --git a/tests/pyplot/test_state_and_figs.py b/tests/pyplot/test_state_and_figs.py index 9039e7c7..39807eb2 100644 --- a/tests/pyplot/test_state_and_figs.py +++ b/tests/pyplot/test_state_and_figs.py @@ -124,6 +124,17 @@ def test_grid_html_has_panels() -> None: assert "Content-Security-Policy" in html +def test_css_grid_html_uses_both_declared_dimensions() -> None: + from xy.pyplot._grid import compose_html + + _fig, ax = plt.subplots() + ax.plot([0, 1], [0, 1]) + chart = ax._build_chart(320, 240) + html = compose_html([chart], nrows=7, ncols=3, suptitle=None) + assert "grid-template-columns: repeat(3, max-content)" in html + assert "grid-template-rows: repeat(7, max-content)" in html + + def test_notebook_repr_isolates_standalone_document_styles() -> None: fig, ax = plt.subplots(figsize=(6.4, 4.8), dpi=100) ax.plot([0, 1], [1, 2]) diff --git a/tests/reflex_adapter/__init__.py b/tests/reflex_adapter/__init__.py deleted file mode 100644 index 7e77b323..00000000 --- a/tests/reflex_adapter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""reflex-xy adapter tests (skipped unless reflex + reflex_xy are installed).""" diff --git a/tests/reflex_adapter/conftest.py b/tests/reflex_adapter/conftest.py deleted file mode 100644 index f9a0d71c..00000000 --- a/tests/reflex_adapter/conftest.py +++ /dev/null @@ -1,38 +0,0 @@ -"""reflex-xy adapter tests. - -These run only when the adapter's dependencies are installed -(`uv pip install -e python/reflex-xy`); the core `xy` suite must -never require Reflex (CLAUDE.md dependency rule), so everything here -importorskips. -""" - -from __future__ import annotations - -import pytest - -reflex = pytest.importorskip("reflex") -pytest.importorskip("reflex_xy") - -import reflex_xy.app as adapter_app # noqa: E402 -from reflex_xy.registry import reset_registry_for_tests # noqa: E402 - - -@pytest.fixture(autouse=True) -def _fresh_registry(): - """Isolate registry + wiring between tests.""" - registry = reset_registry_for_tests() - adapter_app.reset_setup_for_tests() - yield registry - reset_registry_for_tests() - adapter_app.reset_setup_for_tests() - - -@pytest.fixture -def client_token() -> str: - return "11111111-2222-4333-8444-555566667777" - - -def make_router_data(token: str): - import reflex.istate.data as istate_data - - return istate_data.RouterData.from_router_data({"token": token}) diff --git a/tests/test_animation_smoke.py b/tests/test_animation_smoke.py new file mode 100644 index 00000000..59734162 --- /dev/null +++ b/tests/test_animation_smoke.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import importlib.util +import inspect +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def _load_module(): + scripts = ROOT / "scripts" + sys.path.insert(0, str(scripts)) + spec = importlib.util.spec_from_file_location("animation_smoke", scripts / "animation_smoke.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +animation_smoke = _load_module() + + +def _success_title() -> str: + return " ".join(animation_smoke.EXPECTED_ASSERTIONS) + " events=4/4" + + +def _stub_bundle(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + (tmp_path / "standalone.js").write_text("window.xy = {};", encoding="utf-8") + monkeypatch.setattr(animation_smoke, "STATIC", tmp_path) + + +def _stub_browser( + monkeypatch: pytest.MonkeyPatch, + *, + title: str, + returncode: int = 0, +) -> None: + monkeypatch.setattr( + animation_smoke.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace( + stdout=f"{title}", + stderr="synthetic animation diagnostic", + returncode=returncode, + ), + ) + + +def test_animation_fixtures_derive_the_shared_protocol() -> None: + scatter, _ = animation_smoke._payload([(1, 2, 3)]) + errorbar, _ = animation_smoke._errorbar_payload() + bar, _ = animation_smoke._bar_payload([(1, 2, 3)]) + assert {scatter["protocol"], errorbar["protocol"], bar["protocol"]} == { + animation_smoke.PROTOCOL_VERSION + } + for builder in ( + animation_smoke._payload, + animation_smoke._errorbar_payload, + animation_smoke._bar_payload, + ): + assert '"protocol": PROTOCOL_VERSION' in inspect.getsource(builder) + + +def test_animation_smoke_writes_success_evidence(monkeypatch, tmp_path: Path) -> None: + _stub_bundle(monkeypatch, tmp_path) + _stub_browser(monkeypatch, title=_success_title()) + evidence = tmp_path / "artifacts" / "animation.json" + + rc = animation_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 0 + assert payload["status"] == "ok" + assert payload["missing_assertions"] == [] + assert payload["protocol"] == animation_smoke.PROTOCOL_VERSION + + +def test_animation_assertion_failure_is_blocking_and_retained(monkeypatch, tmp_path: Path) -> None: + _stub_bundle(monkeypatch, tmp_path) + _stub_browser(monkeypatch, title=_success_title().replace("frozen=1", "frozen=0")) + evidence = tmp_path / "animation.json" + + rc = animation_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["status"] == "failed" + assert "frozen=1" in payload["missing_assertions"] + assert payload["stderr_tail"] == "synthetic animation diagnostic" + + +def test_animation_assertions_are_matched_as_complete_tokens() -> None: + title = _success_title().replace(" bar=1 ", " bar=0 ") + + evidence = animation_smoke._evidence( + title=title, + chromium="/configured/chromium", + returncode=0, + ) + + assert evidence["status"] == "failed" + assert evidence["missing_assertions"] == ["bar=1"] + + +def test_explicit_missing_chromium_does_not_fall_back(monkeypatch) -> None: + monkeypatch.setattr( + animation_smoke.shutil, + "which", + lambda candidate: "/found/fallback" if candidate == "chromium" else None, + ) + + with pytest.raises(RuntimeError, match="configured chromium not found"): + animation_smoke._chrome("/configured/but/missing/chromium") + + +def test_configured_browser_missing_is_blocking_and_retained(monkeypatch, tmp_path: Path) -> None: + _stub_bundle(monkeypatch, tmp_path) + evidence = tmp_path / "animation-missing-browser.json" + + rc = animation_smoke.main(["/configured/but/missing/chromium", "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["status"] == "failed" + assert payload["chromium_returncode"] is None + assert "configured chromium not found" in payload["title"] + + +def test_browser_startup_failure_is_blocking_and_retained(monkeypatch, tmp_path: Path) -> None: + _stub_bundle(monkeypatch, tmp_path) + + def fail_to_start(*args, **kwargs): + raise OSError("synthetic browser startup failure") + + monkeypatch.setattr(animation_smoke.subprocess, "run", fail_to_start) + evidence = tmp_path / "animation-startup.json" + + rc = animation_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["status"] == "failed" + assert payload["chromium_returncode"] is None + assert "startup failed" in payload["title"] + + +def test_browser_timeout_is_blocking_and_retained(monkeypatch, tmp_path: Path) -> None: + _stub_bundle(monkeypatch, tmp_path) + + def time_out(*args, **kwargs): + raise subprocess.TimeoutExpired( + cmd=args[0], + timeout=kwargs["timeout"], + stderr="synthetic animation timeout diagnostic", + ) + + monkeypatch.setattr(animation_smoke.subprocess, "run", time_out) + evidence = tmp_path / "animation-timeout.json" + + rc = animation_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["status"] == "failed" + assert payload["timed_out"] is True + assert payload["chromium_returncode"] is None + assert payload["stderr_tail"] == "synthetic animation timeout diagnostic" diff --git a/tests/test_batch_export.py b/tests/test_batch_export.py index 0e682363..cf9f2021 100644 --- a/tests/test_batch_export.py +++ b/tests/test_batch_export.py @@ -107,3 +107,44 @@ def render_image(self, html, _width, _height, *, format, scale, quality, transpa assert f"" in seen[0][0] assert seen[0][1] == "png" assert seen[0][2] == 2.0 + + +def test_browser_session_never_silently_downgrades(monkeypatch): + from xy import _chromium + + calls = [] + + class FailingSession: + def __init__(self, executable, *, gl, sandbox): + calls.append((executable, gl, sandbox)) + raise _chromium.ChromiumError("sandbox unavailable") + + monkeypatch.setattr(export, "find_browser", lambda explicit=None: "/fake/chrome") + monkeypatch.setattr(_chromium, "ChromiumSession", FailingSession) + + with pytest.raises( + _chromium.ChromiumError, + match=r"sandboxed Chromium launch failed.*No unsandboxed retry.*sandbox=False", + ): + export._browser_session(gl="software", sandbox=True) + + assert calls == [("/fake/chrome", "software", True)] + + +def test_browser_session_honors_explicit_unsandboxed_opt_in(monkeypatch): + from xy import _chromium + + calls = [] + + class FailingSession: + def __init__(self, executable, *, gl, sandbox): + calls.append((executable, gl, sandbox)) + raise _chromium.ChromiumError("launch failed") + + monkeypatch.setattr(export, "find_browser", lambda explicit=None: "/fake/chrome") + monkeypatch.setattr(_chromium, "ChromiumSession", FailingSession) + + with pytest.raises(_chromium.ChromiumError, match="launch failed"): + export._browser_session(gl="hardware", sandbox=False) + + assert calls == [("/fake/chrome", "hardware", False)] diff --git a/tests/test_benchmark_environment.py b/tests/test_benchmark_environment.py index 893fb98b..482978d5 100644 --- a/tests/test_benchmark_environment.py +++ b/tests/test_benchmark_environment.py @@ -166,6 +166,10 @@ def test_interaction_browser_gates_cover_scatter_and_core_chart_families() -> No "run_worker_probe", "omits Chromium's virtual-time flag", "WORKER_PROBE_TIMEOUT_S = 60", + "worker_terminated", + "worker_cleared", + "root_removed", + "teardown_complete", '"family": "line"', '"family": "histogram"', '"family": "bar"', @@ -174,7 +178,9 @@ def test_interaction_browser_gates_cover_scatter_and_core_chart_families() -> No ] for marker in required_markers: assert marker in bench - assert 'startswith("skipped(")' in smoke + assert '"--allow-worker-skip"' in smoke + assert "WORKER_REQUIRED_TRUE" in smoke + assert "_worker_errors(worker, allow_skip=args.allow_worker_skip)" in smoke def test_interaction_benchmark_completes_gpu_warmup_before_timing() -> None: @@ -193,9 +199,11 @@ def test_dashboard_benchmark_reports_eviction_and_scroll_telemetry() -> None: bench = (ROOT / "benchmarks" / "bench_dashboard.py").read_text(encoding="utf-8") for marker in ( - 'addEventListener("webglcontextlost"', - 'addEventListener("webglcontextrestored"', + 'addEventListener("xy:context_lost"', + "event.detail?.governed === true", + 'addEventListener("xy:context_restored"', "scrollIntoView", + 'dispatchEvent(new PointerEvent("pointerenter"))', "context_lost_chart_ids", "context_restored_chart_ids", "initial_nonblank_chart_ids", @@ -213,7 +221,7 @@ def test_dashboard_benchmark_reports_eviction_and_scroll_telemetry() -> None: # webglcontextlost dispatches as a task, so the probe must yield before # leaving the "create" phase or creation-loop evictions get mislabeled # with whatever phase is current when the queued events finally fire. - creation_loop = bench.index('addEventListener("webglcontextlost"') + creation_loop = bench.index('addEventListener("xy:context_lost"') first_yield = bench.index( "await new Promise((resolve) => setTimeout(resolve, 0));", creation_loop ) @@ -221,20 +229,63 @@ def test_dashboard_benchmark_reports_eviction_and_scroll_telemetry() -> None: assert first_yield < phase_initial +def test_dashboard_probe_budget_scales_with_requested_chart_count(monkeypatch) -> None: + from benchmarks import bench_dashboard + + class DummyFigure: + def build_payload(self): + return {}, b"" + + captured: dict[str, int] = {} + + def fake_probe(html, *, marker, chromium, virtual_time_ms, timeout_s): + del html, marker, chromium + captured["virtual_time_ms"] = virtual_time_ms + captured["timeout_s"] = timeout_s + return {"status": "failed(test sentinel)"} + + monkeypatch.setattr( + bench_dashboard, "_dashboard_figures", lambda count: [DummyFigure() for _ in range(count)] + ) + monkeypatch.setattr(bench_dashboard, "page_for_charts", lambda *args, **kwargs: "") + monkeypatch.setattr(bench_dashboard, "run_json_probe", fake_probe) + monkeypatch.setattr(bench_dashboard, "collect_environment_metadata", lambda **kwargs: {}) + + bench_dashboard.run(chart_counts=[50], chromium="/test/chromium") + + assert captured["virtual_time_ms"] == ( + bench_dashboard.DASHBOARD_BASE_VIRTUAL_TIME_MS + + 50 * bench_dashboard.DASHBOARD_SETTLE_VIRTUAL_TIME_MS_PER_CHART + ) + assert captured["virtual_time_ms"] > 50 * 400 + assert captured["timeout_s"] == bench_dashboard.DASHBOARD_PROBE_TIMEOUT_S + + def test_context_governor_reserves_pending_restores() -> None: """Concurrent visibility callbacks must count restores before their asynchronous ``webglcontextrestored`` events acquire the contexts.""" client = (ROOT / "js" / "src" / "50_chartview.ts").read_text(encoding="utf-8") assert "view._ctxPendingReservation" in client + assert "maxPendingRestores: 1" in client + assert "this._ctxLostPending = true" in client + assert "const activeBudget = Math.max(1, this.budget()" in client constructor = client[client.index(" constructor(") : client.index(" _listen(")] assert 'this.root.textContent = "xy: WebGL2 unavailable in this browser.";' in constructor init_gl = client[client.index(" _initGl(buffer) {") : client.index(" _buildTrace(")] assert "WebGL2 unavailable in this browser" not in init_gl recover = client.index(" _recoverContext() {") - reserve = client.index("XY_CONTEXT_GOVERNOR.reserve(this);", recover) + deferred_reserve = client.index( + "XY_CONTEXT_GOVERNOR.reserve(this, { deferred: true })", recover + ) + release_event_wait = client.index("if (this._ctxReleasedExt && this._ctxLostPending)", recover) restore = client.index("ext.restoreContext();", recover) - assert reserve < restore + assert release_event_wait < deferred_reserve < restore + + pointer = client[client.index('this._listen(this.root, "pointerenter"') :] + promote = pointer.index("this._ctxSeenSeq = XY_CONTEXT_GOVERNOR.seq++;") + recover_pointer = pointer.index("this._recoverContext();") + assert promote < recover_pointer snapshot = client[client.index("_snapshotBeforeRelease()") : recover] draw = snapshot.index("this._drawNow();") diff --git a/tests/test_browser_conformance.py b/tests/test_browser_conformance.py new file mode 100644 index 00000000..110be79f --- /dev/null +++ b/tests/test_browser_conformance.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PROBE = ROOT / "scripts" / "browser_conformance.mjs" + +EXPECTED_MATRIX = [ + { + "id": "direct-linear-scatter-dpr1-reduced", + "tier": "direct", + "family": "scatter", + "dpr": 1, + "motion": "reduce", + "axisClasses": ["linear"], + }, + { + "id": "decimated-log-line-dpr2-motion", + "tier": "decimated", + "family": "line", + "dpr": 2, + "motion": "no-preference", + "axisClasses": ["log"], + }, + { + "id": "direct-category-bar-dpr1-motion", + "tier": "direct", + "family": "bar", + "dpr": 1, + "motion": "no-preference", + "axisClasses": ["category"], + }, + { + "id": "direct-linear-heatmap-dpr2-reduced", + "tier": "direct", + "family": "heatmap", + "dpr": 2, + "motion": "reduce", + "axisClasses": ["linear"], + }, + { + "id": "direct-named-mesh-dpr1-motion", + "tier": "direct", + "family": "mesh", + "dpr": 1, + "motion": "no-preference", + "axisClasses": ["named"], + }, + { + "id": "density-linear-scatter-dpr2-reduced", + "tier": "density", + "family": "scatter", + "dpr": 2, + "motion": "reduce", + "axisClasses": ["linear"], + }, +] + + +def run_probe_option(option: str) -> object: + result = subprocess.run( + ["node", str(PROBE), option], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + timeout=30, + ) + return json.loads(result.stdout) + + +def test_browser_conformance_catalog_is_the_reviewed_bounded_matrix() -> None: + matrix = run_probe_option("--list-cases") + + assert matrix == EXPECTED_MATRIX + assert {case["tier"] for case in matrix} == {"direct", "decimated", "density"} + assert {case["family"] for case in matrix} == { + "scatter", + "line", + "bar", + "heatmap", + "mesh", + } + assert {case["dpr"] for case in matrix} == {1, 2} + assert {case["motion"] for case in matrix} == {"reduce", "no-preference"} + assert {axis for case in matrix for axis in case["axisClasses"]} == { + "linear", + "log", + "category", + "named", + } + + +def test_browser_conformance_negative_controls_are_executable() -> None: + controls = run_probe_option("--self-test") + + assert controls == { + "catalogGapRejected": True, + "corruptedSignatureRejected": True, + "corruptedLayoutRejected": True, + } + + +def test_browser_conformance_retains_failure_evidence(tmp_path: Path) -> None: + evidence_path = tmp_path / "browser-conformance-evidence.json" + result = subprocess.run( + [ + "node", + str(PROBE), + "--browsers=not-an-engine", + "--evidence", + str(evidence_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode != 0 + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + assert evidence["status"] == "failed" + assert "unknown browser not-an-engine" in evidence["error"] + assert len(evidence["matrix"]) == len(EXPECTED_MATRIX) + + +def test_browser_conformance_has_no_skip_path_and_required_engines_fail_loudly() -> None: + source = PROBE.read_text(encoding="utf-8") + + assert ".skip" not in source + assert "WebGL2 unavailable" in source + assert "missing selected engines or WebGL2 fail" in source + assert 'schema: "xy-browser-conformance/v1"' in source diff --git a/tests/test_chart_kind_matrix.py b/tests/test_chart_kind_matrix.py new file mode 100644 index 00000000..2b971560 --- /dev/null +++ b/tests/test_chart_kind_matrix.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest +from scripts import chart_kind_matrix + + +def test_catalog_exactly_covers_shipped_mark_registry() -> None: + registry = chart_kind_matrix.shipped_registry() + chart_kind_matrix.validate_catalog(chart_kind_matrix.CASES, registry) + assert len(registry) == 18 + + +@pytest.mark.parametrize("case", chart_kind_matrix.CASES, ids=lambda case: case.name) +def test_every_catalog_case_has_nonempty_tiered_payload(case: chart_kind_matrix.Case) -> None: + _, spec, payload = chart_kind_matrix.build_case(case) + assert payload + assert tuple(trace["kind"] for trace in spec["traces"]) == case.expected_kinds + assert all(trace["tier"] in {"direct", "decimated", "density"} for trace in spec["traces"]) + assert all(trace["n_points"] > 0 for trace in spec["traces"]) + + +def test_catalog_rejects_missing_registry_kind() -> None: + registry = chart_kind_matrix.shipped_registry() + with pytest.raises(AssertionError, match=r"catalog mismatch.*missing"): + chart_kind_matrix.validate_catalog(chart_kind_matrix.CASES[:-1], registry) + + +def test_payload_oracle_rejects_wrong_expected_kind() -> None: + broken = replace(chart_kind_matrix.CASES[0], expected_kinds=("line",)) + with pytest.raises(AssertionError, match="payload kinds"): + chart_kind_matrix.build_case(broken) + + +def test_pixel_oracle_rejects_blank_render() -> None: + with pytest.raises(AssertionError, match="blank/flat"): + chart_kind_matrix.require_nonblank_pixels("mutant", 0) diff --git a/tests/test_check_pyplot_options.py b/tests/test_check_pyplot_options.py new file mode 100644 index 00000000..15cc32f1 --- /dev/null +++ b/tests/test_check_pyplot_options.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + + +def _load_module(): + path = Path(__file__).resolve().parents[1] / "scripts" / "check_pyplot_options.py" + spec = importlib.util.spec_from_file_location("check_pyplot_options", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +check_pyplot_options = _load_module() + + +def _fixture(tmp_path: Path, source: str) -> tuple[Path, Path, Path]: + source_root = tmp_path / "python" / "xy" / "pyplot" + source_root.mkdir(parents=True) + (source_root / "adapter.py").write_text(source, encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "contracts.py").write_text("def test_contract():\n pass\n", encoding="utf-8") + return source_root, tmp_path / "policy.json", tmp_path + + +def _write_policy(path: Path, noops: list[dict[str, object]]) -> None: + path.write_text( + json.dumps({"schema_version": 1, "noops": noops}, indent=2) + "\n", + encoding="utf-8", + ) + + +def _entry(*options: str) -> dict[str, object]: + return { + "path": "python/xy/pyplot/adapter.py", + "function": "adapter", + "options": list(options), + "rationale": "The compatibility surface deliberately preserves this bounded no-op behavior.", + "test": "tests/contracts.py::test_contract", + } + + +def test_current_pyplot_option_contract_is_exact() -> None: + assert check_pyplot_options.validate() == [] + + +def test_discovery_rejects_new_named_and_assigned_unused_options(tmp_path: Path) -> None: + source_root, _policy, project_root = _fixture( + tmp_path, + """ +def named(value, ignored=None): + return value + +def bare(**kwargs): + kwargs.pop("mode", None) + return 1 + +def assigned(**kwargs): + dropped = kwargs.pop("dpi", None) + return 1 + +def effectful(**kwargs): + mode = kwargs.pop("mode", None) + return mode +""", + ) + discovered = check_pyplot_options.discover_noops(source_root, project_root=project_root) + assert discovered == { + check_pyplot_options.Noop("python/xy/pyplot/adapter.py", "named", "ignored"), + check_pyplot_options.Noop("python/xy/pyplot/adapter.py", "bare", "mode"), + check_pyplot_options.Noop("python/xy/pyplot/adapter.py", "assigned", "dpi"), + } + + +def test_only_reachable_closures_can_consume_an_outer_option(tmp_path: Path) -> None: + source_root, _policy, project_root = _fixture( + tmp_path, + """ +def active(option): + def consume(): + return option + return consume() + +def inactive(option): + def never_called(): + return option + return 1 +""", + ) + discovered = check_pyplot_options.discover_noops(source_root, project_root=project_root) + assert discovered == { + check_pyplot_options.Noop("python/xy/pyplot/adapter.py", "inactive", "option") + } + + +def test_policy_must_exactly_match_discovery(tmp_path: Path) -> None: + source_root, policy, project_root = _fixture( + tmp_path, "def adapter(value, ignored=None):\n return value\n" + ) + _write_policy(policy, [_entry("ignored")]) + assert check_pyplot_options.validate(source_root, policy, project_root=project_root) == [] + + _write_policy(policy, []) + errors = check_pyplot_options.validate(source_root, policy, project_root=project_root) + assert any("no reviewed contract" in error and "ignored" in error for error in errors) + + _write_policy(policy, [_entry("ignored", "stale")]) + errors = check_pyplot_options.validate(source_root, policy, project_root=project_root) + assert any("stale no-op policy" in error and "stale" in error for error in errors) + + +def test_policy_rejects_weak_rationale_and_missing_behavior_test(tmp_path: Path) -> None: + source_root, policy, project_root = _fixture( + tmp_path, "def adapter(value, ignored=None):\n return value\n" + ) + entry = _entry("ignored") + entry["rationale"] = "because" + entry["test"] = "tests/contracts.py::test_missing" + _write_policy(policy, [entry]) + errors = check_pyplot_options.validate(source_root, policy, project_root=project_root) + assert any("substantive rationale" in error for error in errors) + assert any("missing test" in error for error in errors) + + +def test_cli_writes_machine_readable_evidence(tmp_path: Path) -> None: + report = tmp_path / "contract.json" + assert check_pyplot_options.main(["--report", str(report)]) == 0 + evidence = json.loads(report.read_text(encoding="utf-8")) + assert evidence["status"] == "passed" + assert evidence["reviewed_noop_count"] == len(evidence["discovered"]) == 34 + assert evidence["errors"] == [] diff --git a/tests/test_check_required_jobs.py b/tests/test_check_required_jobs.py new file mode 100644 index 00000000..2c196a0c --- /dev/null +++ b/tests/test_check_required_jobs.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + + +def _load_module(): + path = Path(__file__).resolve().parents[1] / "scripts" / "check_required_jobs.py" + spec = importlib.util.spec_from_file_location("check_required_jobs", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +check_required_jobs = _load_module() + + +def _needs(result: str = "success") -> dict[str, dict[str, object]]: + return {name: {"result": result, "outputs": {}} for name in check_required_jobs.HARD_JOBS} + + +def test_required_aggregate_accepts_only_all_success() -> None: + assert "host_integration" in check_required_jobs.HARD_JOBS + assert "reflex_adapter" in check_required_jobs.HARD_JOBS + assert "rust_release" in check_required_jobs.HARD_JOBS + assert "native_parity" in check_required_jobs.HARD_JOBS + assert "javascript_semantics" in check_required_jobs.HARD_JOBS + assert "python_coverage" in check_required_jobs.HARD_JOBS + assert check_required_jobs.evaluate_required_jobs(_needs()) == [] + + +def test_required_aggregate_rejects_every_non_success_conclusion() -> None: + for conclusion in ("failure", "cancelled", "skipped", None, ""): + needs = _needs() + needs["test"]["result"] = conclusion + errors = check_required_jobs.evaluate_required_jobs(needs) + assert any("'test'" in error and repr(conclusion) in error for error in errors) + + +def test_required_aggregate_rejects_missing_and_unexpected_dependencies() -> None: + needs = _needs() + del needs["sdist"] + needs["benchmark"] = {"result": "success"} + + errors = check_required_jobs.evaluate_required_jobs(needs) + + assert any("'sdist' is missing" in error for error in errors) + assert any("unexpected job 'benchmark'" in error for error in errors) + + +def test_required_aggregate_cli_rejects_a_seeded_skip(capsys) -> None: + needs = _needs() + needs["wheels"]["result"] = "skipped" + + assert check_required_jobs.main(["--needs-json", json.dumps(needs)]) == 1 + assert "concluded 'skipped'" in capsys.readouterr().err + + +def test_required_aggregate_cli_rejects_malformed_input(capsys) -> None: + assert check_required_jobs.main(["--needs-json", "[]"]) == 2 + assert "must be an object" in capsys.readouterr().err diff --git a/tests/test_claim_guardrails.py b/tests/test_claim_guardrails.py index b189d219..56b64d91 100644 --- a/tests/test_claim_guardrails.py +++ b/tests/test_claim_guardrails.py @@ -36,6 +36,10 @@ def test_claim_guardrail_accepts_current_public_docs() -> None: assert ( check_claim_guardrails.ROOT / "docs" / "index.md" in check_claim_guardrails._default_paths() ) + assert all( + path in check_claim_guardrails._default_paths() + for path in (check_claim_guardrails.ROOT / "spec").rglob("*.md") + ) def test_claim_guardrail_rejects_broad_fastest_claim(tmp_path: Path) -> None: diff --git a/tests/test_coverage_ratchet.py b/tests/test_coverage_ratchet.py new file mode 100644 index 00000000..d19c44fa --- /dev/null +++ b/tests/test_coverage_ratchet.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + + +def _load_module(): + path = Path(__file__).resolve().parents[1] / "scripts" / "coverage_ratchet.py" + spec = importlib.util.spec_from_file_location("coverage_ratchet", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +coverage_ratchet = _load_module() + + +def _entry( + executed: range | list[int], + missing: range | list[int], + *, + covered_branches: int = 8, + branches: int = 10, +) -> dict[str, object]: + executed_lines = list(executed) + missing_lines = list(missing) + return { + "executed_lines": executed_lines, + "missing_lines": missing_lines, + "excluded_lines": [], + "summary": { + "covered_lines": len(executed_lines), + "num_statements": len(executed_lines) + len(missing_lines), + "num_branches": branches, + "covered_branches": covered_branches, + }, + } + + +def _project(tmp_path: Path) -> tuple[dict[str, object], dict[str, object]]: + paths = ( + "python/xy/core.py", + "python/xy/pyplot/plot.py", + "python/reflex-xy/reflex_xy/component.py", + ) + for path in paths: + target = tmp_path / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + "\n".join(f"value_{line} = {line}" for line in range(1, 11)), encoding="utf-8" + ) + policy: dict[str, object] = { + "source_roots": ["python/xy", "python/reflex-xy/reflex_xy"], + "packages": [ + { + "name": "core", + "path_prefix": "python/xy/", + "exclude_prefixes": ["python/xy/pyplot/"], + "minimum_line_percent": 80.0, + "minimum_branch_percent": 70.0, + }, + { + "name": "pyplot", + "path_prefix": "python/xy/pyplot/", + "exclude_prefixes": [], + "minimum_line_percent": 80.0, + "minimum_branch_percent": 70.0, + }, + { + "name": "adapter", + "path_prefix": "python/reflex-xy/reflex_xy/", + "exclude_prefixes": [], + "minimum_line_percent": 80.0, + "minimum_branch_percent": 70.0, + }, + ], + "modules": [ + { + "path": "python/xy/core.py", + "minimum_line_percent": 80.0, + "minimum_branch_percent": 70.0, + } + ], + "diff": {"minimum_line_percent": 90.0, "missing_coverage_file": "fail"}, + "exclusions": [], + } + coverage: dict[str, object] = { + "files": { + "python/xy/core.py": _entry(range(1, 10), [10]), + "python/xy/pyplot/plot.py": _entry(range(1, 10), [10]), + "python/reflex-xy/reflex_xy/component.py": _entry(range(1, 10), [10]), + } + } + return policy, coverage + + +def test_current_policy_is_structured_and_reviewed() -> None: + policy, errors = coverage_ratchet.load_policy() + assert errors == [] + assert {package["name"] for package in policy["packages"]} == { + "core", + "pyplot", + "reflex_adapter", + } + assert all(len(item["rationale"]) >= 20 for item in policy["exclusions"]) + + +def test_zero_context_diff_parser_returns_exact_added_lines() -> None: + diff = """diff --git a/python/xy/core.py b/python/xy/core.py +--- a/python/xy/core.py ++++ b/python/xy/core.py +@@ -2,0 +3,2 @@ ++first ++second +@@ -9 +11 @@ +-old ++new +diff --git a/python/xy/new.py b/python/xy/new.py +--- /dev/null ++++ b/python/xy/new.py +@@ -0,0 +1,3 @@ ++a ++b ++c +""" + assert coverage_ratchet.parse_changed_lines(diff) == { + "python/xy/core.py": {3, 4, 11}, + "python/xy/new.py": {1, 2, 3}, + } + + +def test_package_module_and_ninety_percent_diff_floors_pass(tmp_path: Path) -> None: + policy, coverage = _project(tmp_path) + changed = {"python/xy/core.py": set(range(1, 11))} + report, errors = coverage_ratchet.evaluate(coverage, policy, changed, project_root=tmp_path) + assert errors == [] + assert report["diff"]["line_percent"] == 90.0 + assert report["diff"]["files"]["python/xy/core.py"]["missing_lines"] == [10] + + +def test_known_coverage_mutations_fail_package_module_and_diff_ratchets( + tmp_path: Path, +) -> None: + policy, coverage = _project(tmp_path) + coverage["files"]["python/xy/core.py"] = _entry(range(1, 9), [9, 10], covered_branches=6) + report, errors = coverage_ratchet.evaluate( + coverage, + policy, + {"python/xy/core.py": set(range(1, 11))}, + project_root=tmp_path, + ) + assert report["status"] == "failed" + assert any("package core branch coverage" in error for error in errors) + assert any("module python/xy/core.py branch coverage" in error for error in errors) + assert any("changed-line coverage 80.00%" in error for error in errors) + + +def test_new_shipped_module_without_measurement_fails_closed(tmp_path: Path) -> None: + policy, coverage = _project(tmp_path) + (tmp_path / "python/xy/unmeasured.py").write_text("value = 1\n", encoding="utf-8") + _report, errors = coverage_ratchet.evaluate(coverage, policy, {}, project_root=tmp_path) + assert any( + "missing shipped Python files" in error and "unmeasured.py" in error for error in errors + ) + + +def test_loader_rejects_non_branch_coverage_and_weak_exclusions(tmp_path: Path) -> None: + coverage_path = tmp_path / "coverage.json" + coverage_path.write_text( + json.dumps({"meta": {"format": 3, "branch_coverage": False}, "files": {}}), + encoding="utf-8", + ) + _coverage, errors = coverage_ratchet.load_coverage(coverage_path) + assert errors == ["coverage JSON must be generated with branch coverage enabled"] + + policy, _errors = coverage_ratchet.load_policy() + policy["exclusions"][0]["rationale"] = "weak" + policy_path = tmp_path / "policy.json" + policy_path.write_text(json.dumps(policy), encoding="utf-8") + _policy, errors = coverage_ratchet.load_policy(policy_path) + assert any("substantive rationale" in error for error in errors) diff --git a/tests/test_dependency_audit.py b/tests/test_dependency_audit.py new file mode 100644 index 00000000..77b3bb5b --- /dev/null +++ b/tests/test_dependency_audit.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import copy +import json +from datetime import date +from pathlib import Path + +import pytest +from scripts import dependency_audit + +ROOT = Path(__file__).resolve().parents[1] +POLICY_PATH = ROOT / "spec" / "testing" / "dependency-audit-policy.json" +REVIEW_DATE = date(2026, 7, 21) + + +def _policy() -> dict: + return json.loads(POLICY_PATH.read_text(encoding="utf-8")) + + +def _write_policy(tmp_path: Path, policy: dict) -> Path: + path = tmp_path / "dependency-audit-policy.json" + path.write_text(json.dumps(policy), encoding="utf-8") + return path + + +def _exception(*, expires: str = "2026-08-20") -> dict: + return { + "id": "GHSA-1234-5678-9abc", + "package": {"ecosystem": "PyPI", "name": "example"}, + "environment": "root-python", + "owner": "@reflex-dev/xy", + "reason": "The affected path is unreachable while the upstream fix is prepared.", + "expires": expires, + } + + +def _raw_report(root: Path, *, vulnerability: dict | None = None) -> dict: + results = [] + for environment in dependency_audit.REQUIRED_ENVIRONMENTS: + package = { + "package": { + "name": "example", + "version": "1.0.0", + "ecosystem": "PyPI", + } + } + if vulnerability is not None and not results: + package["vulnerabilities"] = [vulnerability] + results.append( + { + "source": { + "path": str((root / environment["path"]).resolve()), + "type": "lockfile", + }, + "packages": [package], + } + ) + return {"results": results} + + +def _write_inventory(root: Path) -> None: + for item in dependency_audit.REQUIRED_ENVIRONMENTS + dependency_audit.REQUIRED_EXCLUDED_LOCKS: + path = root / item["path"] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("reviewed lock fixture\n", encoding="utf-8") + + +def test_current_dependency_policy_is_complete() -> None: + policy = dependency_audit.validate_policy(today=REVIEW_DATE) + + assert policy["environments"] == dependency_audit.REQUIRED_ENVIRONMENTS + assert set(policy["fail_severities"]) == dependency_audit.SEVERITIES + assert policy["exceptions"] == [] + + +def test_policy_rejects_omitted_subsystem_lock(tmp_path: Path) -> None: + policy = _policy() + policy["environments"] = policy["environments"][:-1] + + with pytest.raises(dependency_audit.AuditError, match="every required subsystem lock"): + dependency_audit.validate_policy( + _write_policy(tmp_path, policy), root=ROOT, today=REVIEW_DATE + ) + + +def test_policy_rejects_removed_severity_gate(tmp_path: Path) -> None: + policy = _policy() + policy["fail_severities"].remove("UNKNOWN") + + with pytest.raises(dependency_audit.AuditError, match="every reviewed class"): + dependency_audit.validate_policy( + _write_policy(tmp_path, policy), root=ROOT, today=REVIEW_DATE + ) + + +def test_policy_rejects_unreviewed_scanner_checksum(tmp_path: Path) -> None: + policy = _policy() + policy["scanner"]["artifacts"]["linux-x86_64"]["sha256"] = "0" * 64 + + with pytest.raises(dependency_audit.AuditError, match="reviewed version, URLs, and SHA-256"): + dependency_audit.validate_policy( + _write_policy(tmp_path, policy), root=ROOT, today=REVIEW_DATE + ) + + +def test_policy_rejects_new_uninventoried_lock(tmp_path: Path) -> None: + _write_inventory(tmp_path) + extra = tmp_path / "new-subsystem" / "uv.lock" + extra.parent.mkdir() + extra.write_text("unreviewed lock fixture\n", encoding="utf-8") + + with pytest.raises(dependency_audit.AuditError, match="unreviewed committed-style locks"): + dependency_audit.validate_policy(POLICY_PATH, root=tmp_path, today=REVIEW_DATE) + + +def test_policy_rejects_missing_inventoried_lock(tmp_path: Path) -> None: + _write_inventory(tmp_path) + (tmp_path / dependency_audit.REQUIRED_ENVIRONMENTS[0]["path"]).unlink() + + with pytest.raises(dependency_audit.AuditError, match="inventoried locks do not exist"): + dependency_audit.validate_policy(POLICY_PATH, root=tmp_path, today=REVIEW_DATE) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda item: item.pop("owner"), "keys must be exactly"), + (lambda item: item.update(environment="not-a-subsystem"), "identify one of"), + (lambda item: item.update(reason="too short"), "at least 20"), + (lambda item: item.update(expires="2026-07-21"), "expired"), + (lambda item: item.update(expires="2027-07-21"), "no more than 180 days"), + ], +) +def test_policy_rejects_unowned_unreasoned_or_invalid_exceptions( + tmp_path: Path, mutation, message: str +) -> None: + policy = _policy() + item = _exception() + mutation(item) + policy["exceptions"] = [item] + + with pytest.raises(dependency_audit.AuditError, match=message): + dependency_audit.validate_policy( + _write_policy(tmp_path, policy), root=ROOT, today=REVIEW_DATE + ) + + +def test_policy_rejects_duplicate_exact_exceptions(tmp_path: Path) -> None: + policy = _policy() + policy["exceptions"] = [_exception(), copy.deepcopy(_exception())] + + with pytest.raises(dependency_audit.AuditError, match="duplicates"): + dependency_audit.validate_policy( + _write_policy(tmp_path, policy), root=ROOT, today=REVIEW_DATE + ) + + +def test_report_requires_every_environment() -> None: + policy = _policy() + report = _raw_report(ROOT) + report["results"].pop() + + with pytest.raises(dependency_audit.AuditError, match="omitted required environments"): + dependency_audit.evaluate_report(report, policy, root=ROOT) + + +def test_report_requires_nonempty_package_inventory() -> None: + policy = _policy() + report = _raw_report(ROOT) + report["results"][0]["packages"] = [] + + with pytest.raises(dependency_audit.AuditError, match="found no packages"): + dependency_audit.evaluate_report(report, policy, root=ROOT) + + +def test_exact_exception_is_recorded_and_stale_exception_is_rejected() -> None: + policy = _policy() + policy["exceptions"] = [_exception()] + vulnerability = { + "id": "GHSA-1234-5678-9abc", + "summary": "Synthetic finding", + "database_specific": {"severity": "HIGH"}, + } + + environments, findings, unused = dependency_audit.evaluate_report( + _raw_report(ROOT, vulnerability=vulnerability), policy, root=ROOT + ) + + assert len(environments) == len(dependency_audit.REQUIRED_ENVIRONMENTS) + assert findings[0]["severity"] == "HIGH" + assert findings[0]["excepted"] is True + assert unused == [] + + clean_report = _raw_report(ROOT) + _, clean_findings, unused = dependency_audit.evaluate_report(clean_report, policy, root=ROOT) + assert clean_findings == [] + assert unused == policy["exceptions"] + + +def test_exception_does_not_cross_subsystem_boundaries() -> None: + policy = _policy() + exception = _exception() + exception["environment"] = "docs-python" + policy["exceptions"] = [exception] + vulnerability = {"id": "GHSA-1234-5678-9abc", "summary": "Synthetic finding"} + + _, findings, unused = dependency_audit.evaluate_report( + _raw_report(ROOT, vulnerability=vulnerability), policy, root=ROOT + ) + + assert findings[0]["environment"] == "root-python" + assert findings[0]["excepted"] is False + assert unused == policy["exceptions"] + + +def test_unknown_severity_remains_a_reviewed_failure_class() -> None: + policy = _policy() + vulnerability = {"id": "GHSA-1234-5678-9abc", "summary": "No CVSS yet"} + + _, findings, _ = dependency_audit.evaluate_report( + _raw_report(ROOT, vulnerability=vulnerability), policy, root=ROOT + ) + + assert findings[0]["severity"] == "UNKNOWN" + assert findings[0]["severity"] in policy["fail_severities"] diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py index 1dd63f7f..cd648ae7 100644 --- a/tests/test_docs_examples.py +++ b/tests/test_docs_examples.py @@ -475,8 +475,10 @@ def test_production_docs_include_focused_gate_matrix() -> None: "`make check-ci`", "Source distributions and wheels", "`make check-sdist` and `make check-wheel`", - "Production-facing PR", + "Production-facing non-browser change", "`make check-full`", + "the full non-browser local gate", + "it is not equivalent to the browser, host-integration, packaging, cross-platform, or exact-SHA release evidence", ] for marker in required: assert marker in text @@ -505,12 +507,14 @@ def test_docs_name_split_browser_hardening_gates() -> None: step_names = [ "Browser lifecycle smoke (Chromium)", - "Browser visual regression smoke (Chromium)", + "Browser visual health smoke (Chromium)", + "Reviewed visual baseline (Chromium)", "Browser interaction stress smoke (Chromium)", ] scripts = [ "scripts/reflex_lifecycle_smoke.py", - "scripts/visual_regression_smoke.py", + "scripts/visual_health_smoke.py", + "scripts/visual_baseline.py", "scripts/interaction_stress_smoke.py", ] for text in (readme, production, contributing): diff --git a/tests/test_example_apps.py b/tests/test_example_apps.py index bb14d06e..8f9b5f25 100644 --- a/tests/test_example_apps.py +++ b/tests/test_example_apps.py @@ -192,7 +192,7 @@ def lifecycle_mod(): @pytest.fixture(scope="module") def visual_mod(): - return _load(ROOT / "scripts" / "visual_regression_smoke.py", "visual_regression_smoke") + return _load(ROOT / "scripts" / "visual_health_smoke.py", "visual_health_smoke") def test_smokes_cover_the_whole_gallery(lifecycle_mod, visual_mod, charts_mod) -> None: diff --git a/tests/test_fastapi_host_smoke.py b/tests/test_fastapi_host_smoke.py new file mode 100644 index 00000000..14bae5fb --- /dev/null +++ b/tests/test_fastapi_host_smoke.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import struct +import zlib + +import pytest +from scripts import fastapi_host_smoke + + +def _png(rgb: tuple[int, int, int]) -> bytes: + width = height = 2 + raw = b"".join(b"\x00" + bytes(rgb) * width for _ in range(height)) + + def chunk(kind: bytes, body: bytes) -> bytes: + return ( + struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + ) + + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + + +def test_colored_pixel_oracle_counts_saturated_mount() -> None: + count = fastapi_host_smoke.colored_pixels(_png((255, 0, 0)), {"x": 0, "y": 0, "w": 2, "h": 2}) + assert count == 4 + + +def test_mount_oracle_rejects_blank_browser_output() -> None: + with pytest.raises(AssertionError, match="browser mount is blank"): + fastapi_host_smoke.require_ink(0) diff --git a/tests/test_figure.py b/tests/test_figure.py index 8cd467c2..0fed7f28 100644 --- a/tests/test_figure.py +++ b/tests/test_figure.py @@ -1992,7 +1992,7 @@ def fake_run(args, **kwargs): assert "--no-sandbox" in seen[1] -def test_html_to_png_retries_without_sandbox_when_browser_crashes(monkeypatch): +def test_html_to_png_never_silently_downgrades_after_sandbox_failure(monkeypatch): from xy import export seen = [] @@ -2002,21 +2002,43 @@ def test_html_to_png_retries_without_sandbox_when_browser_crashes(monkeypatch): def fake_run(args, **kwargs): del kwargs seen.append(args) - if len(seen) == 2: - shot = next( - arg.removeprefix("--screenshot=") for arg in args if arg.startswith("--screenshot=") - ) - Path(shot).write_bytes(b"\x89PNG\r\n\x1a\nfake") - return export_module.subprocess.CompletedProcess(args, 0, stdout="", stderr="") return export_module.subprocess.CompletedProcess(args, -6, stdout="", stderr="crashed") monkeypatch.setattr(export.subprocess, "run", fake_run) - data = export.html_to_png("", 320, 200) + with pytest.raises( + RuntimeError, + match=r"sandboxed launch produced no screenshot.*No unsandboxed retry was attempted.*sandbox=False", + ): + export.html_to_png("", 320, 200) - assert data == b"\x89PNG\r\n\x1a\nfake" + assert len(seen) == 1 assert "--no-sandbox" not in seen[0] - assert "--no-sandbox" in seen[1] + + +def test_html_to_png_reports_explicit_unsandboxed_launch_failure(monkeypatch): + from xy import export + + seen = [] + monkeypatch.setattr(export, "find_browser", lambda explicit=None: "/fake/chrome") + + def fake_run(args, **kwargs): + del kwargs + seen.append(args) + return export_module.subprocess.CompletedProcess( + args, 17, stdout="", stderr="explicit failure" + ) + + monkeypatch.setattr(export.subprocess, "run", fake_run) + + with pytest.raises( + RuntimeError, + match=r"explicitly unsandboxed launch produced no screenshot \(exit 17\): explicit failure", + ): + export.html_to_png("", 320, 200, sandbox=False) + + assert len(seen) == 1 + assert "--no-sandbox" in seen[0] # --------------------------------------------------------------------------- diff --git a/tests/test_framing_property.py b/tests/test_framing_property.py index f06bab17..310400c1 100644 --- a/tests/test_framing_property.py +++ b/tests/test_framing_property.py @@ -1,13 +1,20 @@ from __future__ import annotations -import pytest +import base64 +import json +import struct +import subprocess +from pathlib import Path -from xy.channel import decode_frame, encode_frame +from hypothesis import given, settings +from hypothesis import strategies as st -hypothesis = pytest.importorskip("hypothesis") -st = pytest.importorskip("hypothesis.strategies") -given = hypothesis.given -settings = hypothesis.settings +from xy.channel import FrameDecodeError, decode_frame, encode_frame + +ROOT = Path(__file__).resolve().parents[1] +CLIENT = ROOT / "python" / "xy" / "static" / "index.js" +HEADER = struct.Struct("<4sBBHIIQ") +U64 = struct.Struct(" int: + return (value + 7) & ~7 + + +def _frame_positions(body: bytes) -> tuple[int, int, list[tuple[int, int, int, int]]]: + _magic, _version, _flags, _header_size, metadata_len, count, _total = HEADER.unpack_from(body) + metadata_end = 24 + metadata_len + position = _align(metadata_end) + buffers: list[tuple[int, int, int, int]] = [] + for _index in range(count): + length_position = position + (length,) = U64.unpack_from(body, length_position) + start = length_position + U64.size + end = start + length + padded_end = _align(end) + buffers.append((length_position, start, end, padded_end)) + position = padded_end + return metadata_end, _align(metadata_end), buffers + + +def _mutations(body: bytes, selector: int, xor_value: int) -> dict[str, bytes]: + metadata_end, metadata_padded_end, buffers = _frame_positions(body) + _magic, version, _flags, header_size, metadata_len, count, _total = HEADER.unpack_from(body) + + def changed(offset: int, value: bytes) -> bytes: + mutated = bytearray(body) + mutated[offset : offset + len(value)] = value + return bytes(mutated) + + metadata_byte = 24 + selector % metadata_len + metadata_mutation = bytearray(body) + metadata_mutation[metadata_byte] ^= xor_value + metadata_padding = bytearray(body) + assert metadata_padded_end > metadata_end + metadata_padding[metadata_end] = xor_value + first_length, _first_start, first_end, _first_padded_end = buffers[0] + (first_size,) = U64.unpack_from(body, first_length) + last_length, _last_start, last_end, last_padded_end = buffers[-1] + assert last_padded_end > last_end + buffer_padding = bytearray(body) + buffer_padding[last_end] = xor_value + + return { + "header_magic": changed(0, bytes([body[0] ^ xor_value])), + "header_version": changed(4, bytes([(version + 1) & 0xFF])), + "header_flags": changed(5, b"\x01"), + "header_size": changed(6, struct.pack(" dict: + try: + decoded = decode_frame(body) + except FrameDecodeError: + return {"ok": False} + return { + "ok": True, + "message": decoded.message, + "buffers": [base64.b64encode(buffer).decode("ascii") for buffer in decoded.buffers], + } + + +def _javascript_decode_results(bodies: list[bytes]) -> list[dict]: + encoded = [base64.b64encode(body).decode("ascii") for body in bodies] + script = f""" + import {{ decodeFrame }} from {CLIENT.as_uri()!r}; + const results = {json.dumps(encoded)}.map((encoded) => {{ + const source = Uint8Array.from(Buffer.from(encoded, "base64")); + try {{ + const decoded = decodeFrame(source.buffer); + return {{ + ok: true, + message: decoded.message, + buffers: decoded.buffers.map((value) => + Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString("base64")), + }}; + }} catch (_error) {{ + return {{ok: false}}; + }} + }}); + process.stdout.write(JSON.stringify(results)); + """ + completed = subprocess.run( + ["node", "--input-type=module", "--eval", script], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + check=True, + ) + return json.loads(completed.stdout) + + +def _javascript_number_model(value: object) -> object: + """Normalize JSON numbers to JavaScript's one IEEE-754 number domain.""" + if isinstance(value, bool) or value is None or isinstance(value, str): + return value + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, list): + return [_javascript_number_model(item) for item in value] + if isinstance(value, dict): + return {key: _javascript_number_model(item) for key, item in value.items()} + raise TypeError(f"value outside the JSON data model: {type(value).__name__}") + + +@settings(max_examples=20, deadline=None) +@given( + value=JSON_VALUES, + payload=st.binary(max_size=64), + selector=st.integers(min_value=0, max_value=4096), + xor_value=st.integers(min_value=1, max_value=255), +) +def test_structural_byte_mutations_reject_or_preserve_python_javascript_parity( + value: object, payload: bytes, selector: int, xor_value: int +) -> None: + message = {"type": "mutation_probe", "value": value, "padding_probe": "x"} + body = encode_frame(message, [payload, b"abc"]) + metadata_end, metadata_padded_end, _buffers = _frame_positions(body) + while metadata_end == metadata_padded_end: + message["padding_probe"] += "x" + body = encode_frame(message, [payload, b"abc"]) + metadata_end, metadata_padded_end, _buffers = _frame_positions(body) + + mutations = _mutations(body, selector, xor_value) + python_results = [_python_decode_result(case) for case in mutations.values()] + javascript_results = _javascript_decode_results(list(mutations.values())) + + assert len(javascript_results) == len(python_results) + for name, python_result, javascript_result in zip( + mutations, python_results, javascript_results, strict=True + ): + assert javascript_result["ok"] is python_result["ok"], name + if python_result["ok"]: + assert _javascript_number_model(javascript_result["message"]) == ( + _javascript_number_model(python_result["message"]) + ), name + assert javascript_result["buffers"] == python_result["buffers"], name diff --git a/tests/test_host_integration_policy.py b/tests/test_host_integration_policy.py new file mode 100644 index 00000000..21ff4e8b --- /dev/null +++ b/tests/test_host_integration_policy.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +from scripts import host_integration_policy + + +def test_host_policy_matches_package_owned_requirements() -> None: + policy = host_integration_policy.load_policy() + assert host_integration_policy.validate_policy(policy) == [] + + +def test_host_policy_rejects_widened_unreviewed_range() -> None: + policy = copy.deepcopy(host_integration_policy.load_policy()) + policy["packages"]["anywidget"]["supported"] = ">=0.9" + errors = host_integration_policy.validate_policy(policy) + assert any("anywidget policy must exactly equal" in error for error in errors) + + +def test_host_policy_rejects_missing_related_dependency() -> None: + policy = copy.deepcopy(host_integration_policy.load_policy()) + policy["hosts"]["fastapi"].remove("httpx") + errors = host_integration_policy.validate_policy(policy) + assert any("host ownership must exactly equal" in error for error in errors) + + +def test_installed_floor_validation_rejects_version_drift() -> None: + policy = host_integration_policy.load_policy() + errors, installed = host_integration_policy.validate_installed( + policy, + "floor", + ["anywidget"], + versions={"anywidget": "0.9.1", "traitlets": "5.14.0"}, + ) + assert installed["anywidget"] == "0.9.1" + assert any("does not satisfy floor selector ==0.9.0" in error for error in errors) + + +def test_installed_validation_rejects_unknown_host() -> None: + errors, installed = host_integration_policy.validate_installed( + host_integration_policy.load_policy(), "latest", ["unknown"] + ) + assert installed == {} + assert errors == ["unknown hosts: ['unknown']"] + + +def test_invalid_policy_still_writes_failure_evidence(tmp_path: Path) -> None: + policy = copy.deepcopy(host_integration_policy.load_policy()) + policy["packages"]["anywidget"]["supported"] = ">=0.9" + report = tmp_path / "versions.json" + + errors = host_integration_policy.write_installed_report( + report, + policy=policy, + profile="latest", + hosts=["anywidget"], + ) + + payload = json.loads(report.read_text(encoding="utf-8")) + assert errors + assert payload["status"] == "failed" + assert payload["installed"] == {} diff --git a/tests/test_interaction_stress_smoke.py b/tests/test_interaction_stress_smoke.py new file mode 100644 index 00000000..8a8a42c1 --- /dev/null +++ b/tests/test_interaction_stress_smoke.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def _load_module(): + scripts = ROOT / "scripts" + sys.path.insert(0, str(scripts)) + spec = importlib.util.spec_from_file_location( + "interaction_stress_smoke", scripts / "interaction_stress_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +interaction_stress_smoke = _load_module() + + +def _healthy_worker() -> dict: + return { + "status": "ok", + "worker_created": True, + "worker_rebinned": True, + "nonblank_pixels": 981, + "x_range_changed": True, + "worker_terminated": True, + "worker_cleared": True, + "root_removed": True, + "teardown_complete": True, + } + + +def _stub_benchmark(monkeypatch: pytest.MonkeyPatch, worker: dict) -> None: + benchmark = SimpleNamespace( + _parse_sizes=lambda raw: [10_000], + run=lambda **kwargs: {"kind": "interaction-browser", "rows": [], "reps": kwargs["reps"]}, + run_worker_probe=lambda **kwargs: dict(worker), + to_markdown=lambda report: "# synthetic interaction report\n", + ) + monkeypatch.setattr(interaction_stress_smoke, "_load_bench_interaction", lambda: benchmark) + monkeypatch.setattr( + interaction_stress_smoke.verify_benchmark_report, + "validate_report", + lambda *args, **kwargs: [], + ) + + +def test_worker_success_evidence_is_required_and_written(monkeypatch, tmp_path: Path) -> None: + worker = _healthy_worker() + _stub_benchmark(monkeypatch, worker) + evidence = tmp_path / "artifacts" / "interaction.json" + + rc = interaction_stress_smoke.main([sys.executable, "--reps", "1", "--json", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 0 + assert payload["standalone_density_worker"] == worker + + +def test_skipped_worker_is_blocking_by_default(monkeypatch, capsys) -> None: + _stub_benchmark(monkeypatch, {"status": "skipped(no chromium)"}) + + rc = interaction_stress_smoke.main([sys.executable, "--reps", "1"]) + + assert rc == 1 + assert "expected 'ok'" in capsys.readouterr().err + + +def test_explicit_local_opt_in_may_skip_unavailable_worker(monkeypatch, capsys) -> None: + _stub_benchmark( + monkeypatch, + {"status": "failed(Playwright is not installed; run make setup-browser or npm install)"}, + ) + + rc = interaction_stress_smoke.main([sys.executable, "--reps", "1", "--allow-worker-skip"]) + + assert rc == 0 + assert "SKIPPED BY EXPLICIT LOCAL OPT-IN" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("field", "bad_value"), + [(field, False) for field in interaction_stress_smoke.WORKER_REQUIRED_TRUE] + + [("nonblank_pixels", 0), ("nonblank_pixels", float("nan"))], +) +def test_incomplete_worker_evidence_is_blocking(monkeypatch, capsys, field, bad_value) -> None: + worker = _healthy_worker() + worker[field] = bad_value + _stub_benchmark(monkeypatch, worker) + + rc = interaction_stress_smoke.main([sys.executable, "--reps", "1"]) + + assert rc == 1 + assert field in capsys.readouterr().err + + +def test_invalid_configured_browser_is_blocking_and_retained(tmp_path: Path) -> None: + evidence = tmp_path / "interaction-missing-browser.json" + + rc = interaction_stress_smoke.main( + ["/configured/but/missing/chromium", "--json", str(evidence)] + ) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["standalone_density_worker"]["status"] == "not-run" + assert "configured chromium not found" in payload["status"] diff --git a/tests/test_pan_zoom_matrix.py b/tests/test_pan_zoom_matrix.py new file mode 100644 index 00000000..656adbb7 --- /dev/null +++ b/tests/test_pan_zoom_matrix.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "pan_zoom_matrix.mjs" + + +def _catalog() -> dict: + proc = subprocess.run( + ["node", str(SCRIPT), "--catalog"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + return json.loads(proc.stdout) + + +def _valid_evidence(profile: str = "full") -> dict: + catalog = _catalog() + selected_ids = catalog["profiles"][profile]["cases"] + selected = [case for case in catalog["cases"] if case["id"] in selected_ids] + coverage = { + field: sorted({value for case in selected for value in case[field]}) + for field in ("actions", "axis_classes", "hosts") + } + cases = [ + { + **case, + "status": "passed", + "assertions": {"semantic": ["semantic"], "layout": ["layout"]}, + } + for case in selected + ] + return { + "schema_version": 1, + "requirement": "TST-NI-011", + "profile": profile, + "status": "passed", + "catalog": catalog, + "coverage": coverage, + "browsers": { + browser: {"status": "passed", "cases": cases} + for browser in catalog["profiles"][profile]["browsers"] + }, + } + + +def _verify(tmp_path: Path, evidence: dict) -> subprocess.CompletedProcess[str]: + path = tmp_path / "evidence.json" + path.write_text(json.dumps(evidence), encoding="utf-8") + return subprocess.run( + ["node", str(SCRIPT), "--verify-evidence", str(path)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def test_catalog_is_bounded_and_covers_the_complete_acceptance_matrix() -> None: + catalog = _catalog() + + assert catalog["requirement"] == "TST-NI-011" + assert len(catalog["cases"]) == 7 + assert {action for case in catalog["cases"] for action in case["actions"]} == { + "drag", + "wheel", + "box", + "toolbar_zoom", + "reset", + } + assert {axis for case in catalog["cases"] for axis in case["axis_classes"]} == { + "linear", + "log", + "reversed", + "category", + "dual", + "named", + } + assert {host for case in catalog["cases"] for host in case["hosts"]} == { + "standalone", + "reflex-live", + "reflex-static", + } + assert catalog["profiles"]["focused"]["browsers"] == [ + "chromium", + "firefox", + "webkit", + ] + + +def test_machine_readable_evidence_validator_accepts_complete_evidence(tmp_path: Path) -> None: + result = _verify(tmp_path, _valid_evidence()) + + assert result.returncode == 0, result.stderr + assert "evidence OK" in result.stdout + + +def test_machine_readable_evidence_validator_rejects_missing_axis_coverage( + tmp_path: Path, +) -> None: + evidence = _valid_evidence() + evidence["coverage"]["axis_classes"].remove("named") + + result = _verify(tmp_path, evidence) + + assert result.returncode != 0 + assert "axis_classes evidence coverage is incomplete" in result.stderr + + +def test_machine_readable_evidence_validator_rejects_failed_case(tmp_path: Path) -> None: + evidence = _valid_evidence() + evidence["browsers"]["chromium"]["cases"][0]["status"] = "failed" + + result = _verify(tmp_path, evidence) + + assert result.returncode != 0 + assert "did not pass" in result.stderr + + +def test_probe_drives_real_input_and_real_reflex_hosts() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + for token in ( + "page.mouse.down()", + "page.mouse.wheel", + "data-xy-modebar-menu-item", + 'for (const id of ["overview", "inline"])', + "window.__xy_views.get(id)", + 'message.type === "density_view"', + ): + assert token in source diff --git a/tests/test_pick_boundary_smoke.py b/tests/test_pick_boundary_smoke.py new file mode 100644 index 00000000..8a0e1a57 --- /dev/null +++ b/tests/test_pick_boundary_smoke.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def _load_module(): + root = Path(__file__).resolve().parents[1] + scripts = root / "scripts" + sys.path.insert(0, str(scripts)) + spec = importlib.util.spec_from_file_location( + "pick_boundary_smoke", scripts / "pick_boundary_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +pick_boundary_smoke = _load_module() + + +def _stub_fixture(monkeypatch, tmp_path: Path, *, title: str, returncode: int = 0) -> None: + (tmp_path / "standalone.js").write_text("window.xy = {};", encoding="utf-8") + monkeypatch.setattr(pick_boundary_smoke, "STATIC", tmp_path) + monkeypatch.setattr( + pick_boundary_smoke, + "build_payload", + lambda: ({"protocol": 4}, b"", {"protocol": 4}, b""), + ) + monkeypatch.setattr( + pick_boundary_smoke.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace( + stdout=f"{title}", + stderr="synthetic browser diagnostic", + returncode=returncode, + ), + ) + + +def test_pick_boundary_smoke_writes_success_evidence(monkeypatch, tmp_path: Path) -> None: + _stub_fixture( + monkeypatch, + tmp_path, + title="XY_OK slots=0,127,253,254,255 big=69999 second=1/0 hoverPickRenders=1 viewRefresh=1", + ) + evidence = tmp_path / "artifacts" / "pick.json" + + rc = pick_boundary_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 0 + assert payload["status"] == "ok" + assert payload["trace_slots"][-1] == 255 + assert payload["large_pick_index"] == 69_999 + + +def test_pick_boundary_smoke_failure_is_blocking_and_retained(monkeypatch, tmp_path: Path) -> None: + _stub_fixture(monkeypatch, tmp_path, title="XY_FAIL slot 255: picked trace 254") + evidence = tmp_path / "pick.json" + + rc = pick_boundary_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["status"] == "failed" + assert "slot 255" in payload["title"] + assert payload["stderr_tail"] == "synthetic browser diagnostic" + + +def test_explicit_missing_chromium_does_not_fall_back(monkeypatch) -> None: + monkeypatch.setattr( + pick_boundary_smoke.shutil, + "which", + lambda candidate: "/found/fallback" if candidate == "chromium" else None, + ) + + with pytest.raises(SystemExit, match="configured chromium not found"): + pick_boundary_smoke.find_chromium("/configured/but/missing/chromium") + + +def test_pick_boundary_timeout_is_blocking_and_retained(monkeypatch, tmp_path: Path) -> None: + _stub_fixture(monkeypatch, tmp_path, title="unused") + + def time_out(*args, **kwargs): + raise subprocess.TimeoutExpired( + cmd=args[0], + timeout=kwargs["timeout"], + stderr="synthetic timeout diagnostic", + ) + + monkeypatch.setattr(pick_boundary_smoke.subprocess, "run", time_out) + evidence = tmp_path / "pick-timeout.json" + + rc = pick_boundary_smoke.main([sys.executable, "--evidence", str(evidence)]) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert rc == 1 + assert payload["status"] == "failed" + assert payload["timed_out"] is True + assert payload["chromium_returncode"] is None + assert "timeout" in payload["title"] + assert payload["stderr_tail"] == "synthetic timeout diagnostic" diff --git a/tests/test_property_figure.py b/tests/test_property_figure.py index a7a16824..1bf14301 100644 --- a/tests/test_property_figure.py +++ b/tests/test_property_figure.py @@ -1,8 +1,11 @@ -"""Property-based tests for the figure builders (hostile-input surface). +"""Property-based tests for every public figure builder. -Hypothesis drives every public builder with adversarial arrays — NaN, ±inf, -huge/tiny magnitudes, empty, constant, unsorted, mismatched lengths — and -checks the invariants the wire contract promises rather than specific values: +Hypothesis drives the complete public-builder catalog through inputs that must +succeed and through classified invalid inputs that must fail cleanly. The core +builders additionally keep their broad hostile-array exploration — NaN, ±inf, +huge/tiny magnitudes, empty, constant, unsorted, and mismatched lengths. The +tests check the invariants the wire contract promises rather than specific +values: P1 build_payload either succeeds or raises ValueError/TypeError — never a crash class (IndexError/ZeroDivisionError/FloatingPointError/...). @@ -17,24 +20,30 @@ P6 every trace records its tier; reduced traces stay screen-bounded. P7 determinism: building twice yields byte-identical blob + equal spec (the anti-shimmer guarantee starts kernel-side). + P8 a late exception after trace insertion restores the exact seeded figure + state, including columns/caches and LOD interaction bookkeeping. -Requires the `hypothesis` dev extra; skipped cleanly where it is absent -(e.g. the no-PyPI sandbox) — CI installs `.[dev]` and runs the full suite. +Hypothesis is a required development dependency. Missing it is a broken test +environment, not a reason to silently skip this contract. """ from __future__ import annotations +import dataclasses import json import math +import struct +import weakref +from dataclasses import dataclass +from typing import Any import numpy as np import pytest +from hypothesis import given, settings +from hypothesis import strategies as st -hyp = pytest.importorskip("hypothesis") -from hypothesis import given, settings # noqa: E402 -from hypothesis import strategies as st # noqa: E402 - -from xy._figure import Figure # noqa: E402 +from xy import interaction +from xy._figure import Figure # Bounded so the whole module stays CI-cheap (< ~30s). COMMON = settings(max_examples=60, deadline=None) @@ -60,6 +69,217 @@ def hostile_array(min_size=0, max_size=200): ACCEPTABLE = (ValueError, TypeError) +PUBLIC_BUILDERS = ( + "line", + "area", + "scatter", + "histogram", + "hist", + "error_band", + "errorbar", + "box", + "violin", + "ecdf", + "hexbin", + "contour", + "step", + "stairs", + "stem", + "segments", + "triangle_mesh", + "bar", + "column", + "heatmap", +) + +VALID_COMMON = settings(max_examples=12, deadline=None) +INVALID_COMMON = settings(max_examples=8, deadline=None) +safe_floats = st.floats( + min_value=-100.0, + max_value=100.0, + allow_nan=False, + allow_infinity=False, + allow_subnormal=False, +) + + +@dataclass(frozen=True) +class Invocation: + """One generated call against a named public builder.""" + + args: tuple[Any, ...] + kwargs: dict[str, Any] + + +@dataclass(frozen=True) +class InvalidInvocation(Invocation): + """A generated rejection with an explicit validation classification.""" + + category: str + match: str + + +def _draw_values(draw: st.DrawFn, n: int) -> np.ndarray: + return np.asarray( + draw(st.lists(safe_floats, min_size=n, max_size=n)), + dtype=np.float64, + ) + + +@st.composite +def _valid_invocation(draw: st.DrawFn, builder: str) -> Invocation: + """Constrained data that is valid by construction for one builder.""" + n = draw(st.integers(min_value=2, max_value=10)) + values = _draw_values(draw, n) + offset = draw(st.floats(min_value=-10.0, max_value=10.0, allow_nan=False)) + spacing = draw(st.floats(min_value=0.25, max_value=4.0, allow_nan=False)) + x = offset + spacing * np.arange(n, dtype=np.float64) + if draw(st.booleans()): + x = x[::-1].copy() # line-like builders must accept and sort valid x + + if builder == "line": + return Invocation((x, values), {"curve": draw(st.sampled_from(("linear", "smooth")))}) + if builder == "area": + base = draw(safe_floats) if draw(st.booleans()) else values - np.abs(_draw_values(draw, n)) + return Invocation((x, values), {"base": base}) + if builder == "scatter": + sizes = np.abs(_draw_values(draw, n)) + 0.25 + colors = _draw_values(draw, n) + return Invocation( + (x, values), + { + "color": colors, + "size": sizes, + "density": draw(st.sampled_from((None, False, True))), + }, + ) + if builder in {"histogram", "hist"}: + return Invocation( + (values,), + { + "bins": draw(st.integers(min_value=1, max_value=12)), + "density": draw(st.booleans()), + "cumulative": draw(st.booleans()), + }, + ) + if builder == "error_band": + extent = np.abs(_draw_values(draw, n)) + 0.01 + return Invocation((x, values - extent, values + extent), {}) + if builder == "errorbar": + extent = np.abs(_draw_values(draw, n)) + axis = draw(st.sampled_from(("xerr", "yerr"))) + return Invocation((x, values), {axis: extent}) + if builder in {"box", "violin"}: + groups = np.asarray([f"group-{index % 3}" for index in range(n)], dtype=object) + kwargs: dict[str, Any] = { + "x": groups, + "orientation": draw(st.sampled_from(("vertical", "horizontal"))), + } + if builder == "box": + kwargs["show_outliers"] = draw(st.booleans()) + else: + kwargs["bins"] = draw(st.integers(min_value=4, max_value=24)) + return Invocation((values,), kwargs) + if builder == "ecdf": + bins = draw(st.one_of(st.none(), st.integers(min_value=1, max_value=12))) + return Invocation((values,), {"bins": bins}) + if builder == "hexbin": + hx = np.linspace(-1.0, 1.0, n, dtype=np.float64) + hy = hx**2 + np.linspace(0.0, 0.5, n, dtype=np.float64) + return Invocation( + (hx, hy), + { + "gridsize": draw(st.integers(min_value=2, max_value=8)), + "range": ((-1.25, 1.25), (-0.25, 1.75)), + "bins": draw(st.sampled_from(("count", "log"))), + }, + ) + if builder == "contour": + rows = draw(st.integers(min_value=2, max_value=5)) + cols = draw(st.integers(min_value=2, max_value=5)) + row = np.linspace(0.0, 1.0, rows, dtype=np.float64)[:, None] + col = np.linspace(0.0, 1.0, cols, dtype=np.float64)[None, :] + z = row + col + draw(st.floats(min_value=-5.0, max_value=5.0, allow_nan=False)) + level = float((np.min(z) + np.max(z)) / 2.0) + return Invocation((z,), {"levels": np.asarray([level]), "filled": draw(st.booleans())}) + if builder == "step": + return Invocation( + (x, values), + {"where": draw(st.sampled_from(("pre", "post", "mid")))}, + ) + if builder == "stairs": + increments = np.abs(_draw_values(draw, n + 1)) + 0.25 + edges = np.cumsum(increments) + return Invocation( + (values, edges), + {"where": draw(st.sampled_from(("pre", "post", "mid")))}, + ) + if builder == "stem": + base = values - np.abs(_draw_values(draw, n)) + return Invocation((x, values), {"base": base, "marker": draw(st.booleans())}) + if builder == "segments": + return Invocation((x, values, x + spacing * 0.5, values + 1.0), {}) + if builder == "triangle_mesh": + return Invocation( + (x, values, x + 0.5, values + 0.25, x + 0.25, values + 1.0), + {}, + ) + if builder in {"bar", "column"}: + categories = np.asarray([f"category-{index}" for index in range(n)], dtype=object) + return Invocation( + (categories, values), + {"orientation": draw(st.sampled_from(("vertical", "horizontal")))}, + ) + if builder == "heatmap": + rows = draw(st.integers(min_value=1, max_value=5)) + cols = draw(st.integers(min_value=1, max_value=5)) + z = _draw_values(draw, rows * cols).reshape(rows, cols) + return Invocation((z,), {}) + raise AssertionError(f"missing valid strategy for {builder}") + + +@st.composite +def _invalid_invocation(draw: st.DrawFn, builder: str) -> InvalidInvocation: + """Generate a documented invalid class for one public builder.""" + n = draw(st.integers(min_value=2, max_value=10)) + x = np.arange(n, dtype=np.float64) + y = _draw_values(draw, n) + short = y[:-1] + + cases: dict[str, InvalidInvocation] = { + "line": InvalidInvocation((x, short), {}, "cardinality", "equal length"), + "area": InvalidInvocation((x, y), {"base": short}, "cardinality", "base must have"), + "scatter": InvalidInvocation( + (x, y), {"color": short}, "channel-cardinality", "color array must be 1-D length" + ), + "histogram": InvalidInvocation((y,), {"bins": 0}, "domain", "bins must be positive"), + "hist": InvalidInvocation((y,), {"bins": 0}, "domain", "bins must be positive"), + "error_band": InvalidInvocation((x, y, short), {}, "cardinality", "upper must have length"), + "errorbar": InvalidInvocation((x, y), {"yerr": -1.0}, "domain", "non-negative"), + "box": InvalidInvocation((y,), {"orientation": "diagonal"}, "enum", "orientation"), + "violin": InvalidInvocation((y,), {"bins": 3}, "domain", "between 4 and 1024"), + "ecdf": InvalidInvocation((y,), {"bins": 0}, "domain", "positive integer"), + "hexbin": InvalidInvocation((x, y), {"gridsize": 0}, "domain", "gridsize"), + "contour": InvalidInvocation((y,), {}, "shape", "2-D matrix"), + "step": InvalidInvocation((x, y), {"where": "sideways"}, "enum", "where"), + "stairs": InvalidInvocation((y, x), {}, "cardinality", "edges must have length"), + "stem": InvalidInvocation((x, y), {"base": short}, "cardinality", "base must have"), + "segments": InvalidInvocation( + (x, y, x, short), {}, "cardinality", "coordinate columns must have equal length" + ), + "triangle_mesh": InvalidInvocation( + (x, y, x, y, x, short), {}, "cardinality", "coordinate columns must have equal length" + ), + "bar": InvalidInvocation((x, y), {"mode": "overlap"}, "enum", "mode"), + "column": InvalidInvocation((x, y), {"orientation": "diagonal"}, "enum", "orientation"), + "heatmap": InvalidInvocation((y,), {}, "shape", "must be 2-D or RGB"), + } + return cases[builder] + + +VALID_STRATEGIES = {builder: _valid_invocation(builder) for builder in PUBLIC_BUILDERS} +INVALID_STRATEGIES = {builder: _invalid_invocation(builder) for builder in PUBLIC_BUILDERS} + def check_payload(fig: Figure) -> None: """P2-P7 on a successfully built figure.""" @@ -72,7 +292,8 @@ def check_payload(fig: Figure) -> None: cols = spec["columns"] for c in cols: assert 0 <= c["byte_offset"] <= len(blob), "P3" - assert c["byte_offset"] + 4 * c["len"] <= len(blob), "P3" + itemsize = {"u8": 1, "u32": 4, "f64": 8}.get(c.get("dtype"), 4) + assert c["byte_offset"] + itemsize * c["len"] <= len(blob), "P3" def col_ref(ref): assert isinstance(ref, int) and 0 <= ref < len(cols), "P3" @@ -108,6 +329,218 @@ def build_or_reject(builder, *args, **kwargs): return fig +@pytest.mark.parametrize("builder", PUBLIC_BUILDERS) +@VALID_COMMON +@given(data=st.data()) +def test_every_public_builder_has_a_valid_strategy(builder, data): + """Every cataloged builder must succeed for data valid by construction.""" + invocation = data.draw(VALID_STRATEGIES[builder], label=builder) + fig = Figure() + result = getattr(fig, builder)(*invocation.args, **invocation.kwargs) + assert result is fig + assert fig.traces, f"{builder} accepted valid data without producing a trace" + check_payload(fig) + + +def test_violin_documented_four_bin_minimum_is_valid(): + fig = Figure().violin([1.0, 2.0, 3.0], bins=4) + + assert len(fig.traces) == 1 + assert len(fig.traces[0].x0) == 4 + check_payload(fig) + + +@pytest.mark.parametrize("builder", PUBLIC_BUILDERS) +@INVALID_COMMON +@given(data=st.data()) +def test_every_public_builder_has_a_classified_invalid_strategy(builder, data): + """Invalid strategies reject predictably and never leave partial state.""" + invocation = data.draw(INVALID_STRATEGIES[builder], label=builder) + assert invocation.category in {"cardinality", "channel-cardinality", "domain", "enum", "shape"} + fig = Figure() + with pytest.raises(ValueError, match=invocation.match): + getattr(fig, builder)(*invocation.args, **invocation.kwargs) + assert fig.traces == [] + assert fig.store.columns == [] + assert fig.store._by_key == {} + assert fig._axis_categories == {} + + +class SyntheticLateFailure(RuntimeError): + """Injected after a trace was inserted, at the latest shared mutation point.""" + + +class _AppendThenFail(list): + def append(self, item: Any) -> None: + super().append(item) + raise SyntheticLateFailure("synthetic failure after trace append") + + +def _freeze(value: Any) -> Any: + """Bitwise, identity-aware snapshot used only for rollback comparison.""" + if value is None or isinstance(value, (str, bytes, bool, int)): + return value + if isinstance(value, float): + return ("float64", struct.pack("=d", value)) + if isinstance(value, np.ndarray): + return ( + "ndarray", + id(value), + value.dtype.str, + value.shape, + value.strides, + value.flags.writeable, + value.tobytes(order="A"), + ) + if isinstance(value, np.generic): + return ("numpy-scalar", value.dtype.str, value.tobytes()) + if isinstance(value, weakref.finalize): + return ("finalize", id(value), value.alive) + if dataclasses.is_dataclass(value) and not isinstance(value, type): + field_names = {field.name for field in dataclasses.fields(value)} + declared = tuple( + (field.name, _freeze(getattr(value, field.name))) for field in dataclasses.fields(value) + ) + # cached_property values and amortized column buffers live outside + # dataclass fields; include them explicitly so this is a cache snapshot. + extras = tuple( + (key, _freeze(item)) for key, item in vars(value).items() if key not in field_names + ) + return (type(value).__qualname__, id(value), declared, extras) + if isinstance(value, dict): + return ( + "dict", + tuple((_freeze(key), _freeze(item)) for key, item in value.items()), + ) + if isinstance(value, (list, tuple)): + return (type(value).__qualname__, tuple(_freeze(item) for item in value)) + return (type(value).__qualname__, id(value), repr(value)) + + +@dataclass(frozen=True) +class _FigureState: + traces: Any + columns: Any + dedup_keys: Any + axes: Any + categories: Any + annotations: Any + interaction: Any + caches: Any + pyramids: Any + drill: Any + + +def _snapshot_figure(fig: Figure) -> _FigureState: + columns = tuple(fig.store.columns) + return _FigureState( + traces=_freeze(tuple(fig.traces)), + columns=_freeze(columns), + dedup_keys=tuple(fig.store._by_key.items()), + axes=_freeze((fig.axis_options, fig._active_axis_ids)), + categories=_freeze(fig._axis_categories), + annotations=_freeze(fig.annotations), + interaction=_freeze(fig.interaction), + caches=_freeze( + tuple( + ( + column._zone, + getattr(column, "_grow", None), + column.ingest_copies, + ) + for column in columns + ) + ), + pyramids=tuple( + ( + trace._pyr_handle, + _freeze(trace._pyr_finalizer), + ) + for trace in fig.traces + ), + drill=_freeze( + tuple((trace.shipped_sel, trace.drill_mode, trace.drill_seq) for trace in fig.traces) + ), + ) + + +def _seed_figure(monkeypatch: pytest.MonkeyPatch) -> tuple[Figure, Any]: + """Create nonempty state across every rollback surface in TST-NI-006.""" + fig = Figure(title="seeded transaction") + seed_x = np.asarray([0.0, 1.0, 2.0, 3.0]) + seed_y = np.asarray([0.0, 1.0, 4.0, 9.0]) + fig.scatter(seed_x, seed_y, name="seed-density", density=True) + seed_trace = fig.traces[0] + fig.bar(np.asarray(["seed-a", "seed-b"], dtype=object), np.asarray([2.0, 3.0])) + fig.set_axis("x2", label="seed secondary", domain=(-5.0, 5.0), side="top") + fig.text(1.0, 4.0, "seed annotation", style={"font_weight": 600}) + fig.interaction = {"mode": "select", "selection": "seeded"} + + # Exercise materialized zone-map cached_properties, not only raw columns. + for column in fig.store.columns: + zone = column.zone + _ = ( + zone.min, + zone.max, + zone.positive_min, + zone.positive_max, + zone.count, + zone.null_count, + ) + + # Build a real, tiny native pyramid. This proves the handle and finalizer + # survive rollback exactly without allocating the production 2048² cache. + monkeypatch.setattr(interaction, "PYRAMID_MIN_POINTS", 1) + monkeypatch.setattr(interaction, "PYRAMID_BASE_DIM", 8) + assert interaction._ensure_pyramid(seed_trace) + seed_trace.shipped_sel = np.asarray([3, 1], dtype=np.uint32) + seed_trace.drill_mode = True + seed_trace.drill_seq = 17 + fig.traces = _AppendThenFail(fig.traces) + return fig, seed_trace + + +ROLLBACK_INVOCATIONS = { + "line": Invocation(([2.0, 0.0, 1.0], [2.0, 0.0, 1.0]), {}), + "area": Invocation(([2.0, 0.0, 1.0], [3.0, 1.0, 2.0]), {"base": [1.0, 0.0, 0.5]}), + "scatter": Invocation(([0.0, 1.0, 2.0], [2.0, 1.0, 3.0]), {"color": [0.1, 0.2, 0.3]}), + "histogram": Invocation(([0.0, 1.0, 2.0, 3.0],), {"bins": 3}), + "hist": Invocation(([0.0, 1.0, 2.0, 3.0],), {"bins": 3}), + "error_band": Invocation(([0.0, 1.0, 2.0], [0.0, 0.5, 1.0], [1.0, 1.5, 2.0]), {}), + "errorbar": Invocation(([0.0, 1.0], [2.0, 3.0]), {"xerr": 0.1, "yerr": 0.2}), + "box": Invocation(([1.0, 2.0, 3.0, 4.0],), {"show_outliers": False}), + "violin": Invocation(([1.0, 2.0, 3.0, 4.0],), {"bins": 8}), + "ecdf": Invocation(([3.0, 1.0, 2.0, 2.0],), {}), + "hexbin": Invocation( + ([0.0, 0.5, 1.0], [0.0, 1.0, 0.5]), + {"gridsize": 4, "range": ((-0.5, 1.5), (-0.5, 1.5))}, + ), + "contour": Invocation((np.asarray([[0.0, 1.0], [2.0, 3.0]]),), {"levels": [1.5]}), + "step": Invocation(([0.0, 1.0, 2.0], [2.0, 1.0, 3.0]), {"where": "mid"}), + "stairs": Invocation(([2.0, 1.0, 3.0], [0.0, 1.0, 2.0, 3.0]), {}), + "stem": Invocation(([0.0, 1.0], [2.0, 3.0]), {"base": [0.5, 1.0]}), + "segments": Invocation(([0.0, 1.0], [2.0, 3.0], [1.0, 2.0], [3.0, 4.0]), {}), + "triangle_mesh": Invocation(([0.0], [0.0], [1.0], [0.0], [0.5], [1.0]), {}), + "bar": Invocation((np.asarray(["new-a", "new-b"]), [1.0, 2.0]), {}), + "column": Invocation((np.asarray(["new-a", "new-b"]), [1.0, 2.0]), {}), + "heatmap": Invocation((np.asarray([[0.0, 1.0], [2.0, 3.0]]),), {}), +} + + +@pytest.mark.parametrize("builder", PUBLIC_BUILDERS) +def test_every_public_builder_rolls_back_an_injected_late_failure(builder, monkeypatch): + """P8: failed builders are exact transactions over an already-live chart.""" + fig, pyramid_trace = _seed_figure(monkeypatch) + before = _snapshot_figure(fig) + invocation = ROLLBACK_INVOCATIONS[builder] + try: + with pytest.raises(SyntheticLateFailure, match="after trace append"): + getattr(fig, builder)(*invocation.args, **invocation.kwargs) + assert _snapshot_figure(fig) == before + finally: + interaction._free_pyramid(pyramid_trace) + + # -- per-builder properties --------------------------------------------------- diff --git a/tests/test_protocol_catalog.py b/tests/test_protocol_catalog.py new file mode 100644 index 00000000..af13f3a8 --- /dev/null +++ b/tests/test_protocol_catalog.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import ast +import base64 +import copy +import json +import subprocess +from pathlib import Path +from typing import Any + +import numpy as np +import pytest + +from xy._figure import DECIMATION_THRESHOLD, Figure +from xy.channel import ChannelCallbacks, decode_frame, encode_frame, handle_message + +ROOT = Path(__file__).resolve().parents[1] +CATALOG_PATH = ROOT / "spec" / "testing" / "protocol-catalog.json" +CHANNEL_PATH = ROOT / "python" / "xy" / "channel.py" +CLIENT = ROOT / "python" / "xy" / "static" / "index.js" +CATALOG = json.loads(CATALOG_PATH.read_text(encoding="utf-8")) +REQUEST_CASES = [ + (request, case_kind, case) + for request in CATALOG["requests"] + for case_kind, case in request["cases"].items() +] + + +def _implemented_request_types(source: str) -> set[str]: + tree = ast.parse(source) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "handle_message" + ) + request_types: set[str] = set() + for node in ast.walk(function): + if not isinstance(node, ast.Compare) or not isinstance(node.left, ast.Name): + continue + if node.left.id != "kind" or len(node.ops) != 1 or len(node.comparators) != 1: + continue + comparator = node.comparators[0] + if isinstance(node.ops[0], ast.Eq) and isinstance(comparator, ast.Constant): + if isinstance(comparator.value, str): + request_types.add(comparator.value) + elif isinstance(node.ops[0], ast.In) and isinstance(comparator, (ast.Set, ast.Tuple)): + request_types.update( + value.value + for value in comparator.elts + if isinstance(value, ast.Constant) and isinstance(value.value, str) + ) + return request_types + + +def _catalog_errors(catalog: dict[str, Any], channel_source: str) -> list[str]: + errors: list[str] = [] + requests = catalog.get("requests") + if not isinstance(requests, list): + return ["requests must be a list"] + names = [request.get("type") for request in requests if isinstance(request, dict)] + if len(names) != len(set(names)): + errors.append("request types must be unique") + implemented = _implemented_request_types(channel_source) + if set(names) != implemented: + errors.append( + f"catalog request types {sorted(names)} do not match implementation {sorted(implemented)}" + ) + expected_cases = set(catalog.get("request_case_kinds", [])) + for request in requests: + if not isinstance(request, dict): + errors.append("request entry must be an object") + continue + actual_cases = set(request.get("cases", {})) + if actual_cases != expected_cases: + errors.append( + f"{request.get('type')} cases {sorted(actual_cases)} do not match " + f"{sorted(expected_cases)}" + ) + replies = catalog.get("replies", {}) + catalog_reply_types = { + case["reply"] + for request in requests + for case in request.get("cases", {}).values() + if "reply" in case + } + if set(replies) != catalog_reply_types: + errors.append( + f"reply catalog {sorted(replies)} does not match generated replies " + f"{sorted(catalog_reply_types)}" + ) + return errors + + +def _figure() -> Figure: + line = np.arange(DECIMATION_THRESHOLD + 1, dtype=np.float64) + points = np.arange(100, dtype=np.float64) + return Figure().line(line, line).scatter(points, points).scatter(points, points, density=True) + + +def _callbacks(fired: list[str]) -> ChannelCallbacks: + def record(name: str): + return lambda _value: fired.append(name) + + return ChannelCallbacks( + on_hover=record("on_hover"), + on_click=record("on_click"), + on_brush=record("on_brush"), + on_select=record("on_select"), + on_view_change=record("on_view_change"), + on_animation_start=record("on_animation_start"), + on_animation_end=record("on_animation_end"), + ) + + +def _assert_reply_contract(message: dict[str, Any], buffers: list[bytes] | None) -> None: + reply = CATALOG["replies"][message["type"]] + assert set(reply["required"]) <= set(message) + policy = reply["buffers"] + if policy == "none": + assert buffers is None + elif policy == "nonempty": + assert buffers + elif policy == "trace_count": + assert len(buffers or []) == len(message["traces"]) + else: # pragma: no cover - catalog schema guard + raise AssertionError(f"unknown buffer policy {policy!r}") + + +def test_protocol_catalog_exactly_covers_dispatcher_and_replies() -> None: + source = CHANNEL_PATH.read_text(encoding="utf-8") + assert _catalog_errors(CATALOG, source) == [] + + +def test_protocol_catalog_oracle_rejects_missing_branch_and_case() -> None: + source = CHANNEL_PATH.read_text(encoding="utf-8") + missing_request = copy.deepcopy(CATALOG) + missing_request["requests"].pop() + assert any( + "do not match implementation" in error for error in _catalog_errors(missing_request, source) + ) + + missing_case = copy.deepcopy(CATALOG) + missing_case["requests"][0]["cases"].pop("wrong_type") + assert any("view cases" in error for error in _catalog_errors(missing_case, source)) + + +@pytest.mark.parametrize( + ("request_entry", "case_kind", "case"), + REQUEST_CASES, + ids=[f"{request['type']}-{case_kind}" for request, case_kind, _case in REQUEST_CASES], +) +def test_catalog_generated_request_matrix( + request_entry: dict[str, Any], case_kind: str, case: dict[str, Any] +) -> None: + assert isinstance(request_entry["type"], str) + fired: list[str] = [] + callbacks = _callbacks(fired) if case_kind == "callback" else ChannelCallbacks() + + result = handle_message(_figure(), case["message"], callbacks=callbacks) + + if "reply" in case: + assert result is not None + message, buffers = result + assert message["type"] == case["reply"] + _assert_reply_contract(message, buffers) + else: + assert case["outcome"] in {"drop", "none"} + assert result is None + assert fired == case.get("callbacks", []) + + +@pytest.mark.parametrize("reply_type", sorted(CATALOG["replies"])) +def test_python_reproduces_and_decodes_committed_golden_frames(reply_type: str) -> None: + golden = CATALOG["replies"][reply_type]["golden"] + buffers = [base64.b64decode(value) for value in golden["buffers_base64"]] + frame = base64.b64decode(golden["frame_base64"]) + + assert encode_frame(golden["message"], buffers) == frame + decoded = decode_frame(frame) + assert decoded.message == golden["message"] + assert [bytes(buffer) for buffer in decoded.buffers] == buffers + assert all(buffer.obj is frame for buffer in decoded.buffers) + + +def test_javascript_decodes_the_same_committed_golden_frames_zero_copy() -> None: + cases = [ + { + "type": reply_type, + "frame": reply["golden"]["frame_base64"], + "message": reply["golden"]["message"], + "buffers": reply["golden"]["buffers_base64"], + } + for reply_type, reply in sorted(CATALOG["replies"].items()) + ] + script = f""" + import {{ decodeFrame }} from {CLIENT.as_uri()!r}; + const cases = {json.dumps(cases)}; + const results = cases.map((item) => {{ + const source = Uint8Array.from(Buffer.from(item.frame, "base64")); + const decoded = decodeFrame(source.buffer); + return {{ + type: item.type, + message: decoded.message, + buffers: decoded.buffers.map((value) => + Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString("base64")), + aligned: decoded.buffers.every((value) => value.byteOffset % 8 === 0), + sameBacking: decoded.buffers.every((value) => value.buffer === source.buffer), + }}; + }}); + process.stdout.write(JSON.stringify(results)); + """ + + completed = subprocess.run( + ["node", "--input-type=module", "--eval", script], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + check=True, + ) + results = json.loads(completed.stdout) + + assert [result["type"] for result in results] == [case["type"] for case in cases] + for result, case in zip(results, cases, strict=True): + assert result["message"] == case["message"] + assert result["buffers"] == case["buffers"] + assert result["aligned"] is True + assert result["sameBacking"] is True diff --git a/tests/test_release_provenance.py b/tests/test_release_provenance.py new file mode 100644 index 00000000..1912cb67 --- /dev/null +++ b/tests/test_release_provenance.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + + +def _load_module(): + path = Path(__file__).resolve().parents[1] / "scripts" / "release_provenance.py" + spec = importlib.util.spec_from_file_location("release_provenance", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +provenance = _load_module() + + +def _manifest(tmp_path: Path): + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + wheel = artifacts / "xy-1.2.3-py3-none-any.whl" + sdist = artifacts / "xy-1.2.3.tar.gz" + wheel.write_bytes(b"wheel") + sdist.write_bytes(b"sdist") + manifest = provenance.create_manifest( + artifacts, + source_sha="a" * 40, + repository="reflex-dev/xy", + workflow_run_id="1234", + tag="v1.2.3", + created_at="2026-07-21T00:00:00+00:00", + ) + return manifest, wheel, sdist + + +def test_release_provenance_round_trip(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + + assert ( + provenance.verify_manifest( + manifest, + [wheel, sdist], + source_sha="a" * 40, + repository="reflex-dev/xy", + workflow_run_id="1234", + tag="v1.2.3", + ) + == [] + ) + assert manifest["schema"] == provenance.SCHEMA + assert {record["name"] for record in manifest["artifacts"]} == {wheel.name, sdist.name} + + +def test_release_provenance_rejects_tampered_artifact(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + wheel.write_bytes(b"mutated wheel") + + errors = provenance.verify_manifest(manifest, [wheel, sdist], source_sha="a" * 40) + + assert any("SHA-256 does not match" in error for error in errors) + assert any("size is" in error for error in errors) + + +def test_release_provenance_rejects_wrong_source_sha(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + + errors = provenance.verify_manifest(manifest, [wheel, sdist], source_sha="b" * 40) + + assert any("source SHA" in error for error in errors) + + +def test_release_provenance_rejects_wrong_identity_metadata(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + + errors = provenance.verify_manifest( + manifest, + [wheel, sdist], + repository="reflex-dev/not-xy", + workflow_run_id="9999", + tag="v9.9.9", + ) + + assert any("repository" in error and "does not match" in error for error in errors) + assert any("workflow run" in error and "does not match" in error for error in errors) + assert any("tag" in error and "does not match" in error for error in errors) + + +def test_release_provenance_rejects_malformed_record_metadata(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + manifest["created_at"] = "2026-07-21" + manifest["artifacts"][0]["path"] = "../escaped.whl" + manifest["artifacts"][0]["size"] = -1 + manifest["artifacts"][0]["sha256"] = "not-a-digest" + + errors = provenance.verify_manifest(manifest, [wheel, sdist]) + + assert any("timezone" in error for error in errors) + assert any("unsafe path" in error for error in errors) + assert any("invalid size" in error for error in errors) + assert any("invalid SHA-256" in error for error in errors) + + +def test_release_provenance_cli_verifies_downloaded_file(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + manifest_path = tmp_path / "release-provenance.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + assert ( + provenance.main( + [ + "verify", + str(manifest_path), + str(wheel), + str(sdist), + "--source-sha", + "a" * 40, + "--repository", + "reflex-dev/xy", + "--workflow-run-id", + "1234", + "--tag", + "v1.2.3", + ] + ) + == 0 + ) + + +def test_release_provenance_rejects_omitted_or_extra_artifacts(tmp_path: Path) -> None: + manifest, wheel, _ = _manifest(tmp_path) + extra = tmp_path / "unrecorded.whl" + extra.write_bytes(b"extra") + + errors = provenance.verify_manifest(manifest, [wheel, extra], source_sha="a" * 40) + + assert any("was not supplied" in error for error in errors) + assert any("absent from provenance" in error for error in errors) + + +def test_release_provenance_rejects_duplicate_manifest_records(tmp_path: Path) -> None: + manifest, wheel, sdist = _manifest(tmp_path) + manifest["artifacts"].append(dict(manifest["artifacts"][0])) + + errors = provenance.verify_manifest(manifest, [wheel, sdist], source_sha="a" * 40) + + assert any("duplicate artifact records" in error for error in errors) diff --git a/tests/test_rendered_label_formats.py b/tests/test_rendered_label_formats.py new file mode 100644 index 00000000..f4d35e88 --- /dev/null +++ b/tests/test_rendered_label_formats.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import numpy as np +import pytest + +import xy +from xy._figure import Figure + + +@pytest.mark.parametrize( + "value", + [ + ".2f", + ",.2f", + "$,.0f", + "€,.2f", + ".1%", + ".0f%", + ".4f s", + ".3f GiB", + ",.0fK", + "$,.0fK", + ], +) +def test_supported_numeric_formats_survive_public_builders(value: str) -> None: + assert xy.x_axis(format=value).format == value + assert xy.y_axis(type_="log", format=value).format == value + assert xy.tooltip(format={"value": value}).format == {"value": value} + + +def test_supported_utc_time_format_survives_axis_and_tooltip_builders() -> None: + value = "%Y-%m-%d %H:%M:%S %b %B" + + assert xy.x_axis(type_="time", format=value).format == value + assert xy.tooltip(format={"when": value}).format == {"when": value} + + fig = Figure() + fig.set_axis("x", type_="time", format=value) + fig.scatter( + np.array(["2024-01-01T00:00:00"], dtype="datetime64[ms]"), + [1.0], + ) + spec, _ = fig.build_payload() + assert spec["x_axis"]["kind"] == "time" + assert spec["x_axis"]["format"] == value + + +@pytest.mark.parametrize( + "value", + [ + "", + "not-a-format", + ".2q", + ".21f", + "USD .2f", + ".2f GiB%", + "%Y-%q", + "%Y-%m-%d %", + ], +) +def test_broken_formats_fail_at_public_python_boundaries(value: str) -> None: + with pytest.raises(ValueError, match=r"supported|UTC time tokens"): + xy.x_axis(format=value) + with pytest.raises(ValueError, match=r"supported|UTC time tokens"): + xy.tooltip(format={"value": value}) + with pytest.raises(ValueError, match=r"supported|UTC time tokens"): + Figure().set_axis("x", format=value) + + +def test_format_kind_mismatches_fail_when_kind_is_known_or_resolved() -> None: + with pytest.raises(ValueError, match="supported numeric format"): + xy.x_axis(type_="linear", format="%Y-%m-%d") + with pytest.raises(ValueError, match="UTC time tokens"): + xy.x_axis(type_="time", format=".2f") + + category_chart = xy.scatter_chart( + xy.scatter(x=np.array(["alpha", "beta"]), y=np.array([1.0, 2.0])), + xy.x_axis(format=".2f"), + ) + with pytest.raises(ValueError, match="not supported for category labels"): + category_chart.figure().build_payload() + + +def test_mutated_component_formats_are_revalidated_before_payload_emission() -> None: + axis = xy.x_axis() + axis.format = "broken-after-construction" + chart = xy.scatter_chart( + xy.scatter(x=np.array([1.0, 2.0]), y=np.array([3.0, 4.0])), + axis, + ) + with pytest.raises(ValueError, match="not a supported"): + chart.figure() + + tooltip = xy.tooltip(fields=["x"]) + tooltip.format = {"x": "broken-after-construction"} + chart = xy.scatter_chart( + xy.scatter(x=np.array([1.0, 2.0]), y=np.array([3.0, 4.0])), + tooltip, + ) + with pytest.raises(ValueError, match="not a supported"): + chart.figure() diff --git a/tests/test_run_pytest_no_skips.py b/tests/test_run_pytest_no_skips.py new file mode 100644 index 00000000..f317df72 --- /dev/null +++ b/tests/test_run_pytest_no_skips.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +def _load_checker(): + path = Path(__file__).resolve().parents[1] / "scripts" / "run_pytest_no_skips.py" + spec = importlib.util.spec_from_file_location("run_pytest_no_skips", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +runner = _load_checker() + + +def test_no_skip_runner_accepts_a_passing_suite(tmp_path: Path) -> None: + test_file = tmp_path / "test_pass.py" + test_file.write_text("def test_ok():\n assert True\n", encoding="utf-8") + + assert runner.main(["-q", str(test_file)]) == 0 + + +def test_no_skip_runner_rejects_runtime_skip(tmp_path: Path) -> None: + test_file = tmp_path / "test_skip.py" + test_file.write_text( + "import pytest\n\ndef test_not_available():\n pytest.skip('host missing')\n", + encoding="utf-8", + ) + + assert runner.main(["-q", str(test_file)]) == 1 + + +def test_no_skip_runner_rejects_collection_skip(tmp_path: Path) -> None: + test_file = tmp_path / "test_collection_skip.py" + test_file.write_text( + "import pytest\npytest.skip('host missing', allow_module_level=True)\n", + encoding="utf-8", + ) + + assert runner.main(["-q", str(test_file)]) == 1 diff --git a/tests/test_runtime_security_smoke.py b/tests/test_runtime_security_smoke.py new file mode 100644 index 00000000..7931e253 --- /dev/null +++ b/tests/test_runtime_security_smoke.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import html +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def _load_module(): + scripts = ROOT / "scripts" + sys.path.insert(0, str(scripts)) + spec = importlib.util.spec_from_file_location( + "runtime_security_smoke", scripts / "runtime_security_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +runtime_security_smoke = _load_module() + + +def _browser_output(*, failures: list[str] | None = None) -> str: + failures = [] if failures is None else failures + report = { + "failures": failures, + "surfaces": [], + "unsafeNodes": [], + "transientUnsafeNodes": [], + "executed": False, + "dialogs": [], + "apiAttempts": [], + "pageErrors": [], + "cspViolations": [{"effectiveDirective": "img-src"}], + "cssApplied": True, + "hostileBackground": True, + "externalResources": [], + } + title = "XY_RUNTIME_SECURITY_OK" if not failures else "XY_RUNTIME_SECURITY_FAIL" + return ( + f"{title}" + f'
'
+        f"{html.escape(json.dumps(report))}
" + ) + + +def _stub_browser(monkeypatch: pytest.MonkeyPatch, *, failures: list[str] | None = None) -> None: + monkeypatch.setattr( + runtime_security_smoke, + "_run_browser", + lambda *args, **kwargs: SimpleNamespace( + stdout=_browser_output(failures=failures), + stderr="synthetic browser diagnostic", + returncode=0, + ), + ) + + +def test_fixture_covers_each_public_text_surface_with_hostile_input() -> None: + sentinel = "http://127.0.0.1:45678/blocked" + + document, expectations = runtime_security_smoke.build_runtime_fixture(sentinel) + + names = {item["name"] for item in expectations} + assert names == { + "title", + "x-axis title", + "y-axis title", + "x tick label", + "y tick label", + "line trace name", + "continuous trace name", + "category", + "annotation", + "legend title", + "colorbar title", + "tooltip title", + "tooltip x field", + "tooltip y field", + "tooltip category field", + "tooltip category value", + } + assert all("