diff --git a/CHANGELOG.md b/CHANGELOG.md index 37d85a18..7069cc88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,17 @@ in the README). that don't use them are byte-identical. ### Changed +- Stable animation `key=` identity encoding now uses one native Rust row scan + for homogeneous fixed-width strings, bytes, booleans, and signed or unsigned + integers, plus finite floating arrays, including non-native-endian NumPy + arrays. Float16/32 values widen exactly to the f64 token contract. It + preserves the existing 64-bit identities and duplicate-row errors; mixed + objects, NUL-containing Python sequences, dates, and non-finite row + diagnostics stay on the conservative Python reference path. Highly padded + Python string/bytes sequences also stay there to bound fixed-width temporary + memory. Fixed-width NumPy strings retain their exact embedded-NUL semantics + natively. + This adds the C ABI v43 transition-key kernel. - Host theme changes made through an ancestor `data-theme` attribute now refresh canvas/SVG paint just like `.dark` class and inline-style changes; DOM chrome continues to follow the cascade automatically. diff --git a/benchmarks/test_codspeed_animation.py b/benchmarks/test_codspeed_animation.py index a56c4161..e64ea225 100644 --- a/benchmarks/test_codspeed_animation.py +++ b/benchmarks/test_codspeed_animation.py @@ -70,10 +70,13 @@ def payload_figures(animation_data): def test_animation_encode_100k_stable_keys(benchmark, animation_data) -> None: + """Track the public list-to-native path, including homogeneous routing.""" _x, _y, keys = animation_data encoded = benchmark(_encode_transition_keys, keys, N, "benchmark key") assert encoded.shape == (N, 2) assert encoded.dtype == np.uint32 + assert encoded[:, 0].flags.c_contiguous + assert encoded[:, 1].flags.c_contiguous def test_animation_plain_payload_100k(benchmark, payload_figures) -> None: diff --git a/python/xy/_native.py b/python/xy/_native.py index a898156c..d7b3153b 100644 --- a/python/xy/_native.py +++ b/python/xy/_native.py @@ -24,7 +24,7 @@ from .config import MAX_CONTOUR_WORK, MAX_SCREEN_DIM -ABI_VERSION = 42 +ABI_VERSION = 43 # Rust reports invalid arguments (and, via the ffi_guard panic shield, any # internal panic) by returning `usize::MAX` from size-returning entry points. @@ -117,6 +117,18 @@ def _load() -> ctypes.CDLL: ctypes.c_void_p, ctypes.c_size_t, ] + lib.xy_transition_keys_fixed.restype = ctypes.c_int32 + lib.xy_transition_keys_fixed.argtypes = [ + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_uint32, + ctypes.c_int32, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t), + ] lib.xy_remap_u8.restype = ctypes.c_int32 lib.xy_remap_u8.argtypes = [ ctypes.c_void_p, @@ -953,6 +965,84 @@ def factorize_unicode1_u8_counts( return codes, unique_indices[:written].copy(), counts[:written].copy() +def transition_keys_fixed( + values: np.ndarray, label: str = "animation key" +) -> Optional[npt.NDArray[np.uint32]]: + """Encode homogeneous fixed-width transition keys in one native row scan. + + The returned ``(N, 2)`` array is Fortran-contiguous: its two ``u32`` + columns are the caller-allocated lo/hi planes written by Rust and can be + shipped independently without another full-size copy. ``None`` asks the + policy layer to use its scalar oracle for an invalid value (for example a + non-finite float or invalid Unicode scalar), preserving its precise + user-facing error. + """ + records = np.asarray(values) + if records.ndim != 1 or records.dtype.hasobject: + raise ValueError("transition key values must be a non-object 1-D array") + + kind_name = records.dtype.kind + width = int(records.dtype.itemsize) + if kind_name == "U" and width > 0 and width % 4 == 0: + kind = 0 + elif kind_name == "S" and width > 0: + kind = 1 + elif kind_name == "b" and width == 1: + kind = 2 + elif kind_name == "i" and width in (1, 2, 4, 8): + kind = 3 + elif kind_name == "u" and width in (1, 2, 4, 8): + kind = 4 + elif kind_name == "f" and width in (2, 4, 8): + # Python canonicalizes NumPy float16/32 scalars through ``.item()``; + # widening to f64 is exact and gives the native kernel the same value. + if width != 8: + records = records.astype(np.float64) + width = 8 + kind = 5 + else: + raise ValueError( + "transition key values must use Unicode, bytes, bool, integer, or float dtype" + ) + + records = np.ascontiguousarray(records) + n = len(records) + result = np.empty((n, 2), dtype=np.uint32, order="F") + if n == 0: + return result + + native_order = "<" if sys.byteorder == "little" else ">" + swap_endian = records.dtype.byteorder not in ("=", "|", native_order) + error_first = ctypes.c_size_t(_USIZE_MAX) + error_index = ctypes.c_size_t(_USIZE_MAX) + status = int( + _lib.xy_transition_keys_fixed( + records.ctypes.data, + n, + width, + kind, + int(swap_endian), + result[:, 0].ctypes.data, + result[:, 1].ctypes.data, + ctypes.byref(error_first), + ctypes.byref(error_index), + ) + ) + if status == 0: + return result + if status == 1: + return None + if status == 2: + if error_first.value >= n or error_index.value >= n: + raise RuntimeError("native transition-key encoder returned invalid row indices") + raise ValueError( + f"{label} contains duplicate value at rows {error_first.value} and {error_index.value}" + ) + if status == 3: + raise ValueError(f"{label} produced an identity digest collision") + raise RuntimeError(f"native transition-key encoder returned unknown status {status}") + + def remap_u8(values: npt.NDArray[np.uint8], mapping: npt.NDArray[np.uint8]) -> None: """Apply a compact categorical codebook permutation in place.""" values = np.asarray(values) diff --git a/python/xy/components.py b/python/xy/components.py index 6f5af4b4..9a48d7d6 100644 --- a/python/xy/components.py +++ b/python/xy/components.py @@ -3850,13 +3850,97 @@ def _transition_key_token(value: Any, index: int) -> bytes: ) +_TRANSITION_KEY_SEQUENCE_MAX_FIXED_BYTES = 256 * 1024 * 1024 +_TRANSITION_KEY_SEQUENCE_MAX_PADDING_RATIO = 8 + + +def _fixed_transition_key_values(value: Any) -> np.ndarray | None: + """Return a conservative homogeneous array for the native key encoder.""" + if type(value) is np.ndarray: + arr = value + elif isinstance(value, (list, tuple)) and value: + first = value[0].item() if isinstance(value[0], np.generic) else value[0] + item_type = type(first) + if item_type not in (str, bytes, bool, int, float): + return None + total_units = 0 + max_units = 0 + for raw in value: + item = raw.item() if isinstance(raw, np.generic) else raw + if type(item) is not item_type: + # In particular, never coerce mixed bool/int or int/float + # keys: their scalar tokens are deliberately type-sensitive. + return None + if (item_type is str and "\x00" in item) or (item_type is bytes and b"\x00" in item): + # Fixed-width storage cannot distinguish padding from an + # explicitly trailing NUL supplied in a Python scalar. + return None + if item_type in (str, bytes): + units = len(item) + total_units += units + max_units = max(max_units, units) + if item_type in (str, bytes): + unit_bytes = 4 if item_type is str else 1 + # NumPy's fixed-width sequence conversion costs N × max width. + # Keep a single long outlier from turning a compact Python list + # into a huge temporary before Rust can scan it. + padded_bytes = len(value) * max(max_units, 1) * unit_bytes + source_bytes = max(total_units * unit_bytes, 1) + if ( + padded_bytes > _TRANSITION_KEY_SEQUENCE_MAX_FIXED_BYTES + or padded_bytes > source_bytes * _TRANSITION_KEY_SEQUENCE_MAX_PADDING_RATIO + ): + return None + try: + arr = np.asarray(value) + except (OverflowError, TypeError, ValueError): + return None + expected_kinds = { + str: {"U"}, + bytes: {"S"}, + bool: {"b"}, + int: {"i", "u"}, + float: {"f"}, + } + if arr.dtype.kind not in expected_kinds[item_type]: + # NumPy may coerce an integer list outside its fixed-width range + # to f64. That would change the type-sensitive ``i:`` token into + # an ``f:`` token, so leave arbitrary-width integers to Python. + return None + else: + return None + if arr.ndim != 1: + return None + if arr.dtype.kind == "U" and arr.dtype.itemsize > 0 and arr.dtype.itemsize % 4 == 0: + return arr + if arr.dtype.kind == "S" and arr.dtype.itemsize > 0: + return arr + if arr.dtype.kind == "b" and arr.dtype.itemsize == 1: + return arr + if arr.dtype.kind in {"i", "u"} and arr.dtype.itemsize in (1, 2, 4, 8): + return arr + if arr.dtype.kind == "f" and arr.dtype.itemsize in (2, 4, 8): + return arr + return None + + def _encode_transition_keys(value: Any, expected: int, label: str) -> np.ndarray: + fixed = _fixed_transition_key_values(value) + if fixed is not None: + if len(fixed) != expected: + raise ValueError(f"{label} must have length {expected}, got {len(fixed)}") + from . import kernels + + encoded = kernels.transition_keys_fixed(fixed, label) + if encoded is not None: + return encoded + arr = np.asarray(value, dtype=object) if arr.ndim != 1: raise ValueError(f"{label} must be one-dimensional") if len(arr) != expected: raise ValueError(f"{label} must have length {expected}, got {len(arr)}") - result = np.empty((expected, 2), dtype=np.uint32) + result = np.empty((expected, 2), dtype=np.uint32, order="F") seen: dict[bytes, int] = {} digests: dict[bytes, bytes] = {} for index, raw in enumerate(arr): diff --git a/python/xy/kernels.py b/python/xy/kernels.py index 520663f2..1b50a2b1 100644 --- a/python/xy/kernels.py +++ b/python/xy/kernels.py @@ -43,6 +43,7 @@ factorize_fixed_u8 = _impl.factorize_fixed_u8 factorize_fixed_u8_counts = _impl.factorize_fixed_u8_counts factorize_unicode1_u8_counts = _impl.factorize_unicode1_u8_counts +transition_keys_fixed = _impl.transition_keys_fixed m4_indices = _impl.m4_indices marching_squares = _impl.marching_squares marching_triangles = _impl.marching_triangles @@ -145,6 +146,7 @@ "stratified_sample_mask", "stratified_sample_range_u8", "streamlines", + "transition_keys_fixed", "triangle_edges", "valid_indices_f64", "vector_segments", diff --git a/scripts/abi_smoke.py b/scripts/abi_smoke.py index 66768ac3..22863c2d 100644 --- a/scripts/abi_smoke.py +++ b/scripts/abi_smoke.py @@ -104,6 +104,18 @@ def load() -> ctypes.CDLL: U64P, ctypes.c_size_t, ] + lib.xy_transition_keys_fixed.restype = ctypes.c_int32 + lib.xy_transition_keys_fixed.argtypes = [ + U8P, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_uint32, + ctypes.c_int32, + U32P, + U32P, + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t), + ] lib.xy_remap_u8.restype = ctypes.c_int32 lib.xy_remap_u8.argtypes = [U8P, ctypes.c_size_t, U8P, ctypes.c_size_t] lib.xy_encode_f32.restype = ctypes.c_int32 @@ -489,6 +501,76 @@ def ok(cond: bool, msg: str) -> None: swapped_count == 4 and list(unicode_codes) == [0, 1, 0, 2, 3], "factorize_unicode1_u8_counts swapped endian", ) + transition_low = array("I", [99, 99]) + transition_high = array("I", [99, 99]) + transition_first = ctypes.c_size_t(size_max) + transition_index = ctypes.c_size_t(size_max) + transition_records = array("B", b"a\0\0a\0b") + status = lib.xy_transition_keys_fixed( + _ptr(transition_records, ctypes.c_uint8), + 2, + 3, + 1, + 0, + _ptr(transition_low, ctypes.c_uint32), + _ptr(transition_high, ctypes.c_uint32), + ctypes.byref(transition_first), + ctypes.byref(transition_index), + ) + ok(status == 0, "transition_keys_fixed success status") + ok( + list(transition_low) == [0x5B3B753B, 0xB1A7FF88] + and list(transition_high) == [0xE1379B39, 0x9D296CBA], + "transition_keys_fixed personalized digest words", + ) + duplicate_records = array("B", b"a\0\0b\0\0a\0\0") + duplicate_low = array("I", [99, 99, 99]) + duplicate_high = array("I", [99, 99, 99]) + status = lib.xy_transition_keys_fixed( + _ptr(duplicate_records, ctypes.c_uint8), + 3, + 3, + 1, + 0, + _ptr(duplicate_low, ctypes.c_uint32), + _ptr(duplicate_high, ctypes.c_uint32), + ctypes.byref(transition_first), + ctypes.byref(transition_index), + ) + ok( + status == 2 and transition_first.value == 0 and transition_index.value == 2, + "transition_keys_fixed duplicate rows", + ) + ok( + lib.xy_transition_keys_fixed( + null_u8, + 0, + 1, + 1, + 0, + null_u32, + null_u32, + ctypes.byref(transition_first), + ctypes.byref(transition_index), + ) + == 0, + "transition_keys_fixed empty/null succeeds", + ) + ok( + lib.xy_transition_keys_fixed( + null_u8, + 1, + 1, + 1, + 0, + null_u32, + null_u32, + ctypes.byref(transition_first), + ctypes.byref(transition_index), + ) + == 1, + "transition_keys_fixed non-empty/null rejected", + ) small_unique = array("I", [99] * 2) ok( lib.xy_factorize_fixed_u8( diff --git a/spec/design/animation.md b/spec/design/animation.md index 53e95138..de758613 100644 --- a/spec/design/animation.md +++ b/spec/design/animation.md @@ -60,13 +60,25 @@ points are dropped. Layouts that cannot share positional buffers record a errorbar marks. It may be an array or a column name resolved through `data=`. `match="key"` requires a key on every effectively keyed mark. -Keys are canonicalized type-sensitively and hashed once in Python to a stable -64-bit identity, shipped as two binary `u32` columns. Strings, finite numbers, -booleans, bytes, dates, datetimes, and NumPy equivalents are supported. -Missing, unsupported, wrong-length, or duplicate values fail during figure -construction. Line-like keys follow the same stable geometry sort as their -coordinates. Errorbar point keys are role-qualified after expansion so the -main segment and caps remain unique and stable. +Keys are canonicalized type-sensitively to a stable 64-bit identity, shipped as +two binary `u32` columns. Homogeneous fixed-width strings, bytes, booleans, +signed or unsigned integers, and finite floating arrays take one bulk native +path, including non-native endian NumPy arrays; f16/f32 values widen exactly to +the f64 token contract. Its identities and first-duplicate row diagnostics are +byte-for-byte compatible with the original Python encoder. +Python remains the conservative reference path for mixed objects, +NUL-containing Python string/bytes sequences, dates, and datetimes, and +supplies the exact row error for non-finite numbers. Python sequences whose +fixed-width conversion would exceed the byte budget or 8× padding ratio also +stay on that path, bounding temporary memory under skewed key lengths. +Fixed-width NumPy `U`/`S` records keep their exact trailing-padding and +embedded-NUL semantics in the native path. This keeps the complete public +grammar—strings, finite numbers, booleans, bytes, dates, datetimes, and NumPy +equivalents—without pushing Python object policy through the C ABI. Missing, +unsupported, wrong-length, or duplicate values fail during figure construction. +Line-like keys follow the same stable geometry sort as their coordinates. +Errorbar point keys are role-qualified after expansion so the main segment and +caps remain unique and stable. The browser builds a bounded key→old-index map. `append` instead matches the old/new x identity and `index` pairs equal positions. Above @@ -151,14 +163,16 @@ an exact deterministic progress without starting a frame loop. ## 7. Verification and performance gates -- `tests/test_animation.py` owns validation, serialization, identity, wire, - sorting, errorbar expansion, and deterministic-export contracts. +- `tests/test_animation.py` owns validation, serialization, native/reference + identity parity and routing, wire, sorting, errorbar expansion, and + deterministic-export contracts. - `scripts/animation_smoke.py` exercises pixel-checked, ghost-free keyed interpolation, explicit partial-match fallback, GPU scratch buffers, rapid replacement, bounded lifetime, lifecycle balance (including destroy), and reduced motion in headless Chrome. -- `benchmarks/test_codspeed_animation.py` attributes key encoding and animated - payload build overhead separately from the plain payload path. +- `benchmarks/test_codspeed_animation.py` attributes the end-to-end native + stable-key encoding row and animated payload build overhead separately from + the plain payload path. - Browser frame/allocation measurements belong to the real-Chrome benchmark lane, not CodSpeed simulation; the animation smoke asserts the hard previous+next allocation bound. diff --git a/spec/design/rust-engine.md b/spec/design/rust-engine.md index d36055b9..e9962b3b 100644 --- a/spec/design/rust-engine.md +++ b/spec/design/rust-engine.md @@ -2,7 +2,7 @@ **Status:** design. Decides what lives in Rust vs Python and how the FFI seam evolves without rewrites. Grounded in the shipped engine: single-dependency -cdylib (`src/lib.rs`, ABI v36; `png` is the one crate, for static export), +cdylib (`src/lib.rs`, ABI v43; `png` is the one crate, for static export), ctypes binding (`_native.py`), and dispatch in `kernels.py`. The native core is required — `kernels.py` raises a clear ImportError when it can't load, with no pure-Python fallback. @@ -27,6 +27,7 @@ clear ImportError when it can't load, with no pure-Python fallback. |---|---|---| | zone maps, encode_f32, m4, bin_2d, bin_2d_mean_color, min_max, histogram_uniform, normalize_f32, range/validity indices, polygon (lasso) selection, local_log_density | Rust (ABI v41) | correct — new equal-length x/y columns use a paired zone-map call with bit-identical per-column reductions; lasso ray casting (`xy_polygon_select`, §34) walks edges inside the point loop instead of one NumPy pass per edge, and buckets edges by y so a point tests only those spanning its row — the answer is the same crossing parity, without the per-edge full-length temporaries that made a 2048-vertex lasso cost ~370 ms over 160k candidates. Bucketing is declined below 16 vertices (the build outweighs the scans it saves) and for a non-finite polygon (slab bounds stop meaning anything); above 16 the slab count tracks the vertex count, capped at 256 and shed further so the CSR never exceeds 2²⁰ entries, which bounds both the allocation and its u32 cursors independently of what a caller passes; full-domain density first paint fuses binning with uniform or counted-u8 overlay sampling while retaining exact standalone outputs; mean-color binning (LOD doc §2) is an integer-only pipeline (checked-in sRGB⇄linear-u16 tables, alpha-weighted u64 sums) so grids are bitwise deterministic across thread counts and platforms; mesh/rectangle validity scans consume only columns not already proven finite by zone metadata | | fixed-width string/bytes/bool factorization | Rust (ABI v36) | correct — compact palettes use a bounded L1-resident codebook with full-record collision checks and emit exact counts; U1 uses a direct Unicode-scalar table with endian support; ≥512k rows probe a prefix then encode disjoint chunks in parallel, merging late labels by canonical first-row order before any retry; Python sees only unique labels and retains display-label ordering policy | +| stable animation-key encoding for homogeneous fixed-width string/bytes/bool/integer/float columns | Rust (ABI v43) | correct — one borrowed row scan emits the two caller-owned u32 identity planes and reports the first duplicate pair; f16/f32 widen exactly to the f64 token contract and non-native arrays carry an explicit endian flag; Python validates shape/length, bounds padded temporaries for skewed string/bytes sequences, assembles exact public errors, and retains the reference encoder for mixed objects, NUL-containing Python sequences, dates, and non-finite row diagnostics | | static display-list raster, row-banded polyline/point/segment paint, batched fill+stroke triangle meshes, affine scatter projection plus typed color/size resolution, density/heatmap colormap and sampling | Rust (ABI v36) | correct — commands borrow f32/u8 payload or canonical spans synchronously; compact stratified sampling reuses factorization counts; batched/banded output is byte-identical | | signal processing: `xy_rfft`, `xy_welch_spectra`, `xy_spectrogram` | Rust (ABI v36) | correct — O(N) transforms over sample columns; window/segment policy stays in Python | | geometry/triangulation: `xy_delaunay_triangles`, `xy_polygon_triangles`, `xy_marching_squares`, `xy_marching_triangles`, `xy_streamlines`, `xy_vector_segments`, `xy_quad_mesh_triangles`, `xy_sector_triangles`, `xy_indexed_triangles`, `xy_triangle_edges` | Rust (ABI v36) | correct — output is screen-bounded index/vertex buffers; level choice and styling stay in Python | diff --git a/spec/process/perf-audit-2026-07-22.md b/spec/process/perf-audit-2026-07-22.md index 455a5156..f73ec2d0 100644 --- a/spec/process/perf-audit-2026-07-22.md +++ b/spec/process/perf-audit-2026-07-22.md @@ -12,8 +12,8 @@ float64 is zero-copy through ctypes; a 10M-point density scatter makes **zero full-size copies** from user array to wire bytes; heavy reductions run in parallel AVX2-dispatched Rust; LOD tiers are computed on demand, not retained; the widget transport is binary with no join copy. The findings are therefore -second-tier: conditional paths (categorical/object dtypes, animation keys, -hover fallbacks) and residual serial kernels. Three kernel findings were fixed +second-tier: conditional paths (categorical/object dtypes and hover fallbacks) +and residual serial kernels. Three kernel findings were fixed in this audit; the rest are recorded as a ranked backlog. ## Fixed in this audit (measured, bit-identical) @@ -128,12 +128,18 @@ there is O(sample), not O(N). ### Medium -3. **Animation `key=` encoding hashes all rows even when discarded** - (`components.py:3629`). Per-row Python `blake2s` over N rows, then - `_transition_entry` falls back to `snap:key-limit` above - `MAX_ANIMATION_MATCH_ROWS` (200k) and throws the digests away. Fix: - short-circuit when the fallback is certain (mind duplicate-key error - semantics), and/or hash fixed-width key buffers natively. +3. **Resolved after this audit — animation `key=` encoding hashed homogeneous + columns row-by-row in Python** (`components.py`). ABI v43 adds a borrowed + fixed-width native encoder for homogeneous string, bytes, bool, integer, + and finite floating keys (f16/f32 widen exactly to f64). It emits both u32 + identity planes in one pass and reports the first duplicate pair so Python + preserves the exact error + contract. Conservative Python fallback retains mixed-object, + NUL-containing Python sequence, date, and non-finite-row diagnostic + semantics; fixed-width NumPy `U`/`S` records keep exact embedded-NUL + semantics natively. + `benchmarks/test_codspeed_animation.py::test_animation_encode_100k_stable_keys` + keeps the original 100k end-to-end row as the regression gate. 4. **Direct-tier traces retain f32 wire buffers for the widget lifetime** (`widget.py:55`, `_payload.py:154`). A near-ceiling direct scatter (500k–2M pts) holds f64 canonical + f32 wire simultaneously (~24 B/pt) diff --git a/src/lib.rs b/src/lib.rs index 2dd2c048..5f2f0b6b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub mod raster; mod simd; pub mod svg; pub mod tiles; +mod transition; use kernels::ZoneMap; @@ -82,7 +83,7 @@ unsafe fn borrowed_byte_spans<'a>( /// ABI version — bumped on any signature change. The Python wrapper checks this /// at load time and refuses a mismatched library loudly (§33 comm-versioning /// rule, applied to the in-process boundary). -pub const ABI_VERSION: u32 = 42; +pub const ABI_VERSION: u32 = 43; const FACTORIZE_CAPACITY_EXCEEDED: usize = usize::MAX - 1; #[no_mangle] @@ -90,6 +91,77 @@ pub extern "C" fn xy_abi_version() -> u32 { ABI_VERSION } +/// Encode homogeneous fixed-width NumPy records as stable animation identity +/// keys. `kind` is 0 for UTF-32 Unicode, 1 for fixed bytes, 2 for bool, 3 for +/// signed integers, 4 for unsigned integers, and 5 for f64. Integer widths are +/// 1/2/4/8 bytes; Unicode width is a positive multiple of four. `swap_endian` +/// must be zero or one. +/// +/// Returns 0 on success, 1 for invalid arguments or scalar data, 2 for a +/// duplicate token, and 3 for a digest collision. For status 2 or 3, +/// `out_error_first` and `out_error_index` receive the prior/current row +/// indices. Invalid scalar data also records its row in both outputs when they +/// are non-null; invalid pointer/dimension calls do not write error outputs. +/// +/// # Safety +/// For non-empty input, `data` addresses `len * width` readable bytes and each +/// key output addresses `len` writable u32s. Error outputs address one writable +/// usize each. Input and output spans must not overlap. +#[no_mangle] +pub unsafe extern "C" fn xy_transition_keys_fixed( + data: *const u8, + len: usize, + width: usize, + kind: u32, + swap_endian: i32, + out_lo: *mut u32, + out_hi: *mut u32, + out_error_first: *mut usize, + out_error_index: *mut usize, +) -> i32 { + if !matches!(swap_endian, 0 | 1) { + return 1; + } + if len == 0 { + return 0; + } + if out_error_first.is_null() || out_error_index.is_null() { + return 1; + } + let byte_len = match len.checked_mul(width) { + Some(value) if width > 0 => value, + _ => return 1, + }; + if data.is_null() || out_lo.is_null() || out_hi.is_null() { + return 1; + } + let data = std::slice::from_raw_parts(data, byte_len); + let low = std::slice::from_raw_parts_mut(out_lo, len); + let high = std::slice::from_raw_parts_mut(out_hi, len); + ffi_guard(1, || { + match transition::encode_fixed_into(data, width, kind, swap_endian != 0, low, high) { + Ok(()) => 0, + Err(transition::TransitionKeyError::Invalid { index }) => { + if let Some(index) = index { + *out_error_first = index; + *out_error_index = index; + } + 1 + } + Err(transition::TransitionKeyError::Duplicate { first, index }) => { + *out_error_first = first; + *out_error_index = index; + 2 + } + Err(transition::TransitionKeyError::Collision { first, index }) => { + *out_error_first = first; + *out_error_index = index; + 3 + } + } + }) +} + /// Serialize parallel f64 screen coordinates into SVG polyline path data. /// Returns the required byte count, or `usize::MAX` for invalid inputs. When /// `out_cap` is too small no bytes are written, allowing callers to retry. @@ -3131,6 +3203,87 @@ mod tests { assert_eq!(ffi_guard(0i32, || 1i32), 1); } + #[test] + fn transition_key_ffi_reports_duplicate_rows_and_invalid_data() { + let values = [7i16, -2, 7]; + let mut low = [0u32; 3]; + let mut high = [0u32; 3]; + let mut first = usize::MAX; + let mut index = usize::MAX; + unsafe { + assert_eq!( + xy_transition_keys_fixed( + values.as_ptr().cast(), + values.len(), + std::mem::size_of::(), + transition::KIND_SIGNED, + 0, + low.as_mut_ptr(), + high.as_mut_ptr(), + &mut first, + &mut index, + ), + 2 + ); + } + assert_eq!((first, index), (0, 2)); + + let nonfinite = f64::INFINITY; + first = usize::MAX; + index = usize::MAX; + unsafe { + assert_eq!( + xy_transition_keys_fixed( + (&nonfinite as *const f64).cast(), + 1, + std::mem::size_of::(), + transition::KIND_FLOAT64, + 0, + low.as_mut_ptr(), + high.as_mut_ptr(), + &mut first, + &mut index, + ), + 1 + ); + } + assert_eq!((first, index), (0, 0)); + } + + #[test] + fn transition_key_ffi_accepts_empty_null_spans_and_rejects_bad_pointers() { + unsafe { + assert_eq!( + xy_transition_keys_fixed( + std::ptr::null(), + 0, + 0, + transition::KIND_BYTES, + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ), + 0 + ); + assert_eq!( + xy_transition_keys_fixed( + std::ptr::null(), + 1, + 1, + transition::KIND_BYTES, + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ), + 1 + ); + } + } + #[test] #[cfg(target_pointer_width = "64")] fn index_emitting_entry_points_reject_u32_overflowing_len() { diff --git a/src/transition.rs b/src/transition.rs new file mode 100644 index 00000000..f48436a0 --- /dev/null +++ b/src/transition.rs @@ -0,0 +1,708 @@ +//! Stable animation identity encoding. +//! +//! The public Python API accepts several scalar key types and defines their +//! identity through a short, type-qualified byte token. This module reproduces +//! those tokens directly from homogeneous NumPy fixed-width storage, hashes +//! them with the same personalized eight-byte BLAKE2s digest, and checks +//! uniqueness without materializing one Python object per row. + +use std::collections::HashMap; + +pub const KIND_UNICODE: u32 = 0; +pub const KIND_BYTES: u32 = 1; +pub const KIND_BOOL: u32 = 2; +pub const KIND_SIGNED: u32 = 3; +pub const KIND_UNSIGNED: u32 = 4; +pub const KIND_FLOAT64: u32 = 5; + +#[derive(Debug, PartialEq, Eq)] +pub enum TransitionKeyError { + Invalid { index: Option }, + Duplicate { first: usize, index: usize }, + Collision { first: usize, index: usize }, +} + +const BLAKE2S_IV: [u32; 8] = [ + 0x6A09_E667, + 0xBB67_AE85, + 0x3C6E_F372, + 0xA54F_F53A, + 0x510E_527F, + 0x9B05_688C, + 0x1F83_D9AB, + 0x5BE0_CD19, +]; + +const BLAKE2S_SIGMA: [[usize; 16]; 10] = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], + [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], + [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8], + [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], + [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9], + [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], + [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10], + [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], + [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0], +]; + +/// Focused incremental BLAKE2s implementation for the transition-key contract. +/// +/// The parameter block is fixed to an eight-byte digest with the seven-byte +/// personalization `xykeyv1` (NUL-padded to BLAKE2s' eight-byte field). +struct Blake2s8 { + state: [u32; 8], + buffer: [u8; 64], + buffered: usize, + count: u64, +} + +impl Blake2s8 { + fn new() -> Self { + let mut parameter = [0u8; 32]; + parameter[0] = 8; // digest length + parameter[2] = 1; // fanout + parameter[3] = 1; // depth + parameter[24..31].copy_from_slice(b"xykeyv1"); + let mut state = BLAKE2S_IV; + for (word, bytes) in state.iter_mut().zip(parameter.as_chunks::<4>().0) { + *word ^= u32::from_le_bytes(*bytes); + } + Self { + state, + buffer: [0; 64], + buffered: 0, + count: 0, + } + } + + fn update(&mut self, mut input: &[u8]) { + if self.buffered != 0 { + let available = 64 - self.buffered; + if input.len() <= available { + self.buffer[self.buffered..self.buffered + input.len()].copy_from_slice(input); + self.buffered += input.len(); + return; + } + self.buffer[self.buffered..].copy_from_slice(&input[..available]); + self.count += 64; + let block = self.buffer; + self.compress(&block, false); + self.buffered = 0; + input = &input[available..]; + } + while input.len() > 64 { + self.count += 64; + let block: &[u8; 64] = input[..64].try_into().expect("complete BLAKE2s block"); + self.compress(block, false); + input = &input[64..]; + } + self.buffer[..input.len()].copy_from_slice(input); + self.buffered = input.len(); + } + + fn finish(mut self) -> [u8; 8] { + self.count += self.buffered as u64; + self.buffer[self.buffered..].fill(0); + let block = self.buffer; + self.compress(&block, true); + let mut digest = [0u8; 8]; + digest[..4].copy_from_slice(&self.state[0].to_le_bytes()); + digest[4..].copy_from_slice(&self.state[1].to_le_bytes()); + digest + } + + fn compress(&mut self, block: &[u8; 64], last: bool) { + let mut message = [0u32; 16]; + for (word, bytes) in message.iter_mut().zip(block.as_chunks::<4>().0) { + *word = u32::from_le_bytes(*bytes); + } + let mut work = [0u32; 16]; + work[..8].copy_from_slice(&self.state); + work[8..].copy_from_slice(&BLAKE2S_IV); + work[12] ^= self.count as u32; + work[13] ^= (self.count >> 32) as u32; + if last { + work[14] = !work[14]; + } + for schedule in BLAKE2S_SIGMA { + blake2s_mix( + &mut work, + 0, + 4, + 8, + 12, + message[schedule[0]], + message[schedule[1]], + ); + blake2s_mix( + &mut work, + 1, + 5, + 9, + 13, + message[schedule[2]], + message[schedule[3]], + ); + blake2s_mix( + &mut work, + 2, + 6, + 10, + 14, + message[schedule[4]], + message[schedule[5]], + ); + blake2s_mix( + &mut work, + 3, + 7, + 11, + 15, + message[schedule[6]], + message[schedule[7]], + ); + blake2s_mix( + &mut work, + 0, + 5, + 10, + 15, + message[schedule[8]], + message[schedule[9]], + ); + blake2s_mix( + &mut work, + 1, + 6, + 11, + 12, + message[schedule[10]], + message[schedule[11]], + ); + blake2s_mix( + &mut work, + 2, + 7, + 8, + 13, + message[schedule[12]], + message[schedule[13]], + ); + blake2s_mix( + &mut work, + 3, + 4, + 9, + 14, + message[schedule[14]], + message[schedule[15]], + ); + } + for index in 0..8 { + self.state[index] ^= work[index] ^ work[index + 8]; + } + } +} + +#[inline(always)] +fn blake2s_mix(work: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, x: u32, y: u32) { + work[a] = work[a].wrapping_add(work[b]).wrapping_add(x); + work[d] = (work[d] ^ work[a]).rotate_right(16); + work[c] = work[c].wrapping_add(work[d]); + work[b] = (work[b] ^ work[c]).rotate_right(12); + work[a] = work[a].wrapping_add(work[b]).wrapping_add(y); + work[d] = (work[d] ^ work[a]).rotate_right(8); + work[c] = work[c].wrapping_add(work[d]); + work[b] = (work[b] ^ work[c]).rotate_right(7); +} + +#[inline] +fn digest_parts(parts: &[&[u8]]) -> [u8; 8] { + let mut hasher = Blake2s8::new(); + for part in parts { + hasher.update(part); + } + hasher.finish() +} + +fn unsigned_decimal(mut value: u64, buffer: &mut [u8; 20]) -> &[u8] { + let mut start = buffer.len(); + loop { + start -= 1; + buffer[start] = b'0' + (value % 10) as u8; + value /= 10; + if value == 0 { + return &buffer[start..]; + } + } +} + +fn signed_decimal(value: i64, buffer: &mut [u8; 21]) -> &[u8] { + let negative = value < 0; + let mut magnitude = value.unsigned_abs(); + let mut start = buffer.len(); + loop { + start -= 1; + buffer[start] = b'0' + (magnitude % 10) as u8; + magnitude /= 10; + if magnitude == 0 { + if negative { + start -= 1; + buffer[start] = b'-'; + } + return &buffer[start..]; + } + } +} + +#[inline] +fn native_u32(bytes: &[u8], swap_endian: bool) -> u32 { + let raw = u32::from_ne_bytes(bytes.try_into().expect("validated u32 width")); + if swap_endian { + raw.swap_bytes() + } else { + raw + } +} + +fn trim_byte_padding(row: &[u8]) -> &[u8] { + let end = row + .iter() + .rposition(|&byte| byte != 0) + .map_or(0, |index| index + 1); + &row[..end] +} + +fn trim_unicode_padding(row: &[u8], swap_endian: bool) -> &[u8] { + let mut end = row.len(); + while end >= 4 && native_u32(&row[end - 4..end], swap_endian) == 0 { + end -= 4; + } + &row[..end] +} + +fn unsigned_value(row: &[u8], swap_endian: bool) -> Option { + Some(match row.len() { + 1 => row[0] as u64, + 2 => { + let raw = u16::from_ne_bytes(row.try_into().ok()?); + if swap_endian { + raw.swap_bytes() as u64 + } else { + raw as u64 + } + } + 4 => { + let raw = u32::from_ne_bytes(row.try_into().ok()?); + if swap_endian { + raw.swap_bytes() as u64 + } else { + raw as u64 + } + } + 8 => { + let raw = u64::from_ne_bytes(row.try_into().ok()?); + if swap_endian { + raw.swap_bytes() + } else { + raw + } + } + _ => return None, + }) +} + +fn signed_value(row: &[u8], swap_endian: bool) -> Option { + Some(match row.len() { + 1 => row[0] as i8 as i64, + 2 => { + let raw = u16::from_ne_bytes(row.try_into().ok()?); + let native = if swap_endian { raw.swap_bytes() } else { raw }; + native as i16 as i64 + } + 4 => { + let raw = u32::from_ne_bytes(row.try_into().ok()?); + let native = if swap_endian { raw.swap_bytes() } else { raw }; + native as i32 as i64 + } + 8 => { + let raw = u64::from_ne_bytes(row.try_into().ok()?); + let native = if swap_endian { raw.swap_bytes() } else { raw }; + native as i64 + } + _ => return None, + }) +} + +fn unicode_digest(row: &[u8], swap_endian: bool) -> Option<[u8; 8]> { + let codepoint_bytes = trim_unicode_padding(row, swap_endian); + let mut hasher = Blake2s8::new(); + hasher.update(b"s:"); + for bytes in codepoint_bytes.as_chunks::<4>().0 { + let scalar = char::from_u32(native_u32(bytes, swap_endian))?; + let mut encoded = [0u8; 4]; + hasher.update(scalar.encode_utf8(&mut encoded).as_bytes()); + } + Some(hasher.finish()) +} + +fn float_digest(bits: u64) -> Option<[u8; 8]> { + let exponent_bits = ((bits >> 52) & 0x7ff) as i32; + if exponent_bits == 0x7ff { + return None; + } + let fraction = bits & 0x000f_ffff_ffff_ffff; + let mut hasher = Blake2s8::new(); + hasher.update(b"f:"); + if bits >> 63 != 0 { + hasher.update(b"-"); + } + if exponent_bits == 0 && fraction == 0 { + hasher.update(b"0x0.0p+0"); + return Some(hasher.finish()); + } + hasher.update(if exponent_bits == 0 { b"0x0." } else { b"0x1." }); + let mut hexadecimal = [0u8; 13]; + for (index, digit) in hexadecimal.iter_mut().enumerate() { + let shift = 48 - index * 4; + let nibble = ((fraction >> shift) & 0xf) as u8; + *digit = if nibble < 10 { + b'0' + nibble + } else { + b'a' + nibble - 10 + }; + } + hasher.update(&hexadecimal); + hasher.update(b"p"); + let exponent = if exponent_bits == 0 { + -1022 + } else { + exponent_bits - 1023 + }; + if exponent >= 0 { + hasher.update(b"+"); + } + let mut exponent_buffer = [0u8; 21]; + hasher.update(signed_decimal(exponent as i64, &mut exponent_buffer)); + Some(hasher.finish()) +} + +fn row_digest(row: &[u8], kind: u32, swap_endian: bool) -> Result<[u8; 8], TransitionKeyError> { + match kind { + KIND_UNICODE => { + unicode_digest(row, swap_endian).ok_or(TransitionKeyError::Invalid { index: None }) + } + KIND_BYTES => Ok(digest_parts(&[b"y:", trim_byte_padding(row)])), + KIND_BOOL => match row { + [0] => Ok(digest_parts(&[b"b:0"])), + [1] => Ok(digest_parts(&[b"b:1"])), + _ => Err(TransitionKeyError::Invalid { index: None }), + }, + KIND_SIGNED => { + let value = signed_value(row, swap_endian) + .ok_or(TransitionKeyError::Invalid { index: None })?; + let mut decimal = [0u8; 21]; + Ok(digest_parts(&[b"i:", signed_decimal(value, &mut decimal)])) + } + KIND_UNSIGNED => { + let value = unsigned_value(row, swap_endian) + .ok_or(TransitionKeyError::Invalid { index: None })?; + let mut decimal = [0u8; 20]; + Ok(digest_parts(&[ + b"i:", + unsigned_decimal(value, &mut decimal), + ])) + } + KIND_FLOAT64 => { + let bits = unsigned_value(row, swap_endian) + .ok_or(TransitionKeyError::Invalid { index: None })?; + float_digest(bits).ok_or(TransitionKeyError::Invalid { index: None }) + } + _ => Err(TransitionKeyError::Invalid { index: None }), + } +} + +fn rows_equal(first: &[u8], second: &[u8], kind: u32, swap_endian: bool) -> bool { + match kind { + KIND_UNICODE => { + let first = trim_unicode_padding(first, swap_endian); + let second = trim_unicode_padding(second, swap_endian); + first.len() == second.len() + && first + .as_chunks::<4>() + .0 + .iter() + .zip(second.as_chunks::<4>().0) + .all(|(a, b)| native_u32(a, swap_endian) == native_u32(b, swap_endian)) + } + KIND_BYTES => trim_byte_padding(first) == trim_byte_padding(second), + KIND_BOOL => first == second, + KIND_SIGNED => signed_value(first, swap_endian) == signed_value(second, swap_endian), + KIND_UNSIGNED | KIND_FLOAT64 => { + unsigned_value(first, swap_endian) == unsigned_value(second, swap_endian) + } + _ => false, + } +} + +fn valid_layout(kind: u32, width: usize) -> bool { + match kind { + KIND_UNICODE => width > 0 && width.is_multiple_of(4), + KIND_BYTES => width > 0, + KIND_BOOL => width == 1, + KIND_SIGNED | KIND_UNSIGNED => matches!(width, 1 | 2 | 4 | 8), + KIND_FLOAT64 => width == 8, + _ => false, + } +} + +/// Encode homogeneous fixed-width rows into Python-compatible transition keys. +/// +/// `data` must contain exactly `out_lo.len() * width` bytes. Outputs have equal +/// lengths and remain caller-owned. Duplicate and digest-collision errors carry +/// both the first conflicting row and the current row. +pub fn encode_fixed_into( + data: &[u8], + width: usize, + kind: u32, + swap_endian: bool, + out_lo: &mut [u32], + out_hi: &mut [u32], +) -> Result<(), TransitionKeyError> { + if !valid_layout(kind, width) + || out_lo.len() != out_hi.len() + || data.len() != out_lo.len().saturating_mul(width) + { + return Err(TransitionKeyError::Invalid { index: None }); + } + let mut seen: HashMap<[u8; 8], usize> = HashMap::with_capacity(out_lo.len()); + for (index, row) in data.chunks_exact(width).enumerate() { + let digest = row_digest(row, kind, swap_endian) + .map_err(|_| TransitionKeyError::Invalid { index: Some(index) })?; + if let Some(&first) = seen.get(&digest) { + let first_row = &data[first * width..(first + 1) * width]; + return Err(if rows_equal(first_row, row, kind, swap_endian) { + TransitionKeyError::Duplicate { first, index } + } else { + TransitionKeyError::Collision { first, index } + }); + } + seen.insert(digest, index); + out_lo[index] = u32::from_le_bytes(digest[..4].try_into().expect("low digest word")); + out_hi[index] = u32::from_le_bytes(digest[4..].try_into().expect("high digest word")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(data: &[u8], len: usize, width: usize, kind: u32, swap: bool) -> Vec<[u32; 2]> { + let mut low = vec![0; len]; + let mut high = vec![0; len]; + encode_fixed_into(data, width, kind, swap, &mut low, &mut high).unwrap(); + low.into_iter().zip(high).map(|(lo, hi)| [lo, hi]).collect() + } + + #[test] + fn blake2s_matches_python_personalized_vectors() { + let vectors: &[(&[u8], [u8; 8])] = &[ + (b"", [0x4e, 0x9e, 0xba, 0xa7, 0x8f, 0x14, 0x9c, 0x5d]), + (b"s:", [0xc5, 0x52, 0x69, 0xcf, 0x55, 0x7a, 0xc4, 0x6f]), + (b"s:a", [0xb5, 0x2d, 0x83, 0x23, 0x79, 0x57, 0x39, 0x2a]), + (b"y:a\0b", [0x88, 0xff, 0xa7, 0xb1, 0xba, 0x6c, 0x29, 0x9d]), + ( + b"i:-9223372036854775808", + [0x2b, 0x78, 0x46, 0x28, 0xf5, 0x8e, 0x4f, 0xcf], + ), + ]; + for &(token, expected) in vectors { + assert_eq!(digest_parts(&[token]), expected, "{token:?}"); + } + } + + #[test] + fn blake2s_streaming_matches_python_across_block_boundaries() { + let vectors = [ + (61, [0x3d, 0xe3, 0xd4, 0x9c, 0x05, 0xb3, 0x59, 0x15]), + (62, [0x57, 0x81, 0xda, 0x6f, 0x90, 0x33, 0xb1, 0x60]), + (63, [0x80, 0xca, 0x70, 0x13, 0x6f, 0x55, 0x55, 0xc5]), + (64, [0xd1, 0x8e, 0xe0, 0xf2, 0x21, 0x08, 0xee, 0x9f]), + (65, [0x6b, 0x19, 0x87, 0x55, 0x8c, 0x12, 0x3b, 0x0b]), + (127, [0xc0, 0x1a, 0x03, 0xc8, 0x2c, 0x90, 0xaf, 0x19]), + (128, [0xbd, 0x94, 0x43, 0xb6, 0xe4, 0x00, 0x84, 0x51]), + (129, [0xda, 0x4d, 0x41, 0xce, 0xc8, 0x00, 0xd4, 0xcb]), + ]; + for (length, expected) in vectors { + let input = vec![b'z'; length]; + assert_eq!(digest_parts(&[&input]), expected, "length {length}"); + + let split = length / 2; + assert_eq!( + digest_parts(&[&input[..split], &input[split..]]), + expected, + "split length {length}" + ); + } + } + + #[test] + fn unicode_tokens_are_utf8_and_trim_only_padding() { + let rows = [ + [0, 0, 0], + [b'a' as u32, 0, 0], + [0x00e9, 0, 0], + [0x1f600, 0, 0], + [b'a' as u32, 0, b'b' as u32], + ]; + let bytes = rows + .iter() + .flatten() + .flat_map(|value| value.to_ne_bytes()) + .collect::>(); + let got = encode(&bytes, rows.len(), 12, KIND_UNICODE, false); + assert_eq!(got[0], [0xcf69_52c5, 0x6fc4_7a55]); // s: + assert_eq!(got[1], [0x2383_2db5, 0x2a39_5779]); // s:a + assert_eq!(got[2], [0xd791_9aa5, 0xd070_c8b1]); // s:é + assert_eq!(got[3], [0x44ec_c8f9, 0x5388_dfc6]); // s:😀 + assert_eq!(got[4], [0x86bf_c964, 0x0f9f_e2d2]); // s:a\0b + } + + #[test] + fn bytes_bool_and_integer_tokens_match_python() { + let byte_rows = [ + b"\0\0\0".as_slice(), + b"a\0\0".as_slice(), + b"a\0b".as_slice(), + ] + .concat(); + let byte_keys = encode(&byte_rows, 3, 3, KIND_BYTES, false); + assert_eq!(byte_keys[0], [0x62f6_fcd3, 0xab0a_595f]); // y: + assert_eq!(byte_keys[1], [0x5b3b_753b, 0xe137_9b39]); // y:a + assert_eq!(byte_keys[2], [0xb1a7_ff88, 0x9d29_6cba]); // y:a\0b + + let bool_keys = encode(&[0, 1], 2, 1, KIND_BOOL, false); + assert_eq!(bool_keys[0], [0xd46a_5a74, 0x9fa0_77d4]); + assert_eq!(bool_keys[1], [0x685f_68b3, 0x242e_6cc2]); + + let signed = [i64::MIN, -1, 0, 127]; + let signed_bytes = signed + .iter() + .flat_map(|value| value.to_ne_bytes()) + .collect::>(); + let signed_keys = encode(&signed_bytes, signed.len(), 8, KIND_SIGNED, false); + assert_eq!(signed_keys[0], [0x2846_782b, 0xcf4f_8ef5]); + assert_eq!(signed_keys[1], [0x74f3_9158, 0x258f_5aee]); + assert_eq!(signed_keys[2], [0x6a0d_2733, 0xd2c8_7bbd]); + assert_eq!(signed_keys[3], [0x6673_71eb, 0xc533_1ccd]); + + let unsigned = u64::MAX.to_ne_bytes(); + assert_eq!( + encode(&unsigned, 1, 8, KIND_UNSIGNED, false)[0], + [0x1f5e_d8a5, 0xf8e0_8beb] + ); + } + + #[test] + fn byte_swapped_scalars_match_native_values() { + let values = [i32::MIN, -17, 2048, i32::MAX]; + let native = values + .iter() + .flat_map(|value| value.to_ne_bytes()) + .collect::>(); + let swapped = values + .iter() + .flat_map(|value| { + let mut bytes = value.to_ne_bytes(); + bytes.reverse(); + bytes + }) + .collect::>(); + assert_eq!( + encode(&native, values.len(), 4, KIND_SIGNED, false), + encode(&swapped, values.len(), 4, KIND_SIGNED, true) + ); + } + + #[test] + fn float_tokens_match_python_hex_and_reject_nonfinite() { + let values = [0.0f64, -0.0, 1.0, 0.5, f64::from_bits(1), f64::MAX]; + let bytes = values + .iter() + .flat_map(|value| value.to_ne_bytes()) + .collect::>(); + let got = encode(&bytes, values.len(), 8, KIND_FLOAT64, false); + let expected_tokens: [&[u8]; 6] = [ + b"f:0x0.0p+0", + b"f:-0x0.0p+0", + b"f:0x1.0000000000000p+0", + b"f:0x1.0000000000000p-1", + b"f:0x0.0000000000001p-1022", + b"f:0x1.fffffffffffffp+1023", + ]; + for (actual, token) in got.iter().zip(expected_tokens) { + let digest = digest_parts(&[token]); + assert_eq!( + *actual, + [ + u32::from_le_bytes(digest[..4].try_into().unwrap()), + u32::from_le_bytes(digest[4..].try_into().unwrap()), + ] + ); + } + let mut low = [0]; + let mut high = [0]; + assert_eq!( + encode_fixed_into( + &f64::NAN.to_ne_bytes(), + 8, + KIND_FLOAT64, + false, + &mut low, + &mut high, + ), + Err(TransitionKeyError::Invalid { index: Some(0) }) + ); + } + + #[test] + fn duplicate_reports_first_and_current_rows() { + let data = [7i16, -2, 7] + .iter() + .flat_map(|value| value.to_ne_bytes()) + .collect::>(); + let mut low = [0; 3]; + let mut high = [0; 3]; + assert_eq!( + encode_fixed_into(&data, 2, KIND_SIGNED, false, &mut low, &mut high), + Err(TransitionKeyError::Duplicate { first: 0, index: 2 }) + ); + + let padded = [b"a\0".as_slice(), b"a\0".as_slice()].concat(); + assert_eq!( + encode_fixed_into(&padded, 2, KIND_BYTES, false, &mut low[..2], &mut high[..2]), + Err(TransitionKeyError::Duplicate { first: 0, index: 1 }) + ); + } + + #[test] + fn invalid_layouts_and_scalar_data_are_rejected() { + let mut low = [0]; + let mut high = [0]; + assert!(matches!( + encode_fixed_into(&[0; 3], 3, KIND_UNICODE, false, &mut low, &mut high), + Err(TransitionKeyError::Invalid { index: None }) + )); + assert!(matches!( + encode_fixed_into(&[2], 1, KIND_BOOL, false, &mut low, &mut high), + Err(TransitionKeyError::Invalid { index: Some(0) }) + )); + let surrogate = 0xd800u32.to_ne_bytes(); + assert!(matches!( + encode_fixed_into(&surrogate, 4, KIND_UNICODE, false, &mut low, &mut high), + Err(TransitionKeyError::Invalid { index: Some(0) }) + )); + } +} diff --git a/tests/test_animation.py b/tests/test_animation.py index 1bbd2355..8484b098 100644 --- a/tests/test_animation.py +++ b/tests/test_animation.py @@ -2,12 +2,16 @@ from __future__ import annotations +import datetime as dt +import hashlib import json import numpy as np import pytest import xy +import xy.components as component_api +from xy import kernels as k def _column(blob: bytes, spec: dict, index: int) -> np.ndarray: @@ -16,6 +20,34 @@ def _column(blob: bytes, spec: dict, index: int) -> np.ndarray: return np.frombuffer(blob, dtype=dtype, count=meta["len"], offset=meta["byte_offset"]) +def _python_transition_key_reference(values) -> np.ndarray: + """The scalar encoder retained as the policy/fallback oracle.""" + arr = np.asarray(values, dtype=object) + result = np.empty((len(arr), 2), dtype=np.uint32, order="F") + seen: dict[bytes, int] = {} + digests: dict[bytes, bytes] = {} + for index, raw in enumerate(arr): + token = component_api._transition_key_token(raw, index) + previous = seen.get(token) + if previous is not None: + raise ValueError( + f"reference key contains duplicate value at rows {previous} and {index}" + ) + seen[token] = index + digest = hashlib.blake2s(token, digest_size=8, person=b"xykeyv1").digest() + collision = digests.get(digest) + if collision is not None and collision != token: + raise ValueError("reference key produced an identity digest collision") + digests[digest] = token + result[index, 0] = int.from_bytes(digest[:4], "little") + result[index, 1] = int.from_bytes(digest[4:], "little") + return result + + +def _swapped(values: np.ndarray) -> np.ndarray: + return values.astype(values.dtype.newbyteorder("S")) + + def test_animation_component_serializes_without_callbacks() -> None: started = lambda event: event # noqa: E731 ended = lambda event: event # noqa: E731 @@ -176,6 +208,187 @@ def test_stable_keys_are_type_sensitive_and_deterministic() -> None: assert first_blob == second_blob +@pytest.mark.parametrize( + "keys", + [ + pytest.param(["", "café", "猫"], id="list-unicode"), + pytest.param([b"", b"ascii", b"\xff"], id="list-bytes"), + # The two-byte ``s:``/``y:`` prefix makes these token lengths + # 62, 63, 64, 65, and 129, spanning BLAKE2s full/final blocks. + pytest.param( + ["a" * size for size in (60, 61, 62, 63, 127)], + id="list-unicode-blake2-block-boundaries", + ), + pytest.param( + [b"a" * size for size in (60, 61, 62, 63, 127)], + id="list-bytes-blake2-block-boundaries", + ), + pytest.param([False, True], id="list-bool"), + pytest.param([-7, 0, 2**40], id="list-int"), + pytest.param([0.0, -0.0, np.nextafter(0.0, 1.0), 1.5], id="list-float"), + pytest.param(np.array(["", "café", "猫"], dtype="U4"), id="numpy-unicode"), + pytest.param( + np.array(["a\x00b", "plain"], dtype="U5"), + id="numpy-unicode-embedded-nul", + ), + pytest.param( + _swapped(np.array(["", "β", "猫"], dtype="U2")), + id="numpy-unicode-swapped", + ), + pytest.param(np.array([b"", b"abc", b"\xff"], dtype="S3"), id="numpy-bytes"), + pytest.param( + np.array([b"a\x00b", b"plain"], dtype="S5"), + id="numpy-bytes-embedded-nul", + ), + pytest.param(np.array([False, True], dtype=np.bool_), id="numpy-bool"), + pytest.param(np.array([-128, 0, 127], dtype=np.int8), id="numpy-int8"), + pytest.param(np.array([-32768, 0, 32767], dtype=np.int16), id="numpy-int16"), + pytest.param( + _swapped(np.array([-(2**31), 0, 2**31 - 1], dtype=np.int32)), + id="numpy-int32-swapped", + ), + pytest.param( + np.array([-(2**63), 0, 2**63 - 1], dtype=np.int64), + id="numpy-int64", + ), + pytest.param(np.array([0, 1, 255], dtype=np.uint8), id="numpy-uint8"), + pytest.param(np.array([0, 1, 65535], dtype=np.uint16), id="numpy-uint16"), + pytest.param( + _swapped(np.array([0, 1, 2**32 - 1], dtype=np.uint32)), + id="numpy-uint32-swapped", + ), + pytest.param( + np.array([0, 1, 2**64 - 1], dtype=np.uint64), + id="numpy-uint64", + ), + pytest.param( + np.array([0.0, -0.0, 0.5], dtype=np.float16), + id="numpy-float16", + ), + pytest.param( + np.array([0.0, -0.0, 1.25], dtype=np.float32), + id="numpy-float32", + ), + pytest.param( + _swapped( + np.array( + [0.0, -0.0, np.nextafter(0.0, 1.0), -2.25], + dtype=np.float64, + ) + ), + id="numpy-float64-swapped-subnormal", + ), + ], +) +def test_native_transition_key_fast_paths_match_python_reference(keys, monkeypatch) -> None: + calls: list[np.ndarray] = [] + native = k.transition_keys_fixed + + def tracked(values: np.ndarray, label: str): + calls.append(values) + return native(values, label) + + monkeypatch.setattr(k, "transition_keys_fixed", tracked) + actual = component_api._encode_transition_keys(keys, len(keys), "parity key") + expected = _python_transition_key_reference(keys) + + assert len(calls) == 1 + np.testing.assert_array_equal(actual, expected) + assert actual.dtype == np.uint32 + assert actual.shape == (len(keys), 2) + assert actual[:, 0].flags.c_contiguous + assert actual[:, 1].flags.c_contiguous + + +@pytest.mark.parametrize( + "keys", + [ + pytest.param([1, "1", True, b"1"], id="mixed-builtins"), + pytest.param(np.array(["a", "b"], dtype=object), id="object-array"), + pytest.param(np.array([1.25, 2.5], dtype=object), id="object-floats"), + pytest.param(["a\x00b", "plain"], id="list-unicode-nul"), + pytest.param([b"a\x00b", b"plain"], id="list-bytes-nul"), + pytest.param( + [dt.date(2024, 1, 1), dt.date(2024, 1, 2)], + id="dates", + ), + pytest.param( + [ + dt.datetime(2024, 1, 1, 12, 30), + dt.datetime(2024, 1, 1, 12, 31), + ], + id="datetimes", + ), + pytest.param([2**100, 2**100 + 1], id="wide-python-ints"), + ], +) +def test_transition_key_object_policy_uses_python_reference(keys, monkeypatch) -> None: + def unexpected_native(_values: np.ndarray, _label: str): + raise AssertionError("object-policy key values must retain the Python oracle") + + monkeypatch.setattr(k, "transition_keys_fixed", unexpected_native) + actual = component_api._encode_transition_keys(keys, len(keys), "fallback key") + np.testing.assert_array_equal(actual, _python_transition_key_reference(keys)) + + +@pytest.mark.parametrize( + "keys", + [ + [f"k{index}" for index in range(16)] + ["z" * 256], + [f"k{index}".encode() for index in range(16)] + [b"z" * 256], + ], +) +def test_skewed_sequence_keys_avoid_fixed_width_memory_amplification(keys, monkeypatch) -> None: + def unexpected_native(_values: np.ndarray, _label: str): + raise AssertionError("skewed sequence keys must retain the Python oracle") + + monkeypatch.setattr(k, "transition_keys_fixed", unexpected_native) + actual = component_api._encode_transition_keys(keys, len(keys), "skewed key") + np.testing.assert_array_equal(actual, _python_transition_key_reference(keys)) + + +@pytest.mark.parametrize( + ("keys", "row"), + [ + (np.array([1.0, np.nan], dtype=np.float64), 1), + (np.array([np.inf, 1.0], dtype=np.float32), 0), + ([-np.inf, 1.0], 0), + ], +) +def test_nonfinite_native_float_keys_retain_exact_python_row_error(keys, row) -> None: + with pytest.raises(ValueError, match=rf"animation key must be finite at row {row}"): + component_api._encode_transition_keys(keys, len(keys), "finite key") + + +@pytest.mark.parametrize( + "keys", + [ + ["a", "b", "a", "b"], + np.array([b"a", b"b", b"a", b"b"], dtype="S1"), + np.array([7, 9, 7, 9], dtype=">i4"), + np.array([1.5, 2.5, 1.5, 2.5], dtype=np.float64), + ], +) +def test_native_transition_key_duplicates_report_first_rows(keys) -> None: + with pytest.raises( + ValueError, + match=r"duplicate key contains duplicate value at rows 0 and 2", + ): + component_api._encode_transition_keys(keys, len(keys), "duplicate key") + + +def test_transition_key_empty_shape_and_length_semantics() -> None: + for keys in ([], np.array([], dtype="U1")): + encoded = component_api._encode_transition_keys(keys, 0, "empty key") + assert encoded.shape == (0, 2) + assert encoded.dtype == np.uint32 + + with pytest.raises(ValueError, match="shape key must be one-dimensional"): + component_api._encode_transition_keys([["a"], ["b"]], 2, "shape key") + with pytest.raises(ValueError, match="length key must have length 3, got 2"): + component_api._encode_transition_keys(["a", "b"], 3, "length key") + + def test_aggregate_tier_records_key_matching_fallback() -> None: chart = xy.scatter_chart( xy.scatter(