Move animation key encoding to Rust

CHANGELOG.md

@@ -44,6 +44,17 @@ in the README).
   that don't use them are byte-identical.
 
 ### Changed
+- Stable animation `key=` identity encoding now uses one native Rust row scan
+  for homogeneous fixed-width strings, bytes, booleans, and signed or unsigned
+  integers, plus finite floating arrays, including non-native-endian NumPy
+  arrays. Float16/32 values widen exactly to the f64 token contract. It
+  preserves the existing 64-bit identities and duplicate-row errors; mixed
+  objects, NUL-containing Python sequences, dates, and non-finite row
+  diagnostics stay on the conservative Python reference path. Highly padded
+  Python string/bytes sequences also stay there to bound fixed-width temporary
+  memory. Fixed-width NumPy strings retain their exact embedded-NUL semantics
+  natively.
+  This adds the C ABI v43 transition-key kernel.
 - Host theme changes made through an ancestor `data-theme` attribute now
   refresh canvas/SVG paint just like `.dark` class and inline-style changes;
   DOM chrome continues to follow the cascade automatically.

benchmarks/test_codspeed_animation.py

@@ -70,10 +70,13 @@ def payload_figures(animation_data):
 
 
 def test_animation_encode_100k_stable_keys(benchmark, animation_data) -> None:
+    """Track the public list-to-native path, including homogeneous routing."""
     _x, _y, keys = animation_data
     encoded = benchmark(_encode_transition_keys, keys, N, "benchmark key")
     assert encoded.shape == (N, 2)
     assert encoded.dtype == np.uint32
+    assert encoded[:, 0].flags.c_contiguous
+    assert encoded[:, 1].flags.c_contiguous
 
 
 def test_animation_plain_payload_100k(benchmark, payload_figures) -> None:

python/xy/_native.py

@@ -24,7 +24,7 @@
 
 from .config import MAX_CONTOUR_WORK, MAX_SCREEN_DIM
 
-ABI_VERSION = 42
+ABI_VERSION = 43
 
 # Rust reports invalid arguments (and, via the ffi_guard panic shield, any
 # internal panic) by returning `usize::MAX` from size-returning entry points.
@@ -117,6 +117,18 @@ def _load() -> ctypes.CDLL:
         ctypes.c_void_p,
         ctypes.c_size_t,
     ]
+    lib.xy_transition_keys_fixed.restype = ctypes.c_int32
+    lib.xy_transition_keys_fixed.argtypes = [
+        ctypes.c_void_p,
+        ctypes.c_size_t,
+        ctypes.c_size_t,
+        ctypes.c_uint32,
+        ctypes.c_int32,
+        ctypes.c_void_p,
+        ctypes.c_void_p,
+        ctypes.POINTER(ctypes.c_size_t),
+        ctypes.POINTER(ctypes.c_size_t),
+    ]
     lib.xy_remap_u8.restype = ctypes.c_int32
     lib.xy_remap_u8.argtypes = [
         ctypes.c_void_p,
@@ -953,6 +965,84 @@ def factorize_unicode1_u8_counts(
     return codes, unique_indices[:written].copy(), counts[:written].copy()
 
 
+def transition_keys_fixed(
+    values: np.ndarray, label: str = "animation key"
+) -> Optional[npt.NDArray[np.uint32]]:
+    """Encode homogeneous fixed-width transition keys in one native row scan.
+
+    The returned ``(N, 2)`` array is Fortran-contiguous: its two ``u32``
+    columns are the caller-allocated lo/hi planes written by Rust and can be
+    shipped independently without another full-size copy. ``None`` asks the
+    policy layer to use its scalar oracle for an invalid value (for example a
+    non-finite float or invalid Unicode scalar), preserving its precise
+    user-facing error.
+    """
+    records = np.asarray(values)
+    if records.ndim != 1 or records.dtype.hasobject:
+        raise ValueError("transition key values must be a non-object 1-D array")
+
+    kind_name = records.dtype.kind
+    width = int(records.dtype.itemsize)
+    if kind_name == "U" and width > 0 and width % 4 == 0:
+        kind = 0
+    elif kind_name == "S" and width > 0:
+        kind = 1
+    elif kind_name == "b" and width == 1:
+        kind = 2
+    elif kind_name == "i" and width in (1, 2, 4, 8):
+        kind = 3
+    elif kind_name == "u" and width in (1, 2, 4, 8):
+        kind = 4
+    elif kind_name == "f" and width in (2, 4, 8):
+        # Python canonicalizes NumPy float16/32 scalars through ``.item()``;
+        # widening to f64 is exact and gives the native kernel the same value.
+        if width != 8:
+            records = records.astype(np.float64)
+            width = 8
+        kind = 5
+    else:
+        raise ValueError(
+            "transition key values must use Unicode, bytes, bool, integer, or float dtype"
+        )
+
+    records = np.ascontiguousarray(records)
+    n = len(records)
+    result = np.empty((n, 2), dtype=np.uint32, order="F")
+    if n == 0:
+        return result
+
+    native_order = "<" if sys.byteorder == "little" else ">"
+    swap_endian = records.dtype.byteorder not in ("=", "|", native_order)
+    error_first = ctypes.c_size_t(_USIZE_MAX)
+    error_index = ctypes.c_size_t(_USIZE_MAX)
+    status = int(
+        _lib.xy_transition_keys_fixed(
+            records.ctypes.data,
+            n,
+            width,
+            kind,
+            int(swap_endian),
+            result[:, 0].ctypes.data,
+            result[:, 1].ctypes.data,
+            ctypes.byref(error_first),
+            ctypes.byref(error_index),
+        )
+    )
+    if status == 0:
+        return result
+    if status == 1:
+        return None
+    if status == 2:
+        if error_first.value >= n or error_index.value >= n:
+            raise RuntimeError("native transition-key encoder returned invalid row indices")
+        raise ValueError(
+            f"{label} contains duplicate value at rows {error_first.value} and {error_index.value}"
+        )
+    if status == 3:
+        raise ValueError(f"{label} produced an identity digest collision")
+    raise RuntimeError(f"native transition-key encoder returned unknown status {status}")
+
+
 def remap_u8(values: npt.NDArray[np.uint8], mapping: npt.NDArray[np.uint8]) -> None:
     """Apply a compact categorical codebook permutation in place."""
     values = np.asarray(values)

python/xy/components.py

@@ -3850,13 +3850,97 @@ def _transition_key_token(value: Any, index: int) -> bytes:
     )
 
 
+_TRANSITION_KEY_SEQUENCE_MAX_FIXED_BYTES = 256 * 1024 * 1024
+_TRANSITION_KEY_SEQUENCE_MAX_PADDING_RATIO = 8
+
+
+def _fixed_transition_key_values(value: Any) -> np.ndarray | None:
+    """Return a conservative homogeneous array for the native key encoder."""
+    if type(value) is np.ndarray:
+        arr = value
+    elif isinstance(value, (list, tuple)) and value:
+        first = value[0].item() if isinstance(value[0], np.generic) else value[0]
+        item_type = type(first)
+        if item_type not in (str, bytes, bool, int, float):
+            return None
+        total_units = 0
+        max_units = 0
+        for raw in value:
+            item = raw.item() if isinstance(raw, np.generic) else raw
+            if type(item) is not item_type:
+                # In particular, never coerce mixed bool/int or int/float
+                # keys: their scalar tokens are deliberately type-sensitive.
+                return None
+            if (item_type is str and "\x00" in item) or (item_type is bytes and b"\x00" in item):
+                # Fixed-width storage cannot distinguish padding from an
+                # explicitly trailing NUL supplied in a Python scalar.
+                return None
+            if item_type in (str, bytes):
+                units = len(item)
+                total_units += units
+                max_units = max(max_units, units)
+        if item_type in (str, bytes):
+            unit_bytes = 4 if item_type is str else 1
+            # NumPy's fixed-width sequence conversion costs N × max width.
+            # Keep a single long outlier from turning a compact Python list
+            # into a huge temporary before Rust can scan it.
+            padded_bytes = len(value) * max(max_units, 1) * unit_bytes
+            source_bytes = max(total_units * unit_bytes, 1)
+            if (
+                padded_bytes > _TRANSITION_KEY_SEQUENCE_MAX_FIXED_BYTES
+                or padded_bytes > source_bytes * _TRANSITION_KEY_SEQUENCE_MAX_PADDING_RATIO
+            ):
+                return None
+        try:
+            arr = np.asarray(value)
+        except (OverflowError, TypeError, ValueError):
+            return None
+        expected_kinds = {
+            str: {"U"},
+            bytes: {"S"},
+            bool: {"b"},
+            int: {"i", "u"},
+            float: {"f"},
+        }
+        if arr.dtype.kind not in expected_kinds[item_type]:
+            # NumPy may coerce an integer list outside its fixed-width range
+            # to f64. That would change the type-sensitive ``i:`` token into
+            # an ``f:`` token, so leave arbitrary-width integers to Python.
+            return None
+    else:
+        return None
+    if arr.ndim != 1:
+        return None
+    if arr.dtype.kind == "U" and arr.dtype.itemsize > 0 and arr.dtype.itemsize % 4 == 0:
+        return arr
+    if arr.dtype.kind == "S" and arr.dtype.itemsize > 0:
+        return arr
+    if arr.dtype.kind == "b" and arr.dtype.itemsize == 1:
+        return arr
+    if arr.dtype.kind in {"i", "u"} and arr.dtype.itemsize in (1, 2, 4, 8):
+        return arr
+    if arr.dtype.kind == "f" and arr.dtype.itemsize in (2, 4, 8):
+        return arr
+    return None
+
+
 def _encode_transition_keys(value: Any, expected: int, label: str) -> np.ndarray:
+    fixed = _fixed_transition_key_values(value)
+    if fixed is not None:
+        if len(fixed) != expected:
+            raise ValueError(f"{label} must have length {expected}, got {len(fixed)}")
+        from . import kernels
+
+        encoded = kernels.transition_keys_fixed(fixed, label)
+        if encoded is not None:
+            return encoded
+
     arr = np.asarray(value, dtype=object)
     if arr.ndim != 1:
         raise ValueError(f"{label} must be one-dimensional")
     if len(arr) != expected:
         raise ValueError(f"{label} must have length {expected}, got {len(arr)}")
-    result = np.empty((expected, 2), dtype=np.uint32)
+    result = np.empty((expected, 2), dtype=np.uint32, order="F")
     seen: dict[bytes, int] = {}
     digests: dict[bytes, bytes] = {}
     for index, raw in enumerate(arr):

python/xy/kernels.py

@@ -43,6 +43,7 @@
 factorize_fixed_u8 = _impl.factorize_fixed_u8
 factorize_fixed_u8_counts = _impl.factorize_fixed_u8_counts
 factorize_unicode1_u8_counts = _impl.factorize_unicode1_u8_counts
+transition_keys_fixed = _impl.transition_keys_fixed
 m4_indices = _impl.m4_indices
 marching_squares = _impl.marching_squares
 marching_triangles = _impl.marching_triangles
@@ -145,6 +146,7 @@
     "stratified_sample_mask",
     "stratified_sample_range_u8",
     "streamlines",
+    "transition_keys_fixed",
     "triangle_edges",
     "valid_indices_f64",
     "vector_segments",

scripts/abi_smoke.py

@@ -104,6 +104,18 @@ def load() -> ctypes.CDLL:
         U64P,
         ctypes.c_size_t,
     ]
+    lib.xy_transition_keys_fixed.restype = ctypes.c_int32
+    lib.xy_transition_keys_fixed.argtypes = [
+        U8P,
+        ctypes.c_size_t,
+        ctypes.c_size_t,
+        ctypes.c_uint32,
+        ctypes.c_int32,
+        U32P,
+        U32P,
+        ctypes.POINTER(ctypes.c_size_t),
+        ctypes.POINTER(ctypes.c_size_t),
+    ]
     lib.xy_remap_u8.restype = ctypes.c_int32
     lib.xy_remap_u8.argtypes = [U8P, ctypes.c_size_t, U8P, ctypes.c_size_t]
     lib.xy_encode_f32.restype = ctypes.c_int32
@@ -489,6 +501,76 @@ def ok(cond: bool, msg: str) -> None:
         swapped_count == 4 and list(unicode_codes) == [0, 1, 0, 2, 3],
         "factorize_unicode1_u8_counts swapped endian",
     )
+    transition_low = array("I", [99, 99])
+    transition_high = array("I", [99, 99])
+    transition_first = ctypes.c_size_t(size_max)
+    transition_index = ctypes.c_size_t(size_max)
+    transition_records = array("B", b"a\0\0a\0b")
+    status = lib.xy_transition_keys_fixed(
+        _ptr(transition_records, ctypes.c_uint8),
+        2,
+        3,
+        1,
+        0,
+        _ptr(transition_low, ctypes.c_uint32),
+        _ptr(transition_high, ctypes.c_uint32),
+        ctypes.byref(transition_first),
+        ctypes.byref(transition_index),
+    )
+    ok(status == 0, "transition_keys_fixed success status")
+    ok(
+        list(transition_low) == [0x5B3B753B, 0xB1A7FF88]
+        and list(transition_high) == [0xE1379B39, 0x9D296CBA],
+        "transition_keys_fixed personalized digest words",
+    )
+    duplicate_records = array("B", b"a\0\0b\0\0a\0\0")
+    duplicate_low = array("I", [99, 99, 99])
+    duplicate_high = array("I", [99, 99, 99])
+    status = lib.xy_transition_keys_fixed(
+        _ptr(duplicate_records, ctypes.c_uint8),
+        3,
+        3,
+        1,
+        0,
+        _ptr(duplicate_low, ctypes.c_uint32),
+        _ptr(duplicate_high, ctypes.c_uint32),
+        ctypes.byref(transition_first),
+        ctypes.byref(transition_index),
+    )
+    ok(
+        status == 2 and transition_first.value == 0 and transition_index.value == 2,
+        "transition_keys_fixed duplicate rows",
+    )
+    ok(
+        lib.xy_transition_keys_fixed(
+            null_u8,
+            0,
+            1,
+            1,
+            0,
+            null_u32,
+            null_u32,
+            ctypes.byref(transition_first),
+            ctypes.byref(transition_index),
+        )
+        == 0,
+        "transition_keys_fixed empty/null succeeds",
+    )
+    ok(
+        lib.xy_transition_keys_fixed(
+            null_u8,
+            1,
+            1,
+            1,
+            0,
+            null_u32,
+            null_u32,
+            ctypes.byref(transition_first),
+            ctypes.byref(transition_index),
+        )
+        == 1,
+        "transition_keys_fixed non-empty/null rejected",
+    )
     small_unique = array("I", [99] * 2)
     ok(
         lib.xy_factorize_fixed_u8(