From 15eb4907a0fab3d281b3e824b4332e59eb0ab93b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 29 Jul 2026 23:23:31 +0200 Subject: [PATCH] re-add XXH64 to read borg 1.x integrity data, #9935 XXH64 (and the XXH64FileHashingWrapper) were removed in #9672 / #9750, which switched borg 2.x file integrity to SHA256. But borg 1.x wrote XXH64 checksums into the repo index/hints integrity data, so we need XXH64 to verify those files when reading a borg 1.x (legacy) repository during `borg transfer`. Rather than re-introducing the external "xxhash" PyPI package (and the libxxhash system dependency on msys2 that #9750 dropped), add a small, dependency-free, streaming XXH64 implementation in cython to crypto.low_level. It is only used on the read path; borg 2.x native repos keep using SHA256. XXH64 is non-cryptographic and must not be used as a security mechanism. Tests use the official xxHash sanity-check vectors (the test buffer generator is transcribed verbatim from xxHash tests/sanity_test.c) and cover all code paths, streaming vs one-shot, and a legacy XXH64 integrity round-trip through IntegrityCheckedFile. Co-Authored-By: Claude Opus 4.8 --- src/borg/crypto/file_integrity.py | 15 +- src/borg/crypto/low_level.pyi | 14 ++ src/borg/crypto/low_level.pyx | 171 ++++++++++++++++++ .../testsuite/crypto/file_integrity_test.py | 64 ++++++- src/borg/testsuite/crypto/low_level_test.py | 105 +++++++++++ 5 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 src/borg/testsuite/crypto/low_level_test.py diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py index 489b34a6fe..3d16776e0d 100644 --- a/src/borg/crypto/file_integrity.py +++ b/src/borg/crypto/file_integrity.py @@ -5,6 +5,7 @@ from collections.abc import Callable from pathlib import Path +from .low_level import XXH64 from ..helpers import IntegrityError from ..logger import create_logger @@ -110,7 +111,19 @@ class SHA256FileHashingWrapper(FileHashingWrapper): FACTORY = hashlib.sha256 -SUPPORTED_ALGORITHMS = {SHA256FileHashingWrapper.ALGORITHM: SHA256FileHashingWrapper} +class XXH64FileHashingWrapper(FileHashingWrapper): + # This only exists to support `borg transfer` from borg 1.x repos, see #9935: + # borg 1.x wrote XXH64 integrity data for the repo index/hints files, so we need + # XXH64 to verify those when reading a borg 1.x (legacy) repository. borg 2.x writes + # SHA256 (see SHA256FileHashingWrapper), so XXH64 is only ever used for reading here. + ALGORITHM = "XXH64" + FACTORY = XXH64 + + +SUPPORTED_ALGORITHMS = { + SHA256FileHashingWrapper.ALGORITHM: SHA256FileHashingWrapper, + XXH64FileHashingWrapper.ALGORITHM: XXH64FileHashingWrapper, +} class FileIntegrityError(IntegrityError): diff --git a/src/borg/crypto/low_level.pyi b/src/borg/crypto/low_level.pyi index 9d5e4c0566..4ac4448aa7 100644 --- a/src/borg/crypto/low_level.pyi +++ b/src/borg/crypto/low_level.pyi @@ -13,6 +13,20 @@ def hmac_sha256(key: bytes, data: bytes) -> bytes: ... def blake2b_256(key: bytes, data: bytes) -> bytes: ... def blake2b_128(data: bytes) -> bytes: ... +# XXH64: only used to read borg 1.x repo integrity data on `borg transfer`, see #9935. +def xxh64(data: bytes, seed: int = 0) -> bytes: ... + +class XXH64: + """Streaming XXH64 hasher (non-cryptographic), compatible with the "xxhash" package's xxh64. + + Only exists to read borg 1.x repo integrity data on `borg transfer`, see #9935. + """ + + def __init__(self, data: bytes = b"", seed: int = 0) -> None: ... + def update(self, data: bytes) -> None: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + # Exception classes class CryptoError(Exception): """Malfunction in the crypto module.""" diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 149ea1d738..7c0e659f0e 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -825,3 +825,174 @@ cdef class CSPRNG: # Swap items[i] and items[j] items[i], items[j] = items[j], items[i] + + +# XXH64: a pure-cython, dependency-free implementation of the (non-cryptographic) xxHash-64 hash. +# +# This only exists to support `borg transfer` from borg 1.x repos, see #9935: +# borg 1.x stored XXH64 checksums in the repository's index/hints integrity data, so we need +# XXH64 to verify those files when opening a borg 1.x (legacy) repository. It is not used for +# anything in borg 2.x native repos and MUST NOT be used as a security mechanism. +# +# Reference: https://github.com/Cyan4973/xxHash (algorithm is in the public domain / BSD-2-Clause). +# The digest uses the canonical (big-endian) representation, matching what borg 1.x wrote +# (it used the "xxhash" PyPI package, whose digest()/hexdigest() are big-endian). + +cdef extern from *: + """ + #include + /* xxHash-64 prime constants, defined as real uint64_t so there is no ambiguity + about the width/signedness of these (large) literals. */ + static const uint64_t BORG_XXH64_P1 = 0x9E3779B185EBCA87ULL; + static const uint64_t BORG_XXH64_P2 = 0xC2B2AE3D27D4EB4FULL; + static const uint64_t BORG_XXH64_P3 = 0x165667B19E3779F9ULL; + static const uint64_t BORG_XXH64_P4 = 0x85EBCA77C2B2AE63ULL; + static const uint64_t BORG_XXH64_P5 = 0x27D4EB2F165667C5ULL; + """ + const uint64_t BORG_XXH64_P1 + const uint64_t BORG_XXH64_P2 + const uint64_t BORG_XXH64_P3 + const uint64_t BORG_XXH64_P4 + const uint64_t BORG_XXH64_P5 + + +cdef inline uint64_t _xxh_rotl(uint64_t x, int r) noexcept: + return (x << r) | (x >> (64 - r)) + + +cdef inline uint64_t _xxh_round(uint64_t acc, uint64_t inp) noexcept: + acc += inp * BORG_XXH64_P2 + acc = _xxh_rotl(acc, 31) + acc *= BORG_XXH64_P1 + return acc + + +cdef inline uint64_t _xxh_merge(uint64_t acc, uint64_t val) noexcept: + acc ^= _xxh_round(0, val) + acc = acc * BORG_XXH64_P1 + BORG_XXH64_P4 + return acc + + +# read 64/32 bits little-endian, byte-wise, so this is correct on both little- and big-endian hosts. +cdef inline uint64_t _xxh_read64(const uint8_t *p) noexcept: + return (p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24) | + (p[4] << 32) | (p[5] << 40) | (p[6] << 48) | (p[7] << 56)) + + +cdef inline uint32_t _xxh_read32(const uint8_t *p) noexcept: + return (p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24)) + + +cdef class XXH64: + """ + Streaming XXH64 hasher with an interface compatible with the "xxhash" PyPI package's xxh64 + (as used by borg 1.x): XXH64([data], [seed]) then .update(data) and .digest()/.hexdigest(). + + See the comment above: this exists only to support `borg transfer` from borg 1.x repos, #9935. + """ + cdef uint64_t v1, v2, v3, v4 + cdef uint64_t total_len + cdef uint64_t seed + cdef uint8_t mem[32] + cdef unsigned int memsize + + def __init__(self, data=b"", seed=0): + # coerce seed into a C uint64_t first, so the setup arithmetic below wraps in C + # (mod 2**64) instead of overflowing as an unbounded python int. + cdef uint64_t s = seed + self.seed = s + self.v1 = s + BORG_XXH64_P1 + BORG_XXH64_P2 + self.v2 = s + BORG_XXH64_P2 + self.v3 = s + self.v4 = s - BORG_XXH64_P1 + self.total_len = 0 + self.memsize = 0 + if data: + self.update(data) + + def update(self, data): + cdef Py_buffer view + PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) + try: + self._update( view.buf, view.len) + finally: + PyBuffer_Release(&view) + + cdef void _update(self, const uint8_t *p, Py_ssize_t length) noexcept: + cdef const uint8_t *end = p + length + cdef const uint8_t *limit + cdef unsigned int fill + self.total_len += length + if self.memsize + length < 32: + # not enough (even together with buffered data) for a full 32-byte stripe: just buffer it. + memcpy(&self.mem[self.memsize], p, length) + self.memsize += length + return + if self.memsize > 0: + # complete and process the buffered partial stripe first. + fill = 32 - self.memsize + memcpy(&self.mem[self.memsize], p, fill) + self.v1 = _xxh_round(self.v1, _xxh_read64(&self.mem[0])) + self.v2 = _xxh_round(self.v2, _xxh_read64(&self.mem[8])) + self.v3 = _xxh_round(self.v3, _xxh_read64(&self.mem[16])) + self.v4 = _xxh_round(self.v4, _xxh_read64(&self.mem[24])) + p += fill + self.memsize = 0 + # process full 32-byte stripes directly from the input. + limit = end - 32 + while p <= limit: + self.v1 = _xxh_round(self.v1, _xxh_read64(p)); p += 8 + self.v2 = _xxh_round(self.v2, _xxh_read64(p)); p += 8 + self.v3 = _xxh_round(self.v3, _xxh_read64(p)); p += 8 + self.v4 = _xxh_round(self.v4, _xxh_read64(p)); p += 8 + # buffer the remaining (< 32) bytes for the next update()/digest(). + if p < end: + memcpy(&self.mem[0], p, end - p) + self.memsize = (end - p) + + def digest(self): + """Return the digest as 8 bytes, in canonical (big-endian) representation.""" + cdef uint64_t h + cdef const uint8_t *p = &self.mem[0] + cdef const uint8_t *end = &self.mem[self.memsize] + cdef uint8_t out[8] + cdef int i + if self.total_len >= 32: + h = _xxh_rotl(self.v1, 1) + _xxh_rotl(self.v2, 7) + _xxh_rotl(self.v3, 12) + _xxh_rotl(self.v4, 18) + h = _xxh_merge(h, self.v1) + h = _xxh_merge(h, self.v2) + h = _xxh_merge(h, self.v3) + h = _xxh_merge(h, self.v4) + else: + h = self.seed + BORG_XXH64_P5 + h += self.total_len + while p + 8 <= end: + h ^= _xxh_round(0, _xxh_read64(p)) + h = _xxh_rotl(h, 27) * BORG_XXH64_P1 + BORG_XXH64_P4 + p += 8 + if p + 4 <= end: + h ^= _xxh_read32(p) * BORG_XXH64_P1 + h = _xxh_rotl(h, 23) * BORG_XXH64_P2 + BORG_XXH64_P3 + p += 4 + while p < end: + h ^= p[0] * BORG_XXH64_P5 + h = _xxh_rotl(h, 11) * BORG_XXH64_P1 + p += 1 + # final avalanche + h ^= h >> 33 + h *= BORG_XXH64_P2 + h ^= h >> 29 + h *= BORG_XXH64_P3 + h ^= h >> 32 + for i in range(8): + out[i] = (h >> (56 - 8 * i)) # big-endian + return PyBytes_FromStringAndSize( out, 8) + + def hexdigest(self): + """Return the digest as a 16-character hex string (canonical, big-endian).""" + return self.digest().hex() + + +def xxh64(data, seed=0): + """One-shot XXH64: return the 8-byte canonical (big-endian) digest of *data*.""" + return XXH64(data, seed).digest() diff --git a/src/borg/testsuite/crypto/file_integrity_test.py b/src/borg/testsuite/crypto/file_integrity_test.py index 5db5569004..13cf1feb30 100644 --- a/src/borg/testsuite/crypto/file_integrity_test.py +++ b/src/borg/testsuite/crypto/file_integrity_test.py @@ -4,7 +4,8 @@ import pytest from ...crypto.file_integrity import DetachedIntegrityCheckedFile, FileIntegrityError, IntegrityCheckedFile -from ...crypto.file_integrity import SHA256FileHashingWrapper +from ...crypto.file_integrity import SHA256FileHashingWrapper, XXH64FileHashingWrapper, SUPPORTED_ALGORITHMS +from ...crypto.low_level import XXH64 from ...platform import SyncFile @@ -177,3 +178,64 @@ def test_impure_hash_write(self): # pure_hash=False appends the file length ("11" in this case) at exit expected_hash = hashlib.sha256(data + b"11").hexdigest() assert wrapper.hexdigest() == expected_hash + + +class TestXXH64FileHashingWrapper: + # XXH64 support only exists to read borg 1.x repos during `borg transfer`, see #9935. + def test_registered(self): + assert SUPPORTED_ALGORITHMS["XXH64"] is XXH64FileHashingWrapper + assert XXH64FileHashingWrapper.ALGORITHM == "XXH64" + + def test_pure_hash_write(self): + bio = io.BytesIO() + data = b"hello world" + with XXH64FileHashingWrapper(bio, write=True, pure_hash=True) as wrapper: + wrapper.write(data) + assert bio.getvalue() == data + assert wrapper.hexdigest() == XXH64(data).hexdigest() + + def test_pure_hash_read(self): + data = b"hello world" + bio = io.BytesIO(data) + with XXH64FileHashingWrapper(bio, write=False, pure_hash=True) as wrapper: + assert wrapper.read() == data + assert wrapper.hexdigest() == XXH64(data).hexdigest() + + def test_impure_hash_write(self): + bio = io.BytesIO() + data = b"hello world" + with XXH64FileHashingWrapper(bio, write=True, pure_hash=False) as wrapper: + wrapper.write(data) + # pure_hash=False appends the file length ("11" in this case) at exit + assert wrapper.hexdigest() == XXH64(data + b"11").hexdigest() + + +class TestReadLegacyXXH64Integrity: + # simulate reading a borg 1.x file whose integrity data uses the XXH64 algorithm, #9935. + # borg 2.x writes SHA256, so to produce an XXH64 integrity file we make the writer use the + # XXH64 wrapper (as borg 1.x did), then read it back through the normal (SHA256-defaulting) + # reader, which must pick XXH64 from SUPPORTED_ALGORITHMS based on the stored algorithm. + def _write_xxh64_protected(self, tmpdir, payload, monkeypatch): + import borg.crypto.file_integrity as fi + + monkeypatch.setattr(fi, "SHA256FileHashingWrapper", XXH64FileHashingWrapper) + path = str(tmpdir.join("file")) + with IntegrityCheckedFile(path, write=True) as fd: + fd.write(payload) + integrity_data = fd.integrity_data + monkeypatch.undo() + assert '"algorithm": "XXH64"' in integrity_data + return path, integrity_data + + def test_verify_ok(self, tmpdir, monkeypatch): + path, integrity_data = self._write_xxh64_protected(tmpdir, b"borg 1.x payload", monkeypatch) + with IntegrityCheckedFile(path, write=False, integrity_data=integrity_data) as fd: + assert fd.read() == b"borg 1.x payload" + + def test_verify_detects_corruption(self, tmpdir, monkeypatch): + path, integrity_data = self._write_xxh64_protected(tmpdir, b"borg 1.x payload", monkeypatch) + with open(path, "ab") as fd: + fd.write(b" tampered") + with pytest.raises(FileIntegrityError): + with IntegrityCheckedFile(path, write=False, integrity_data=integrity_data) as fd: + fd.read() diff --git a/src/borg/testsuite/crypto/low_level_test.py b/src/borg/testsuite/crypto/low_level_test.py new file mode 100644 index 0000000000..c75618a02c --- /dev/null +++ b/src/borg/testsuite/crypto/low_level_test.py @@ -0,0 +1,105 @@ +"""Tests for borg.crypto.low_level.XXH64. + +XXH64 only exists to support ``borg transfer`` from borg 1.x repos (see #9935), where it is +needed to verify the XXH64 integrity data that borg 1.x wrote for its repo index/hints files. + +The reference vectors below are the official xxHash sanity-check vectors: the test buffer is +produced by the exact PRNG generator from the xxHash test suite +(https://github.com/Cyan4973/xxHash, tests/sanity_test.c) and the expected digests are the +canonical (big-endian) XXH64 results for that buffer, cross-checked against the reference +xxHash implementation. +""" + +import pytest + +from ...crypto.low_level import XXH64, xxh64 + +# xxHash test-buffer generator, transcribed verbatim from tests/sanity_test.c: +# XXH_U64 byteGen = PRIME32; +# for each byte: buffer[i] = (U8)(byteGen >> 56); byteGen *= PRIME64; +_PRIME32 = 2654435761 +_PRIME64 = 11400714785074694797 +_MASK64 = (1 << 64) - 1 +_SANITY_BUFFER_SIZE = 4096 + 64 + 1 # 4161 + + +def _sanity_buffer(size): + buf = bytearray(size) + byte_gen = _PRIME32 + for i in range(size): + buf[i] = (byte_gen >> 56) & 0xFF + byte_gen = (byte_gen * _PRIME64) & _MASK64 + return bytes(buf) + + +SANITY_BUFFER = _sanity_buffer(_SANITY_BUFFER_SIZE) + +# (length, seed, expected canonical hexdigest) for prefixes of SANITY_BUFFER. +# The lengths deliberately exercise all code paths: empty, the 1-byte tail loop, the 4-byte +# and 1-byte tails (<32 bytes -> seed+PRIME5 path), exactly one 32-byte stripe, multiple +# stripes plus tails, and the full multi-KiB buffer. Seeds are 0 and PRIME32 (0x9e3779b1). +XXH64_VECTORS = [ + (0, 0x00000000, "ef46db3751d8e999"), + (1, 0x00000000, "e934a84adb052768"), + (14, 0x00000000, "8282dcc4994e35c8"), + (32, 0x00000000, "18b216492bb44b70"), + (95, 0x00000000, "ff9f46bdcc644624"), + (_SANITY_BUFFER_SIZE, 0x00000000, "cd3d6df2db509a75"), + (0, 0x9E3779B1, "ac75fda2929b17ef"), + (1, 0x9E3779B1, "5014607643a9b4c3"), + (14, 0x9E3779B1, "c3bd6bf63deb6df0"), + (32, 0x9E3779B1, "b3f33bdf93ade409"), + (95, 0x9E3779B1, "72e75a560ef624a3"), + (_SANITY_BUFFER_SIZE, 0x9E3779B1, "2394cb79e97368d1"), +] + + +@pytest.mark.parametrize("length, seed, expected", XXH64_VECTORS) +def test_official_vectors(length, seed, expected): + data = SANITY_BUFFER[:length] + # one-shot via the constructor and via the module-level helper + assert XXH64(data, seed).hexdigest() == expected + assert XXH64(data, seed).digest() == bytes.fromhex(expected) + assert xxh64(data, seed) == bytes.fromhex(expected) + + +def test_empty(): + # the single most widely published XXH64 constant + assert XXH64().hexdigest() == "ef46db3751d8e999" + assert XXH64(b"").hexdigest() == "ef46db3751d8e999" + + +def test_default_seed_is_zero(): + assert XXH64(SANITY_BUFFER).hexdigest() == XXH64(SANITY_BUFFER, 0).hexdigest() + + +def test_digest_is_stable_and_bytes(): + h = XXH64(b"borg") + d1 = h.digest() + d2 = h.digest() + assert isinstance(d1, bytes) and len(d1) == 8 + assert d1 == d2 # digest() can be called repeatedly without changing state + assert h.hexdigest() == d1.hex() + + +@pytest.mark.parametrize("length, seed, expected", XXH64_VECTORS) +@pytest.mark.parametrize("chunk", [1, 3, 7, 8, 16, 31, 32, 33, 64]) +def test_streaming_matches_oneshot(length, seed, expected, chunk): + # feeding the data in arbitrary chunk sizes (crossing the internal 32-byte stripe + # boundary in every possible alignment) must reproduce the one-shot digest. + data = SANITY_BUFFER[:length] + h = XXH64(seed=seed) + for off in range(0, len(data), chunk): + h.update(data[off : off + chunk]) + assert h.hexdigest() == expected + + +def test_accepts_buffer_protocol(): + data = SANITY_BUFFER[:100] + expected = XXH64(data).hexdigest() + assert XXH64(bytearray(data)).hexdigest() == expected + assert XXH64(memoryview(data)).hexdigest() == expected + h = XXH64() + h.update(bytearray(data[:50])) + h.update(memoryview(data[50:])) + assert h.hexdigest() == expected