From bc8b3c07721b348b684f76812641a0f933f4377e Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Tue, 14 Jul 2026 17:25:16 +0530 Subject: [PATCH 1/4] check: track a set of checked packs for partial checks Replace the single last-pack-checked marker with a persisted set of checked pack ids, so a pack added between partial runs is verified regardless of how its content-sha256 name sorts. --- src/borg/repository.py | 81 +++++++++++++++++++-------- src/borg/testsuite/repository_test.py | 50 ++++++++++++++++- 2 files changed, 108 insertions(+), 23 deletions(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 580507519f..0967820056 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1,10 +1,13 @@ +import io import os import sys import time -from collections import defaultdict +from collections import defaultdict, namedtuple from pathlib import Path from hashlib import sha256 +from borghash import HashTableNT + from borgstore.store import Store from borgstore.backends.rest import REST, ssh_cmd from borgstore.store import ObjectNotFound as StoreObjectNotFound, ReadRangeError @@ -29,6 +32,18 @@ logger = create_logger(__name__) +# check() records the packs verified in the current check cycle in CHECKED_PACKS, a HashTableNT mapping +# pack_id (32 bytes) -> (timestamp, result). Partial checks (--max-duration) skip packs already in this +# set. Pack names are content sha256s with no meaningful order, so skipping is a set lookup. +CHECKED_PACKS = "cache/checked-packs" +CheckedPackEntry = namedtuple("CheckedPackEntry", "timestamp result") +CheckedPackEntryFormatT = namedtuple("CheckedPackEntryFormatT", "timestamp result") +CheckedPackEntryFormat = CheckedPackEntryFormatT(timestamp="Q", result="B") # unix ts (uint64), result 1=ok 0=corrupt + + +def new_checked_packs_table(): + return HashTableNT(key_size=32, value_type=CheckedPackEntry, value_format=CheckedPackEntryFormat) + def repo_lister(repository, *, limit=None): marker = None @@ -53,7 +68,7 @@ def borg_permissions(permissions): return { "": "lr", "archives": "lrw", - "cache": "lrwWD", # WD for last-pack-checked, ... + "cache": "lrwWD", # WD for checked-packs, ... "config": "lrW", # W for manifest "index": "lrwWD", # WD for index/ (merge/compaction of incremental indexes) "keys": "lr", @@ -678,23 +693,19 @@ def store_list(namespace): partial = bool(max_duration) assert not (repair and partial) mode = "partial" if partial else "full" - LAST_PACK_CHECKED = "cache/last-pack-checked" logger.info(f"Starting {mode} repository check") if partial: - # continue a past partial check (if any) or from a checkpoint or start one from beginning - try: - last_pack_checked = self.store.load(LAST_PACK_CHECKED).decode() - except StoreObjectNotFound: - last_pack_checked = "" + # resume the cycle: packs already in the set are skipped below. + checked_packs = self._load_checked_packs() else: - # start from the beginning and also forget about any potential past partial checks - last_pack_checked = "" + # a full check verifies every pack; start a new cycle. + checked_packs = new_checked_packs_table() try: - self.store.delete(LAST_PACK_CHECKED) + self.store.delete(CHECKED_PACKS) except StoreObjectNotFound: pass - if last_pack_checked: - logger.info(f"Skipping to packs after {last_pack_checked}.") + if len(checked_packs): + logger.info(f"Continuing check cycle, {len(checked_packs)} packs already checked.") else: logger.info("Starting from beginning.") t_start = time.monotonic() @@ -725,28 +736,30 @@ def store_list(namespace): for info in pack_infos: self._lock_refresh() pack_pi.show(increase=1) # advance for every pack, including ones a partial resume skips below - key = "packs/%s" % info.name - if key <= last_pack_checked: # needs sorted keys + pack_id = hex_to_bin(info.name) + if pack_id in checked_packs: # already verified this cycle continue pack_files += 1 - if not verify("packs", info.name): + ok = verify("packs", info.name) + if not ok: pack_errors += 1 # repair (salvage into a new pack, fix index) is not implemented yet + checked_packs[pack_id] = CheckedPackEntry(timestamp=int(time.time()), result=int(ok)) now = time.monotonic() if now > t_last_checkpoint + 300: # checkpoint every 5 mins t_last_checkpoint = now - logger.info(f"Checkpointing at pack {key}.") - self.store.store(LAST_PACK_CHECKED, key.encode()) + logger.info(f"Checkpointing at pack {info.name}.") + self._store_checked_packs(checked_packs) if partial and now > t_start + max_duration: - logger.info(f"Finished partial repository check, last pack checked is {key}.") - self.store.store(LAST_PACK_CHECKED, key.encode()) + logger.info(f"Finished partial repository check, {len(checked_packs)} packs checked so far.") + self._store_checked_packs(checked_packs) break else: - # the pack scan reached the end (no partial timeout): the check is complete, drop the checkpoint. + # scanned all packs without hitting the time limit: the cycle is done, drop the set. if pack_infos: pack_pi.show(current=len(pack_infos)) # finish at 100% logger.info("Finished checking packs.") try: - self.store.delete(LAST_PACK_CHECKED) + self.store.delete(CHECKED_PACKS) except StoreObjectNotFound: pass pack_pi.finish() @@ -765,6 +778,30 @@ def store_list(namespace): logger.error(f"Finished {mode} repository check, errors found.") return objs_errors == 0 or repair + def _load_checked_packs(self): + """Load the CHECKED_PACKS set. Return an empty table if it is missing, corrupted or unreadable.""" + try: + data = self.store.load(CHECKED_PACKS) + except StoreObjectNotFound: + return new_checked_packs_table() + # the trailing 32 bytes are a sha256 over the rest; a mismatch means the blob rotted. + if len(data) < 32 or sha256(data[:-32]).digest() != data[-32:]: + logger.warning("Ignoring corrupted checked-packs set.") + return new_checked_packs_table() + try: + with io.BytesIO(data[:-32]) as f: + return HashTableNT.read(f) + except ValueError: + logger.warning("Ignoring unreadable checked-packs set.") + return new_checked_packs_table() + + def _store_checked_packs(self, checked_packs): + """Store the CHECKED_PACKS set with a sha256 appended over its content.""" + with io.BytesIO() as f: + checked_packs.write(f) + data = f.getvalue() + self.store.store(CHECKED_PACKS, data + sha256(data).digest()) + def list(self, limit=None, marker=None): """ list infos starting from after id . diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index f9e4125979..5a244ea92a 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -5,7 +5,16 @@ import pytest from ..helpers import IntegrityError, Location, bin_to_hex from ..hashindex import ChunkIndex -from ..repository import Repository, MAX_DATA_SIZE, rest_serve_command, PackWriter, PackReader +from ..repository import ( + Repository, + MAX_DATA_SIZE, + rest_serve_command, + PackWriter, + PackReader, + CHECKED_PACKS, + CheckedPackEntry, + new_checked_packs_table, +) from ..repoobj import RepoObj, OBJ_MAGIC, OBJ_VERSION from .hashindex_test import H @@ -979,6 +988,45 @@ def test_check_intact_multi_object_pack_passes(tmp_path): assert repository.check(repair=False) is True +def test_check_checked_packs_roundtrip(tmp_path): + # the set survives a store/load round-trip; a rotted blob loads as empty. + with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository: + table = new_checked_packs_table() + table[H(1)] = CheckedPackEntry(timestamp=123, result=1) + table[H(2)] = CheckedPackEntry(timestamp=456, result=0) + repository._store_checked_packs(table) + + loaded = repository._load_checked_packs() + assert len(loaded) == 2 + assert H(1) in loaded and H(2) in loaded + assert tuple(loaded[H(2)]) == (456, 0) + + corrupted = bytearray(repository.store.load(CHECKED_PACKS)) + corrupted[0] ^= 0xFF # break the appended sha256 + repository.store.store(CHECKED_PACKS, bytes(corrupted)) + assert len(repository._load_checked_packs()) == 0 + + +def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path): + # a partial check verifies a new pack even when its id sorts before an already-checked pack. + intact = fchunk(b"INTACT", chunk_id=H(1)) + intact_id = sha256(intact).digest() + with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository: + repository.store_store("packs/" + bin_to_hex(intact_id), intact) + + # mark the intact pack as already checked in this cycle. + table = new_checked_packs_table() + table[intact_id] = CheckedPackEntry(timestamp=1, result=1) + repository._store_checked_packs(table) + + # add a corrupt pack (content does not hash to its name) whose id sorts before intact_id. + early_id = b"\x00" * 32 + assert bin_to_hex(early_id) < bin_to_hex(intact_id) + repository.store_store("packs/" + bin_to_hex(early_id), b"CORRUPT-does-not-match-name") + + assert repository.check(repair=False, max_duration=3600) is False + + def test_check_progress_covers_packs_and_index(tmp_path, monkeypatch): # check() uses a separate progress indicator for index/ and for packs/. Each one is sized to its own # namespace and driven to 100% by a final show(current=total). A fake indicator records the wiring From 9543c1411c945d50bb649cf2ad6fcb5bb24b58d2 Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Tue, 14 Jul 2026 17:45:37 +0530 Subject: [PATCH 2/4] check: re-verify packs recorded as corrupt instead of skipping them Split the vertical import and trim an irrelevant line from the checked-packs comment. --- src/borg/repository.py | 7 ++-- src/borg/testsuite/repository_test.py | 51 +++++++++++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 0967820056..5534923a5b 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -33,8 +33,8 @@ logger = create_logger(__name__) # check() records the packs verified in the current check cycle in CHECKED_PACKS, a HashTableNT mapping -# pack_id (32 bytes) -> (timestamp, result). Partial checks (--max-duration) skip packs already in this -# set. Pack names are content sha256s with no meaningful order, so skipping is a set lookup. +# pack_id (32 bytes) -> (timestamp, result). A partial check (--max-duration) skips packs recorded intact +# and re-verifies packs recorded corrupt. CHECKED_PACKS = "cache/checked-packs" CheckedPackEntry = namedtuple("CheckedPackEntry", "timestamp result") CheckedPackEntryFormatT = namedtuple("CheckedPackEntryFormatT", "timestamp result") @@ -737,7 +737,8 @@ def store_list(namespace): self._lock_refresh() pack_pi.show(increase=1) # advance for every pack, including ones a partial resume skips below pack_id = hex_to_bin(info.name) - if pack_id in checked_packs: # already verified this cycle + entry = checked_packs.get(pack_id) + if entry is not None and entry.result: # verified intact earlier in this cycle continue pack_files += 1 ok = verify("packs", info.name) diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index 5a244ea92a..f5f87f724b 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -5,16 +5,8 @@ import pytest from ..helpers import IntegrityError, Location, bin_to_hex from ..hashindex import ChunkIndex -from ..repository import ( - Repository, - MAX_DATA_SIZE, - rest_serve_command, - PackWriter, - PackReader, - CHECKED_PACKS, - CheckedPackEntry, - new_checked_packs_table, -) +from ..repository import Repository, MAX_DATA_SIZE, rest_serve_command, PackWriter, PackReader +from ..repository import CHECKED_PACKS, CheckedPackEntry, new_checked_packs_table from ..repoobj import RepoObj, OBJ_MAGIC, OBJ_VERSION from .hashindex_test import H @@ -1027,6 +1019,45 @@ def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path): assert repository.check(repair=False, max_duration=3600) is False +def test_check_partial_rechecks_pack_recorded_corrupt(tmp_path): + # a pack recorded corrupt earlier in the cycle is re-verified, so the corruption keeps being reported. + with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository: + corrupt_id = H(1) # stored content does not hash to this name + repository.store_store("packs/" + bin_to_hex(corrupt_id), b"CORRUPT-does-not-match-name") + + table = new_checked_packs_table() + table[corrupt_id] = CheckedPackEntry(timestamp=1, result=0) + repository._store_checked_packs(table) + + assert repository.check(repair=False, max_duration=3600) is False + + +def test_check_partial_clears_recorded_corruption_when_intact(tmp_path, monkeypatch): + # a pack recorded corrupt is re-verified, not skipped: assert True alone would also hold if the + # pack were skipped outright, so this spies on store.hash to confirm verify() actually ran on it. + intact = fchunk(b"INTACT", chunk_id=H(1)) + intact_id = sha256(intact).digest() + with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository: + pack_key = "packs/" + bin_to_hex(intact_id) + repository.store_store(pack_key, intact) + + table = new_checked_packs_table() + table[intact_id] = CheckedPackEntry(timestamp=1, result=0) # stale corrupt record + repository._store_checked_packs(table) + + hashed_keys = [] + orig_hash = repository.store.hash + + def spy_hash(key): + hashed_keys.append(key) + return orig_hash(key) + + monkeypatch.setattr(repository.store, "hash", spy_hash) + + assert repository.check(repair=False, max_duration=3600) is True + assert pack_key in hashed_keys # verify() ran on the pack; it was not skipped + + def test_check_progress_covers_packs_and_index(tmp_path, monkeypatch): # check() uses a separate progress indicator for index/ and for packs/. Each one is sized to its own # namespace and driven to 100% by a final show(current=total). A fake indicator records the wiring From 996331b2cc1da9e9cf95e027db7f67de48a0ced3 Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Wed, 15 Jul 2026 01:00:33 +0530 Subject: [PATCH 3/4] check: move checked-packs helpers into a PackTracker class Also shrink the partial-check checkpoint window from 5 minutes to 1 minute so a Ctrl-C between checkpoints redoes at most a minute of verification work. --- src/borg/repository.py | 123 ++++++++++++++------------ src/borg/testsuite/repository_test.py | 43 ++++----- 2 files changed, 89 insertions(+), 77 deletions(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 5534923a5b..d89a9d51a0 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -32,18 +32,6 @@ logger = create_logger(__name__) -# check() records the packs verified in the current check cycle in CHECKED_PACKS, a HashTableNT mapping -# pack_id (32 bytes) -> (timestamp, result). A partial check (--max-duration) skips packs recorded intact -# and re-verifies packs recorded corrupt. -CHECKED_PACKS = "cache/checked-packs" -CheckedPackEntry = namedtuple("CheckedPackEntry", "timestamp result") -CheckedPackEntryFormatT = namedtuple("CheckedPackEntryFormatT", "timestamp result") -CheckedPackEntryFormat = CheckedPackEntryFormatT(timestamp="Q", result="B") # unix ts (uint64), result 1=ok 0=corrupt - - -def new_checked_packs_table(): - return HashTableNT(key_size=32, value_type=CheckedPackEntry, value_format=CheckedPackEntryFormat) - def repo_lister(repository, *, limit=None): marker = None @@ -248,6 +236,60 @@ def iter_headers(self): offset += obj_size +class PackTracker: + """Packs check() has verified in the current cycle, mapping pack_id (32 bytes) -> (timestamp, result). + + is_intact() drives the skip in a partial check: intact packs are skipped, packs recorded corrupt are + re-verified. save() appends a sha256 over the serialized table; load() drops a blob whose sha256 does + not match, so a rotted set re-checks everything instead of skipping an unverified pack. + """ + + NAME = "cache/checked-packs" + Entry = namedtuple("Entry", "timestamp result") + EntryFormatT = namedtuple("EntryFormatT", "timestamp result") + _EntryFormat = EntryFormatT(timestamp="Q", result="B") # unix ts, 1=ok 0=corrupt + + def __init__(self, store): + self.store = store + self.table = HashTableNT(key_size=32, value_type=self.Entry, value_format=self._EntryFormat) + + def __len__(self): + return len(self.table) + + def is_intact(self, pack_id): + entry = self.table.get(pack_id) + return entry is not None and bool(entry.result) + + def record(self, pack_id, ok): + self.table[pack_id] = self.Entry(timestamp=int(time.time()), result=int(ok)) + + def load(self): + try: + data = self.store.load(self.NAME) + except StoreObjectNotFound: + return + if len(data) < 32 or sha256(data[:-32]).digest() != data[-32:]: + logger.warning("Ignoring corrupted checked-packs set.") + return + try: + with io.BytesIO(data[:-32]) as f: + self.table = HashTableNT.read(f) + except ValueError: + logger.warning("Ignoring unreadable checked-packs set.") + + def save(self): + with io.BytesIO() as f: + self.table.write(f) + data = f.getvalue() + self.store.store(self.NAME, data + sha256(data).digest()) + + def clear(self): + try: + self.store.delete(self.NAME) + except StoreObjectNotFound: + pass + + class Repository: """borgstore-based key/value store.""" @@ -694,18 +736,13 @@ def store_list(namespace): assert not (repair and partial) mode = "partial" if partial else "full" logger.info(f"Starting {mode} repository check") + tracker = PackTracker(self.store) if partial: - # resume the cycle: packs already in the set are skipped below. - checked_packs = self._load_checked_packs() + tracker.load() # resume the cycle: packs already recorded intact are skipped below. else: - # a full check verifies every pack; start a new cycle. - checked_packs = new_checked_packs_table() - try: - self.store.delete(CHECKED_PACKS) - except StoreObjectNotFound: - pass - if len(checked_packs): - logger.info(f"Continuing check cycle, {len(checked_packs)} packs already checked.") + tracker.clear() # a full check verifies every pack; start a new cycle. + if len(tracker): + logger.info(f"Continuing check cycle, {len(tracker)} packs already checked.") else: logger.info("Starting from beginning.") t_start = time.monotonic() @@ -737,32 +774,28 @@ def store_list(namespace): self._lock_refresh() pack_pi.show(increase=1) # advance for every pack, including ones a partial resume skips below pack_id = hex_to_bin(info.name) - entry = checked_packs.get(pack_id) - if entry is not None and entry.result: # verified intact earlier in this cycle + if tracker.is_intact(pack_id): # verified intact earlier in this cycle continue pack_files += 1 ok = verify("packs", info.name) if not ok: pack_errors += 1 # repair (salvage into a new pack, fix index) is not implemented yet - checked_packs[pack_id] = CheckedPackEntry(timestamp=int(time.time()), result=int(ok)) + tracker.record(pack_id, ok) now = time.monotonic() - if now > t_last_checkpoint + 300: # checkpoint every 5 mins + if now > t_last_checkpoint + 60: # checkpoint every minute t_last_checkpoint = now logger.info(f"Checkpointing at pack {info.name}.") - self._store_checked_packs(checked_packs) + tracker.save() if partial and now > t_start + max_duration: - logger.info(f"Finished partial repository check, {len(checked_packs)} packs checked so far.") - self._store_checked_packs(checked_packs) + logger.info(f"Finished partial repository check, {len(tracker)} packs checked so far.") + tracker.save() break else: # scanned all packs without hitting the time limit: the cycle is done, drop the set. if pack_infos: pack_pi.show(current=len(pack_infos)) # finish at 100% logger.info("Finished checking packs.") - try: - self.store.delete(CHECKED_PACKS) - except StoreObjectNotFound: - pass + tracker.clear() pack_pi.finish() else: # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572). @@ -779,30 +812,6 @@ def store_list(namespace): logger.error(f"Finished {mode} repository check, errors found.") return objs_errors == 0 or repair - def _load_checked_packs(self): - """Load the CHECKED_PACKS set. Return an empty table if it is missing, corrupted or unreadable.""" - try: - data = self.store.load(CHECKED_PACKS) - except StoreObjectNotFound: - return new_checked_packs_table() - # the trailing 32 bytes are a sha256 over the rest; a mismatch means the blob rotted. - if len(data) < 32 or sha256(data[:-32]).digest() != data[-32:]: - logger.warning("Ignoring corrupted checked-packs set.") - return new_checked_packs_table() - try: - with io.BytesIO(data[:-32]) as f: - return HashTableNT.read(f) - except ValueError: - logger.warning("Ignoring unreadable checked-packs set.") - return new_checked_packs_table() - - def _store_checked_packs(self, checked_packs): - """Store the CHECKED_PACKS set with a sha256 appended over its content.""" - with io.BytesIO() as f: - checked_packs.write(f) - data = f.getvalue() - self.store.store(CHECKED_PACKS, data + sha256(data).digest()) - def list(self, limit=None, marker=None): """ list infos starting from after id . diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index f5f87f724b..c583857892 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -6,7 +6,7 @@ from ..helpers import IntegrityError, Location, bin_to_hex from ..hashindex import ChunkIndex from ..repository import Repository, MAX_DATA_SIZE, rest_serve_command, PackWriter, PackReader -from ..repository import CHECKED_PACKS, CheckedPackEntry, new_checked_packs_table +from ..repository import PackTracker from ..repoobj import RepoObj, OBJ_MAGIC, OBJ_VERSION from .hashindex_test import H @@ -983,20 +983,23 @@ def test_check_intact_multi_object_pack_passes(tmp_path): def test_check_checked_packs_roundtrip(tmp_path): # the set survives a store/load round-trip; a rotted blob loads as empty. with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository: - table = new_checked_packs_table() - table[H(1)] = CheckedPackEntry(timestamp=123, result=1) - table[H(2)] = CheckedPackEntry(timestamp=456, result=0) - repository._store_checked_packs(table) + tracker = PackTracker(repository.store) + tracker.table[H(1)] = PackTracker.Entry(timestamp=123, result=1) + tracker.table[H(2)] = PackTracker.Entry(timestamp=456, result=0) + tracker.save() - loaded = repository._load_checked_packs() + loaded = PackTracker(repository.store) + loaded.load() assert len(loaded) == 2 - assert H(1) in loaded and H(2) in loaded - assert tuple(loaded[H(2)]) == (456, 0) + assert H(1) in loaded.table and H(2) in loaded.table + assert tuple(loaded.table[H(2)]) == (456, 0) - corrupted = bytearray(repository.store.load(CHECKED_PACKS)) + corrupted = bytearray(repository.store.load(PackTracker.NAME)) corrupted[0] ^= 0xFF # break the appended sha256 - repository.store.store(CHECKED_PACKS, bytes(corrupted)) - assert len(repository._load_checked_packs()) == 0 + repository.store.store(PackTracker.NAME, bytes(corrupted)) + rotted = PackTracker(repository.store) + rotted.load() + assert len(rotted) == 0 def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path): @@ -1007,9 +1010,9 @@ def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path): repository.store_store("packs/" + bin_to_hex(intact_id), intact) # mark the intact pack as already checked in this cycle. - table = new_checked_packs_table() - table[intact_id] = CheckedPackEntry(timestamp=1, result=1) - repository._store_checked_packs(table) + tracker = PackTracker(repository.store) + tracker.record(intact_id, ok=True) + tracker.save() # add a corrupt pack (content does not hash to its name) whose id sorts before intact_id. early_id = b"\x00" * 32 @@ -1025,9 +1028,9 @@ def test_check_partial_rechecks_pack_recorded_corrupt(tmp_path): corrupt_id = H(1) # stored content does not hash to this name repository.store_store("packs/" + bin_to_hex(corrupt_id), b"CORRUPT-does-not-match-name") - table = new_checked_packs_table() - table[corrupt_id] = CheckedPackEntry(timestamp=1, result=0) - repository._store_checked_packs(table) + tracker = PackTracker(repository.store) + tracker.record(corrupt_id, ok=False) + tracker.save() assert repository.check(repair=False, max_duration=3600) is False @@ -1041,9 +1044,9 @@ def test_check_partial_clears_recorded_corruption_when_intact(tmp_path, monkeypa pack_key = "packs/" + bin_to_hex(intact_id) repository.store_store(pack_key, intact) - table = new_checked_packs_table() - table[intact_id] = CheckedPackEntry(timestamp=1, result=0) # stale corrupt record - repository._store_checked_packs(table) + tracker = PackTracker(repository.store) + tracker.record(intact_id, ok=False) # stale corrupt record + tracker.save() hashed_keys = [] orig_hash = repository.store.hash From a1b88e2dc7f371ac879b192831d88b2bf9c36041 Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Wed, 15 Jul 2026 02:09:42 +0530 Subject: [PATCH 4/4] check: reduce checkpoint interval to 30 min and fix data-structures docs --- docs/internals/data-structures.rst | 9 ++++++--- src/borg/repository.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index 84a80f6f36..6a91997613 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -31,12 +31,15 @@ config/ the repository version encoded as decimal number text manifest some data about the repository, binary - last-pack-checked - repository check progress (partial checks, full checks' checkpointing), - key of last pack checked as text space-reserve.N purely random binary data to reserve space, e.g. for disk-full emergencies +cache/ + checked-packs + repository check progress (partial checks, full checks' checkpointing), + the set of packs checked so far this cycle (pack key -> timestamp, result), + as a hashtable with an appended integrity hash + There is a list of pointers to archive objects in this directory: archives/ diff --git a/src/borg/repository.py b/src/borg/repository.py index d89a9d51a0..1c263be374 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -782,7 +782,7 @@ def store_list(namespace): pack_errors += 1 # repair (salvage into a new pack, fix index) is not implemented yet tracker.record(pack_id, ok) now = time.monotonic() - if now > t_last_checkpoint + 60: # checkpoint every minute + if now > t_last_checkpoint + 30 * 60: # checkpoint every 30 minutes t_last_checkpoint = now logger.info(f"Checkpointing at pack {info.name}.") tracker.save()