diff --git a/src/borg/archiver/compact_cmd.py b/src/borg/archiver/compact_cmd.py index 5cbe996c76..dd16cc1832 100644 --- a/src/borg/archiver/compact_cmd.py +++ b/src/borg/archiver/compact_cmd.py @@ -11,7 +11,7 @@ from ..helpers.argparsing import ArgumentParser from ..constants import * # NOQA from ..hashindex import ChunkIndex -from ..helpers import set_ec, EXIT_ERROR, format_file_size, bin_to_hex +from ..helpers import set_ec, EXIT_ERROR, Error, sig_int, format_file_size, bin_to_hex from ..helpers import ProgressIndicatorPercent from ..manifest import Manifest from ..repository import Repository @@ -50,6 +50,8 @@ def garbage_collect(self): self.report_and_delete() if not self.dry_run: self.save_chunk_index() + if sig_int: # raise after saving, so a Ctrl-C still leaves a valid index + raise Error("Got Ctrl-C / SIGINT.") self.cleanup_files_cache() logger.info("Finished compaction / garbage collection...") @@ -248,19 +250,19 @@ def compact_packs(self): delete_chunkindex_from_repo(self.repository) # Pass 2: collect object ids only for the affected packs (a subset, not the whole index) - keep = {pid: set() for pid in rewrite_packs} # survivors to copy forward, per pack - drop = {pid: set() for pid in rewrite_packs} # unused objects in those same packs - forget = [] # ids living in fully-unused packs we delete outright + keep = defaultdict(set) # survivors to copy forward, per rewritten pack + drop = defaultdict(set) # unused objects in those same packs + forget = defaultdict(set) # ids in fully-unused packs we delete outright for id, entry in self.chunks.iteritems(): pid = entry.pack_id if pid in rewrite_packs: (keep if entry.flags & ChunkIndex.F_USED else drop)[pid].add(id) elif pid in drop_packs: - forget.append(id) + forget[pid].add(id) # count what we remove: every object of a dropped pack, plus the unused objects cut from # rewritten packs. unused objects in below-threshold packs stay, so they don't count. - deleted = len(forget) + sum(len(ids) for ids in drop.values()) + deleted = sum(len(ids) for ids in forget.values()) + sum(len(ids) for ids in drop.values()) logger.info(f"Deleting {deleted} unused objects...") pi = ProgressIndicatorPercent( total=len(drop_packs) + len(rewrite_packs), @@ -269,17 +271,22 @@ def compact_packs(self): msgid="compact.compact_packs", ) progress = 0 + # self.chunks stays consistent with the store after each pack, so Ctrl-C can stop between packs for pid in drop_packs: + if sig_int: + break try: self.repository.store_delete("packs/" + bin_to_hex(pid)) except StoreObjectNotFound: # happens when a stale chunk index references an already deleted pack (#9850) logger.warning(f"Pack {bin_to_hex(pid)} to delete was already gone.") + for id in forget[pid]: # drop the deleted pack's index entries + del self.chunks[id] progress += 1 pi.show(progress) # report after the work, so the final pack lands on 100% - for id in forget: - del self.chunks[id] # their pack file is gone, so drop their index entries too for pid in rewrite_packs: + if sig_int: + break # chunks=self.chunks: the index updates (repoint kept objects, remove dropped ones) # must land in the index that save_chunk_index() persists (#9850). self.repository.compact_pack(pid, keep_ids=keep[pid], drop_ids=drop[pid], chunks=self.chunks) diff --git a/src/borg/testsuite/archiver/compact_cmd_test.py b/src/borg/testsuite/archiver/compact_cmd_test.py index df5185c3bc..dd96cb290c 100644 --- a/src/borg/testsuite/archiver/compact_cmd_test.py +++ b/src/borg/testsuite/archiver/compact_cmd_test.py @@ -4,7 +4,7 @@ import pytest from ...constants import * # NOQA -from ...helpers import get_cache_dir, bin_to_hex +from ...helpers import get_cache_dir, bin_to_hex, sig_int, Error from ...hashindex import ChunkIndex from ...repository import Repository from ...cache import files_cache_name, discover_files_cache_names, list_chunkindex_hashes @@ -150,6 +150,58 @@ def interrupt(): assert fd.read() == content +def test_compact_soft_interrupt_persists_valid_index(archivers, request, monkeypatch): + """One Ctrl-C during pack deletion stops at the next pack boundary, saves a chunk index that + still matches the repository, and exits with an error (#9830). A later compact finishes the rest.""" + from ...archiver.compact_cmd import ArchiveGarbageCollector + + archiver = request.getfixturevalue(archivers) + monkeypatch.setenv("BORG_PACK_MAX_COUNT", "1") # one object per pack -> several packs to stop between + + for i in range(3): + create_regular_file(archiver.input_path, f"file{i}", contents=os.urandom(1024)) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "archive", "input") + cmd(archiver, "delete", "-a", "archive") + + repository = open_repository(archiver) + with repository: + pack_names_before = {info.name for info in repository.store_list("packs")} + assert len(pack_names_before) >= 2 # need several packs to observe an early stop + + manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) + gc = ArchiveGarbageCollector(repository, manifest, stats=False, iec=False, threshold=10) + + original_store_delete = repository.store_delete + calls = [] + + def store_delete_then_interrupt(name, **kwargs): + original_store_delete(name, **kwargs) + if name.startswith("packs/"): # only real pack deletes, not the earlier archive/index deletes + calls.append(name) + if len(calls) == 1: + sig_int._sig_int_triggered = True # one Ctrl-C after the first pack is deleted + + monkeypatch.setattr(repository, "store_delete", store_delete_then_interrupt) + try: + with pytest.raises(Error, match="Got Ctrl-C"): + gc.garbage_collect() + finally: + sig_int._sig_int_triggered = False # reset the global flag for the following tests + + # every persisted index entry points at a pack that still exists + repository = open_repository(archiver) + with repository: + assert list_chunkindex_hashes(repository) != [] + pack_names_after = {info.name for info in repository.store_list("packs")} + assert 0 < len(pack_names_after) < len(pack_names_before) # some packs deleted, some left + for id, entry in repository.chunks.iteritems(): + assert bin_to_hex(entry.pack_id) in pack_names_after + + output = cmd(archiver, "compact", "-v", exit_code=0) + assert "Finished compaction" in output + + def test_compact_packs_respects_threshold(tmp_path): # Two multi-object packs in one repo, then pack-level compaction at a 40% threshold. The pack that # wastes 2/3 of its bytes is rewritten down to its single survivor (and its old file deleted); the