compact: stop cleanly at a pack boundary on Ctrl-C, keep the saved index valid

src/borg/archiver/compact_cmd.py

@@ -11,7 +11,7 @@
 from ..helpers.argparsing import ArgumentParser
 from ..constants import *  # NOQA
 from ..hashindex import ChunkIndex
-from ..helpers import set_ec, EXIT_ERROR, format_file_size, bin_to_hex
+from ..helpers import set_ec, EXIT_ERROR, Error, sig_int, format_file_size, bin_to_hex
 from ..helpers import ProgressIndicatorPercent
 from ..manifest import Manifest
 from ..repository import Repository
@@ -50,6 +50,8 @@ def garbage_collect(self):
         self.report_and_delete()
         if not self.dry_run:
             self.save_chunk_index()
+            if sig_int:  # raise after saving, so a Ctrl-C still leaves a valid index
+                raise Error("Got Ctrl-C / SIGINT.")
             self.cleanup_files_cache()
         logger.info("Finished compaction / garbage collection...")
 
@@ -248,19 +250,19 @@ def compact_packs(self):
         delete_chunkindex_from_repo(self.repository)
 
         # Pass 2: collect object ids only for the affected packs (a subset, not the whole index)
-        keep = {pid: set() for pid in rewrite_packs}  # survivors to copy forward, per pack
-        drop = {pid: set() for pid in rewrite_packs}  # unused objects in those same packs
-        forget = []  # ids living in fully-unused packs we delete outright
+        keep = defaultdict(set)  # survivors to copy forward, per rewritten pack
+        drop = defaultdict(set)  # unused objects in those same packs
+        forget = defaultdict(set)  # ids in fully-unused packs we delete outright
         for id, entry in self.chunks.iteritems():
             pid = entry.pack_id
             if pid in rewrite_packs:
                 (keep if entry.flags & ChunkIndex.F_USED else drop)[pid].add(id)
             elif pid in drop_packs:
-                forget.append(id)
+                forget[pid].add(id)
 
         # count what we remove: every object of a dropped pack, plus the unused objects cut from
         # rewritten packs. unused objects in below-threshold packs stay, so they don't count.
-        deleted = len(forget) + sum(len(ids) for ids in drop.values())
+        deleted = sum(len(ids) for ids in forget.values()) + sum(len(ids) for ids in drop.values())
         logger.info(f"Deleting {deleted} unused objects...")
         pi = ProgressIndicatorPercent(
             total=len(drop_packs) + len(rewrite_packs),
@@ -269,17 +271,22 @@ def compact_packs(self):
             msgid="compact.compact_packs",
         )
         progress = 0
+        # self.chunks stays consistent with the store after each pack, so Ctrl-C can stop between packs
         for pid in drop_packs:
+            if sig_int:
+                break
             try:
                 self.repository.store_delete("packs/" + bin_to_hex(pid))
             except StoreObjectNotFound:
                 # happens when a stale chunk index references an already deleted pack (#9850)
                 logger.warning(f"Pack {bin_to_hex(pid)} to delete was already gone.")
+            for id in forget[pid]:  # drop the deleted pack's index entries
+                del self.chunks[id]
             progress += 1
             pi.show(progress)  # report after the work, so the final pack lands on 100%
-        for id in forget:
-            del self.chunks[id]  # their pack file is gone, so drop their index entries too
         for pid in rewrite_packs:
+            if sig_int:
+                break
             # chunks=self.chunks: the index updates (repoint kept objects, remove dropped ones)
             # must land in the index that save_chunk_index() persists (#9850).
             self.repository.compact_pack(pid, keep_ids=keep[pid], drop_ids=drop[pid], chunks=self.chunks)

src/borg/testsuite/archiver/compact_cmd_test.py

@@ -4,7 +4,7 @@
 import pytest
 
 from ...constants import *  # NOQA
-from ...helpers import get_cache_dir, bin_to_hex
+from ...helpers import get_cache_dir, bin_to_hex, sig_int, Error
 from ...hashindex import ChunkIndex
 from ...repository import Repository
 from ...cache import files_cache_name, discover_files_cache_names, list_chunkindex_hashes
@@ -150,6 +150,58 @@ def interrupt():
             assert fd.read() == content
 
 
+def test_compact_soft_interrupt_persists_valid_index(archivers, request, monkeypatch):
+    """One Ctrl-C during pack deletion stops at the next pack boundary, saves a chunk index that
+    still matches the repository, and exits with an error (#9830). A later compact finishes the rest."""
+    from ...archiver.compact_cmd import ArchiveGarbageCollector
+
+    archiver = request.getfixturevalue(archivers)
+    monkeypatch.setenv("BORG_PACK_MAX_COUNT", "1")  # one object per pack -> several packs to stop between
+
+    for i in range(3):
+        create_regular_file(archiver.input_path, f"file{i}", contents=os.urandom(1024))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "archive", "input")
+    cmd(archiver, "delete", "-a", "archive")
+
+    repository = open_repository(archiver)
+    with repository:
+        pack_names_before = {info.name for info in repository.store_list("packs")}
+        assert len(pack_names_before) >= 2  # need several packs to observe an early stop
+
+        manifest = Manifest.load(repository, (Manifest.Operation.DELETE,))
+        gc = ArchiveGarbageCollector(repository, manifest, stats=False, iec=False, threshold=10)
+
+        original_store_delete = repository.store_delete
+        calls = []
+
+        def store_delete_then_interrupt(name, **kwargs):
+            original_store_delete(name, **kwargs)
+            if name.startswith("packs/"):  # only real pack deletes, not the earlier archive/index deletes
+                calls.append(name)
+                if len(calls) == 1:
+                    sig_int._sig_int_triggered = True  # one Ctrl-C after the first pack is deleted
+
+        monkeypatch.setattr(repository, "store_delete", store_delete_then_interrupt)
+        try:
+            with pytest.raises(Error, match="Got Ctrl-C"):
+                gc.garbage_collect()
+        finally:
+            sig_int._sig_int_triggered = False  # reset the global flag for the following tests
+
+    # every persisted index entry points at a pack that still exists
+    repository = open_repository(archiver)
+    with repository:
+        assert list_chunkindex_hashes(repository) != []
+        pack_names_after = {info.name for info in repository.store_list("packs")}
+        assert 0 < len(pack_names_after) < len(pack_names_before)  # some packs deleted, some left
+        for id, entry in repository.chunks.iteritems():
+            assert bin_to_hex(entry.pack_id) in pack_names_after
+
+    output = cmd(archiver, "compact", "-v", exit_code=0)
+    assert "Finished compaction" in output
+
+
 def test_compact_packs_respects_threshold(tmp_path):
     # Two multi-object packs in one repo, then pack-level compaction at a 40% threshold. The pack that
     # wastes 2/3 of its bytes is rewritten down to its single survivor (and its old file deleted); the