check: report missing chunks grouped as chunk -> files -> archives, #9218

src/borg/archive.py

@@ -1809,6 +1809,11 @@ def __next__(self):
 
 
 class ArchiveChecker:
+    # Bound how many missing file chunks rebuild_archives buffers for its end-of-run report,
+    # so checking a badly damaged repo with very many missing chunks can not exhaust memory.
+    MAX_MISSING_CHUNKS = 10000  # max. distinct missing chunk ids kept for the report
+    MAX_REFS_PER_CHUNK = 100  # max. referencing files kept per missing chunk
+
     def __init__(self):
         self.error_found = False
         self.key = None
@@ -2075,6 +2080,29 @@ def rebuild_archives(
     ):
         """Analyze and rebuild archives, expecting some damage and trying to make stuff consistent again."""
 
+        # Missing file chunks, collected during the per-archive checks and reported grouped as
+        # chunk -> files -> archives after all archives were analyzed. Bounded by
+        # MAX_MISSING_CHUNKS / MAX_REFS_PER_CHUNK.
+        missing_chunks = {}  # chunk_id -> [size, {path: {archive_name}}]
+        missing_chunks_truncated = False  # True once the MAX_MISSING_CHUNKS cap was hit
+        missing_refs_truncated = set()  # chunk_ids whose MAX_REFS_PER_CHUNK cap was hit
+
+        def record_missing_chunk(archive_name, path, chunk_id, size):
+            nonlocal missing_chunks_truncated
+            entry = missing_chunks.get(chunk_id)
+            if entry is None:
+                if len(missing_chunks) >= self.MAX_MISSING_CHUNKS:
+                    missing_chunks_truncated = True
+                    return
+                entry = missing_chunks[chunk_id] = [size, {}]
+            refs = entry[1]
+            if path in refs:
+                refs[path].add(archive_name)
+            elif len(refs) < self.MAX_REFS_PER_CHUNK:
+                refs[path] = {archive_name}
+            else:
+                missing_refs_truncated.add(chunk_id)
+
         def add_callback(chunk):
             id_ = self.key.id_hash(chunk)
             cdata = self.repo_objs.format(id_, {}, chunk, ro_type=ROBJ_ARCHIVE_STREAM)
@@ -2091,16 +2119,17 @@ def add_reference(id_, size, cdata):
                     self.chunks.update_pack_info(pack_results)
 
         def verify_file_chunks(archive_name, item):
-            """Verifies that all file chunks are present. Missing file chunks will be logged."""
+            """Verify that all of a file's chunks are present, collecting any missing ones for the report."""
             offset = 0
             for chunk in item.chunks:
                 chunk_id, size = chunk
                 if chunk_id not in self.chunks:
-                    logger.error(
+                    logger.debug(
                         "{}: {}: Missing file chunk detected (Byte {}-{}, Chunk {}).".format(
                             archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id)
                         )
                     )
+                    record_missing_chunk(archive_name, item.path, chunk_id, size)
                     self.error_found = True
                 offset += size
             if "size" in item:
@@ -2114,6 +2143,21 @@ def verify_file_chunks(archive_name, item):
                         )
                     )
 
+        def report_missing_chunks():
+            """Report the collected missing chunks, grouped as chunk -> files -> archives."""
+            if not missing_chunks:
+                return
+            logger.error("The following chunks are missing in the repository:")
+            for chunk_id, (size, refs) in missing_chunks.items():
+                logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {size:,} bytes")
+                for path in sorted(refs):
+                    archive_names = ", ".join(sorted(refs[path]))
+                    logger.error(f"    - {path}: {archive_names}")
+                if chunk_id in missing_refs_truncated:
+                    logger.error(f"    - ... (only the first {self.MAX_REFS_PER_CHUNK} files are listed)")
+            if missing_chunks_truncated:
+                logger.error(f"... (only the first {self.MAX_MISSING_CHUNKS} missing chunks are listed)")
+
         def robust_iterator(archive):
             """Iterates through all archive items
 
@@ -2271,6 +2315,7 @@ def valid_item(obj):
                 if archive_id != new_archive_id:
                     self.manifest.archives.delete_by_id(archive_id)
         pi.finish()
+        report_missing_chunks()
 
     def finish(self):
         if self.repair:

src/borg/testsuite/archiver/check_cmd_test.py

@@ -6,7 +6,7 @@
 
 import pytest
 
-from ...archive import ChunkBuffer
+from ...archive import ChunkBuffer, ArchiveChecker
 from ...constants import *  # NOQA
 from ...helpers import bin_to_hex, msgpack, CommandError
 from ...manifest import Manifest
@@ -169,9 +169,17 @@ def test_missing_file_chunk(archivers, request):
             pytest.fail("should not happen")  # convert 'fail'
 
     output = cmd(archiver, "check", exit_code=1)
-    assert "Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    # archive1 and archive2 share src_file, so the missing chunk appears once, with both archives
+    # listed on its single reference line.
+    assert output.count(bin_to_hex(killed_chunk.id)) == 1
+    ref_lines = [line for line in output.splitlines() if src_file in line]
+    assert len(ref_lines) == 1
+    assert "archive1" in ref_lines[0] and "archive2" in ref_lines[0]
     output = cmd(archiver, "check", "--repair", exit_code=0)
-    assert "Missing file chunk detected" in output  # repair is not changing anything, just reporting.
+    # repair is not changing anything, just reporting.
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(killed_chunk.id) in output
 
     # check does not modify the chunks list.
     for archive_name in ("archive1", "archive2"):
@@ -191,7 +199,34 @@ def test_missing_file_chunk(archivers, request):
 
     # check should not complain anymore about missing chunks:
     output = cmd(archiver, "check", "-v", "--repair", exit_code=0)
-    assert "Missing file chunk detected" not in output
+    assert "The following chunks are missing in the repository:" not in output
+
+
+def test_missing_file_chunk_report_truncated(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+
+    # remove several distinct file chunks, so more missing chunks exist than the (patched) report limit.
+    archive, repository = open_archive(archiver.repository_path, "archive1")
+    killed_ids = []
+    with repository:
+        for item in archive.iter_items():
+            if "chunks" not in item or not item.chunks:
+                continue
+            chunk_id = item.chunks[-1].id
+            if chunk_id not in killed_ids:
+                repository.delete(chunk_id)
+                killed_ids.append(chunk_id)
+            if len(killed_ids) >= 3:
+                break
+    assert len(killed_ids) >= 2  # need several distinct missing chunks to exercise truncation
+
+    # cap the report to a single chunk, so the remaining missing chunks are truncated.
+    with patch.object(ArchiveChecker, "MAX_MISSING_CHUNKS", 1):
+        output = cmd(archiver, "check", exit_code=1)
+    assert "The following chunks are missing in the repository:" in output
+    assert output.count("- Chunk ") == 1  # only one chunk is detailed
+    assert "only the first 1 missing chunks are listed" in output  # the rest are noted as truncated
 
 
 def test_missing_archive_item_chunk(archivers, request):
@@ -474,11 +509,14 @@ def test_verify_data(archivers, request, init_args):
     # repair will find the defect chunk and remove it
     output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0)
     assert f"{bin_to_hex(chunk.id)}, integrity error" in output
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(chunk.id) in output
+    assert src_file in output
 
     # run with --verify-data again, it will notice the missing chunk.
     output = cmd(archiver, "check", "--archives-only", "--verify-data", exit_code=1)
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(chunk.id) in output
 
 
 @pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none"]])
@@ -506,12 +544,15 @@ def test_corrupted_file_chunk(archivers, request, init_args):
     # repair: the defect chunk will be removed.
     output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0)
     assert f"{bin_to_hex(chunk.id)}, integrity error" in output
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(chunk.id) in output
+    assert src_file in output
 
     # run normal check again
     cmd(archiver, "check", "--repository-only", exit_code=0)
     output = cmd(archiver, "check", "--archives-only", exit_code=1)
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert src_file in output
 
 
 @pytest.mark.skip(