diff --git a/src/borg/archive.py b/src/borg/archive.py index 4af2d47f18..ebbd9a7aea 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1809,6 +1809,11 @@ def __next__(self): class ArchiveChecker: + # Bound how many missing file chunks rebuild_archives buffers for its end-of-run report, + # so checking a badly damaged repo with very many missing chunks can not exhaust memory. + MAX_MISSING_CHUNKS = 10000 # max. distinct missing chunk ids kept for the report + MAX_REFS_PER_CHUNK = 100 # max. referencing files kept per missing chunk + def __init__(self): self.error_found = False self.key = None @@ -2075,6 +2080,29 @@ def rebuild_archives( ): """Analyze and rebuild archives, expecting some damage and trying to make stuff consistent again.""" + # Missing file chunks, collected during the per-archive checks and reported grouped as + # chunk -> files -> archives after all archives were analyzed. Bounded by + # MAX_MISSING_CHUNKS / MAX_REFS_PER_CHUNK. + missing_chunks = {} # chunk_id -> [size, {path: {archive_name}}] + missing_chunks_truncated = False # True once the MAX_MISSING_CHUNKS cap was hit + missing_refs_truncated = set() # chunk_ids whose MAX_REFS_PER_CHUNK cap was hit + + def record_missing_chunk(archive_name, path, chunk_id, size): + nonlocal missing_chunks_truncated + entry = missing_chunks.get(chunk_id) + if entry is None: + if len(missing_chunks) >= self.MAX_MISSING_CHUNKS: + missing_chunks_truncated = True + return + entry = missing_chunks[chunk_id] = [size, {}] + refs = entry[1] + if path in refs: + refs[path].add(archive_name) + elif len(refs) < self.MAX_REFS_PER_CHUNK: + refs[path] = {archive_name} + else: + missing_refs_truncated.add(chunk_id) + def add_callback(chunk): id_ = self.key.id_hash(chunk) cdata = self.repo_objs.format(id_, {}, chunk, ro_type=ROBJ_ARCHIVE_STREAM) @@ -2091,16 +2119,17 @@ def add_reference(id_, size, cdata): self.chunks.update_pack_info(pack_results) def verify_file_chunks(archive_name, item): - """Verifies that all file chunks are present. Missing file chunks will be logged.""" + """Verify that all of a file's chunks are present, collecting any missing ones for the report.""" offset = 0 for chunk in item.chunks: chunk_id, size = chunk if chunk_id not in self.chunks: - logger.error( + logger.debug( "{}: {}: Missing file chunk detected (Byte {}-{}, Chunk {}).".format( archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id) ) ) + record_missing_chunk(archive_name, item.path, chunk_id, size) self.error_found = True offset += size if "size" in item: @@ -2114,6 +2143,21 @@ def verify_file_chunks(archive_name, item): ) ) + def report_missing_chunks(): + """Report the collected missing chunks, grouped as chunk -> files -> archives.""" + if not missing_chunks: + return + logger.error("The following chunks are missing in the repository:") + for chunk_id, (size, refs) in missing_chunks.items(): + logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {size:,} bytes") + for path in sorted(refs): + archive_names = ", ".join(sorted(refs[path])) + logger.error(f" - {path}: {archive_names}") + if chunk_id in missing_refs_truncated: + logger.error(f" - ... (only the first {self.MAX_REFS_PER_CHUNK} files are listed)") + if missing_chunks_truncated: + logger.error(f"... (only the first {self.MAX_MISSING_CHUNKS} missing chunks are listed)") + def robust_iterator(archive): """Iterates through all archive items @@ -2271,6 +2315,7 @@ def valid_item(obj): if archive_id != new_archive_id: self.manifest.archives.delete_by_id(archive_id) pi.finish() + report_missing_chunks() def finish(self): if self.repair: diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 1e91ea84ba..f3531bf46f 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -6,7 +6,7 @@ import pytest -from ...archive import ChunkBuffer +from ...archive import ChunkBuffer, ArchiveChecker from ...constants import * # NOQA from ...helpers import bin_to_hex, msgpack, CommandError from ...manifest import Manifest @@ -169,9 +169,17 @@ def test_missing_file_chunk(archivers, request): pytest.fail("should not happen") # convert 'fail' output = cmd(archiver, "check", exit_code=1) - assert "Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + # archive1 and archive2 share src_file, so the missing chunk appears once, with both archives + # listed on its single reference line. + assert output.count(bin_to_hex(killed_chunk.id)) == 1 + ref_lines = [line for line in output.splitlines() if src_file in line] + assert len(ref_lines) == 1 + assert "archive1" in ref_lines[0] and "archive2" in ref_lines[0] output = cmd(archiver, "check", "--repair", exit_code=0) - assert "Missing file chunk detected" in output # repair is not changing anything, just reporting. + # repair is not changing anything, just reporting. + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(killed_chunk.id) in output # check does not modify the chunks list. for archive_name in ("archive1", "archive2"): @@ -191,7 +199,34 @@ def test_missing_file_chunk(archivers, request): # check should not complain anymore about missing chunks: output = cmd(archiver, "check", "-v", "--repair", exit_code=0) - assert "Missing file chunk detected" not in output + assert "The following chunks are missing in the repository:" not in output + + +def test_missing_file_chunk_report_truncated(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + + # remove several distinct file chunks, so more missing chunks exist than the (patched) report limit. + archive, repository = open_archive(archiver.repository_path, "archive1") + killed_ids = [] + with repository: + for item in archive.iter_items(): + if "chunks" not in item or not item.chunks: + continue + chunk_id = item.chunks[-1].id + if chunk_id not in killed_ids: + repository.delete(chunk_id) + killed_ids.append(chunk_id) + if len(killed_ids) >= 3: + break + assert len(killed_ids) >= 2 # need several distinct missing chunks to exercise truncation + + # cap the report to a single chunk, so the remaining missing chunks are truncated. + with patch.object(ArchiveChecker, "MAX_MISSING_CHUNKS", 1): + output = cmd(archiver, "check", exit_code=1) + assert "The following chunks are missing in the repository:" in output + assert output.count("- Chunk ") == 1 # only one chunk is detailed + assert "only the first 1 missing chunks are listed" in output # the rest are noted as truncated def test_missing_archive_item_chunk(archivers, request): @@ -474,11 +509,14 @@ def test_verify_data(archivers, request, init_args): # repair will find the defect chunk and remove it output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0) assert f"{bin_to_hex(chunk.id)}, integrity error" in output - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(chunk.id) in output + assert src_file in output # run with --verify-data again, it will notice the missing chunk. output = cmd(archiver, "check", "--archives-only", "--verify-data", exit_code=1) - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(chunk.id) in output @pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none"]]) @@ -506,12 +544,15 @@ def test_corrupted_file_chunk(archivers, request, init_args): # repair: the defect chunk will be removed. output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0) assert f"{bin_to_hex(chunk.id)}, integrity error" in output - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(chunk.id) in output + assert src_file in output # run normal check again cmd(archiver, "check", "--repository-only", exit_code=0) output = cmd(archiver, "check", "--archives-only", exit_code=1) - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert src_file in output @pytest.mark.skip(