diff --git a/docs/changes.rst b/docs/changes.rst index d8c550cacf..6047cbadb0 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -162,6 +162,19 @@ Compatibility notes: Change Log 2.x ============== +Version 2.0.0b23 (not released yet) +----------------------------------- + +New features: + +- analyze: report the deduplicated size of a set of archives, #5741. + For the considered (matching) set of archives, ``borg analyze`` now reports the + deduplicated size of the set (union of referenced chunks, counted once) and the + exclusive size of the set (chunks referenced only by the set, i.e. the space that + deleting the whole set would free). Chunk ids and sizes are read from the + per-archive references cache maintained by ``borg compact``, so unchanged archives + usually do not need to be opened. + Version 2.0.0b22 (2026-07-22) ----------------------------- diff --git a/docs/usage/analyze.rst.inc b/docs/usage/analyze.rst.inc index 7b548dcc1c..dd652d549b 100644 --- a/docs/usage/analyze.rst.inc +++ b/docs/usage/analyze.rst.inc @@ -12,6 +12,10 @@ borg analyze .. class:: borg-options-table + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | **options** | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--by-name`` | decompose the whole repository by archive name (not combinable with archive filters) | +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ | .. class:: borg-common-opt-ref | | | @@ -48,6 +52,10 @@ borg analyze + options + --by-name decompose the whole repository by archive name (not combinable with archive filters) + + :ref:`common_options` | @@ -65,20 +73,62 @@ borg analyze Description ~~~~~~~~~~~ -Analyze archives to find "hot spots". +Analyze the archives matching the usual archive selection options (e.g. ``-a series_name``). + +**Deduplicated size of a set of archives** + +For the considered (matching) set of archives, ``borg analyze`` reports the *source* +(uncompressed source data) and *stored* (compressed, as stored in the repository) size, +plus the *compression* factor relating the two (stored / source): + +- the *deduplicated size of the set*: the summed size of the union of chunks the + considered archives reference (chunks shared *within* the set are counted once); +- the *exclusive size of the set*: the summed size of chunks referenced *only* by the + considered set and by no other archive - i.e. the space that deleting the whole set + would free (assuming the other archives stay). + +Additionally, the *unreferenced chunks* are reported: chunks in the repository that no +non-deleted archive references, i.e. what ``borg compact`` could free in the current +state of the repository. Only their stored size is known - a chunk's source size is +only recorded in the archives that reference it. Note that chunks belonging to +soft-deleted archives count as unreferenced here, although ``borg compact`` keeps them +when the repository is damaged (so ``borg undelete`` stays possible). + +If no archive filter is given, all archives are considered - every referenced chunk is +then trivially exclusive to them, so only the repository-wide deduplicated size is shown. + +The stored sizes come from the repository's chunk index; the chunk membership and the +source sizes come from the per-archive references cache that ``borg compact`` +maintains, so unchanged archives usually do not need to be opened. + +**Decomposition by archive name (--by-name)** + +With ``--by-name``, the whole repository is decomposed by archive name instead. Archives +sharing a name form a series, so a name usually groups all backups of one source; for +old-style archives that do not form a series, each name is just one archive. + +Every chunk is counted in exactly one row, so the rows add up to the repository's +deduplicated size: + +- one row per archive name, showing what is *exclusive* to it: no archive of another name + references these chunks, so deleting all archives of that name would free them; +- one row for the chunks shared by two or more names; +- one row for the unreferenced chunks (see above). -``borg analyze`` relies on the usual archive matching options to select the -archives that should be considered for analysis (e.g. ``-a series_name``). -Then it iterates over all matching archives, over all contained files, and -collects information about chunks stored in all directories it encounters. +This answers "which name costs how much, and what would I get back by dropping it" for +all names at once, in a single pass over the archives. As the shared and unreferenced +rows can only be determined by looking at every archive, ``--by-name`` always covers the +whole repository and cannot be combined with archive filters. -It considers chunk IDs and their plaintext sizes (we do not have the compressed -size in the repository easily available) and adds up the sizes of added and removed -chunks per direct parent directory, and outputs a list of "directory: size". +**Hot spots** -You can use that list to find directories with a lot of "activity" — maybe -some of these are temporary or cache directories you forgot to exclude. +If at least two archives match, ``borg analyze`` additionally iterates over all matching +archives, over all contained files, and collects information about chunks stored in all +directories it encounters. It considers chunk IDs and their plaintext sizes and adds up +the sizes of added and removed chunks per direct parent directory, and outputs a list of +"directory: size". -To avoid including these unwanted directories in your backups, you can carefully -exclude them in ``borg create`` (for future backups) or use ``borg recreate`` -to recreate existing archives without them. \ No newline at end of file +You can use that list to find directories with a lot of "activity" — maybe some of these +are temporary or cache directories you forgot to exclude. To avoid including these unwanted +directories in your backups, you can carefully exclude them in ``borg create`` (for future +backups) or use ``borg recreate`` to recreate existing archives without them. \ No newline at end of file diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index 17e72e94e5..fec796cb84 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -81,11 +81,9 @@ Description This command displays detailed information about the specified archive. -Please note that the deduplicated sizes of the individual archives do not add -up to the deduplicated size of the repository ("all archives"), because the two -mean different things: - -This archive / deduplicated size = amount of data stored ONLY for this archive -= unique chunks of this archive. -All archives / deduplicated size = amount of data stored in the repository -= all chunks in the repository. \ No newline at end of file +The original size shown here is the total size of the archive's source data +(uncompressed, counting duplicate content per occurrence). + +Deduplicated sizes are not shown here (computing them per archive is expensive). +For the deduplicated size of a set of archives, use ``borg analyze``; for the +repository-wide deduplicated size, use ``borg compact --stats``. \ No newline at end of file diff --git a/src/borg/archiver/analyze_cmd.py b/src/borg/archiver/analyze_cmd.py index 7608c93e9d..8aa811648f 100644 --- a/src/borg/archiver/analyze_cmd.py +++ b/src/borg/archiver/analyze_cmd.py @@ -3,8 +3,9 @@ from ._common import with_repository, define_archive_filters_group from ..archive import Archive +from ..cache import get_archive_references, list_archive_reference_caches from ..constants import * # NOQA -from ..helpers import bin_to_hex, Error +from ..helpers import bin_to_hex, Error, format_file_size, use_iec_units from ..helpers import ProgressIndicatorPercent from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest @@ -14,6 +15,22 @@ logger = create_logger() +# Ephemeral (never persisted) user flag bits, used only while computing deduplication sizes. +# ChunkIndex.M_USER covers bits 0..23; bits 0..2 are taken by F_USED/F_COMPRESS/F_PENDING. +# +# Set mode (a set of archives is considered): F_CONSIDERED marks a chunk referenced by the +# considered set, F_REST one referenced by the rest of the archives (see issue #5741 and +# ThomasWaldmann's proposal there). +F_CONSIDERED = 2**3 +F_REST = 2**4 +# By-name mode: bits 5..22 hold the owning archive name (as index + 1, so 0 means "referenced by no +# archive"), bit 23 marks a chunk referenced by more than one name. This decomposes the repository +# in a single pass: each chunk is either exclusive to one name, shared, or unreferenced. +OWNER_SHIFT = 5 +OWNER_MASK = 2**18 - 1 # bits 5..22 +F_MULTI = 2**23 +MAX_NAMES = OWNER_MASK - 1 # owner values are name index + 1 + class ArchiveAnalyzer: def __init__(self, args, repository, manifest): @@ -21,21 +38,260 @@ def __init__(self, args, repository, manifest): self.repository = repository assert isinstance(repository, Repository) self.manifest = manifest + self.iec = use_iec_units() self.difference_by_path = defaultdict(int) # directory path -> count of chunks changed def analyze(self): logger.info("Starting archives analysis...") - self.analyze_archives() - self.report() + if self.args.by_name: + # the decomposition is inherently repository-wide: "shared" and "unreferenced" can only + # be determined by looking at every archive, so archive filters must not be applied. + filters = ["match_archives", "first", "last", "older", "newer", "oldest", "newest"] + if any(getattr(self.args, name, None) for name in filters): + raise Error("--by-name analyzes the whole repository and cannot be combined with archive filters.") + self.analyze_by_name() + else: + considered_infos = self.manifest.archives.list_considering(self.args) + if not considered_infos: + raise Error("No archives match the given selection criteria.") + self.analyze_dedup_size(considered_infos) + if len(considered_infos) >= 2: + self.analyze_hotspots(considered_infos) + self.report_hotspots() + else: + logger.info("Skipping hot-spot analysis (needs at least 2 matching archives).") logger.info("Finished archives analysis.") - def analyze_archives(self) -> None: - """Analyze all archives matching the given selection criteria.""" - archive_infos = self.manifest.archives.list_considering(self.args) - num_archives = len(archive_infos) - if num_archives < 2: - raise Error("Need at least 2 archives to analyze.") + def fmt(self, value): + return format_file_size(value, iec=self.iec) + + def factor(self, stored, plaintext): + """The compression factor stored/plaintext, or n/a if there is no plaintext size to relate to + (unreferenced chunks, or chunks that are all archive metadata, which is recorded with size 0). + """ + return f"{stored / plaintext:.2f}" if plaintext else "n/a" + + def mark_references(self, archive_infos, update_flags): + """Walk the given archives' referenced chunks, updating their entry in the repository chunk + index via update_flags(flags) -> flags. Also fills in the plaintext size, which is 0 in the + repo index (it is only recorded in the archives referencing a chunk). Returns the number of + referenced chunks that are missing from the index. + + The chunk ids and plaintext sizes come from the per-archive references cache (see borg + compact), so unchanged archives are not opened/decrypted at all. + """ + index = self.repository.chunks + cached_hex_ids = list_archive_reference_caches(self.repository) + missing = 0 + num = len(archive_infos) + pi = ProgressIndicatorPercent( + total=num, msg="Computing referenced chunks %3.1f%%", step=0.1, msgid="analyze.dedup_size" + ) + for i, info in enumerate(archive_infos): + references = get_archive_references( + self.repository, self.manifest, info.id, cached=bin_to_hex(info.id) in cached_hex_ids, iec=self.iec + ) + for id, ref in references.ids.items(): + entry = index.get(id) + if entry is None: + missing += 1 # referenced by an archive but absent from the repo index + continue + # _replace keeps pack_id/obj_offset/obj_size (esp. the stored size obj_size) intact. + index[id] = entry._replace(flags=update_flags(entry.flags), size=ref.size or entry.size) + pi.show(i + 1) + pi.finish() + return missing + + def analyze_by_name(self) -> None: + """Decompose the whole repository by archive name. + + Archives sharing a name form a series, so a name usually groups all backups of one source; + for old-style archives that do not form a series, each name is just one archive. + + Every chunk falls into exactly one bucket: exclusive to a single name (no archive of another + name references it, so deleting all archives of that name would free it), shared by two or + more names, or referenced by no non-deleted archive at all (reclaimable by borg compact). + The buckets add up to the repository's deduplicated size. + + This needs only a single pass over all archives: each chunk records the name that first + referenced it, and a reference from a different name sets the F_MULTI bit. + """ + all_infos = self.manifest.archives.list() # non-deleted archives + if not all_infos: + raise Error("The repository does not contain any archives.") + archives_per_name: dict[str, int] = defaultdict(int) + for info in all_infos: + archives_per_name[info.name] += 1 + names = sorted(archives_per_name) + if len(names) > MAX_NAMES: + raise Error(f"Too many distinct archive names ({len(names)}) to decompose, limit is {MAX_NAMES}.") + owner_of = {name: i + 1 for i, name in enumerate(names)} # 0 means "unreferenced" + + missing = 0 + for info in all_infos: + owner = owner_of[info.name] + + def update_flags(flags, owner=owner): + previous = (flags >> OWNER_SHIFT) & OWNER_MASK + if previous == 0: # first name referencing this chunk becomes its owner + return flags | (owner << OWNER_SHIFT) + if previous != owner: # a second, different name references it: shared + return flags | F_MULTI + return flags + + missing += self.mark_references([info], update_flags) + + exclusive = {name: [0, 0] for name in names} # archive name -> [source size, stored size] + shared = [0, 0] + unref_count = unref_stored = total_count = 0 + for id, entry in self.repository.chunks.iteritems(): + total_count += 1 + owner = (entry.flags >> OWNER_SHIFT) & OWNER_MASK + if owner == 0: + unref_count += 1 + unref_stored += entry.obj_size + continue + bucket = shared if entry.flags & F_MULTI else exclusive[names[owner - 1]] + bucket[0] += entry.size + bucket[1] += entry.obj_size + + if missing: + logger.warning(f"{missing} chunk(s) referenced by archives are missing from the repository index.") + + total_plaintext = shared[0] + sum(v[0] for v in exclusive.values()) + total_stored = shared[1] + sum(v[1] for v in exclusive.values()) + width = min(max([30] + [len(name) for name in names]), 60) + + def row(label, archives, source, stored, *, source_known=True): + sizes = f"{self.fmt(source) if source_known else 'n/a':>14}{self.fmt(stored):>14}" + ratio = self.factor(stored, source) if source_known else "n/a" + print(f"{label:<{width}}{archives:>10}{sizes}{ratio:>13}") + + print() + print("Repository decomposition by archive name") + print("=" * (width + 51)) + print(f"{len(all_infos)} archive(s) with {len(names)} distinct name(s)") + print() + print(f"{'name':<{width}}{'archives':>10}{'source':>14}{'stored':>14}{'compression':>13}") + # biggest exclusive consumer first - that is what one would act on + for name in sorted(names, key=lambda n: exclusive[n][1], reverse=True): + source, stored = exclusive[name] + row(name, archives_per_name[name], source, stored) + row("(shared by 2+ names)", "", shared[0], shared[1]) + row("(unreferenced)", "", 0, unref_stored, source_known=False) + print("-" * (width + 51)) + row("total (deduplicated)", len(all_infos), total_plaintext, total_stored) + print() + print(f"Unreferenced: {unref_count} of {total_count} chunks in the repository index.") + print() + print("Each chunk is counted in exactly one row, so the rows add up to the total.") + print("A name row shows what is exclusive to it: no archive of another name references these") + print("chunks, so deleting all archives of that name would free them. Chunks used by several") + print("names are counted in the shared row, not against any single name.") + print() + print(f"{'source':<12} = uncompressed source data size (each chunk counted once)") + print(f"{'stored':<12} = deduplicated size as stored in the repository (compressed)") + print(f"{'compression':<12} = stored / source (1.00 = not compressible)") + print(f"{'unreferenced':<12} = referenced by no non-deleted archive; borg compact could free") + print(f"{'':<12} these. Their source size (n/a) is not known: it is only recorded") + print(f"{'':<12} in the archives referencing a chunk. Chunks of soft-deleted") + print(f"{'':<12} archives count as unreferenced (borg compact keeps them if the") + print(f"{'':<12} repository is damaged).") + def analyze_dedup_size(self, considered_infos) -> None: + """Compute and report the deduplicated size of the considered set of archives. + + For both the plaintext (uncompressed source) size and the stored (compressed, as stored in + the repository) size, two figures are reported: + + - the set's deduplicated size: the summed size of the union of chunks the considered archives + reference (chunks shared *within* the set are counted once); + - the set's exclusive size: the summed size of chunks referenced *only* by the considered set + and not by any other (rest) archive - i.e. the space that deleting the whole set would free + (assuming the rest of the archives stay). + + As the considered set plus the rest covers all non-deleted archives, the chunks that end up + with neither flag are referenced by no non-deleted archive at all: what `borg compact` could + free in the current state of the repository. These are reported as well; only their stored + size is known, as a chunk's plaintext size is only recorded in the archives referencing it. + + Following the model in issue #5741, we flag each chunk as referenced by the considered set + (F_CONSIDERED) and/or by the rest of the archives (F_REST). Rather than building a second + index, we reuse the repository's own chunk index (self.repository.chunks): it already holds + every chunk's stored size (obj_size). We OR in the membership flag bits and fill in the + plaintext size (which is 0 in the repo index) from the per-archive references cache. These + in-memory mutations are never persisted: write_chunkindex_to_repo() zeroes flags and size, + and close() only serializes F_NEW entries (there are none here). + """ + considered_ids = {info.id for info in considered_infos} + all_infos = self.manifest.archives.list() # non-deleted archives; the rest = all - considered + rest_infos = [info for info in all_infos if info.id not in considered_ids] + + missing = self.mark_references(considered_infos, lambda flags: flags | F_CONSIDERED) + missing += self.mark_references(rest_infos, lambda flags: flags | F_REST) + + set_plaintext = set_stored = excl_plaintext = excl_stored = 0 + unref_count = unref_stored = total_count = 0 + for id, entry in self.repository.chunks.iteritems(): + total_count += 1 + flags = entry.flags + if flags & F_CONSIDERED: + set_plaintext += entry.size + set_stored += entry.obj_size + if not (flags & F_REST): + excl_plaintext += entry.size + excl_stored += entry.obj_size + elif not (flags & F_REST): + # neither flag: this chunk is referenced by no non-deleted archive at all. + unref_count += 1 + unref_stored += entry.obj_size + + if missing: + logger.warning(f"{missing} chunk(s) referenced by archives are missing from the repository index.") + + # with no archive left over, the considered set is the whole repository: every referenced + # chunk is trivially exclusive to it, so that line would just repeat the deduplicated size. + whole_repo = not rest_infos + + def row(label, source, stored, *, source_known=True): + sizes = f"{self.fmt(source) if source_known else 'n/a':>14}{self.fmt(stored):>14}" + ratio = self.factor(stored, source) if source_known else "n/a" + print(f"{label:<26}{sizes}{ratio:>13}") + + print() + if whole_repo: + print("Deduplicated size of the whole repository") + print("=" * 67) + print(f"Archives: {len(all_infos)} (all archives in the repository)") + else: + print(f"Deduplicated size of the {len(considered_infos)} considered archive(s)") + print("=" * 67) + print(f"Considered archives: {len(considered_infos)} (of {len(all_infos)} in the repository)") + print() + print(f"{'':26}{'source':>14}{'stored':>14}{'compression':>13}") + row("Deduplicated size:" if whole_repo else "Deduplicated size of set:", set_plaintext, set_stored) + if not whole_repo: + row("Exclusive size of set:", excl_plaintext, excl_stored) + row("Unreferenced chunks:", 0, unref_stored, source_known=False) + print() + print(f"Unreferenced: {unref_count} of {total_count} chunks in the repository index.") + print() + if whole_repo: + print(f"{'source':<12} = uncompressed source data size (each chunk counted once)") + else: + print(f"{'source':<12} = uncompressed source data size (chunks shared within the set counted once)") + print(f"{'stored':<12} = deduplicated size as stored in the repository (compressed)") + print(f"{'compression':<12} = stored / source (1.00 = not compressible)") + if not whole_repo: + print(f"{'exclusive':<12} = chunks referenced only by this set; would be freed by deleting the set") + print(f"{'unreferenced':<12} = chunks referenced by no non-deleted archive; borg compact could free") + print(f"{'':<12} these. Their source size (n/a) is not known: it is only recorded in") + print(f"{'':<12} the archives referencing a chunk. Chunks of soft-deleted archives") + print(f"{'':<12} count as unreferenced (borg compact keeps them if the repo is damaged).") + + def analyze_hotspots(self, archive_infos) -> None: + """Analyze all considered archives to find directory "hot spots" (chunk churn per directory).""" + num_archives = len(archive_infos) pi = ProgressIndicatorPercent( total=num_archives, msg="Analyzing archives %3.1f%%", step=0.1, msgid="analyze.analyze_archives" ) @@ -86,7 +342,7 @@ def analyze_path_change(path): if directory_path not in base: analyze_path_change(directory_path) - def report(self): + def report_hotspots(self): print() print("chunks added or removed by directory path") print("=========================================") @@ -106,25 +362,73 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser): analyze_epilog = process_epilog( """ - Analyze archives to find "hot spots". + Analyze the archives matching the usual archive selection options (e.g. ``-a series_name``). + + **Deduplicated size of a set of archives** + + For the considered (matching) set of archives, ``borg analyze`` reports the *source* + (uncompressed source data) and *stored* (compressed, as stored in the repository) size, + plus the *compression* factor relating the two (stored / source): + + - the *deduplicated size of the set*: the summed size of the union of chunks the + considered archives reference (chunks shared *within* the set are counted once); + - the *exclusive size of the set*: the summed size of chunks referenced *only* by the + considered set and by no other archive - i.e. the space that deleting the whole set + would free (assuming the other archives stay). + + Additionally, the *unreferenced chunks* are reported: chunks in the repository that no + non-deleted archive references, i.e. what ``borg compact`` could free in the current + state of the repository. Only their stored size is known - a chunk's source size is + only recorded in the archives that reference it. Note that chunks belonging to + soft-deleted archives count as unreferenced here, although ``borg compact`` keeps them + when the repository is damaged (so ``borg undelete`` stays possible). + + If no archive filter is given, all archives are considered - every referenced chunk is + then trivially exclusive to them, so only the repository-wide deduplicated size is shown. - ``borg analyze`` relies on the usual archive matching options to select the - archives that should be considered for analysis (e.g. ``-a series_name``). - Then it iterates over all matching archives, over all contained files, and - collects information about chunks stored in all directories it encounters. + The stored sizes come from the repository's chunk index; the chunk membership and the + source sizes come from the per-archive references cache that ``borg compact`` + maintains, so unchanged archives usually do not need to be opened. - It considers chunk IDs and their plaintext sizes (we do not have the compressed - size in the repository easily available) and adds up the sizes of added and removed - chunks per direct parent directory, and outputs a list of "directory: size". + **Decomposition by archive name (--by-name)** - You can use that list to find directories with a lot of "activity" — maybe - some of these are temporary or cache directories you forgot to exclude. + With ``--by-name``, the whole repository is decomposed by archive name instead. Archives + sharing a name form a series, so a name usually groups all backups of one source; for + old-style archives that do not form a series, each name is just one archive. - To avoid including these unwanted directories in your backups, you can carefully - exclude them in ``borg create`` (for future backups) or use ``borg recreate`` - to recreate existing archives without them. + Every chunk is counted in exactly one row, so the rows add up to the repository's + deduplicated size: + + - one row per archive name, showing what is *exclusive* to it: no archive of another name + references these chunks, so deleting all archives of that name would free them; + - one row for the chunks shared by two or more names; + - one row for the unreferenced chunks (see above). + + This answers "which name costs how much, and what would I get back by dropping it" for + all names at once, in a single pass over the archives. As the shared and unreferenced + rows can only be determined by looking at every archive, ``--by-name`` always covers the + whole repository and cannot be combined with archive filters. + + **Hot spots** + + If at least two archives match, ``borg analyze`` additionally iterates over all matching + archives, over all contained files, and collects information about chunks stored in all + directories it encounters. It considers chunk IDs and their plaintext sizes and adds up + the sizes of added and removed chunks per direct parent directory, and outputs a list of + "directory: size". + + You can use that list to find directories with a lot of "activity" — maybe some of these + are temporary or cache directories you forgot to exclude. To avoid including these unwanted + directories in your backups, you can carefully exclude them in ``borg create`` (for future + backups) or use ``borg recreate`` to recreate existing archives without them. """ ) subparser = ArgumentParser(parents=[common_parser], description=self.do_analyze.__doc__, epilog=analyze_epilog) subparsers.add_subcommand("analyze", subparser, help="analyze archives") + subparser.add_argument( + "--by-name", + dest="by_name", + action="store_true", + help="decompose the whole repository by archive name (not combinable with archive filters)", + ) define_archive_filters_group(subparser) diff --git a/src/borg/archiver/compact_cmd.py b/src/borg/archiver/compact_cmd.py index c5a15eed05..a00ec10402 100644 --- a/src/borg/archiver/compact_cmd.py +++ b/src/borg/archiver/compact_cmd.py @@ -1,15 +1,13 @@ -from collections import defaultdict, namedtuple -import hashlib -import io +from collections import defaultdict from pathlib import Path -from borghash import HashTableNT from borgstore.store import ItemInfo, ObjectNotFound as StoreObjectNotFound from ._common import with_repository from ..archive import Archive from ..cache import write_chunkindex_to_repo, build_chunkindex_from_repo, delete_chunkindex_from_repo from ..cache import files_cache_name, discover_files_cache_names +from ..cache import get_archive_references, list_archive_reference_caches, cleanup_archive_reference_caches from ..helpers import get_cache_dir from ..helpers.argparsing import ArgumentParser from ..constants import * # NOQA @@ -24,20 +22,6 @@ logger = create_logger() -# per-archive cache of the objects an archive references, stored in the repo as -# cache/referenced-by-archive.. it lets a following compact skip re-scanning an -# unchanged archive's items. the blob is: file_count (uint64 LE), content_size (uint64 LE), a -# serialized HashTableNT mapping object id (32 bytes) -> plaintext object size (uint32), and a -# sha256 of all of that appended for integrity. -REFERENCED_BY_ARCHIVE = "referenced-by-archive." # name prefix within the "cache" store namespace -ArchiveReferenceEntry = namedtuple("ArchiveReferenceEntry", "size") -ArchiveReferenceEntryFormatT = namedtuple("ArchiveReferenceEntryFormatT", "size") -ArchiveReferenceEntryFormat = ArchiveReferenceEntryFormatT(size="I") # uint32 plaintext size -# what an archive references: the objects to mark used (id -> size) plus the tallies compact reports. -# file_count and content_size are counted per occurrence (matching a full scan), so they cannot be -# derived from the deduplicated ids table and are cached alongside it. -ArchiveReferences = namedtuple("ArchiveReferences", "file_count content_size ids") - class ArchiveGarbageCollector: def __init__(self, repository, manifest, *, stats, iec, threshold, dry_run=False): @@ -158,11 +142,11 @@ def analyze_archives(self) -> tuple[set, int, int, int]: missing_chunks: set[bytes] = set() archive_infos = self.manifest.archives.list(sort_by=["ts"]) num_archives = len(archive_infos) - cached_hex_ids = self.list_archive_reference_caches() + cached_hex_ids = list_archive_reference_caches(self.repository) if not self.dry_run: # drop the reference caches of archives that do not exist anymore. valid_hex_ids = {bin_to_hex(info.id) for info in archive_infos} - self.cleanup_archive_reference_caches(cached_hex_ids - valid_hex_ids) + cleanup_archive_reference_caches(self.repository, cached_hex_ids - valid_hex_ids) pi = ProgressIndicatorPercent( total=num_archives, msg="Computing used chunks %3.1f%%", step=0.1, msgid="compact.analyze_archives" ) @@ -171,7 +155,14 @@ def analyze_archives(self) -> tuple[set, int, int, int]: logger.info( f"Analyzing archive {info.name} {info.ts.astimezone()} {bin_to_hex(info.id)} ({i + 1}/{num_archives})" ) - references = self.get_archive_references(info.id, cached=bin_to_hex(info.id) in cached_hex_ids) + references = get_archive_references( + self.repository, + self.manifest, + info.id, + cached=bin_to_hex(info.id) in cached_hex_ids, + iec=self.iec, + store=not self.dry_run, + ) total_files += references.file_count # every fs object counts, not just regular files total_size += references.content_size # original, uncompressed file content size for id, entry in references.ids.items(): @@ -181,98 +172,6 @@ def analyze_archives(self) -> tuple[set, int, int, int]: pi.finish() return missing_chunks, total_files, total_size, num_archives - def get_archive_references(self, archive_id: bytes, *, cached: bool) -> ArchiveReferences: - """Return what the archive references, read from its per-archive cache in the repo if present, - else computed by scanning the archive and then cached for next time. - - On a cache hit the archive is not opened at all (that is the point of the cache): loading an - Archive fetches and decrypts its metadata and item-metadata objects, which is exactly the work - we want to skip for an unchanged archive. - """ - references = self.load_archive_references(archive_id) if cached else None - if references is None: - references = self.scan_archive_references(archive_id) - if not self.dry_run: - self.store_archive_references(archive_id, references) - return references - - def scan_archive_references(self, archive_id: bytes) -> ArchiveReferences: - """Open the archive and scan its items, collecting the objects it references (id -> plaintext - size) plus its source file count and content size (both counted per occurrence, like a full - scan). Opening the archive fetches and decrypts its metadata and item-metadata objects.""" - archive = Archive(self.manifest, archive_id, iec=self.iec) - ids = HashTableNT(key_size=32, value_type=ArchiveReferenceEntry, value_format=ArchiveReferenceEntryFormat) - # archive metadata objects: only their ids matter for GC, their content size is unknown here - # and not part of the source data size, so record them with size 0. - ids[archive.id] = ArchiveReferenceEntry(size=0) - for id in archive.metadata.item_ptrs: - ids[id] = ArchiveReferenceEntry(size=0) - for id in archive.metadata.items: - ids[id] = ArchiveReferenceEntry(size=0) - file_count, content_size = 0, 0 - for item in archive.iter_items(): - file_count += 1 # every fs object counts, not just regular files - if "chunks" in item: - for id, size in item.chunks: - content_size += size # original, uncompressed content size, counted per occurrence - ids[id] = ArchiveReferenceEntry(size=size) - return ArchiveReferences(file_count=file_count, content_size=content_size, ids=ids) - - @staticmethod - def archive_reference_cache_name(archive_id: bytes) -> str: - """The store name of an archive's reference cache (well within borgstore's name length limit).""" - return f"cache/{REFERENCED_BY_ARCHIVE}{bin_to_hex(archive_id)}" - - def list_archive_reference_caches(self) -> set[str]: - """Return the set of archive ids (hex) that currently have a reference cache in the repo.""" - hex_ids = set() - for info in self.repository.store_list("cache"): # store_list yields ItemInfo namedtuples - if info.name.startswith(REFERENCED_BY_ARCHIVE): - hex_ids.add(info.name[len(REFERENCED_BY_ARCHIVE) :]) - return hex_ids - - def load_archive_references(self, archive_id: bytes) -> ArchiveReferences | None: - """Load and verify an archive's references cache; return it, or None if it is missing/corrupted.""" - try: - data = self.repository.store_load(self.archive_reference_cache_name(archive_id)) - except StoreObjectNotFound: - return None - # the serialized blob has a sha256 of its content appended (the store name cannot also carry it, - # as borgstore's name length limit is too small for archive id hex + sha256 hex). a mismatch means - # the cache is corrupted; we then return None so the caller falls back to scanning the archive. - hex_id = bin_to_hex(archive_id) - if len(data) < 16 + 32 or hashlib.sha256(data[:-32]).digest() != data[-32:]: - logger.warning(f"Ignoring corrupted references cache of archive {hex_id}.") - return None - try: - with io.BytesIO(data[:-32]) as f: - file_count = int.from_bytes(f.read(8), "little") - content_size = int.from_bytes(f.read(8), "little") - ids = HashTableNT.read(f) - except ValueError: - logger.warning(f"Ignoring unreadable references cache of archive {hex_id}.") - return None - return ArchiveReferences(file_count=file_count, content_size=content_size, ids=ids) - - def store_archive_references(self, archive_id: bytes, references: ArchiveReferences) -> None: - """Serialize the references (a small header plus the id->size table, with a sha256 appended).""" - with io.BytesIO() as f: - f.write(references.file_count.to_bytes(8, "little")) - f.write(references.content_size.to_bytes(8, "little")) - references.ids.write(f) - data = f.getvalue() - data += hashlib.sha256(data).digest() - self.repository.store_store(self.archive_reference_cache_name(archive_id), data) - - def cleanup_archive_reference_caches(self, stale_hex_ids: set[str]) -> None: - """Delete reference caches belonging to archives that are not in the archives list anymore.""" - for hex_id in stale_hex_ids: - try: - self.repository.store_delete(f"cache/{REFERENCED_BY_ARCHIVE}{hex_id}") - except StoreObjectNotFound: - pass - logger.debug(f"Removed {len(stale_hex_ids)} stale archive references caches.") - def report_and_delete(self): if self.missing_chunks: logger.error(f"Repository has {len(self.missing_chunks)} missing objects!") diff --git a/src/borg/archiver/info_cmd.py b/src/borg/archiver/info_cmd.py index fad9a52201..1405805c71 100644 --- a/src/borg/archiver/info_cmd.py +++ b/src/borg/archiver/info_cmd.py @@ -68,14 +68,12 @@ def build_parser_info(self, subparsers, common_parser, mid_common_parser): """ This command displays detailed information about the specified archive. - Please note that the deduplicated sizes of the individual archives do not add - up to the deduplicated size of the repository ("all archives"), because the two - mean different things: + The original size shown here is the total size of the archive's source data + (uncompressed, counting duplicate content per occurrence). - This archive / deduplicated size = amount of data stored ONLY for this archive - = unique chunks of this archive. - All archives / deduplicated size = amount of data stored in the repository - = all chunks in the repository. + Deduplicated sizes are not shown here (computing them per archive is expensive). + For the deduplicated size of a set of archives, use ``borg analyze``; for the + repository-wide deduplicated size, use ``borg compact --stats``. """ ) subparser = ArgumentParser(parents=[common_parser], description=self.do_info.__doc__, epilog=info_epilog) diff --git a/src/borg/cache.py b/src/borg/cache.py index e500b116a2..1d818e7b33 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -10,6 +10,7 @@ from pathlib import Path from time import perf_counter +from borghash import HashTableNT from borgstore.backends.errors import PermissionDenied from .logger import create_logger @@ -896,6 +897,123 @@ def build_chunkindex_from_repo( return chunks +# per-archive cache of the objects an archive references, stored in the repo as +# cache/referenced-by-archive.. it lets a following compact or analyze skip re-scanning +# an unchanged archive's items. the blob is: file_count (uint64 LE), content_size (uint64 LE), a +# serialized HashTableNT mapping object id (32 bytes) -> plaintext object size (uint32), and a +# sha256 of all of that appended for integrity. +REFERENCED_BY_ARCHIVE = "referenced-by-archive." # name prefix within the "cache" store namespace +ArchiveReferenceEntry = namedtuple("ArchiveReferenceEntry", "size") +ArchiveReferenceEntryFormatT = namedtuple("ArchiveReferenceEntryFormatT", "size") +ArchiveReferenceEntryFormat = ArchiveReferenceEntryFormatT(size="I") # uint32 plaintext size +# what an archive references: the objects to mark used (id -> size) plus the tallies compact reports. +# file_count and content_size are counted per occurrence (matching a full scan), so they cannot be +# derived from the deduplicated ids table and are cached alongside it. +ArchiveReferences = namedtuple("ArchiveReferences", "file_count content_size ids") + + +def archive_reference_cache_name(archive_id: bytes) -> str: + """The store name of an archive's reference cache (well within borgstore's name length limit).""" + return f"cache/{REFERENCED_BY_ARCHIVE}{bin_to_hex(archive_id)}" + + +def list_archive_reference_caches(repository) -> set: + """Return the set of archive ids (hex) that currently have a reference cache in the repo.""" + hex_ids = set() + for info in repository.store_list("cache"): # store_list yields ItemInfo namedtuples + if info.name.startswith(REFERENCED_BY_ARCHIVE): + hex_ids.add(info.name[len(REFERENCED_BY_ARCHIVE) :]) + return hex_ids + + +def load_archive_references(repository, archive_id: bytes): + """Load and verify an archive's references cache; return it, or None if it is missing/corrupted.""" + try: + data = repository.store_load(archive_reference_cache_name(archive_id)) + except StoreObjectNotFound: + return None + # the serialized blob has a sha256 of its content appended (the store name cannot also carry it, + # as borgstore's name length limit is too small for archive id hex + sha256 hex). a mismatch means + # the cache is corrupted; we then return None so the caller falls back to scanning the archive. + hex_id = bin_to_hex(archive_id) + if len(data) < 16 + 32 or hashlib.sha256(data[:-32]).digest() != data[-32:]: + logger.warning(f"Ignoring corrupted references cache of archive {hex_id}.") + return None + try: + with io.BytesIO(data[:-32]) as f: + file_count = int.from_bytes(f.read(8), "little") + content_size = int.from_bytes(f.read(8), "little") + ids = HashTableNT.read(f) + except ValueError: + logger.warning(f"Ignoring unreadable references cache of archive {hex_id}.") + return None + return ArchiveReferences(file_count=file_count, content_size=content_size, ids=ids) + + +def store_archive_references(repository, archive_id: bytes, references) -> None: + """Serialize the references (a small header plus the id->size table, with a sha256 appended).""" + with io.BytesIO() as f: + f.write(references.file_count.to_bytes(8, "little")) + f.write(references.content_size.to_bytes(8, "little")) + references.ids.write(f) + data = f.getvalue() + data += hashlib.sha256(data).digest() + repository.store_store(archive_reference_cache_name(archive_id), data) + + +def cleanup_archive_reference_caches(repository, stale_hex_ids: set) -> None: + """Delete reference caches belonging to archives that are not in the archives list anymore.""" + for hex_id in stale_hex_ids: + try: + repository.store_delete(f"cache/{REFERENCED_BY_ARCHIVE}{hex_id}") + except StoreObjectNotFound: + pass + logger.debug(f"Removed {len(stale_hex_ids)} stale archive references caches.") + + +def scan_archive_references(manifest, archive_id: bytes, *, iec: bool = False): + """Open the archive and scan its items, collecting the objects it references (id -> plaintext + size) plus its source file count and content size (both counted per occurrence, like a full + scan). Opening the archive fetches and decrypts its metadata and item-metadata objects.""" + from .archive import Archive # avoid circular import (archive.py imports from cache.py) + + archive = Archive(manifest, archive_id, iec=iec) + ids = HashTableNT(key_size=32, value_type=ArchiveReferenceEntry, value_format=ArchiveReferenceEntryFormat) + # archive metadata objects: only their ids matter for GC, their content size is unknown here + # and not part of the source data size, so record them with size 0. + ids[archive.id] = ArchiveReferenceEntry(size=0) + for id in archive.metadata.item_ptrs: + ids[id] = ArchiveReferenceEntry(size=0) + for id in archive.metadata.items: + ids[id] = ArchiveReferenceEntry(size=0) + file_count, content_size = 0, 0 + for item in archive.iter_items(): + file_count += 1 # every fs object counts, not just regular files + if "chunks" in item: + for id, size in item.chunks: + content_size += size # original, uncompressed content size, counted per occurrence + ids[id] = ArchiveReferenceEntry(size=size) + return ArchiveReferences(file_count=file_count, content_size=content_size, ids=ids) + + +def get_archive_references( + repository, manifest, archive_id: bytes, *, cached: bool, iec: bool = False, store: bool = True +): + """Return what the archive references, read from its per-archive cache in the repo if present, + else computed by scanning the archive and (when *store*) cached for next time. + + On a cache hit the archive is not opened at all (that is the point of the cache): loading an + Archive fetches and decrypts its metadata and item-metadata objects, which is exactly the work + we want to skip for an unchanged archive. + """ + references = load_archive_references(repository, archive_id) if cached else None + if references is None: + references = scan_archive_references(manifest, archive_id, iec=iec) + if store: + store_archive_references(repository, archive_id, references) + return references + + class ChunksMixin: """ Chunks index related code for misc. Cache implementations. diff --git a/src/borg/testsuite/archiver/analyze_cmd_test.py b/src/borg/testsuite/archiver/analyze_cmd_test.py index eb6f6463c7..7afd3f3797 100644 --- a/src/borg/testsuite/archiver/analyze_cmd_test.py +++ b/src/borg/testsuite/archiver/analyze_cmd_test.py @@ -1,6 +1,10 @@ import pathlib +import re + +import pytest from ...constants import * # NOQA +from ...helpers import Error from . import cmd, generate_archiver_tests, RK_ENCRYPTION pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local") # NOQA @@ -39,3 +43,152 @@ def analyze_archives(): create_archive() assert "/input: 7" in analyze_archives() # 2nd/3rd archives added 2, 4th archive removed 1 + + +def test_analyze_dedup_size(archivers, request): + """borg analyze reports the deduplicated and exclusive size of a considered set of archives (#5741).""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + + # each small file becomes a single chunk of exactly its plaintext size (1000 bytes). + # "shared" is identical in the userA and userB archives, so it deduplicates across the sets. + (input_path / "shared").write_text("s" * 1000) + (input_path / "a_only").write_text("a" * 1000) + cmd(archiver, "create", "userA-1", archiver.input_path) + cmd(archiver, "create", "userA-2", archiver.input_path) # same content as userA-1 + + (input_path / "a_only").unlink() + (input_path / "b_only").write_text("b" * 1000) + cmd(archiver, "create", "userB-1", archiver.input_path) + + # set = the two userA archives; rest = userB-1. + # union of userA chunks = {shared, a_only} -> 2000 B deduplicated size of the set. + # exclusive = chunks only in the set = {a_only} -> 1000 B (shared is also in userB-1). + output = cmd(archiver, "analyze", "-a", "sh:userA-*") + assert "Considered archives: 2 (of 3 in the repository)" in output + assert re.search(r"Deduplicated size of set:\s*2\.00 kB", output) + assert re.search(r"Exclusive size of set:\s*1\.00 kB", output) + # every chunk is referenced by one of the three archives + assert re.search(r"Unreferenced: 0 of \d+ chunks", output) + + # after deleting userB-1, the userA archives are all archives there are: nothing is left over + # to compare against, so this is reported as the whole repository, without the (then trivially + # equal) exclusive size. + cmd(archiver, "delete", "-a", "sh:userB-*") + output = cmd(archiver, "analyze", "-a", "sh:userA-*") + assert "Deduplicated size of the whole repository" in output + assert "Archives: 2 (all archives in the repository)" in output + assert re.search(r"Deduplicated size:\s*2\.00 kB", output) + assert "Exclusive size" not in output + # userB-1 is soft-deleted now, so the chunks only it referenced (b_only and its + # metadata objects) are referenced by no non-deleted archive anymore. + assert re.search(r"Unreferenced: [1-9]\d* of \d+ chunks", output) + + +def test_analyze_whole_repository(archivers, request): + """Without an archive filter, all archives are considered: report the repository as a whole.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + (input_path / "file1").write_text("x" * 1000) + cmd(archiver, "create", "one", archiver.input_path) + (input_path / "file2").write_text("y" * 1000) + cmd(archiver, "create", "two", archiver.input_path) + + output = cmd(archiver, "analyze") + assert "Deduplicated size of the whole repository" in output + assert "Archives: 2 (all archives in the repository)" in output + assert re.search(r"Deduplicated size:\s*2\.00 kB", output) + # every chunk is trivially exclusive to "all archives", so that line is suppressed + assert "Exclusive size" not in output + + +def test_analyze_by_name(archivers, request): + """--by-name decomposes the repository into per-name exclusive, shared and unreferenced.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + + # name "alpha" (2 archives, a series) and name "beta" (1 archive) share "shared"; + # each has one unique file. + (input_path / "shared").write_text("s" * 1000) + (input_path / "a_only").write_text("a" * 1000) + cmd(archiver, "create", "alpha", archiver.input_path) + cmd(archiver, "create", "alpha", archiver.input_path) # same name = same series + + (input_path / "a_only").unlink() + (input_path / "b_only").write_text("b" * 1000) + cmd(archiver, "create", "beta", archiver.input_path) + + output = cmd(archiver, "analyze", "--by-name") + assert "3 archive(s) with 2 distinct name(s)" in output + # exclusive to each name is its own 1000 B file; "shared" is used by archives of both names + assert re.search(r"^alpha\s+2\s+1\.00 kB", output, re.MULTILINE) + assert re.search(r"^beta\s+1\s+1\.00 kB", output, re.MULTILINE) + assert re.search(r"\(shared by 2\+ names\)\s+1\.00 kB", output) + # the rows add up: 1000 (alpha) + 1000 (beta) + 1000 (shared) = 3000 B + assert re.search(r"total \(deduplicated\)\s+3\s+3\.00 kB", output) + + +def test_analyze_by_name_rejects_filters(archivers, request): + """--by-name is repository-wide, so combining it with an archive filter is an error.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + (input_path / "file1").write_text("x" * 1000) + cmd(archiver, "create", "one", archiver.input_path) + + with pytest.raises(Error, match="cannot be combined with archive filters"): + cmd(archiver, "analyze", "--by-name", "-a", "sh:one") + + +def test_analyze_unreferenced_chunks(archivers, request): + """analyze reports chunks referenced by no non-deleted archive - what compact could free.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + (input_path / "keep").write_text("k" * 1000) + cmd(archiver, "create", "keeper", archiver.input_path) + (input_path / "gone").write_text("g" * 1000) + cmd(archiver, "create", "goner", archiver.input_path) + + # nothing deleted yet: all chunks are referenced + output = cmd(archiver, "analyze", "-a", "sh:keeper") + assert re.search(r"Unreferenced chunks:\s*n/a\s*0 B", output) + assert re.search(r"Unreferenced: 0 of \d+ chunks", output) + + # deleting "goner" orphans its chunks; compact has not run yet, so they are still there + cmd(archiver, "delete", "-a", "sh:goner") + output = cmd(archiver, "analyze", "-a", "sh:keeper") + match = re.search(r"Unreferenced: (\d+) of \d+ chunks", output) + assert match and int(match.group(1)) > 0 + + # compact frees exactly those chunks, so afterwards nothing is unreferenced anymore + cmd(archiver, "compact") + output = cmd(archiver, "analyze", "-a", "sh:keeper") + assert re.search(r"Unreferenced: 0 of \d+ chunks", output) + + +def test_analyze_dedup_size_single_archive(archivers, request): + """A single matching archive still reports dedup sizes and skips the >=2-archive hot-spot report.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + (input_path / "file1").write_text("x" * 1000) + cmd(archiver, "create", "only-1", archiver.input_path) + # a second archive that is not selected, so the selection is a real subset + (input_path / "file2").write_text("y" * 1000) + cmd(archiver, "create", "other-1", archiver.input_path) + + output = cmd(archiver, "analyze", "-a", "sh:only-*") + assert "Considered archives: 1 (of 2 in the repository)" in output + assert re.search(r"Deduplicated size of set:\s*1\.00 kB", output) + # file1 is also in other-1, so nothing is exclusive to only-1 + assert re.search(r"Exclusive size of set:\s*0 B", output)