diff --git a/src/borg/archive.py b/src/borg/archive.py index b17685429c..c144d886e3 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -29,8 +29,8 @@ from .helpers import parse_timestamp, to_localtime from .helpers import format_time, format_timedelta, format_file_size, file_status from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates -from .helpers import decode_dict, StableDict -from .helpers import int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import StableDict +from .helpers import bin_to_hex from .helpers import ProgressIndicatorPercent, log_multi from .helpers import PathPrefixPattern, FnmatchPattern from .helpers import consume @@ -964,44 +964,37 @@ def unpack_next(): class ArchiveChecker: + def __init__(self, args, repository): + self.args = args + self.repository = repository - def __init__(self): self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, archive=None, last=None, prefix=None, verify_data=False, - save_space=False): - """Perform a set of checks on 'repository' - - :param repair: enable repair mode, write updated or corrected data into repository - :param archive: only check this archive - :param last: only check this number of recent archives - :param prefix: only check archives with this prefix - :param verify_data: integrity verification of data referenced by archives - :param save_space: Repository.commit(save_space) - """ - logger.info('Starting archive consistency check...') - self.check_all = archive is None and last is None and prefix is None - self.repair = repair - self.repository = repository - self.init_chunks() - self.key = self.identify_key(repository) - if verify_data: - self.verify_data() + self.chunks = self.init_chunks() + self.key = self.identify_key(self.repository) if Manifest.MANIFEST_ID not in self.chunks: logger.error("Repository manifest not found!") self.error_found = True self.manifest = self.rebuild_manifest() else: - self.manifest, _ = Manifest.load(repository, key=self.key) - self.rebuild_refcounts(archive=archive, last=last, prefix=prefix) - self.orphan_chunks_check() - self.finish(save_space=save_space) - if self.error_found: - logger.error('Archive consistency check complete, problems found.') - else: - logger.info('Archive consistency check complete, no problems found.') - return self.repair or not self.error_found + self.manifest, _ = Manifest.load(self.repository, key=self.key) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is None and self.args.repair: + self.manifest.write() + self.repository.commit(save_space=self.args.save_space) + + def check(self, archive_info): + """Perform a set of checks on 'repository' + + :param archive_info: check this archive + :type archive_info: :class:`borg.helpers.ArchiveInfo` + """ + self.rebuild_refcounts(archive_info) def init_chunks(self): """Fetch a list of all object keys from repository @@ -1009,7 +1002,7 @@ def init_chunks(self): # Explicitly set the initial hash table capacity to avoid performance issues # due to hash table "resonance" capacity = int(len(self.repository) * 1.35 + 1) # > len * 1.0 / HASH_MAX_LOAD (see _hashindex.c) - self.chunks = ChunkIndex(capacity) + chunks = ChunkIndex(capacity) marker = None while True: result = self.repository.list(limit=10000, marker=marker) @@ -1018,7 +1011,8 @@ def init_chunks(self): marker = result[-1] init_entry = ChunkIndexEntry(refcount=0, size=0, csize=0) for id_ in result: - self.chunks[id_] = init_entry + chunks[id_] = init_entry + return chunks def identify_key(self, repository): try: @@ -1039,7 +1033,6 @@ def verify_data(self): try: encrypted_data = self.repository.get(chunk_id) except Repository.ObjectNotFound: - self.error_found = True errors += 1 logger.error('chunk %s not found', bin_to_hex(chunk_id)) continue @@ -1047,12 +1040,12 @@ def verify_data(self): _chunk_id = None if chunk_id == Manifest.MANIFEST_ID else chunk_id _, data = self.key.decrypt(_chunk_id, encrypted_data) except IntegrityError as integrity_error: - self.error_found = True errors += 1 logger.error('chunk %s, integrity error: %s', bin_to_hex(chunk_id), integrity_error) pi.finish() log = logger.error if errors else logger.info log('Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.', count, errors) + self.error_found |= bool(errors) def rebuild_manifest(self): """Rebuild the manifest object if it is missing @@ -1095,10 +1088,11 @@ def valid_archive(obj): logger.info('Manifest rebuild complete.') return manifest - def rebuild_refcounts(self, archive=None, last=None, prefix=None): + def rebuild_refcounts(self, archive_info): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected + :type archive_info: :class:`borg.helpers.ArchiveInfo` """ # Exclude the manifest from chunks del self.chunks[Manifest.MANIFEST_ID] @@ -1119,7 +1113,7 @@ def add_reference(id_, size, csize, cdata=None): except KeyError: assert cdata is not None self.chunks[id_] = ChunkIndexEntry(refcount=1, size=size, csize=csize) - if self.repair: + if self.args.repair: self.repository.put(id_, cdata) def verify_file_chunks(item): @@ -1228,73 +1222,45 @@ def valid_item(obj): raise i += 1 - if archive is None: - # we need last N or all archives - archive_infos = self.manifest.archives.list(sort_by='ts', reverse=True) - if prefix is not None: - archive_infos = [info for info in archive_infos if info.name.startswith(prefix)] - num_archives = len(archive_infos) - end = None if last is None else min(num_archives, last) - else: - # we only want one specific archive - info = self.manifest.archives.get(archive) - if info is None: - logger.error("Archive '%s' not found.", archive) - archive_infos = [] - else: - archive_infos = [info] - num_archives = 1 - end = 1 - with cache_if_remote(self.repository) as repository: - for i, info in enumerate(archive_infos[:end]): - logger.info('Analyzing archive {} ({}/{})'.format(info.name, num_archives - i, num_archives)) - archive_id = info.id - if archive_id not in self.chunks: - logger.error('Archive metadata block is missing!') - self.error_found = True - del self.manifest.archives[info.name] - continue - mark_as_possibly_superseded(archive_id) - cdata = self.repository.get(archive_id) - _, data = self.key.decrypt(archive_id, cdata) - archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) - if archive.version != 1: - raise Exception('Unknown archive metadata version') - archive.cmdline = [safe_decode(arg) for arg in archive.cmdline] - items_buffer = ChunkBuffer(self.key) - items_buffer.write_chunk = add_callback - for item in robust_iterator(archive): - if 'chunks' in item: - verify_file_chunks(item) - items_buffer.add(item) - items_buffer.flush(flush=True) - for previous_item_id in archive.items: - mark_as_possibly_superseded(previous_item_id) - archive.items = items_buffer.chunks - data = msgpack.packb(archive.as_dict(), unicode_errors='surrogateescape') - new_archive_id = self.key.id_hash(data) - cdata = self.key.encrypt(Chunk(data)) - add_reference(new_archive_id, len(data), len(cdata), cdata) - self.manifest.archives[info.name] = (new_archive_id, info.ts) - - def orphan_chunks_check(self): - if self.check_all: - unused = {id_ for id_, entry in self.chunks.iteritems() if entry.refcount == 0} - orphaned = unused - self.possibly_superseded - if orphaned: - logger.error('{} orphaned objects found!'.format(len(orphaned))) + archive_id = archive_info.id + if archive_id not in self.chunks: + logger.error('Archive metadata block is missing!') self.error_found = True - if self.repair: - for id_ in unused: - self.repository.delete(id_) - else: - logger.info('Orphaned objects check skipped (needs all archives checked).') + del self.manifest.archives[archive_info.name] + return + mark_as_possibly_superseded(archive_id) + cdata = self.repository.get(archive_id) + _, data = self.key.decrypt(archive_id, cdata) + archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) + if archive.version != 1: + raise Exception('Unknown archive metadata version') + archive.cmdline = [safe_decode(arg) for arg in archive.cmdline] + items_buffer = ChunkBuffer(self.key) + items_buffer.write_chunk = add_callback + for item in robust_iterator(archive): + if 'chunks' in item: + verify_file_chunks(item) + items_buffer.add(item) + items_buffer.flush(flush=True) + for previous_item_id in archive.items: + mark_as_possibly_superseded(previous_item_id) + archive.items = items_buffer.chunks + data = msgpack.packb(archive.as_dict(), unicode_errors='surrogateescape') + new_archive_id = self.key.id_hash(data) + cdata = self.key.encrypt(Chunk(data)) + add_reference(new_archive_id, len(data), len(cdata), cdata) + self.manifest.archives[archive_info.name] = (new_archive_id, archive_info.ts) - def finish(self, save_space=False): - if self.repair: - self.manifest.write() - self.repository.commit(save_space=save_space) + def orphan_chunks_check(self): + unused = {id_ for id_, entry in self.chunks.iteritems() if entry.refcount == 0} + orphaned = unused - self.possibly_superseded + if orphaned: + logger.error('{} orphaned objects found!'.format(len(orphaned))) + self.error_found = True + if self.args.repair: + for id_ in unused: + self.repository.delete(id_) class ArchiveRecreater: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 619dbd7e97..5ff1f1e7b5 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -16,6 +16,7 @@ from binascii import unhexlify from datetime import datetime from itertools import zip_longest +from operator import attrgetter from .logger import create_logger, setup_logging logger = create_logger() @@ -28,7 +29,8 @@ from .constants import * # NOQA from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError -from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec, PrefixSpec +from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec +from .helpers import prefix_spec, sort_by_spec, HUMAN_SORT_KEYS from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive from .helpers import safe_encode, remove_surrogates, bin_to_hex from .helpers import prune_within, prune_split @@ -202,18 +204,55 @@ def do_check(self, args, repository): if not yes(msg, false_msg="Aborting.", truish=('YES', ), env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): return EXIT_ERROR - if args.repo_only and args.verify_data: - self.print_error("--repository-only and --verify-data contradict each other. Please select one.") + + if args.repo_only and any((args.verify_data, args.first, args.last)): + self.print_error("--repository-only contradicts --first, --last " + "and --verify-data arguments.") return EXIT_ERROR + if not args.archives_only: if not repository.check(repair=args.repair, save_space=args.save_space): return EXIT_WARNING - if not args.repo_only and not ArchiveChecker().check( - repository, repair=args.repair, archive=args.location.archive, - last=args.last, prefix=args.prefix, verify_data=args.verify_data, - save_space=args.save_space): - return EXIT_WARNING - return EXIT_SUCCESS + + if not args.repo_only: + logger.info('Starting archive consistency check...') + with ArchiveChecker(args, repository) as archive_checker: + manifest = archive_checker.manifest + all_archives_count = len(manifest.archives.list()) + + if args.verify_data: + archive_checker.verify_data() + + if args.location.archive: + archive_info = manifest.archives.get(args.location.archive) + if archive_info is None: + logger.error('Archive %s not found.' % args.location.archive) + return EXIT_ERROR + archives = (archive_info,) + elif any((args.first, args.last, args.prefix)): + archives = self._get_archives_slice(args, manifest) + else: + archives = manifest.archives.list(prefix=args.prefix) + + for i, archive_info in enumerate(archives, 1): + logger.info('Analyzing archive {} ({}/{}):'.format(archive_info.name, i, len(archives))) + archive_checker.check(archive_info) + + if len(archives) == all_archives_count: + logger.info('Looking for orphaned chunks...') + archive_checker.orphan_chunks_check() + + if archive_checker.error_found: + msg = 'Archive consistency check complete, problems found.' + if args.repair: + logger.info(msg[:-1] + ' and fixed.') + else: + logger.error(msg) + self.exit_code = EXIT_WARNING + else: + logger.info('Archive consistency check complete, no problems found.') + + return self.exit_code @with_repository() def do_change_passphrase(self, args, repository, manifest, key): @@ -729,45 +768,72 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): - """Delete an existing repository or archive""" + """Delete an existing repository or archives""" + if any((args.first, args.last, args.prefix)): + return self._delete_archives(args, repository) if args.location.archive: - manifest, key = Manifest.load(repository) - with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - stats = Statistics() - archive.delete(stats, progress=args.progress, forced=args.forced) - manifest.write() - repository.commit(save_space=args.save_space) - cache.commit() - logger.info("Archive deleted.") - if args.stats: - log_multi(DASHES, - STATS_HEADER, - stats.summary.format(label='Deleted data:', stats=stats), - str(cache), - DASHES, logger=logging.getLogger('borg.output.stats')) + return self._delete_archive(args, repository) else: - if not args.cache_only: - msg = [] - try: - manifest, key = Manifest.load(repository) - except NoManifestError: - msg.append("You requested to completely DELETE the repository *including* all archives it may contain.") - msg.append("This repository seems to have no manifest, so we can't tell anything about its contents.") - else: - msg.append("You requested to completely DELETE the repository *including* all archives it contains:") - for archive_info in manifest.archives.list(sort_by='ts'): - msg.append(format_archive(archive_info)) - msg.append("Type 'YES' if you understand this and want to continue: ") - msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", truish=('YES', ), - env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): - self.exit_code = EXIT_ERROR - return self.exit_code - repository.destroy() - logger.info("Repository deleted.") - Cache.destroy(repository) - logger.info("Cache deleted.") + return self._delete_repository(args, repository) + + def _delete_archive(self, args, repository, manifest=None, key=None): + """Delete a single archive""" + if manifest is None: + manifest, key = Manifest.load(repository) + with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) + stats = Statistics() + archive.delete(stats, progress=args.progress, forced=args.forced) + manifest.write() + repository.commit(save_space=args.save_space) + cache.commit() + logger.info("Archive deleted.") + if args.stats: + log_multi(DASHES, + STATS_HEADER, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES, logger=logging.getLogger('borg.output.stats')) + return self.exit_code + + def _delete_archives(self, args, repository): + """Delete multiple archives""" + manifest, key = Manifest.load(repository) + archives = self._get_archives_slice(args, manifest) + for i, archive_info in enumerate(archives, 1): + logger.info('Deleting {} ({}/{}):'.format(archive_info.name, i, len(archives))) + args.location.archive = archive_info.name + self._delete_archive(args, repository, manifest, key) + if self.exit_code: + break + return self.exit_code + + def _delete_repository(self, args, repository): + """Delete a repository""" + if not args.cache_only: + msg = [] + try: + manifest, key = Manifest.load(repository) + except NoManifestError: + msg.append("You requested to completely DELETE the repository *including* all archives it may " + "contain.") + msg.append("This repository seems to have no manifest, so we can't tell anything about its " + "contents.") + else: + msg.append("You requested to completely DELETE the repository *including* all archives it " + "contains:") + for archive_info in manifest.archives.list(sort_by='ts'): + msg.append(format_archive(archive_info)) + msg.append("Type 'YES' if you understand this and want to continue: ") + msg = '\n'.join(msg) + if not yes(msg, false_msg="Aborting.", truish=('YES',), + env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): + self.exit_code = EXIT_ERROR + return self.exit_code + repository.destroy() + logger.info("Repository deleted.") + Cache.destroy(repository) + logger.info("Cache deleted.") return self.exit_code @with_repository() @@ -809,65 +875,103 @@ def write(bytestring): else: write = sys.stdout.buffer.write - if args.location.archive: - matcher, _ = self.build_matcher(args.excludes, args.paths) - with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache, - consider_part_files=args.consider_part_files) - - if args.format is not None: - format = args.format - elif args.short: - format = "{path}{NL}" - else: - format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" - formatter = ItemFormatter(archive, format) - - for item in archive.iter_items(lambda item: matcher.match(item.path)): - write(safe_encode(formatter.format_item(item))) + if any((args.first, args.last, args.prefix)): + return self._list_archives(args, repository, manifest, key, write) + elif args.location.archive: + return self._list_archive(args, repository, manifest, key, write) else: + return self._list_repository(args, manifest, write) + + def _list_archive(self, args, repository, manifest, key, write): + matcher, _ = self.build_matcher(args.excludes, args.paths) + with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache, + consider_part_files=args.consider_part_files) if args.format is not None: format = args.format elif args.short: - format = "{archive}{NL}" + format = "{path}{NL}" else: - format = "{archive:<36} {time} [{id}]{NL}" - formatter = ArchiveFormatter(format) + format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" + formatter = ItemFormatter(archive, format) - for archive_info in manifest.archives.list(sort_by='ts'): - if args.prefix and not archive_info.name.startswith(args.prefix): - continue - write(safe_encode(formatter.format_item(archive_info))) + for item in archive.iter_items(lambda item: matcher.match(item.path)): + write(safe_encode(formatter.format_item(item))) + return self.exit_code + + def _list_archives(self, args, repository, manifest, key, write): + archives = self._get_archives_slice(args, manifest) + for i, archive_info in enumerate(archives, 1): + write('Contents of {} ({}/{}):'.format(archive_info.name, i, len(archives)).encode()) + args.location.archive = archive_info.name + self._list_archive(args, repository, manifest, key, write) + if self.exit_code: + break + if len(archives) - i > 0: + write(b'\n') + return self.exit_code + + def _list_repository(self, args, manifest, write): + if args.format is not None: + format = args.format + elif args.short: + format = "{archive}{NL}" + else: + format = "{archive:<36} {time} [{id}]{NL}" + formatter = ArchiveFormatter(format) + + for archive_info in manifest.archives.list(sort_by=args.sort_by, prefix=args.prefix): + write(safe_encode(formatter.format_item(archive_info))) return self.exit_code @with_repository(cache=True) def do_info(self, args, repository, manifest, key, cache): """Show archive details such as disk space used""" + if any((args.first, args.last, args.prefix)): + return self._info_archives(args, repository, manifest, key, cache) + elif args.location.archive: + return self._info_archive(args, repository, manifest, key, cache) + else: + return self._info_repository(cache) + + def _info_archive(self, args, repository, manifest, key, cache): def format_cmdline(cmdline): return remove_surrogates(' '.join(shlex.quote(x) for x in cmdline)) - if args.location.archive: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache, - consider_part_files=args.consider_part_files) - stats = archive.calc_stats(cache) - print('Archive name: %s' % archive.name) - print('Archive fingerprint: %s' % archive.fpr) - print('Comment: %s' % archive.metadata.get('comment', '')) - print('Hostname: %s' % archive.metadata.hostname) - print('Username: %s' % archive.metadata.username) - print('Time (start): %s' % format_time(to_localtime(archive.ts))) - print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) - print('Duration: %s' % archive.duration_from_meta) - print('Number of files: %d' % stats.nfiles) - print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) - print(DASHES) - print(STATS_HEADER) - print(str(stats)) - print(str(cache)) - else: - print(STATS_HEADER) - print(str(cache)) + archive = Archive(repository, key, manifest, args.location.archive, cache=cache, + consider_part_files=args.consider_part_files) + stats = archive.calc_stats(cache) + print('Archive name: %s' % archive.name) + print('Archive fingerprint: %s' % archive.fpr) + print('Comment: %s' % archive.metadata.get('comment', '')) + print('Hostname: %s' % archive.metadata.hostname) + print('Username: %s' % archive.metadata.username) + print('Time (start): %s' % format_time(to_localtime(archive.ts))) + print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) + print('Duration: %s' % archive.duration_from_meta) + print('Number of files: %d' % stats.nfiles) + print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) + print(DASHES) + print(STATS_HEADER) + print(str(stats)) + print(str(cache)) + return self.exit_code + + def _info_archives(self, args, repository, manifest, key, cache): + archives = self._get_archives_slice(args, manifest) + for i, archive_info in enumerate(archives, 1): + args.location.archive = archive_info.name + self._info_archive(args, repository, manifest, key, cache) + if self.exit_code: + break + if len(archives) - i > 0: + print() + return self.exit_code + + def _info_repository(self, cache): + print(STATS_HEADER) + print(str(cache)) return self.exit_code @with_repository(exclusive=True) @@ -1470,14 +1574,10 @@ def build_parser(self, prog=None): subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') - subparser.add_argument('--last', dest='last', - type=int, default=None, metavar='N', - help='only check last N archives (Default: all)') - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, - help='only consider archive names starting with this prefix') subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, help="""show progress display while checking""") + self.add_archives_filter_args(subparser) change_passphrase_epilog = textwrap.dedent(""" The key files used for repository encryption are optionally passphrase @@ -1780,6 +1880,7 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') + self.add_archives_filter_args(subparser) list_epilog = textwrap.dedent(""" This command lists the contents of a repository or an archive. @@ -1806,8 +1907,6 @@ def build_parser(self, prog=None): subparser.add_argument('--format', '--list-format', dest='format', type=str, help="""specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""") - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, - help='only consider archive names starting with this prefix') subparser.add_argument('-e', '--exclude', dest='excludes', type=parse_pattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') @@ -1819,6 +1918,7 @@ def build_parser(self, prog=None): help='repository/archive to list contents of') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to list; patterns are supported') + self.add_archives_filter_args(subparser) mount_epilog = textwrap.dedent(""" This command mounts an archive as a FUSE filesystem. This can be useful for @@ -1879,6 +1979,7 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), help='archive or repository to display information about') + self.add_archives_filter_args(subparser) break_lock_epilog = textwrap.dedent(""" This command breaks the repository and cache locks. @@ -1967,7 +2068,7 @@ def build_parser(self, prog=None): help='number of monthly archives to keep') subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, help='number of yearly archives to keep') - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', help='only consider archive names starting with this prefix') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, @@ -2280,6 +2381,22 @@ def build_parser(self, prog=None): help='hex object ID(s) to delete from the repo') return parser + @staticmethod + def add_archives_filter_args(subparser): + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', + help='only consider archive names starting with this prefix') + + sort_by_default = 'timestamp' + subparser.add_argument('--sort-by', dest='sort_by', type=sort_by_spec, default=sort_by_default, + help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' + .format(HUMAN_SORT_KEYS, sort_by_default)) + + group = subparser.add_mutually_exclusive_group() + group.add_argument('--first', dest='first', metavar='N', default=0, type=int, + help='delete N first archives') + group.add_argument('--last', dest='last', metavar='N', default=0, type=int, + help='delete N last archives') + def get_args(self, argv, cmd): """usually, just returns argv, except if we deal with a ssh forced command for borg serve.""" result = self.parse_args(argv[1:]) @@ -2342,6 +2459,28 @@ def run(self, args): logger.warning("Using a pure-python msgpack! This will result in lower performance.") return args.func(args) + def _get_archives_slice(self, args, manifest): + if args.location.archive: + logger.error('The options --prefix, --first and --last can only used on repository targets.') + self.exit_code = EXIT_ERROR + return [] + + archives = manifest.archives.list(prefix=args.prefix) + + if not archives: + logger.error('There are no archives.') + self.exit_code = EXIT_ERROR + return [] + + for sortkey in reversed(args.sort_by.split(',')): + archives.sort(key=attrgetter(sortkey)) + if args.last: + archives.reverse() + + n = args.first or args.last or len(archives) + + return archives[:n] + def sig_info_handler(signum, stack): # pragma: no cover """search the stack for infos about the currently processed file and print them""" diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 3d30692f25..4879ed0885 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -142,9 +142,12 @@ def __delitem__(self, name): name = safe_encode(name) del self._archives[name] - def list(self, sort_by=None, reverse=False): - # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts - archives = self.values() # [self[name] for name in self] + def list(self, sort_by=None, reverse=False, prefix=''): + """ Inexpensive Archive.list_archives replacement if we just need .name, .id, .ts + + :rtype: A :class:`list` of :class:`borg.helpers.ArchiveInfo` instances + """ + archives = [x for x in self.values() if x.name.startswith(prefix)] if sort_by is not None: archives = sorted(archives, key=attrgetter(sort_by), reverse=reverse) return archives @@ -557,10 +560,6 @@ def CompressionSpec(s): raise ValueError -def PrefixSpec(s): - return replace_placeholders(s) - - def dir_is_cachedir(path): """Determines whether the specified path is a cache directory (and therefore should potentially be excluded from the backup) according to @@ -633,6 +632,18 @@ def replace_placeholders(text): } return format_line(text, data) +prefix_spec = replace_placeholders + + +HUMAN_SORT_KEYS = ['timestamp'] + list(ArchiveInfo._fields) +HUMAN_SORT_KEYS.remove('ts') + +def sort_by_spec(text): + for token in text.split(','): + if token not in HUMAN_SORT_KEYS: + raise ValueError('Invalid sort key: %s' % token) + return text.replace('timestamp', 'ts') + def safe_timestamp(item_timestamp_ns): try: diff --git a/src/borg/repository.py b/src/borg/repository.py index 9eebd90e6a..2dbde2f901 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -685,8 +685,6 @@ def report_error(msg): return not error_found or repair def rollback(self, cleanup=False): - """ - """ if cleanup: self.io.cleanup(self.io.get_segments_transaction_id()) self.index = None diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 9d68e3eea2..49996974c2 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -57,6 +57,9 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): except subprocess.CalledProcessError as e: output = e.output ret = e.returncode + except SystemExit as e: + output = '' + ret = e.code return ret, os.fsdecode(output) else: stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr @@ -953,6 +956,8 @@ def test_info(self): assert 'All archives:' in info_repo info_archive = self.cmd('info', self.repository_location + '::test') assert 'Archive name: test\n' in info_archive + info_archive = self.cmd('info', '--first', '1', self.repository_location) + assert 'Archive name: test\n' in info_archive def test_comment(self): self.create_regular_file('file1', size=1024 * 80) @@ -979,8 +984,13 @@ def test_delete(self): self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') self.cmd('create', self.repository_location + '::test.2', 'input') + self.cmd('create', self.repository_location + '::test.3', 'input') + self.cmd('create', self.repository_location + '::another_test.1', 'input') + self.cmd('create', self.repository_location + '::another_test.2', 'input') self.cmd('extract', '--dry-run', self.repository_location + '::test') self.cmd('extract', '--dry-run', self.repository_location + '::test.2') + self.cmd('delete', '--prefix', 'another_', self.repository_location) + self.cmd('delete', '--last', '1', self.repository_location) self.cmd('delete', self.repository_location + '::test') self.cmd('extract', '--dry-run', self.repository_location + '::test.2') output = self.cmd('delete', '--stats', self.repository_location + '::test.2') @@ -1772,6 +1782,13 @@ def test_recreate_list_output(self): self.assert_not_in("input/file1", output) self.assert_not_in("x input/file5", output) + def test_bad_filters(self): + self.cmd('init', self.repository_location) + self.cmd('delete', '--last', '1', self.repository_location, exit_code=2) + self.cmd('create', self.repository_location + '::test', 'input') + self.cmd('delete', '--first', '1', '--last', '1', self.repository_location, fork=True, exit_code=2) + self.cmd('delete', '--last', '1', self.repository_location + '::test', exit_code=2) + @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase): @@ -1817,21 +1834,26 @@ def setUp(self): self.create_src_archive('archive2') def test_check_usage(self): - output = self.cmd('check', '-v', '--progress', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--progress', self.repository_location) self.assert_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) self.assert_in('Checking segments', output) # reset logging to new process default to avoid need for fork=True on next check logging.getLogger('borg.output.progress').setLevel(logging.NOTSET) - output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--repository-only', self.repository_location) self.assert_in('Starting repository check', output) self.assert_not_in('Starting archive consistency check', output) self.assert_not_in('Checking segments', output) - output = self.cmd('check', '-v', '--archives-only', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--archives-only', self.repository_location) self.assert_not_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) - output = self.cmd('check', '-v', '--archives-only', '--prefix=archive2', self.repository_location, exit_code=0) + output = self.cmd('check', '--repository-only', '--verify-data', self.repository_location, exit_code=2) + self.assert_in('contradicts', output) + output = self.cmd('check', '-v', '--archives-only', '--prefix=archive2', self.repository_location) self.assert_not_in('archive1', output) + output = self.cmd('check', '-v', self.repository_location + '::archive1') + self.assert_in('Starting archive consistency check', output) + self.assert_not_in('archive2', output) def test_missing_file_chunk(self): archive, repository = self.open_archive('archive1')