From 62098afe71eca7f9353bfd2144145f95128fdfce Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 23 Jul 2016 21:02:55 +0200 Subject: [PATCH 01/13] Extends delete cmd w/ --oldest & --latest --- src/borg/archiver.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 619dbd7e97..5e171e12d5 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -730,8 +730,26 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): """Delete an existing repository or archive""" + manifest = None + + if args.oldest or args.latest: + if args.location.archive: + logger.error('The options --oldest and --latest have no effect on archive targets.') + return EXIT_ERROR + else: + manifest, key = Manifest.load(repository) + archives = manifest.list_archive_infos('ts', reverse=args.latest) + if archives: + archive_name = archives[0].name + logger.info('Resolved archive: %s' % archive_name) + args.location.archive = archive_name + else: + logger.error('There are no archives.') + return EXIT_ERROR + if args.location.archive: - manifest, key = Manifest.load(repository) + if manifest is None: + manifest, key = Manifest.load(repository) with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = Statistics() @@ -752,10 +770,13 @@ def do_delete(self, args, repository): try: manifest, key = Manifest.load(repository) except NoManifestError: - msg.append("You requested to completely DELETE the repository *including* all archives it may contain.") - msg.append("This repository seems to have no manifest, so we can't tell anything about its contents.") + msg.append("You requested to completely DELETE the repository *including* all archives it may " + "contain.") + msg.append("This repository seems to have no manifest, so we can't tell anything about its " + "contents.") else: - msg.append("You requested to completely DELETE the repository *including* all archives it contains:") + msg.append("You requested to completely DELETE the repository *including* all archives it " + "contains:") for archive_info in manifest.archives.list(sort_by='ts'): msg.append(format_archive(archive_info)) msg.append("Type 'YES' if you understand this and want to continue: ") From f761ade236b219314f2fe29709d6e0a6fa858bb8 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 24 Jul 2016 00:40:15 +0200 Subject: [PATCH 02/13] Uncomplexifies `Archiver.do_delete` --- src/borg/archiver.py | 123 ++++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 55 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 5e171e12d5..fe3372f1df 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -730,65 +730,78 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): """Delete an existing repository or archive""" - manifest = None - if args.oldest or args.latest: - if args.location.archive: - logger.error('The options --oldest and --latest have no effect on archive targets.') - return EXIT_ERROR - else: - manifest, key = Manifest.load(repository) - archives = manifest.list_archive_infos('ts', reverse=args.latest) - if archives: - archive_name = archives[0].name - logger.info('Resolved archive: %s' % archive_name) - args.location.archive = archive_name - else: - logger.error('There are no archives.') - return EXIT_ERROR + return self._delete_archives(args, repository) + if args.location.archive: + return self._delete_archive(args, repository) + else: + return self._delete_repository(args, repository) + + def _delete_archive(self, args, repository, manifest=None): + """Delete a single archive""" + if manifest is None: + manifest, key = Manifest.load(repository) + with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) + stats = Statistics() + archive.delete(stats, progress=args.progress, forced=args.forced) + manifest.write() + repository.commit(save_space=args.save_space) + cache.commit() + logger.info("Archive deleted.") + if args.stats: + log_multi(DASHES, + STATS_HEADER, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES, logger=logging.getLogger('borg.output.stats')) + return self.exit_code + def _delete_archives(self, args, repository): + """Delete multiple archives""" if args.location.archive: - if manifest is None: - manifest, key = Manifest.load(repository) - with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - stats = Statistics() - archive.delete(stats, progress=args.progress, forced=args.forced) - manifest.write() - repository.commit(save_space=args.save_space) - cache.commit() - logger.info("Archive deleted.") - if args.stats: - log_multi(DASHES, - STATS_HEADER, - stats.summary.format(label='Deleted data:', stats=stats), - str(cache), - DASHES, logger=logging.getLogger('borg.output.stats')) + logger.error('The options --oldest and --latest have no effect on archive targets.') + self.exit_code = EXIT_ERROR else: - if not args.cache_only: - msg = [] - try: - manifest, key = Manifest.load(repository) - except NoManifestError: - msg.append("You requested to completely DELETE the repository *including* all archives it may " - "contain.") - msg.append("This repository seems to have no manifest, so we can't tell anything about its " - "contents.") - else: - msg.append("You requested to completely DELETE the repository *including* all archives it " - "contains:") - for archive_info in manifest.archives.list(sort_by='ts'): - msg.append(format_archive(archive_info)) - msg.append("Type 'YES' if you understand this and want to continue: ") - msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", truish=('YES', ), - env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): - self.exit_code = EXIT_ERROR - return self.exit_code - repository.destroy() - logger.info("Repository deleted.") - Cache.destroy(repository) - logger.info("Cache deleted.") + manifest, key = Manifest.load(repository) + archives = manifest.list_archive_infos('ts', reverse=args.latest)[0] + for archive in archives: + logger.info('Deleting %s' % archive.name) + args.location.archive = archive.name + self._delete_archive(args, repository, manifest) + if self.exit_code: + break + else: + logger.error('There are no archives.') + self.exit_code = EXIT_ERROR + return self.exit_code + + def _delete_repository(self, args, repository): + """Delete a repository""" + if not args.cache_only: + msg = [] + try: + manifest, key = Manifest.load(repository) + except NoManifestError: + msg.append("You requested to completely DELETE the repository *including* all archives it may " + "contain.") + msg.append("This repository seems to have no manifest, so we can't tell anything about its " + "contents.") + else: + msg.append("You requested to completely DELETE the repository *including* all archives it " + "contains:") + for archive_info in manifest.archives.list(sort_by='ts'): + msg.append(format_archive(archive_info)) + msg.append("Type 'YES' if you understand this and want to continue: ") + msg = '\n'.join(msg) + if not yes(msg, false_msg="Aborting.", truish=('YES',), + env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): + self.exit_code = EXIT_ERROR + return self.exit_code + repository.destroy() + logger.info("Repository deleted.") + Cache.destroy(repository) + logger.info("Cache deleted.") return self.exit_code @with_repository() From baa59df2f7a01dec3a37dd36ce72d7ceb4677879 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 24 Jul 2016 01:52:54 +0200 Subject: [PATCH 03/13] Makes delete --oldest and --latest take a quantity --- src/borg/archiver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index fe3372f1df..300aa392f7 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -764,7 +764,8 @@ def _delete_archives(self, args, repository): self.exit_code = EXIT_ERROR else: manifest, key = Manifest.load(repository) - archives = manifest.list_archive_infos('ts', reverse=args.latest)[0] + n = args.oldest or args.latest + archives = manifest.list_archive_infos('ts', reverse=args.latest)[:n] for archive in archives: logger.info('Deleting %s' % archive.name) args.location.archive = archive.name @@ -1814,6 +1815,11 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') + group = subparser.add_mutually_exclusive_group() + group.add_argument('--oldest', dest='oldest', metavar='n', default=0, type=int, + help='delete n oldest archives') + group.add_argument('--latest', dest='latest', metavar='n', default=0, type=int, + help='delete n latest archives') list_epilog = textwrap.dedent(""" This command lists the contents of a repository or an archive. From 0b50b7d567bdabafbaf148f626fb7aa68db99b7b Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 7 Aug 2016 21:03:44 +0200 Subject: [PATCH 04/13] feedback changes --- src/borg/archiver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 300aa392f7..c63807ed37 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -766,15 +766,15 @@ def _delete_archives(self, args, repository): manifest, key = Manifest.load(repository) n = args.oldest or args.latest archives = manifest.list_archive_infos('ts', reverse=args.latest)[:n] + if not archives: + logger.error('There are no archives.') + self.exit_code = EXIT_ERROR for archive in archives: logger.info('Deleting %s' % archive.name) args.location.archive = archive.name self._delete_archive(args, repository, manifest) if self.exit_code: break - else: - logger.error('There are no archives.') - self.exit_code = EXIT_ERROR return self.exit_code def _delete_repository(self, args, repository): @@ -1816,9 +1816,9 @@ def build_parser(self, prog=None): type=location_validator(), help='archive or repository to delete') group = subparser.add_mutually_exclusive_group() - group.add_argument('--oldest', dest='oldest', metavar='n', default=0, type=int, + group.add_argument('--oldest', dest='oldest', metavar='N', default=0, type=int, help='delete n oldest archives') - group.add_argument('--latest', dest='latest', metavar='n', default=0, type=int, + group.add_argument('--latest', dest='latest', metavar='N', default=0, type=int, help='delete n latest archives') list_epilog = textwrap.dedent(""" From f1c8e8408a7e09e81a57064e7efd6c2159336b7d Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 13 Aug 2016 14:29:23 +0200 Subject: [PATCH 05/13] Adds arcives slice selction for info and list --- src/borg/archiver.py | 174 ++++++++++++++++++++++++++++--------------- 1 file changed, 112 insertions(+), 62 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index c63807ed37..c92a2eed6c 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -729,7 +729,7 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): - """Delete an existing repository or archive""" + """Delete an existing repository or archives""" if args.oldest or args.latest: return self._delete_archives(args, repository) if args.location.archive: @@ -759,22 +759,14 @@ def _delete_archive(self, args, repository, manifest=None): def _delete_archives(self, args, repository): """Delete multiple archives""" - if args.location.archive: - logger.error('The options --oldest and --latest have no effect on archive targets.') - self.exit_code = EXIT_ERROR - else: - manifest, key = Manifest.load(repository) - n = args.oldest or args.latest - archives = manifest.list_archive_infos('ts', reverse=args.latest)[:n] - if not archives: - logger.error('There are no archives.') - self.exit_code = EXIT_ERROR - for archive in archives: - logger.info('Deleting %s' % archive.name) - args.location.archive = archive.name - self._delete_archive(args, repository, manifest) - if self.exit_code: - break + manifest, key = Manifest.load(repository) + archives = self._get_archives_slice(args, manifest) + for i, archive in enumerate(archives): + logger.info('Deleting {} ({}/{}):'.format(archive.name, i+1, len(archives))) + args.location.archive = archive.name + self._delete_archive(args, repository, manifest) + if self.exit_code: + break return self.exit_code def _delete_repository(self, args, repository): @@ -844,30 +836,50 @@ def write(bytestring): else: write = sys.stdout.buffer.write - if args.location.archive: - matcher, _ = self.build_matcher(args.excludes, args.paths) - with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache, - consider_part_files=args.consider_part_files) - - if args.format is not None: - format = args.format - elif args.short: - format = "{path}{NL}" - else: - format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" - formatter = ItemFormatter(archive, format) - - for item in archive.iter_items(lambda item: matcher.match(item.path)): - write(safe_encode(formatter.format_item(item))) + if args.oldest or args.latest: + return self._list_archives(args, repository, manifest, key, write) + elif args.location.archive: + return self._list_archive(args, repository, manifest, key, write) else: + return self._list_repository(args, manifest, write) + + def _list_archive(self, args, repository, manifest, key, write): + matcher, _ = self.build_matcher(args.excludes, args.paths) + with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache, + consider_part_files=args.consider_part_files) if args.format is not None: format = args.format elif args.short: - format = "{archive}{NL}" + format = "{path}{NL}" else: - format = "{archive:<36} {time} [{id}]{NL}" - formatter = ArchiveFormatter(format) + format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" + formatter = ItemFormatter(archive, format) + + for item in archive.iter_items(lambda item: matcher.match(item.path)): + write(safe_encode(formatter.format_item(item))) + return self.exit_code + + def _list_archives(self, args, repository, manifest, key, write): + archives = self._get_archives_slice(args, manifest) + for i, archive in enumerate(archives): + write('Contents of {} ({}/{}):'.format(archive.name, i+1, len(archives))) + args.location.archive = archive.name + self._list_archive(args, repository, manifest, key, write) + if self.exit_code: + break + if len(archives) - i > 1: + write('\n') + return self.exit_code + + def _list_repository(self, args, manifest, write): + if args.format is not None: + format = args.format + elif args.short: + format = "{archive}{NL}" + else: + format = "{archive:<36} {time} [{id}]{NL}" + formatter = ArchiveFormatter(format) for archive_info in manifest.archives.list(sort_by='ts'): if args.prefix and not archive_info.name.startswith(args.prefix): @@ -879,30 +891,50 @@ def write(bytestring): @with_repository(cache=True) def do_info(self, args, repository, manifest, key, cache): """Show archive details such as disk space used""" + if args.oldest or args.latest: + return self._info_archives(args, repository, manifest, key, cache) + elif args.location.archive: + return self._info_archive(args, repository, manifest, key, cache) + else: + return self._info_repository(cache) + + def _info_archive(self, args, repository, manifest, key, cache): def format_cmdline(cmdline): return remove_surrogates(' '.join(shlex.quote(x) for x in cmdline)) - if args.location.archive: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache, - consider_part_files=args.consider_part_files) - stats = archive.calc_stats(cache) - print('Archive name: %s' % archive.name) - print('Archive fingerprint: %s' % archive.fpr) - print('Comment: %s' % archive.metadata.get('comment', '')) - print('Hostname: %s' % archive.metadata.hostname) - print('Username: %s' % archive.metadata.username) - print('Time (start): %s' % format_time(to_localtime(archive.ts))) - print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) - print('Duration: %s' % archive.duration_from_meta) - print('Number of files: %d' % stats.nfiles) - print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) - print(DASHES) - print(STATS_HEADER) - print(str(stats)) - print(str(cache)) - else: - print(STATS_HEADER) - print(str(cache)) + archive = Archive(repository, key, manifest, args.location.archive, cache=cache, + consider_part_files=args.consider_part_files) + stats = archive.calc_stats(cache) + print('Archive name: %s' % archive.name) + print('Archive fingerprint: %s' % archive.fpr) + print('Comment: %s' % archive.metadata.get('comment', '')) + print('Hostname: %s' % archive.metadata.hostname) + print('Username: %s' % archive.metadata.username) + print('Time (start): %s' % format_time(to_localtime(archive.ts))) + print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) + print('Duration: %s' % archive.duration_from_meta) + print('Number of files: %d' % stats.nfiles) + print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) + print(DASHES) + print(STATS_HEADER) + print(str(stats)) + print(str(cache)) + return self.exit_code + + def _info_archives(self, args, repository, manifest, key, cache): + archives = self._get_archives_slice(args, manifest) + for i, archive in enumerate(archives): + args.location.archive = archive.name + self._info_archive(args, repository, manifest, key, cache) + if self.exit_code: + break + if len(archives) - i > 1: + print('\n') + return self.exit_code + + def _info_repository(self, cache): + print(STATS_HEADER) + print(str(cache)) return self.exit_code @with_repository(exclusive=True) @@ -1815,11 +1847,7 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') - group = subparser.add_mutually_exclusive_group() - group.add_argument('--oldest', dest='oldest', metavar='N', default=0, type=int, - help='delete n oldest archives') - group.add_argument('--latest', dest='latest', metavar='N', default=0, type=int, - help='delete n latest archives') + self.add_archives_slice_selection_args(subparser) list_epilog = textwrap.dedent(""" This command lists the contents of a repository or an archive. @@ -1859,6 +1887,7 @@ def build_parser(self, prog=None): help='repository/archive to list contents of') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to list; patterns are supported') + self.add_archives_slice_selection_args(subparser) mount_epilog = textwrap.dedent(""" This command mounts an archive as a FUSE filesystem. This can be useful for @@ -1919,6 +1948,7 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), help='archive or repository to display information about') + self.add_archives_slice_selection_args(subparser) break_lock_epilog = textwrap.dedent(""" This command breaks the repository and cache locks. @@ -2320,6 +2350,14 @@ def build_parser(self, prog=None): help='hex object ID(s) to delete from the repo') return parser + @staticmethod + def add_archives_slice_selection_args(subparser): + group = subparser.add_mutually_exclusive_group() + group.add_argument('--oldest', dest='oldest', metavar='N', default=0, type=int, + help='delete n oldest archives') + group.add_argument('--latest', dest='latest', metavar='N', default=0, type=int, + help='delete n latest archives') + def get_args(self, argv, cmd): """usually, just returns argv, except if we deal with a ssh forced command for borg serve.""" result = self.parse_args(argv[1:]) @@ -2382,6 +2420,18 @@ def run(self, args): logger.warning("Using a pure-python msgpack! This will result in lower performance.") return args.func(args) + def _get_archives_slice(self, args, manifest): + if args.location.archive: + logger.error('The options --oldest and --latest must not be used on archive targets.') + self.exit_code = EXIT_ERROR + return [] + n = args.oldest or args.latest + archives = manifest.list_archive_infos('ts', reverse=args.latest)[:n] + if not archives: + logger.error('There are no archives.') + self.exit_code = EXIT_ERROR + return archives + def sig_info_handler(signum, stack): # pragma: no cover """search the stack for infos about the currently processed file and print them""" From b8c6604de9aaaf2e2610718b1998e01feac6f43b Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 21 Aug 2016 15:36:38 +0200 Subject: [PATCH 06/13] feedback changes --- src/borg/archiver.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index c92a2eed6c..0921d94022 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -761,8 +761,8 @@ def _delete_archives(self, args, repository): """Delete multiple archives""" manifest, key = Manifest.load(repository) archives = self._get_archives_slice(args, manifest) - for i, archive in enumerate(archives): - logger.info('Deleting {} ({}/{}):'.format(archive.name, i+1, len(archives))) + for i, archive in enumerate(archives, 1): + logger.info('Deleting {} ({}/{}):'.format(archive.name, i, len(archives))) args.location.archive = archive.name self._delete_archive(args, repository, manifest) if self.exit_code: @@ -862,14 +862,14 @@ def _list_archive(self, args, repository, manifest, key, write): def _list_archives(self, args, repository, manifest, key, write): archives = self._get_archives_slice(args, manifest) - for i, archive in enumerate(archives): - write('Contents of {} ({}/{}):'.format(archive.name, i+1, len(archives))) + for i, archive in enumerate(archives, 1): + write('Contents of {} ({}/{}):'.format(archive.name, i, len(archives))) args.location.archive = archive.name self._list_archive(args, repository, manifest, key, write) if self.exit_code: break if len(archives) - i > 1: - write('\n') + write() return self.exit_code def _list_repository(self, args, manifest, write): @@ -881,10 +881,10 @@ def _list_repository(self, args, manifest, write): format = "{archive:<36} {time} [{id}]{NL}" formatter = ArchiveFormatter(format) - for archive_info in manifest.archives.list(sort_by='ts'): - if args.prefix and not archive_info.name.startswith(args.prefix): - continue - write(safe_encode(formatter.format_item(archive_info))) + for archive_info in manifest.archives.list(sort_by='ts'): + if args.prefix and not archive_info.name.startswith(args.prefix): + continue + write(safe_encode(formatter.format_item(archive_info))) return self.exit_code @@ -923,13 +923,13 @@ def format_cmdline(cmdline): def _info_archives(self, args, repository, manifest, key, cache): archives = self._get_archives_slice(args, manifest) - for i, archive in enumerate(archives): + for i, archive in enumerate(archives, 1): args.location.archive = archive.name self._info_archive(args, repository, manifest, key, cache) if self.exit_code: break - if len(archives) - i > 1: - print('\n') + if len(archives) - i: + print() return self.exit_code def _info_repository(self, cache): @@ -2354,9 +2354,9 @@ def build_parser(self, prog=None): def add_archives_slice_selection_args(subparser): group = subparser.add_mutually_exclusive_group() group.add_argument('--oldest', dest='oldest', metavar='N', default=0, type=int, - help='delete n oldest archives') + help='delete N oldest archives') group.add_argument('--latest', dest='latest', metavar='N', default=0, type=int, - help='delete n latest archives') + help='delete N latest archives') def get_args(self, argv, cmd): """usually, just returns argv, except if we deal with a ssh forced command for borg serve.""" @@ -2422,11 +2422,12 @@ def run(self, args): def _get_archives_slice(self, args, manifest): if args.location.archive: - logger.error('The options --oldest and --latest must not be used on archive targets.') + logger.error('The options --oldest and --latest can only used on repository targets.') self.exit_code = EXIT_ERROR return [] n = args.oldest or args.latest - archives = manifest.list_archive_infos('ts', reverse=args.latest)[:n] + assert n > 0 + archives = manifest.list_archive_infos('ts', reverse=bool(args.latest))[:n] if not archives: logger.error('There are no archives.') self.exit_code = EXIT_ERROR From ec01058ca1916c2daeb968d1c82521952dc29c68 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 21 Aug 2016 18:19:38 +0200 Subject: [PATCH 07/13] Adds `--sort-by` option - also ensures that `Archives.list` returns a list --- src/borg/archiver.py | 57 ++++++++++++++++++++++++++++---------------- src/borg/helpers.py | 6 ++++- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0921d94022..65328fbb89 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -15,7 +15,8 @@ import traceback from binascii import unhexlify from datetime import datetime -from itertools import zip_longest +from itertools import permutations, zip_longest +from operator import attrgetter from .logger import create_logger, setup_logging logger = create_logger() @@ -28,7 +29,8 @@ from .constants import * # NOQA from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError -from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec, PrefixSpec +from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec +from .helpers import PrefixSpec, sort_by_spec from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive from .helpers import safe_encode, remove_surrogates, bin_to_hex from .helpers import prune_within, prune_split @@ -730,7 +732,7 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): """Delete an existing repository or archives""" - if args.oldest or args.latest: + if args.first or args.last: return self._delete_archives(args, repository) if args.location.archive: return self._delete_archive(args, repository) @@ -759,7 +761,7 @@ def _delete_archive(self, args, repository, manifest=None): def _delete_archives(self, args, repository): """Delete multiple archives""" - manifest, key = Manifest.load(repository) + manifest, _ = Manifest.load(repository) archives = self._get_archives_slice(args, manifest) for i, archive in enumerate(archives, 1): logger.info('Deleting {} ({}/{}):'.format(archive.name, i, len(archives))) @@ -836,7 +838,7 @@ def write(bytestring): else: write = sys.stdout.buffer.write - if args.oldest or args.latest: + if args.first or args.last: return self._list_archives(args, repository, manifest, key, write) elif args.location.archive: return self._list_archive(args, repository, manifest, key, write) @@ -881,9 +883,7 @@ def _list_repository(self, args, manifest, write): format = "{archive:<36} {time} [{id}]{NL}" formatter = ArchiveFormatter(format) - for archive_info in manifest.archives.list(sort_by='ts'): - if args.prefix and not archive_info.name.startswith(args.prefix): - continue + for archive_info in manifest.archives.list(sort_by=args.sort_by): write(safe_encode(formatter.format_item(archive_info))) return self.exit_code @@ -891,7 +891,7 @@ def _list_repository(self, args, manifest, write): @with_repository(cache=True) def do_info(self, args, repository, manifest, key, cache): """Show archive details such as disk space used""" - if args.oldest or args.latest: + if args.first or args.last: return self._info_archives(args, repository, manifest, key, cache) elif args.location.archive: return self._info_archive(args, repository, manifest, key, cache) @@ -1537,10 +1537,7 @@ def build_parser(self, prog=None): subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') - subparser.add_argument('--last', dest='last', - type=int, default=None, metavar='N', - help='only check last N archives (Default: all)') - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', help='only consider archive names starting with this prefix') subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, @@ -2352,11 +2349,21 @@ def build_parser(self, prog=None): @staticmethod def add_archives_slice_selection_args(subparser): + valid_sort_keys = ('timestamp', 'name') + sort_by_default = 'timestamp' + sort_by_choices = [] + for r in range(len(valid_sort_keys)): + sort_by_choices.extend([','.join(x) for x in permutations(valid_sort_keys, r+1)]) + + subparser.add_argument('--sort-by', dest='sort_by', type=sort_by_spec, + choices=sort_by_choices, default=sort_by_default, + help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' + .format(valid_sort_keys, sort_by_default)) group = subparser.add_mutually_exclusive_group() - group.add_argument('--oldest', dest='oldest', metavar='N', default=0, type=int, - help='delete N oldest archives') - group.add_argument('--latest', dest='latest', metavar='N', default=0, type=int, - help='delete N latest archives') + group.add_argument('--first', dest='first', metavar='N', default=0, type=int, + help='delete N first archives') + group.add_argument('--last', dest='last', metavar='N', default=0, type=int, + help='delete N last archives') def get_args(self, argv, cmd): """usually, just returns argv, except if we deal with a ssh forced command for borg serve.""" @@ -2422,16 +2429,24 @@ def run(self, args): def _get_archives_slice(self, args, manifest): if args.location.archive: - logger.error('The options --oldest and --latest can only used on repository targets.') + logger.error('The options --first and --last can only used on repository targets.') self.exit_code = EXIT_ERROR return [] - n = args.oldest or args.latest + n = args.first or args.last assert n > 0 - archives = manifest.list_archive_infos('ts', reverse=bool(args.latest))[:n] + + archives = manifest.archives.list() + if not archives: logger.error('There are no archives.') self.exit_code = EXIT_ERROR - return archives + return [] + + for sortkey in reversed(args.sort_by.split(',')): + archives.sort(key=attrgetter(sortkey)) + if args.last: + archives.reverse() + return archives[:n] def sig_info_handler(signum, stack): # pragma: no cover diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 3d30692f25..ac31eb7c70 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -144,7 +144,7 @@ def __delitem__(self, name): def list(self, sort_by=None, reverse=False): # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts - archives = self.values() # [self[name] for name in self] + archives = list(self.values()) # [self[name] for name in self] if sort_by is not None: archives = sorted(archives, key=attrgetter(sort_by), reverse=reverse) return archives @@ -634,6 +634,10 @@ def replace_placeholders(text): return format_line(text, data) +def sort_by_spec(text): + return text.replace('timestamp', 'ts') + + def safe_timestamp(item_timestamp_ns): try: return datetime.fromtimestamp(bigint_to_int(item_timestamp_ns) / 1e9) From 8f20aee19b49d8ac414f5c6c19e25140bb51aae7 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Mon, 22 Aug 2016 21:30:38 +0200 Subject: [PATCH 08/13] Adds prefix argument to helpers.Archives.list also renames function PrefixSpec to prefix_spec --- src/borg/archiver.py | 15 +++++++++------ src/borg/helpers.py | 10 ++++------ src/borg/repository.py | 2 -- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 65328fbb89..b8b2e32f5b 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -30,7 +30,7 @@ from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec -from .helpers import PrefixSpec, sort_by_spec +from .helpers import prefix_spec, sort_by_spec from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive from .helpers import safe_encode, remove_surrogates, bin_to_hex from .helpers import prune_within, prune_split @@ -883,7 +883,7 @@ def _list_repository(self, args, manifest, write): format = "{archive:<36} {time} [{id}]{NL}" formatter = ArchiveFormatter(format) - for archive_info in manifest.archives.list(sort_by=args.sort_by): + for archive_info in manifest.archives.list(sort_by=args.sort_by, prefix=args.prefix): write(safe_encode(formatter.format_item(archive_info))) return self.exit_code @@ -1537,7 +1537,10 @@ def build_parser(self, prog=None): subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') - subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', + subparser.add_argument('--last', dest='last', + type=int, default=None, metavar='N', + help='only check last N archives (Default: all)') + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, help='only consider archive names starting with this prefix') subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, @@ -1871,7 +1874,7 @@ def build_parser(self, prog=None): subparser.add_argument('--format', '--list-format', dest='format', type=str, help="""specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""") - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', help='only consider archive names starting with this prefix') subparser.add_argument('-e', '--exclude', dest='excludes', type=parse_pattern, action='append', @@ -2034,7 +2037,7 @@ def build_parser(self, prog=None): help='number of monthly archives to keep') subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, help='number of yearly archives to keep') - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', help='only consider archive names starting with this prefix') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, @@ -2435,7 +2438,7 @@ def _get_archives_slice(self, args, manifest): n = args.first or args.last assert n > 0 - archives = manifest.archives.list() + archives = manifest.archives.list(prefix=args.prefix) if not archives: logger.error('There are no archives.') diff --git a/src/borg/helpers.py b/src/borg/helpers.py index ac31eb7c70..8f84f374a1 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -142,9 +142,9 @@ def __delitem__(self, name): name = safe_encode(name) del self._archives[name] - def list(self, sort_by=None, reverse=False): + def list(self, sort_by=None, reverse=False, prefix=''): # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts - archives = list(self.values()) # [self[name] for name in self] + archives = [x for x in self.values() if x.name.startswith(prefix)] if sort_by is not None: archives = sorted(archives, key=attrgetter(sort_by), reverse=reverse) return archives @@ -557,10 +557,6 @@ def CompressionSpec(s): raise ValueError -def PrefixSpec(s): - return replace_placeholders(s) - - def dir_is_cachedir(path): """Determines whether the specified path is a cache directory (and therefore should potentially be excluded from the backup) according to @@ -633,6 +629,8 @@ def replace_placeholders(text): } return format_line(text, data) +prefix_spec = replace_placeholders + def sort_by_spec(text): return text.replace('timestamp', 'ts') diff --git a/src/borg/repository.py b/src/borg/repository.py index 9eebd90e6a..2dbde2f901 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -685,8 +685,6 @@ def report_error(msg): return not error_found or repair def rollback(self, cleanup=False): - """ - """ if cleanup: self.io.cleanup(self.io.get_segments_transaction_id()) self.index = None From 303c479e935eef29fb03339cd86acf61ab2ed359 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Mon, 22 Aug 2016 21:37:47 +0200 Subject: [PATCH 09/13] Implements check usage with --first and --last refactors ArchiveChecker to be used as context manager --- src/borg/archive.py | 164 +++++++++++++-------------------- src/borg/archiver.py | 81 +++++++++++----- src/borg/testsuite/archiver.py | 13 ++- 3 files changed, 132 insertions(+), 126 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index b17685429c..18a3116428 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -964,44 +964,37 @@ def unpack_next(): class ArchiveChecker: + def __init__(self, args, repository): + self.args = args + self.repository = repository - def __init__(self): self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, archive=None, last=None, prefix=None, verify_data=False, - save_space=False): - """Perform a set of checks on 'repository' - - :param repair: enable repair mode, write updated or corrected data into repository - :param archive: only check this archive - :param last: only check this number of recent archives - :param prefix: only check archives with this prefix - :param verify_data: integrity verification of data referenced by archives - :param save_space: Repository.commit(save_space) - """ - logger.info('Starting archive consistency check...') - self.check_all = archive is None and last is None and prefix is None - self.repair = repair - self.repository = repository - self.init_chunks() - self.key = self.identify_key(repository) - if verify_data: - self.verify_data() + self.chunks = self.init_chunks() + self.key = self.identify_key(self.repository) if Manifest.MANIFEST_ID not in self.chunks: logger.error("Repository manifest not found!") self.error_found = True self.manifest = self.rebuild_manifest() else: - self.manifest, _ = Manifest.load(repository, key=self.key) - self.rebuild_refcounts(archive=archive, last=last, prefix=prefix) - self.orphan_chunks_check() - self.finish(save_space=save_space) - if self.error_found: - logger.error('Archive consistency check complete, problems found.') - else: - logger.info('Archive consistency check complete, no problems found.') - return self.repair or not self.error_found + self.manifest, _ = Manifest.load(self.repository, key=self.key) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is None and self.args.repair: + self.manifest.write() + self.repository.commit(save_space=self.args.save_space) + + def check(self, archive_info): + """Perform a set of checks on 'repository' + + :param archive_info: check this archive + :type archive_info: :class:`borg.helpers.ArchiveInfo` + """ + self.rebuild_refcounts(archive_info) def init_chunks(self): """Fetch a list of all object keys from repository @@ -1009,7 +1002,7 @@ def init_chunks(self): # Explicitly set the initial hash table capacity to avoid performance issues # due to hash table "resonance" capacity = int(len(self.repository) * 1.35 + 1) # > len * 1.0 / HASH_MAX_LOAD (see _hashindex.c) - self.chunks = ChunkIndex(capacity) + chunks = ChunkIndex(capacity) marker = None while True: result = self.repository.list(limit=10000, marker=marker) @@ -1018,7 +1011,8 @@ def init_chunks(self): marker = result[-1] init_entry = ChunkIndexEntry(refcount=0, size=0, csize=0) for id_ in result: - self.chunks[id_] = init_entry + chunks[id_] = init_entry + return chunks def identify_key(self, repository): try: @@ -1039,7 +1033,6 @@ def verify_data(self): try: encrypted_data = self.repository.get(chunk_id) except Repository.ObjectNotFound: - self.error_found = True errors += 1 logger.error('chunk %s not found', bin_to_hex(chunk_id)) continue @@ -1047,12 +1040,12 @@ def verify_data(self): _chunk_id = None if chunk_id == Manifest.MANIFEST_ID else chunk_id _, data = self.key.decrypt(_chunk_id, encrypted_data) except IntegrityError as integrity_error: - self.error_found = True errors += 1 logger.error('chunk %s, integrity error: %s', bin_to_hex(chunk_id), integrity_error) pi.finish() log = logger.error if errors else logger.info log('Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.', count, errors) + self.error_found |= bool(errors) def rebuild_manifest(self): """Rebuild the manifest object if it is missing @@ -1095,10 +1088,11 @@ def valid_archive(obj): logger.info('Manifest rebuild complete.') return manifest - def rebuild_refcounts(self, archive=None, last=None, prefix=None): + def rebuild_refcounts(self, archive_info): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected + :type archive_info: :class:`borg.helpers.ArchiveInfo` """ # Exclude the manifest from chunks del self.chunks[Manifest.MANIFEST_ID] @@ -1119,7 +1113,7 @@ def add_reference(id_, size, csize, cdata=None): except KeyError: assert cdata is not None self.chunks[id_] = ChunkIndexEntry(refcount=1, size=size, csize=csize) - if self.repair: + if self.args.repair: self.repository.put(id_, cdata) def verify_file_chunks(item): @@ -1228,73 +1222,45 @@ def valid_item(obj): raise i += 1 - if archive is None: - # we need last N or all archives - archive_infos = self.manifest.archives.list(sort_by='ts', reverse=True) - if prefix is not None: - archive_infos = [info for info in archive_infos if info.name.startswith(prefix)] - num_archives = len(archive_infos) - end = None if last is None else min(num_archives, last) - else: - # we only want one specific archive - info = self.manifest.archives.get(archive) - if info is None: - logger.error("Archive '%s' not found.", archive) - archive_infos = [] - else: - archive_infos = [info] - num_archives = 1 - end = 1 - with cache_if_remote(self.repository) as repository: - for i, info in enumerate(archive_infos[:end]): - logger.info('Analyzing archive {} ({}/{})'.format(info.name, num_archives - i, num_archives)) - archive_id = info.id - if archive_id not in self.chunks: - logger.error('Archive metadata block is missing!') - self.error_found = True - del self.manifest.archives[info.name] - continue - mark_as_possibly_superseded(archive_id) - cdata = self.repository.get(archive_id) - _, data = self.key.decrypt(archive_id, cdata) - archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) - if archive.version != 1: - raise Exception('Unknown archive metadata version') - archive.cmdline = [safe_decode(arg) for arg in archive.cmdline] - items_buffer = ChunkBuffer(self.key) - items_buffer.write_chunk = add_callback - for item in robust_iterator(archive): - if 'chunks' in item: - verify_file_chunks(item) - items_buffer.add(item) - items_buffer.flush(flush=True) - for previous_item_id in archive.items: - mark_as_possibly_superseded(previous_item_id) - archive.items = items_buffer.chunks - data = msgpack.packb(archive.as_dict(), unicode_errors='surrogateescape') - new_archive_id = self.key.id_hash(data) - cdata = self.key.encrypt(Chunk(data)) - add_reference(new_archive_id, len(data), len(cdata), cdata) - self.manifest.archives[info.name] = (new_archive_id, info.ts) - - def orphan_chunks_check(self): - if self.check_all: - unused = {id_ for id_, entry in self.chunks.iteritems() if entry.refcount == 0} - orphaned = unused - self.possibly_superseded - if orphaned: - logger.error('{} orphaned objects found!'.format(len(orphaned))) + archive_id = archive_info.id + if archive_id not in self.chunks: + logger.error('Archive metadata block is missing!') self.error_found = True - if self.repair: - for id_ in unused: - self.repository.delete(id_) - else: - logger.info('Orphaned objects check skipped (needs all archives checked).') + del self.manifest.archives[archive_info.name] + return + mark_as_possibly_superseded(archive_id) + cdata = self.repository.get(archive_id) + _, data = self.key.decrypt(archive_id, cdata) + archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) + if archive.version != 1: + raise Exception('Unknown archive metadata version') + archive.cmdline = [safe_decode(arg) for arg in archive.cmdline] + items_buffer = ChunkBuffer(self.key) + items_buffer.write_chunk = add_callback + for item in robust_iterator(archive): + if 'chunks' in item: + verify_file_chunks(item) + items_buffer.add(item) + items_buffer.flush(flush=True) + for previous_item_id in archive.items: + mark_as_possibly_superseded(previous_item_id) + archive.items = items_buffer.chunks + data = msgpack.packb(archive.as_dict(), unicode_errors='surrogateescape') + new_archive_id = self.key.id_hash(data) + cdata = self.key.encrypt(Chunk(data)) + add_reference(new_archive_id, len(data), len(cdata), cdata) + self.manifest.archives[archive_info.name] = (new_archive_id, archive_info.ts) - def finish(self, save_space=False): - if self.repair: - self.manifest.write() - self.repository.commit(save_space=save_space) + def orphan_chunks_check(self): + unused = {id_ for id_, entry in self.chunks.iteritems() if entry.refcount == 0} + orphaned = unused - self.possibly_superseded + if orphaned: + logger.error('{} orphaned objects found!'.format(len(orphaned))) + self.error_found = True + if self.args.repair: + for id_ in unused: + self.repository.delete(id_) class ArchiveRecreater: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index b8b2e32f5b..159cd28b1a 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -204,18 +204,55 @@ def do_check(self, args, repository): if not yes(msg, false_msg="Aborting.", truish=('YES', ), env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): return EXIT_ERROR - if args.repo_only and args.verify_data: - self.print_error("--repository-only and --verify-data contradict each other. Please select one.") + + if args.repo_only and any((args.verify_data, args.first, args.last)): + self.print_error("--repository-only contradicts --first, --last " + "and --verify-data arguments.") return EXIT_ERROR + if not args.archives_only: if not repository.check(repair=args.repair, save_space=args.save_space): return EXIT_WARNING - if not args.repo_only and not ArchiveChecker().check( - repository, repair=args.repair, archive=args.location.archive, - last=args.last, prefix=args.prefix, verify_data=args.verify_data, - save_space=args.save_space): - return EXIT_WARNING - return EXIT_SUCCESS + + if not args.repo_only: + logger.info('Starting archive consistency check...') + with ArchiveChecker(args, repository) as archive_checker: + manifest = archive_checker.manifest + all_archives_count = len(manifest.archives.list()) + + if args.verify_data: + archive_checker.verify_data() + + if args.location.archive: + archive_info = manifest.archives.get(args.location.archive) + if archive_info is None: + logger.error('Archive %s not found.' % args.location.archive) + return EXIT_ERROR + archives = (archive_info,) + elif any((args.first, args.last, args.prefix)): + archives = self._get_archives_slice(args, manifest) + else: + archives = manifest.archives.list(prefix=args.prefix) + + for i, archive_info in enumerate(archives, 1): + logger.info('Analyzing archive {} ({}/{}):'.format(archive_info.name, i, len(archives))) + archive_checker.check(archive_info) + + if len(archives) == all_archives_count: + logger.info('Looking for orphaned chunks...') + archive_checker.orphan_chunks_check() + + if archive_checker.error_found: + msg = 'Archive consistency check complete, problems found.' + if args.repair: + logger.info(msg[:-1] + ' and fixed.') + else: + logger.error(msg) + self.exit_code = EXIT_WARNING + else: + logger.info('Archive consistency check complete, no problems found.') + + return self.exit_code @with_repository() def do_change_passphrase(self, args, repository, manifest, key): @@ -1537,14 +1574,10 @@ def build_parser(self, prog=None): subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') - subparser.add_argument('--last', dest='last', - type=int, default=None, metavar='N', - help='only check last N archives (Default: all)') - subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, - help='only consider archive names starting with this prefix') subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, help="""show progress display while checking""") + self.add_archives_filter_args(subparser) change_passphrase_epilog = textwrap.dedent(""" The key files used for repository encryption are optionally passphrase @@ -1847,7 +1880,7 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') - self.add_archives_slice_selection_args(subparser) + self.add_archives_filter_args(subparser) list_epilog = textwrap.dedent(""" This command lists the contents of a repository or an archive. @@ -1874,8 +1907,6 @@ def build_parser(self, prog=None): subparser.add_argument('--format', '--list-format', dest='format', type=str, help="""specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""") - subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', - help='only consider archive names starting with this prefix') subparser.add_argument('-e', '--exclude', dest='excludes', type=parse_pattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') @@ -1887,7 +1918,7 @@ def build_parser(self, prog=None): help='repository/archive to list contents of') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to list; patterns are supported') - self.add_archives_slice_selection_args(subparser) + self.add_archives_filter_args(subparser) mount_epilog = textwrap.dedent(""" This command mounts an archive as a FUSE filesystem. This can be useful for @@ -1948,7 +1979,7 @@ def build_parser(self, prog=None): subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), help='archive or repository to display information about') - self.add_archives_slice_selection_args(subparser) + self.add_archives_filter_args(subparser) break_lock_epilog = textwrap.dedent(""" This command breaks the repository and cache locks. @@ -2351,17 +2382,20 @@ def build_parser(self, prog=None): return parser @staticmethod - def add_archives_slice_selection_args(subparser): + def add_archives_filter_args(subparser): + subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', + help='only consider archive names starting with this prefix') + valid_sort_keys = ('timestamp', 'name') sort_by_default = 'timestamp' sort_by_choices = [] for r in range(len(valid_sort_keys)): sort_by_choices.extend([','.join(x) for x in permutations(valid_sort_keys, r+1)]) - subparser.add_argument('--sort-by', dest='sort_by', type=sort_by_spec, choices=sort_by_choices, default=sort_by_default, help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' .format(valid_sort_keys, sort_by_default)) + group = subparser.add_mutually_exclusive_group() group.add_argument('--first', dest='first', metavar='N', default=0, type=int, help='delete N first archives') @@ -2432,11 +2466,9 @@ def run(self, args): def _get_archives_slice(self, args, manifest): if args.location.archive: - logger.error('The options --first and --last can only used on repository targets.') + logger.error('The options --prefix, --first and --last can only used on repository targets.') self.exit_code = EXIT_ERROR return [] - n = args.first or args.last - assert n > 0 archives = manifest.archives.list(prefix=args.prefix) @@ -2449,6 +2481,9 @@ def _get_archives_slice(self, args, manifest): archives.sort(key=attrgetter(sortkey)) if args.last: archives.reverse() + + n = args.first or args.last or len(archives) + return archives[:n] diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 9d68e3eea2..1e775ab5cf 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1817,21 +1817,26 @@ def setUp(self): self.create_src_archive('archive2') def test_check_usage(self): - output = self.cmd('check', '-v', '--progress', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--progress', self.repository_location) self.assert_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) self.assert_in('Checking segments', output) # reset logging to new process default to avoid need for fork=True on next check logging.getLogger('borg.output.progress').setLevel(logging.NOTSET) - output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--repository-only', self.repository_location) self.assert_in('Starting repository check', output) self.assert_not_in('Starting archive consistency check', output) self.assert_not_in('Checking segments', output) - output = self.cmd('check', '-v', '--archives-only', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--archives-only', self.repository_location) self.assert_not_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) - output = self.cmd('check', '-v', '--archives-only', '--prefix=archive2', self.repository_location, exit_code=0) + output = self.cmd('check', '--repository-only', '--verify-data', self.repository_location, exit_code=2) + self.assert_in('contradicts', output) + output = self.cmd('check', '-v', '--archives-only', '--prefix=archive2', self.repository_location) self.assert_not_in('archive1', output) + output = self.cmd('check', '-v', self.repository_location + '::archive1') + self.assert_in('Starting archive consistency check', output) + self.assert_not_in('archive2', output) def test_missing_file_chunk(self): archive, repository = self.open_archive('archive1') From ef923f9a2c0c20fd444b0e53ca50d7c9bba40092 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Tue, 23 Aug 2016 21:38:59 +0200 Subject: [PATCH 10/13] Tests and fixes for delete usage w/ archives filter --- src/borg/archiver.py | 12 ++++++------ src/borg/testsuite/archiver.py | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 159cd28b1a..cad99082a4 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -776,7 +776,7 @@ def do_delete(self, args, repository): else: return self._delete_repository(args, repository) - def _delete_archive(self, args, repository, manifest=None): + def _delete_archive(self, args, repository, manifest=None, key=None): """Delete a single archive""" if manifest is None: manifest, key = Manifest.load(repository) @@ -798,12 +798,12 @@ def _delete_archive(self, args, repository, manifest=None): def _delete_archives(self, args, repository): """Delete multiple archives""" - manifest, _ = Manifest.load(repository) + manifest, key = Manifest.load(repository) archives = self._get_archives_slice(args, manifest) - for i, archive in enumerate(archives, 1): - logger.info('Deleting {} ({}/{}):'.format(archive.name, i, len(archives))) - args.location.archive = archive.name - self._delete_archive(args, repository, manifest) + for i, archive_info in enumerate(archives, 1): + logger.info('Deleting {} ({}/{}):'.format(archive_info.name, i, len(archives))) + args.location.archive = archive_info.name + self._delete_archive(args, repository, manifest, key) if self.exit_code: break return self.exit_code diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 1e775ab5cf..328302fc84 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -979,8 +979,13 @@ def test_delete(self): self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') self.cmd('create', self.repository_location + '::test.2', 'input') + self.cmd('create', self.repository_location + '::test.3', 'input') + self.cmd('create', self.repository_location + '::another_test.1', 'input') + self.cmd('create', self.repository_location + '::another_test.2', 'input') self.cmd('extract', '--dry-run', self.repository_location + '::test') self.cmd('extract', '--dry-run', self.repository_location + '::test.2') + self.cmd('delete', '--prefix', 'another_', self.repository_location) + self.cmd('delete', '--last', '1', self.repository_location) self.cmd('delete', self.repository_location + '::test') self.cmd('extract', '--dry-run', self.repository_location + '::test.2') output = self.cmd('delete', '--stats', self.repository_location + '::test.2') From 0403e44dcc2eda597d345dd158c43dde0f2b360e Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Tue, 23 Aug 2016 21:39:49 +0200 Subject: [PATCH 11/13] Tests for archives filter arguments --- src/borg/testsuite/archiver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 328302fc84..0e53ef21fa 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -57,6 +57,9 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): except subprocess.CalledProcessError as e: output = e.output ret = e.returncode + except SystemExit as e: + output = '' + ret = e.code return ret, os.fsdecode(output) else: stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr @@ -1777,6 +1780,13 @@ def test_recreate_list_output(self): self.assert_not_in("input/file1", output) self.assert_not_in("x input/file5", output) + def test_bad_filters(self): + self.cmd('init', self.repository_location) + self.cmd('delete', '--last', '1', self.repository_location, exit_code=2) + self.cmd('create', self.repository_location + '::test', 'input') + self.cmd('delete', '--first', '1', '--last', '1', self.repository_location, fork=True, exit_code=2) + self.cmd('delete', '--last', '1', self.repository_location + '::test', exit_code=2) + @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase): From e3b2558270474623132b42e4ea53bf27aa23a9dc Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Tue, 23 Aug 2016 21:40:36 +0200 Subject: [PATCH 12/13] Assorted cleanup --- src/borg/archive.py | 4 ++-- src/borg/archiver.py | 22 +++++++++++----------- src/borg/helpers.py | 5 ++++- src/borg/testsuite/archiver.py | 2 ++ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 18a3116428..c144d886e3 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -29,8 +29,8 @@ from .helpers import parse_timestamp, to_localtime from .helpers import format_time, format_timedelta, format_file_size, file_status from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates -from .helpers import decode_dict, StableDict -from .helpers import int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import StableDict +from .helpers import bin_to_hex from .helpers import ProgressIndicatorPercent, log_multi from .helpers import PathPrefixPattern, FnmatchPattern from .helpers import consume diff --git a/src/borg/archiver.py b/src/borg/archiver.py index cad99082a4..69874ce9c6 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -769,7 +769,7 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): """Delete an existing repository or archives""" - if args.first or args.last: + if any((args.first, args.last, args.prefix)): return self._delete_archives(args, repository) if args.location.archive: return self._delete_archive(args, repository) @@ -875,7 +875,7 @@ def write(bytestring): else: write = sys.stdout.buffer.write - if args.first or args.last: + if any((args.first, args.last, args.prefix)): return self._list_archives(args, repository, manifest, key, write) elif args.location.archive: return self._list_archive(args, repository, manifest, key, write) @@ -901,14 +901,14 @@ def _list_archive(self, args, repository, manifest, key, write): def _list_archives(self, args, repository, manifest, key, write): archives = self._get_archives_slice(args, manifest) - for i, archive in enumerate(archives, 1): - write('Contents of {} ({}/{}):'.format(archive.name, i, len(archives))) - args.location.archive = archive.name + for i, archive_info in enumerate(archives, 1): + write('Contents of {} ({}/{}):'.format(archive_info.name, i, len(archives)).encode()) + args.location.archive = archive_info.name self._list_archive(args, repository, manifest, key, write) if self.exit_code: break - if len(archives) - i > 1: - write() + if len(archives) - i > 0: + write(b'\n') return self.exit_code def _list_repository(self, args, manifest, write): @@ -928,7 +928,7 @@ def _list_repository(self, args, manifest, write): @with_repository(cache=True) def do_info(self, args, repository, manifest, key, cache): """Show archive details such as disk space used""" - if args.first or args.last: + if any((args.first, args.last, args.prefix)): return self._info_archives(args, repository, manifest, key, cache) elif args.location.archive: return self._info_archive(args, repository, manifest, key, cache) @@ -960,12 +960,12 @@ def format_cmdline(cmdline): def _info_archives(self, args, repository, manifest, key, cache): archives = self._get_archives_slice(args, manifest) - for i, archive in enumerate(archives, 1): - args.location.archive = archive.name + for i, archive_info in enumerate(archives, 1): + args.location.archive = archive_info.name self._info_archive(args, repository, manifest, key, cache) if self.exit_code: break - if len(archives) - i: + if len(archives) - i > 0: print() return self.exit_code diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 8f84f374a1..b4427ff841 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -143,7 +143,10 @@ def __delitem__(self, name): del self._archives[name] def list(self, sort_by=None, reverse=False, prefix=''): - # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts + """ Inexpensive Archive.list_archives replacement if we just need .name, .id, .ts + + :rtype: A :class:`list` of :class:`borg.helpers.ArchiveInfo` instances + """ archives = [x for x in self.values() if x.name.startswith(prefix)] if sort_by is not None: archives = sorted(archives, key=attrgetter(sort_by), reverse=reverse) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 0e53ef21fa..49996974c2 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -956,6 +956,8 @@ def test_info(self): assert 'All archives:' in info_repo info_archive = self.cmd('info', self.repository_location + '::test') assert 'Archive name: test\n' in info_archive + info_archive = self.cmd('info', '--first', '1', self.repository_location) + assert 'Archive name: test\n' in info_archive def test_comment(self): self.create_regular_file('file1', size=1024 * 80) From 3bda57eb1667a63be92020edb36df0a96bfe99ff Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Tue, 23 Aug 2016 22:26:16 +0200 Subject: [PATCH 13/13] Simplifies --sort-by parsing --- src/borg/archiver.py | 13 ++++--------- src/borg/helpers.py | 6 ++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 69874ce9c6..5ff1f1e7b5 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -15,7 +15,7 @@ import traceback from binascii import unhexlify from datetime import datetime -from itertools import permutations, zip_longest +from itertools import zip_longest from operator import attrgetter from .logger import create_logger, setup_logging @@ -30,7 +30,7 @@ from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec -from .helpers import prefix_spec, sort_by_spec +from .helpers import prefix_spec, sort_by_spec, HUMAN_SORT_KEYS from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive from .helpers import safe_encode, remove_surrogates, bin_to_hex from .helpers import prune_within, prune_split @@ -2386,15 +2386,10 @@ def add_archives_filter_args(subparser): subparser.add_argument('-P', '--prefix', dest='prefix', type=prefix_spec, default='', help='only consider archive names starting with this prefix') - valid_sort_keys = ('timestamp', 'name') sort_by_default = 'timestamp' - sort_by_choices = [] - for r in range(len(valid_sort_keys)): - sort_by_choices.extend([','.join(x) for x in permutations(valid_sort_keys, r+1)]) - subparser.add_argument('--sort-by', dest='sort_by', type=sort_by_spec, - choices=sort_by_choices, default=sort_by_default, + subparser.add_argument('--sort-by', dest='sort_by', type=sort_by_spec, default=sort_by_default, help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' - .format(valid_sort_keys, sort_by_default)) + .format(HUMAN_SORT_KEYS, sort_by_default)) group = subparser.add_mutually_exclusive_group() group.add_argument('--first', dest='first', metavar='N', default=0, type=int, diff --git a/src/borg/helpers.py b/src/borg/helpers.py index b4427ff841..4879ed0885 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -635,7 +635,13 @@ def replace_placeholders(text): prefix_spec = replace_placeholders +HUMAN_SORT_KEYS = ['timestamp'] + list(ArchiveInfo._fields) +HUMAN_SORT_KEYS.remove('ts') + def sort_by_spec(text): + for token in text.split(','): + if token not in HUMAN_SORT_KEYS: + raise ValueError('Invalid sort key: %s' % token) return text.replace('timestamp', 'ts')