From e90f44aac8d8f19015320e8ed793bf3d74b38c87 Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Mon, 16 Jan 2023 12:28:18 +0300 Subject: [PATCH 1/7] Support for date-based matching during archive listing --- - Added arguments --older --newer --newest --oldest to argparser - Added parser to detect relative time markers i.e. 1d/1m - Added an in-house month-offseter - Edited manifest to initially filter by date if markers are present - Added test cases to test date matching arguments --- src/borg/archive.py | 14 +++++--- src/borg/archiver/_common.py | 32 ++++++++++++++++- src/borg/archiver/check_cmd.py | 4 +++ src/borg/helpers/time.py | 46 ++++++++++++++++++++++-- src/borg/manifest.py | 40 ++++++++++++++++++--- src/borg/testsuite/archiver/__init__.py | 5 +-- src/borg/testsuite/archiver/check_cmd.py | 35 ++++++++++++++++++ 7 files changed, 162 insertions(+), 14 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 3fb51b141d..650742bca0 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1692,7 +1692,8 @@ def __init__(self): self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, first=0, last=0, sort_by="", match=None, verify_data=False): + def check(self, repository, repair=False, first=0, last=0, sort_by="", match=None, older=None, oldest=None, + newer=None, newest=None, verify_data=False): """Perform a set of checks on 'repository' :param repair: enable repair mode, write updated or corrected data into repository @@ -1724,7 +1725,8 @@ def check(self, repository, repair=False, first=0, last=0, sort_by="", match=Non self.error_found = True del self.chunks[Manifest.MANIFEST_ID] self.manifest = self.rebuild_manifest() - self.rebuild_refcounts(match=match, first=first, last=last, sort_by=sort_by) + self.rebuild_refcounts(match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, + newer=newer, newest=newest) self.orphan_chunks_check() self.finish() if self.error_found: @@ -1919,7 +1921,8 @@ def valid_archive(obj): logger.info("Manifest rebuild complete.") return manifest - def rebuild_refcounts(self, first=0, last=0, sort_by="", match=None): + def rebuild_refcounts(self, first=0, last=0, sort_by="", match=None, older=None, oldest=None, newer=None, + newest=None): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected @@ -2113,8 +2116,9 @@ def valid_item(obj): i += 1 sort_by = sort_by.split(",") - if any((first, last, match)): - archive_infos = self.manifest.archives.list(sort_by=sort_by, match=match, first=first, last=last) + if any((first, last, match, older, newer, newest, oldest)): + archive_infos = self.manifest.archives.list(sort_by=sort_by, match=match, first=first, last=last, + oldest=oldest, newest=newest, older=older, newer=newer) if match and not archive_infos: logger.warning("--match-archives %s does not match any archives", match) if first and len(archive_infos) < first: diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 7b504d46ca..990f194724 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -353,7 +353,7 @@ def define_exclusion_group(subparser, **kwargs): return exclude_group -def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): +def define_archive_filters_group(subparser, *, sort_by=True, first_last=True, oldest_newest=True, older_newer=True): filters_group = subparser.add_argument_group( "Archive filters", "Archive filters can be applied to repository targets." ) @@ -399,6 +399,36 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help="consider last N archives after other filters were applied", ) + if oldest_newest: + group = filters_group.add_mutually_exclusive_group() + group.add_argument( + "--oldest", + metavar="Nd", + dest="oldest", + help="consider archives N-days after the oldest archive's timestamp", + ) + group.add_argument( + "--newest", + metavar="Nd", + dest="newest", + help="consider archives N-days before the newest archive's timestamp", + ) + + if older_newer: + group = filters_group.add_mutually_exclusive_group() + group.add_argument( + "--older", + metavar="Nd", + dest="older", + help="consider archives older than N-days ago", + ) + group.add_argument( + "--newer", + metavar="Nd", + dest="newer", + help="consider archives between now and N-days ago", + ) + return filters_group diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index a183eebbc4..6831480101 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -57,6 +57,10 @@ def do_check(self, args, repository): sort_by=args.sort_by or "ts", match=args.match_archives, verify_data=args.verify_data, + oldest=args.oldest, + newest=args.newest, + older=args.older, + newer=args.newer ): return EXIT_WARNING return EXIT_SUCCESS diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index e3aa8fa35b..ab9274ba6a 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -1,6 +1,6 @@ import os -from datetime import datetime, timezone - +import re +from datetime import datetime, timezone, timedelta def parse_timestamp(timestamp, tzinfo=timezone.utc): """Parse a ISO 8601 timestamp string. @@ -109,6 +109,48 @@ def format_timedelta(td): return txt +def calculate_relative_offset(format_string, from_date, earlier=False): + """ Calculates offset based on a relative marker. 7d (7 days), 8m (8 months) + earlier: whether offset should be calculated to an earlier time. + """ + if from_date is None: + from_date = archive_ts_now().date() + + if format_string is None: + return from_date + + day_offset_regex = re.compile(r'\d+d') + month_offset_regex = re.compile(r'\d+m') + + day_offset_str = day_offset_regex.search(format_string) + if day_offset_str is not None: + day_offset = int(day_offset_str.group()[:-1]) + day_offset *= -1 if earlier else 1 + return from_date + timedelta(days=day_offset) + + month_offset_str = month_offset_regex.search(format_string) + if month_offset_str is not None: + month_offset = int(month_offset_str.group()[:-1]) + month_offset *= -1 if earlier else 1 + return offset_n_months(from_date, month_offset) + + return from_date + + +def offset_n_months(from_date, n_months): + # Calculate target month and year by getting completed total_months until target_month + total_months = (from_date.year * 12) + from_date.month + n_months + target_year = (total_months - 1) // 12 + target_month = (total_months % 12) or 12 + + # calculate the max days of the target month by subtracting a day from the next month + next_month_month = ((total_months + 1) % 12) or 12 + next_month_year = (total_months + 1) // 12 + max_days_in_month = (datetime(next_month_year, next_month_month, 1) - timedelta(1)).day + + return datetime(day=min(from_date.day, max_days_in_month), month=target_month, year=target_year).replace(tzinfo=timezone.utc) + + class OutputTimestamp: def __init__(self, ts: datetime): self.ts = ts diff --git a/src/borg/manifest.py b/src/borg/manifest.py index 807a550d1e..6ef0447e1f 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -3,7 +3,7 @@ import os.path import re from collections import abc, namedtuple -from datetime import datetime, timedelta, timezone +from datetime import datetime, date, timedelta, timezone from operator import attrgetter from typing import Sequence, FrozenSet @@ -14,7 +14,7 @@ from .constants import * # NOQA from .helpers.datastruct import StableDict from .helpers.parseformat import bin_to_hex -from .helpers.time import parse_timestamp +from .helpers.time import parse_timestamp, calculate_relative_offset, archive_ts_now from .helpers.errors import Error from .patterns import get_regex_from_pattern from .repoobj import RepoObj @@ -34,6 +34,29 @@ class MandatoryFeatureUnsupported(Error): AI_HUMAN_SORT_KEYS.remove("ts") +def filter_archives_by_date(archives, oldest=None, newest=None, older=None, newer=None): + def get_first_and_last_archive_dates(archives_list): + dates = [x.ts.date() for x in archives_list] + return min(dates), max(dates) + + today = archive_ts_now().date() + first_archive_date, last_archive_date = get_first_and_last_archive_dates(archives) + + until_date = calculate_relative_offset(older, from_date=today, earlier=True) if older is not None else last_archive_date + from_date = calculate_relative_offset(newer, from_date=today, earlier=True) if newer is not None else first_archive_date + archives = [x for x in archives if from_date <= x.ts.date() <= until_date] + + first_archive_date, last_archive_date = get_first_and_last_archive_dates(archives) + if oldest: + oldest_date = calculate_relative_offset(oldest, from_date=first_archive_date, earlier=False) + archives = [x for x in archives if x.ts.date() <= oldest_date] + if newest: + newest_date = calculate_relative_offset(newest, from_date=last_archive_date, earlier=True) + archives = [x for x in archives if x.ts.date() >= newest_date ] + + return archives + + class Archives(abc.MutableMapping): """ Nice wrapper around the archives dict, making sure only valid types/values get in @@ -82,7 +105,11 @@ def list( consider_checkpoints=True, first=None, last=None, - reverse=False + reverse=False, + oldest=None, + newest=None, + older=None, + newer=None ): """ Return list of ArchiveInfo instances according to the parameters. @@ -98,9 +125,14 @@ def list( """ if isinstance(sort_by, (str, bytes)): raise TypeError("sort_by must be a sequence of str") + + archives = self.values() + if any([oldest, newest, older, newer]) and len(archives) > 0: + archives = filter_archives_by_date(archives, oldest=oldest, newest=newest, newer=newer, older=older) + regex = get_regex_from_pattern(match or "re:.*") regex = re.compile(regex + match_end) - archives = [x for x in self.values() if regex.match(x.name) is not None] + archives = [x for x in archives if regex.match(x.name) is not None] if not consider_checkpoints: archives = [x for x in archives if ".checkpoint" not in x.name] for sortkey in reversed(sort_by): diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index b1c4fcf40b..04b2b930b6 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -21,6 +21,7 @@ from ...helpers import Location from ...helpers import EXIT_SUCCESS from ...helpers import bin_to_hex +from ...helpers import archive_ts_now from ...manifest import Manifest from ...logger import setup_logging from ...remote import RemoteRepository @@ -172,8 +173,8 @@ def cmd(self, *args, **kw): output = empty.join(line for line in output.splitlines(keepends=True) if pp_msg not in line) return output - def create_src_archive(self, name): - self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", name, src_dir) + def create_src_archive(self, name, ts=archive_ts_now().isoformat()): + self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", f"--timestamp={ts}", name, src_dir) def open_archive(self, name): repository = Repository(self.repository_path, exclusive=True) diff --git a/src/borg/testsuite/archiver/check_cmd.py b/src/borg/testsuite/archiver/check_cmd.py index b7f74247cf..c73319231a 100644 --- a/src/borg/testsuite/archiver/check_cmd.py +++ b/src/borg/testsuite/archiver/check_cmd.py @@ -54,6 +54,41 @@ def test_check_usage(self): self.assert_not_in("archive1", output) self.assert_in("archive2", output) + def test_date_matching(self): + shutil.rmtree(self.repository_path) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + earliest_ts = "2022-11-20T23:59:59" + ts_in_between = "2022-12-18T23:59:59" + latest_between = "2023-01-17T23:59:59" + self.create_src_archive("archive1", ts=earliest_ts) + self.create_src_archive("archive2", ts=ts_in_between) + self.create_src_archive("archive3", ts=latest_between) + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--oldest=1m", exit_code=0 + ) + self.assert_in("archive1", output) + self.assert_in("archive2", output) + self.assert_not_in("archive3", output) + + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--newest=1m", exit_code=0 + ) + self.assert_in("archive2", output) + self.assert_in("archive3", output) + self.assert_not_in("archive1", output) + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--newer=1d", exit_code=0 + ) + self.assert_in("archive3", output) + self.assert_not_in("archive1", output) + self.assert_not_in("archive2", output) + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--older=1d", exit_code=0 + ) + self.assert_in("archive1", output) + self.assert_in("archive2", output) + self.assert_not_in("archive3", output) + def test_missing_file_chunk(self): archive, repository = self.open_archive("archive1") with repository: From 1aeba67e34b36a72a4e2ec20228ae7f96e3052ac Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Tue, 17 Jan 2023 09:39:43 +0300 Subject: [PATCH 2/7] Date-based listing now considers datetime instead of date --- - Added validators for argparse to reject unrecognized relative time markers - Calculate relative offset now raises exception on invalid markers - Match-archives now preceeds date-based filtering - Cleaned up regex references in time helper --- src/borg/archive.py | 38 ++++++++++++++++++----- src/borg/archiver/_common.py | 6 +++- src/borg/archiver/check_cmd.py | 2 +- src/borg/helpers/__init__.py | 9 +++++- src/borg/helpers/parseformat.py | 11 +++++++ src/borg/helpers/time.py | 39 ++++++++++++------------ src/borg/manifest.py | 28 +++++++++-------- src/borg/testsuite/archiver/__init__.py | 6 ++-- src/borg/testsuite/archiver/check_cmd.py | 3 ++ 9 files changed, 97 insertions(+), 45 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 650742bca0..a236659df5 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1692,8 +1692,20 @@ def __init__(self): self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, first=0, last=0, sort_by="", match=None, older=None, oldest=None, - newer=None, newest=None, verify_data=False): + def check( + self, + repository, + repair=False, + first=0, + last=0, + sort_by="", + match=None, + older=None, + oldest=None, + newer=None, + newest=None, + verify_data=False, + ): """Perform a set of checks on 'repository' :param repair: enable repair mode, write updated or corrected data into repository @@ -1725,8 +1737,9 @@ def check(self, repository, repair=False, first=0, last=0, sort_by="", match=Non self.error_found = True del self.chunks[Manifest.MANIFEST_ID] self.manifest = self.rebuild_manifest() - self.rebuild_refcounts(match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, - newer=newer, newest=newest) + self.rebuild_refcounts( + match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, newer=newer, newest=newest + ) self.orphan_chunks_check() self.finish() if self.error_found: @@ -1921,8 +1934,9 @@ def valid_archive(obj): logger.info("Manifest rebuild complete.") return manifest - def rebuild_refcounts(self, first=0, last=0, sort_by="", match=None, older=None, oldest=None, newer=None, - newest=None): + def rebuild_refcounts( + self, first=0, last=0, sort_by="", match=None, older=None, oldest=None, newer=None, newest=None + ): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected @@ -2117,8 +2131,16 @@ def valid_item(obj): sort_by = sort_by.split(",") if any((first, last, match, older, newer, newest, oldest)): - archive_infos = self.manifest.archives.list(sort_by=sort_by, match=match, first=first, last=last, - oldest=oldest, newest=newest, older=older, newer=newer) + archive_infos = self.manifest.archives.list( + sort_by=sort_by, + match=match, + first=first, + last=last, + oldest=oldest, + newest=newest, + older=older, + newer=newer, + ) if match and not archive_infos: logger.warning("--match-archives %s does not match any archives", match) if first and len(archive_infos) < first: diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 990f194724..1c7718d251 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -8,7 +8,7 @@ from ..constants import * # NOQA from ..cache import Cache, assert_secure from ..helpers import Error -from ..helpers import SortBySpec, positive_int_validator, location_validator, Location +from ..helpers import SortBySpec, positive_int_validator, location_validator, Location, relative_time_marker_validator from ..helpers.nanorst import rst_to_terminal from ..manifest import Manifest, AI_HUMAN_SORT_KEYS from ..patterns import PatternMatcher @@ -404,12 +404,14 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True, ol group.add_argument( "--oldest", metavar="Nd", + type=relative_time_marker_validator, dest="oldest", help="consider archives N-days after the oldest archive's timestamp", ) group.add_argument( "--newest", metavar="Nd", + type=relative_time_marker_validator, dest="newest", help="consider archives N-days before the newest archive's timestamp", ) @@ -419,12 +421,14 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True, ol group.add_argument( "--older", metavar="Nd", + type=relative_time_marker_validator, dest="older", help="consider archives older than N-days ago", ) group.add_argument( "--newer", metavar="Nd", + type=relative_time_marker_validator, dest="newer", help="consider archives between now and N-days ago", ) diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 6831480101..35ee29a210 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -60,7 +60,7 @@ def do_check(self, args, repository): oldest=args.oldest, newest=args.newest, older=args.older, - newer=args.newer + newer=args.newer, ): return EXIT_WARNING return EXIT_SUCCESS diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 36f6aa5c07..157a409840 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -25,7 +25,14 @@ from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal from .parseformat import format_line, replace_placeholders, PlaceholderError from .parseformat import format_archive, parse_stringified_list, clean_lines -from .parseformat import Location, location_validator, archivename_validator, comment_validator, text_validator +from .parseformat import ( + Location, + location_validator, + archivename_validator, + comment_validator, + text_validator, + relative_time_marker_validator, +) from .parseformat import BaseFormatter, ArchiveFormatter, ItemFormatter, file_status from .parseformat import swidth_slice, ellipsis_truncate from .parseformat import BorgJsonEncoder, basic_json_data, json_print, json_dump, prepare_dump_dict diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 6864319311..0ac5ea9782 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -536,6 +536,17 @@ def validator(text): return validator +def relative_time_marker_validator(text: str): + day_offset_regex = r"^\d+d$" + month_offset_regex = r"^\d+m$" + day_match = re.compile(day_offset_regex).search(text) + month_match = re.compile(month_offset_regex).search(text) + if not day_match and not month_match: + raise argparse.ArgumentTypeError(f"Invalid relative time marker used: {text}") + else: + return text + + def text_validator(*, name, max_length, min_length=0, invalid_ctrl_chars="\0", invalid_chars="", no_blanks=False): def validator(text): assert isinstance(text, str) diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index ab9274ba6a..1953e315e0 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -2,6 +2,7 @@ import re from datetime import datetime, timezone, timedelta + def parse_timestamp(timestamp, tzinfo=timezone.utc): """Parse a ISO 8601 timestamp string. @@ -110,31 +111,29 @@ def format_timedelta(td): def calculate_relative_offset(format_string, from_date, earlier=False): - """ Calculates offset based on a relative marker. 7d (7 days), 8m (8 months) - earlier: whether offset should be calculated to an earlier time. + """Calculates offset based on a relative marker. 7d (7 days), 8m (8 months) + earlier: whether offset should be calculated to an earlier time. """ if from_date is None: from_date = archive_ts_now().date() - if format_string is None: - return from_date - - day_offset_regex = re.compile(r'\d+d') - month_offset_regex = re.compile(r'\d+m') + if format_string is not None: + day_offset_regex = re.compile(r"(?P\d+)d") + month_offset_regex = re.compile(r"(?P\d+)+m") - day_offset_str = day_offset_regex.search(format_string) - if day_offset_str is not None: - day_offset = int(day_offset_str.group()[:-1]) - day_offset *= -1 if earlier else 1 - return from_date + timedelta(days=day_offset) + day_offset_match = day_offset_regex.search(format_string) + if day_offset_match: + day_offset = int(day_offset_match.group("offset")) + day_offset *= -1 if earlier else 1 + return from_date + timedelta(days=day_offset) - month_offset_str = month_offset_regex.search(format_string) - if month_offset_str is not None: - month_offset = int(month_offset_str.group()[:-1]) - month_offset *= -1 if earlier else 1 - return offset_n_months(from_date, month_offset) + month_offset_match = month_offset_regex.search(format_string) + if month_offset_match: + month_offset = int(month_offset_match.group("offset")) + month_offset *= -1 if earlier else 1 + return offset_n_months(from_date, month_offset) - return from_date + raise ValueError(f"{format_string} is not a recognized relative time marker") def offset_n_months(from_date, n_months): @@ -148,7 +147,9 @@ def offset_n_months(from_date, n_months): next_month_year = (total_months + 1) // 12 max_days_in_month = (datetime(next_month_year, next_month_month, 1) - timedelta(1)).day - return datetime(day=min(from_date.day, max_days_in_month), month=target_month, year=target_year).replace(tzinfo=timezone.utc) + return datetime(day=min(from_date.day, max_days_in_month), month=target_month, year=target_year).replace( + tzinfo=from_date.tzinfo + ) class OutputTimestamp: diff --git a/src/borg/manifest.py b/src/borg/manifest.py index 6ef0447e1f..e9aefc2526 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -3,7 +3,7 @@ import os.path import re from collections import abc, namedtuple -from datetime import datetime, date, timedelta, timezone +from datetime import datetime, timedelta, timezone from operator import attrgetter from typing import Sequence, FrozenSet @@ -36,23 +36,27 @@ class MandatoryFeatureUnsupported(Error): def filter_archives_by_date(archives, oldest=None, newest=None, older=None, newer=None): def get_first_and_last_archive_dates(archives_list): - dates = [x.ts.date() for x in archives_list] + dates = [x.ts for x in archives_list] return min(dates), max(dates) - today = archive_ts_now().date() + today = archive_ts_now() first_archive_date, last_archive_date = get_first_and_last_archive_dates(archives) - until_date = calculate_relative_offset(older, from_date=today, earlier=True) if older is not None else last_archive_date - from_date = calculate_relative_offset(newer, from_date=today, earlier=True) if newer is not None else first_archive_date - archives = [x for x in archives if from_date <= x.ts.date() <= until_date] - + until_date = ( + calculate_relative_offset(older, from_date=today, earlier=True) if older is not None else last_archive_date + ) + from_date = ( + calculate_relative_offset(newer, from_date=today, earlier=True) if newer is not None else first_archive_date + ) + archives = [x for x in archives if from_date <= x.ts <= until_date] + first_archive_date, last_archive_date = get_first_and_last_archive_dates(archives) if oldest: oldest_date = calculate_relative_offset(oldest, from_date=first_archive_date, earlier=False) - archives = [x for x in archives if x.ts.date() <= oldest_date] + archives = [x for x in archives if x.ts <= oldest_date] if newest: newest_date = calculate_relative_offset(newest, from_date=last_archive_date, earlier=True) - archives = [x for x in archives if x.ts.date() >= newest_date ] + archives = [x for x in archives if x.ts >= newest_date] return archives @@ -127,12 +131,12 @@ def list( raise TypeError("sort_by must be a sequence of str") archives = self.values() - if any([oldest, newest, older, newer]) and len(archives) > 0: - archives = filter_archives_by_date(archives, oldest=oldest, newest=newest, newer=newer, older=older) - regex = get_regex_from_pattern(match or "re:.*") regex = re.compile(regex + match_end) archives = [x for x in archives if regex.match(x.name) is not None] + + if any([oldest, newest, older, newer]) and len(archives) > 0: + archives = filter_archives_by_date(archives, oldest=oldest, newest=newest, newer=newer, older=older) if not consider_checkpoints: archives = [x for x in archives if ".checkpoint" not in x.name] for sortkey in reversed(sort_by): diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 04b2b930b6..4f0f2f62e6 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -21,7 +21,6 @@ from ...helpers import Location from ...helpers import EXIT_SUCCESS from ...helpers import bin_to_hex -from ...helpers import archive_ts_now from ...manifest import Manifest from ...logger import setup_logging from ...remote import RemoteRepository @@ -173,8 +172,9 @@ def cmd(self, *args, **kw): output = empty.join(line for line in output.splitlines(keepends=True) if pp_msg not in line) return output - def create_src_archive(self, name, ts=archive_ts_now().isoformat()): - self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", f"--timestamp={ts}", name, src_dir) + def create_src_archive(self, name, ts=None): + timestamp_arg = f"--timestamp={ts}" if ts else "" + self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", timestamp_arg, name, src_dir) def open_archive(self, name): repository = Repository(self.repository_path, exclusive=True) diff --git a/src/borg/testsuite/archiver/check_cmd.py b/src/borg/testsuite/archiver/check_cmd.py index c73319231a..dff2572d20 100644 --- a/src/borg/testsuite/archiver/check_cmd.py +++ b/src/borg/testsuite/archiver/check_cmd.py @@ -63,6 +63,9 @@ def test_date_matching(self): self.create_src_archive("archive1", ts=earliest_ts) self.create_src_archive("archive2", ts=ts_in_between) self.create_src_archive("archive3", ts=latest_between) + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--oldest=23e", exit_code=2 + ) output = self.cmd( f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--oldest=1m", exit_code=0 ) From 3d9f200c5f91aebae7be8919adbb6b40b0770b45 Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Wed, 18 Jan 2023 17:17:11 +0300 Subject: [PATCH 3/7] Formatting fixes --- - Fixed inefficient regex for relative time marker matching - Fixed import formatting - Renamed 'dates' to 'ts' in calculate_relative_offset - Modified relative time marker validator to match using single regex --- src/borg/helpers/__init__.py | 13 +++--------- src/borg/helpers/parseformat.py | 8 +++----- src/borg/helpers/time.py | 2 +- src/borg/manifest.py | 36 +++++++++++++++------------------ 4 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 157a409840..3dbd357ddd 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -22,17 +22,10 @@ from .parseformat import remove_surrogates, eval_escapes, decode_dict, positive_int_validator, interval from .parseformat import SortBySpec, ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper from .parseformat import format_file_size, parse_file_size, FileSize, parse_storage_quota -from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal -from .parseformat import format_line, replace_placeholders, PlaceholderError +from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator +from .parseformat import format_line, replace_placeholders, PlaceholderError, relative_time_marker_validator from .parseformat import format_archive, parse_stringified_list, clean_lines -from .parseformat import ( - Location, - location_validator, - archivename_validator, - comment_validator, - text_validator, - relative_time_marker_validator, -) +from .parseformat import location_validator, archivename_validator, comment_validator from .parseformat import BaseFormatter, ArchiveFormatter, ItemFormatter, file_status from .parseformat import swidth_slice, ellipsis_truncate from .parseformat import BorgJsonEncoder, basic_json_data, json_print, json_dump, prepare_dump_dict diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 0ac5ea9782..f4739a85c1 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -537,11 +537,9 @@ def validator(text): def relative_time_marker_validator(text: str): - day_offset_regex = r"^\d+d$" - month_offset_regex = r"^\d+m$" - day_match = re.compile(day_offset_regex).search(text) - month_match = re.compile(month_offset_regex).search(text) - if not day_match and not month_match: + time_marker_regex = r"^\d+[md]$" + match = re.compile(time_marker_regex).search(text) + if not match: raise argparse.ArgumentTypeError(f"Invalid relative time marker used: {text}") else: return text diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 1953e315e0..379a89e8ef 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -119,7 +119,7 @@ def calculate_relative_offset(format_string, from_date, earlier=False): if format_string is not None: day_offset_regex = re.compile(r"(?P\d+)d") - month_offset_regex = re.compile(r"(?P\d+)+m") + month_offset_regex = re.compile(r"(?P\d+)m") day_offset_match = day_offset_regex.search(format_string) if day_offset_match: diff --git a/src/borg/manifest.py b/src/borg/manifest.py index e9aefc2526..7c15eb310f 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -35,28 +35,24 @@ class MandatoryFeatureUnsupported(Error): def filter_archives_by_date(archives, oldest=None, newest=None, older=None, newer=None): - def get_first_and_last_archive_dates(archives_list): - dates = [x.ts for x in archives_list] - return min(dates), max(dates) - - today = archive_ts_now() - first_archive_date, last_archive_date = get_first_and_last_archive_dates(archives) - - until_date = ( - calculate_relative_offset(older, from_date=today, earlier=True) if older is not None else last_archive_date - ) - from_date = ( - calculate_relative_offset(newer, from_date=today, earlier=True) if newer is not None else first_archive_date - ) - archives = [x for x in archives if from_date <= x.ts <= until_date] - - first_archive_date, last_archive_date = get_first_and_last_archive_dates(archives) + def get_first_and_last_archive_ts(archives_list): + timestamps = [x.ts for x in archives_list] + return min(timestamps), max(timestamps) + + now = archive_ts_now() + earliest_ts, latest_ts = get_first_and_last_archive_ts(archives) + + until_ts = calculate_relative_offset(older, from_date=now, earlier=True) if older is not None else latest_ts + from_ts = calculate_relative_offset(newer, from_date=now, earlier=True) if newer is not None else earliest_ts + archives = [x for x in archives if from_ts <= x.ts <= until_ts] + + earliest_ts, latest_ts = get_first_and_last_archive_ts(archives) if oldest: - oldest_date = calculate_relative_offset(oldest, from_date=first_archive_date, earlier=False) - archives = [x for x in archives if x.ts <= oldest_date] + until_ts = calculate_relative_offset(oldest, from_date=earliest_ts, earlier=False) + archives = [x for x in archives if x.ts <= until_ts] if newest: - newest_date = calculate_relative_offset(newest, from_date=last_archive_date, earlier=True) - archives = [x for x in archives if x.ts >= newest_date] + from_ts = calculate_relative_offset(newest, from_date=latest_ts, earlier=True) + archives = [x for x in archives if x.ts >= from_ts] return archives From c0ab0a4dff8cbe4145179510687e91209fe37424 Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Thu, 19 Jan 2023 11:33:10 +0300 Subject: [PATCH 4/7] - Refactored ts offset function variables _date -> _ts - Modified create_src_archive to branch when timestamp flag is present --- src/borg/helpers/time.py | 51 ++++++++++++------------- src/borg/testsuite/archiver/__init__.py | 6 ++- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 379a89e8ef..5657190924 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -110,46 +110,45 @@ def format_timedelta(td): return txt -def calculate_relative_offset(format_string, from_date, earlier=False): +def calculate_relative_offset(format_string, from_ts, earlier=False): """Calculates offset based on a relative marker. 7d (7 days), 8m (8 months) earlier: whether offset should be calculated to an earlier time. """ - if from_date is None: - from_date = archive_ts_now().date() + if from_ts is None: + from_ts = archive_ts_now() if format_string is not None: - day_offset_regex = re.compile(r"(?P\d+)d") - month_offset_regex = re.compile(r"(?P\d+)m") + offset_regex = re.compile(r"(?P\d+)(?P[md])") + match = offset_regex.search(format_string) - day_offset_match = day_offset_regex.search(format_string) - if day_offset_match: - day_offset = int(day_offset_match.group("offset")) - day_offset *= -1 if earlier else 1 - return from_date + timedelta(days=day_offset) + if match: + unit = match.group("unit") + offset = int(match.group("offset")) + offset *= -1 if earlier else 1 - month_offset_match = month_offset_regex.search(format_string) - if month_offset_match: - month_offset = int(month_offset_match.group("offset")) - month_offset *= -1 if earlier else 1 - return offset_n_months(from_date, month_offset) + if unit == 'd': + return from_ts + timedelta(days=offset) + elif unit == 'm': + return offset_n_months(from_ts, offset) + + raise ValueError(f"Invalid relative ts offset format: {format_string}") - raise ValueError(f"{format_string} is not a recognized relative time marker") +def offset_n_months(from_ts, n_months): + def get_month_and_year_from_total(total_completed_months): + month = (total_completed_months % 12) + 1 + year = total_completed_months // 12 + return month, year -def offset_n_months(from_date, n_months): # Calculate target month and year by getting completed total_months until target_month - total_months = (from_date.year * 12) + from_date.month + n_months - target_year = (total_months - 1) // 12 - target_month = (total_months % 12) or 12 + total_months = (from_ts.year * 12) + from_ts.month + n_months - 1 + target_month, target_year = get_month_and_year_from_total(total_months) # calculate the max days of the target month by subtracting a day from the next month - next_month_month = ((total_months + 1) % 12) or 12 - next_month_year = (total_months + 1) // 12 - max_days_in_month = (datetime(next_month_year, next_month_month, 1) - timedelta(1)).day + following_month, year_of_following_month = get_month_and_year_from_total(total_months + 1) + max_days_in_month = (datetime(year_of_following_month, following_month, 1) - timedelta(1)).day - return datetime(day=min(from_date.day, max_days_in_month), month=target_month, year=target_year).replace( - tzinfo=from_date.tzinfo - ) + return datetime(day=min(from_ts.day, max_days_in_month), month=target_month, year=target_year).replace(tzinfo=from_ts.tzinfo) class OutputTimestamp: diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 4f0f2f62e6..2656be8656 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -173,8 +173,10 @@ def cmd(self, *args, **kw): return output def create_src_archive(self, name, ts=None): - timestamp_arg = f"--timestamp={ts}" if ts else "" - self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", timestamp_arg, name, src_dir) + if ts: + self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", f"--timestamp={ts}", name, src_dir) + else: + self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", name, src_dir) def open_archive(self, name): repository = Repository(self.repository_path, exclusive=True) From 65b28973c584a25a0dafacbb4055bfdc1f1180c8 Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Fri, 20 Jan 2023 13:03:31 +0300 Subject: [PATCH 5/7] Fn formatting and docstring update --- - Updated check/list functions to have same order of older/newer ... - Updated archive docstring to reflect parameters - Updated manifest docstring to reflect order of exec of parameters - Updated argparser group help description --- src/borg/archive.py | 8 +++++--- src/borg/archiver/_common.py | 8 ++++---- src/borg/helpers/time.py | 10 ++++++---- src/borg/manifest.py | 17 ++++++++++++----- src/borg/testsuite/archiver/__init__.py | 4 +++- src/borg/testsuite/archiver/check_cmd.py | 3 +-- 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index a236659df5..c70343f91a 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1701,8 +1701,8 @@ def check( sort_by="", match=None, older=None, - oldest=None, newer=None, + oldest=None, newest=None, verify_data=False, ): @@ -1711,10 +1711,12 @@ def check( :param repair: enable repair mode, write updated or corrected data into repository :param first/last/sort_by: only check this number of first/last archives ordered by sort_by :param match: only check archives matching this pattern + :param older/newer: only check archives older/newer than timedelta from now + :param oldest/newest: only check archives older/newer than timedelta from oldest/newest archive timestamp :param verify_data: integrity verification of data referenced by archives """ logger.info("Starting archive consistency check...") - self.check_all = not any((first, last, match)) + self.check_all = not any((first, last, match, older, newer, oldest, newest)) self.repair = repair self.repository = repository self.init_chunks() @@ -1935,7 +1937,7 @@ def valid_archive(obj): return manifest def rebuild_refcounts( - self, first=0, last=0, sort_by="", match=None, older=None, oldest=None, newer=None, newest=None + self, first=0, last=0, sort_by="", match=None, older=None, newer=None, oldest=None, newest=None ): """Rebuild object reference counts by walking the metadata diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 1c7718d251..c8eb80da8b 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -406,14 +406,14 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True, ol metavar="Nd", type=relative_time_marker_validator, dest="oldest", - help="consider archives N-days after the oldest archive's timestamp", + help="consider archives N [days/months] after the oldest archive's timestamp", ) group.add_argument( "--newest", metavar="Nd", type=relative_time_marker_validator, dest="newest", - help="consider archives N-days before the newest archive's timestamp", + help="consider archives N [days/months] before the newest archive's timestamp", ) if older_newer: @@ -423,14 +423,14 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True, ol metavar="Nd", type=relative_time_marker_validator, dest="older", - help="consider archives older than N-days ago", + help="consider archives older than N [days/months] ago", ) group.add_argument( "--newer", metavar="Nd", type=relative_time_marker_validator, dest="newer", - help="consider archives between now and N-days ago", + help="consider archives between now and N [days/months] ago", ) return filters_group diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 5657190924..774e3d5a17 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -126,11 +126,11 @@ def calculate_relative_offset(format_string, from_ts, earlier=False): offset = int(match.group("offset")) offset *= -1 if earlier else 1 - if unit == 'd': + if unit == "d": return from_ts + timedelta(days=offset) - elif unit == 'm': + elif unit == "m": return offset_n_months(from_ts, offset) - + raise ValueError(f"Invalid relative ts offset format: {format_string}") @@ -148,7 +148,9 @@ def get_month_and_year_from_total(total_completed_months): following_month, year_of_following_month = get_month_and_year_from_total(total_months + 1) max_days_in_month = (datetime(year_of_following_month, following_month, 1) - timedelta(1)).day - return datetime(day=min(from_ts.day, max_days_in_month), month=target_month, year=target_year).replace(tzinfo=from_ts.tzinfo) + return datetime(day=min(from_ts.day, max_days_in_month), month=target_month, year=target_year).replace( + tzinfo=from_ts.tzinfo + ) class OutputTimestamp: diff --git a/src/borg/manifest.py b/src/borg/manifest.py index 7c15eb310f..af61024d10 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -34,7 +34,7 @@ class MandatoryFeatureUnsupported(Error): AI_HUMAN_SORT_KEYS.remove("ts") -def filter_archives_by_date(archives, oldest=None, newest=None, older=None, newer=None): +def filter_archives_by_date(archives, older=None, newer=None, oldest=None, newest=None): def get_first_and_last_archive_ts(archives_list): timestamps = [x.ts for x in archives_list] return min(timestamps), max(timestamps) @@ -106,18 +106,25 @@ def list( first=None, last=None, reverse=False, - oldest=None, - newest=None, older=None, - newer=None + newer=None, + oldest=None, + newest=None ): """ Return list of ArchiveInfo instances according to the parameters. - First match *match* (considering *match_end*), then *sort_by*. + First match *match* (considering *match_end*) + then filter by timestamp considering *older* and *newer* followed by a second pass to + filter the result considering *oldest* and *newest* + then *sort_by*. + Apply *first* and *last* filters, and then possibly *reverse* the list. *sort_by* is a list of sort keys applied in reverse order. + *newer* and *older* are relative time markers that indicate offset from now + *newest* and *oldest* are relative time markers that indicate offset from newest/oldest archive's timestamp + Note: for better robustness, all filtering / limiting parameters must default to "not limit / not filter", so a FULL archive list is produced by a simple .list(). diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 2656be8656..b10cee76f7 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -174,7 +174,9 @@ def cmd(self, *args, **kw): def create_src_archive(self, name, ts=None): if ts: - self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", f"--timestamp={ts}", name, src_dir) + self.cmd( + f"--repo={self.repository_location}", "create", "--compression=lz4", f"--timestamp={ts}", name, src_dir + ) else: self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", name, src_dir) diff --git a/src/borg/testsuite/archiver/check_cmd.py b/src/borg/testsuite/archiver/check_cmd.py index dff2572d20..2a3eb910ed 100644 --- a/src/borg/testsuite/archiver/check_cmd.py +++ b/src/borg/testsuite/archiver/check_cmd.py @@ -59,10 +59,9 @@ def test_date_matching(self): self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) earliest_ts = "2022-11-20T23:59:59" ts_in_between = "2022-12-18T23:59:59" - latest_between = "2023-01-17T23:59:59" self.create_src_archive("archive1", ts=earliest_ts) self.create_src_archive("archive2", ts=ts_in_between) - self.create_src_archive("archive3", ts=latest_between) + self.create_src_archive("archive3") output = self.cmd( f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--oldest=23e", exit_code=2 ) From 9b6c84f3e7600f57499eb67067118b3df2b51a32 Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Mon, 23 Jan 2023 13:20:32 +0300 Subject: [PATCH 6/7] Resolved use of keyword argument that doesn't exist --- - Revisited argparser group for newer / older - Revisited docstring for archives list method --- src/borg/archiver/_common.py | 16 ++++++++-------- src/borg/helpers/time.py | 3 ++- src/borg/manifest.py | 18 ++++++++---------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index c8eb80da8b..b2bab8a494 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -403,34 +403,34 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True, ol group = filters_group.add_mutually_exclusive_group() group.add_argument( "--oldest", - metavar="Nd", + metavar="TIMESTAMP", type=relative_time_marker_validator, dest="oldest", - help="consider archives N [days/months] after the oldest archive's timestamp", + help="consider archives between the oldest archive's timestamp and the TIMESTAMP offset. e.g. 3d 7m", ) group.add_argument( "--newest", - metavar="Nd", + metavar="TIMESTAMP", type=relative_time_marker_validator, dest="newest", - help="consider archives N [days/months] before the newest archive's timestamp", + help="consider archives between the newest archive's timestamp and the TIMESTAMP offset. e.g. 3d 7m", ) if older_newer: group = filters_group.add_mutually_exclusive_group() group.add_argument( "--older", - metavar="Nd", + metavar="TIMESTAMP", type=relative_time_marker_validator, dest="older", - help="consider archives older than N [days/months] ago", + help="consider archives older than (now - TIMESTAMP). e.g. 3d 7m", ) group.add_argument( "--newer", - metavar="Nd", + metavar="TIMESTAMP", type=relative_time_marker_validator, dest="newer", - help="consider archives between now and N [days/months] ago", + help="consider archives after (now - TIMESTAMP). e.g. 3d 7m", ) return filters_group diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 774e3d5a17..d402bf0f52 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -111,7 +111,8 @@ def format_timedelta(td): def calculate_relative_offset(format_string, from_ts, earlier=False): - """Calculates offset based on a relative marker. 7d (7 days), 8m (8 months) + """ + Calculates offset based on a relative marker. 7d (7 days), 8m (8 months) earlier: whether offset should be calculated to an earlier time. """ if from_ts is None: diff --git a/src/borg/manifest.py b/src/borg/manifest.py index af61024d10..0ad4a2fe3f 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -42,16 +42,16 @@ def get_first_and_last_archive_ts(archives_list): now = archive_ts_now() earliest_ts, latest_ts = get_first_and_last_archive_ts(archives) - until_ts = calculate_relative_offset(older, from_date=now, earlier=True) if older is not None else latest_ts - from_ts = calculate_relative_offset(newer, from_date=now, earlier=True) if newer is not None else earliest_ts + until_ts = calculate_relative_offset(older, now, earlier=True) if older is not None else latest_ts + from_ts = calculate_relative_offset(newer, now, earlier=True) if newer is not None else earliest_ts archives = [x for x in archives if from_ts <= x.ts <= until_ts] earliest_ts, latest_ts = get_first_and_last_archive_ts(archives) if oldest: - until_ts = calculate_relative_offset(oldest, from_date=earliest_ts, earlier=False) + until_ts = calculate_relative_offset(oldest, earliest_ts, earlier=False) archives = [x for x in archives if x.ts <= until_ts] if newest: - from_ts = calculate_relative_offset(newest, from_date=latest_ts, earlier=True) + from_ts = calculate_relative_offset(newest, latest_ts, earlier=True) archives = [x for x in archives if x.ts >= from_ts] return archives @@ -114,16 +114,14 @@ def list( """ Return list of ArchiveInfo instances according to the parameters. - First match *match* (considering *match_end*) - then filter by timestamp considering *older* and *newer* followed by a second pass to - filter the result considering *oldest* and *newest* - then *sort_by*. + First match *match* (considering *match_end*), then filter by timestamp considering *older* and *newer*. + Second, follow with a filter considering *oldest* and *newest*, then sort by the given *sort_by* argument. Apply *first* and *last* filters, and then possibly *reverse* the list. *sort_by* is a list of sort keys applied in reverse order. - *newer* and *older* are relative time markers that indicate offset from now - *newest* and *oldest* are relative time markers that indicate offset from newest/oldest archive's timestamp + *newer* and *older* are relative time markers that indicate offset from now. + *newest* and *oldest* are relative time markers that indicate offset from newest/oldest archive's timestamp. Note: for better robustness, all filtering / limiting parameters must default to From f88cc5a74a6be79d0170f7b3277b1ef49ed4d483 Mon Sep 17 00:00:00 2001 From: Michael Deyaso Date: Mon, 23 Jan 2023 15:56:53 +0300 Subject: [PATCH 7/7] Fixed faulty test case --- src/borg/testsuite/archiver/check_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/testsuite/archiver/check_cmd.py b/src/borg/testsuite/archiver/check_cmd.py index 2a3eb910ed..bf0d48286d 100644 --- a/src/borg/testsuite/archiver/check_cmd.py +++ b/src/borg/testsuite/archiver/check_cmd.py @@ -75,8 +75,8 @@ def test_date_matching(self): output = self.cmd( f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--newest=1m", exit_code=0 ) - self.assert_in("archive2", output) self.assert_in("archive3", output) + self.assert_not_in("archive2", output) self.assert_not_in("archive1", output) output = self.cmd( f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--newer=1d", exit_code=0