From 205bcf48e746791759786808a08c4c506b66c20d Mon Sep 17 00:00:00 2001 From: Ted Kaemming Date: Thu, 8 Aug 2019 19:55:33 -0700 Subject: [PATCH 1/2] fix: `get_partitions_to_optimize` should return a Sequence, not Iterable Similar to GH-403, this maintains the Python 2 behavior. --- snuba/optimize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snuba/optimize.py b/snuba/optimize.py index f1d28155b3e..4425c564c0f 100644 --- a/snuba/optimize.py +++ b/snuba/optimize.py @@ -69,7 +69,7 @@ def get_partitions_to_optimize(clickhouse, database, table, before=None): parts = [util.decode_part_str(part) for part, count in active_parts] if before: - parts = filter(lambda p: (p[0] + timedelta(days=6 - p[0].weekday())) < before, parts) + parts = [p for p in parts if (p[0] + timedelta(days=6 - p[0].weekday())) < before] return parts From beec9221f0eb7fa3fe47534d0e9a6b6274e163c7 Mon Sep 17 00:00:00 2001 From: Ted Kaemming Date: Thu, 8 Aug 2019 19:59:57 -0700 Subject: [PATCH 2/2] annotate --- snuba/optimize.py | 3 ++- snuba/util.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/snuba/optimize.py b/snuba/optimize.py index 4425c564c0f..a2af187a95f 100644 --- a/snuba/optimize.py +++ b/snuba/optimize.py @@ -1,4 +1,5 @@ from datetime import timedelta +from typing import Sequence import logging from snuba import util @@ -13,7 +14,7 @@ def run_optimize(clickhouse, database, table, before=None): return len(parts) -def get_partitions_to_optimize(clickhouse, database, table, before=None): +def get_partitions_to_optimize(clickhouse, database, table, before=None) -> Sequence[util.Part]: engine = clickhouse.execute(""" SELECT engine FROM system.tables diff --git a/snuba/util.py b/snuba/util.py index 29fc4d15951..a3c6fbb5713 100644 --- a/snuba/util.py +++ b/snuba/util.py @@ -9,6 +9,7 @@ from functools import wraps from hashlib import md5 from itertools import chain, groupby +from typing import NamedTuple import jsonschema import logging import numbers @@ -636,7 +637,12 @@ def wrapper(*args, **kwargs): return decorator -def decode_part_str(part_str): +class Part(NamedTuple): + date: datetime + retention_days: int + + +def decode_part_str(part_str: str) -> Part: match = PART_RE.match(part_str) if not match: raise ValueError("Unknown part name/format: " + str(part_str)) @@ -644,7 +650,7 @@ def decode_part_str(part_str): date_str, retention_days = match.groups() date = datetime.strptime(date_str, '%Y-%m-%d') - return (date, int(retention_days)) + return Part(date, int(retention_days)) def force_bytes(s):