From 6a392ac38f53515c39a8a7da6c1da268e266fcf1 Mon Sep 17 00:00:00 2001 From: Charles Larivier Date: Sun, 6 Feb 2022 21:57:18 -0500 Subject: [PATCH 01/16] feat(mbql): add Query to compile MBQL Signed-off-by: Charles Larivier --- src/metabase/mbql/__init__.py | 1 + src/metabase/mbql/aggregations.py | 32 ++++++++++++++++++++++ src/metabase/mbql/base.py | 18 +++++++++++++ src/metabase/mbql/groupby.py | 44 +++++++++++++++++++++++++++++++ src/metabase/mbql/query.py | 21 +++++++++++++++ 5 files changed, 116 insertions(+) create mode 100644 src/metabase/mbql/__init__.py create mode 100644 src/metabase/mbql/aggregations.py create mode 100644 src/metabase/mbql/base.py create mode 100644 src/metabase/mbql/groupby.py create mode 100644 src/metabase/mbql/query.py diff --git a/src/metabase/mbql/__init__.py b/src/metabase/mbql/__init__.py new file mode 100644 index 0000000..97206e2 --- /dev/null +++ b/src/metabase/mbql/__init__.py @@ -0,0 +1 @@ +from metabase.mbql.query import Query diff --git a/src/metabase/mbql/aggregations.py b/src/metabase/mbql/aggregations.py new file mode 100644 index 0000000..bddfb71 --- /dev/null +++ b/src/metabase/mbql/aggregations.py @@ -0,0 +1,32 @@ +from metabase.mbql.base import Mbql + + +class Aggregation(Mbql): + mbql: str + + def compile(self): + return [self.mbql] + + +class ColumnAggregation(Aggregation): + def __init__(self, field_id: int): + self.field_id = field_id + + def compile(self): + return [self.mbql, ["field", self.field_id, None]] + + +class Count(Aggregation): + mbql = "count" + + +class Sum(ColumnAggregation): + mbql = "sum" + + +class Max(ColumnAggregation): + mbql = "max" + + +class Min(ColumnAggregation): + mbql = "min" diff --git a/src/metabase/mbql/base.py b/src/metabase/mbql/base.py new file mode 100644 index 0000000..da3aede --- /dev/null +++ b/src/metabase/mbql/base.py @@ -0,0 +1,18 @@ +from typing import List + + +class Mbql: + def compile(self): + raise NotImplementedError() + + def __repr__(self): + return str(self.compile()) + + +class Field(Mbql): + def __init__(self, id: int, option=None): + self.id = id + self.option = option + + def compile(self) -> List: + return ["field", self.id, self.option] diff --git a/src/metabase/mbql/groupby.py b/src/metabase/mbql/groupby.py new file mode 100644 index 0000000..0142bf2 --- /dev/null +++ b/src/metabase/mbql/groupby.py @@ -0,0 +1,44 @@ +from enum import Enum + +from metabase.mbql.base import Field + + +class TemporalOption(Enum): + MINUTE = {"temporal-unit": "minute"} + HOUR = {"temporal-unit": "hour"} + DAY = {"temporal-unit": "day"} + WEEK = {"temporal-unit": "week"} + MONTH = {"temporal-unit": "month"} + QUARTER = {"temporal-unit": "quarter"} + YEAR = {"temporal-unit": "year"} + MINUTE_OF_HOUR = {"temporal-unit": "minute-of-hour"} + HOUR_OF_DAY = {"temporal-unit": "hour-of-day"} + DAY_OF_WEEK = {"temporal-unit": "day-of-week"} + DAY_OF_MONTH = {"temporal-unit": "day-of-month"} + DAY_OF_YEAR = {"temporal-unit": "day-of-year"} + WEEK_OF_YEAR = {"temporal-unit": "week-of-year"} + MONTH_OF_YEAR = {"temporal-unit": "month-of-year"} + QUARTER_OF_YEAR = {"temporal-unit": "quarter-of-year"} + + +class BinOption(Enum): + AUTO = {"binning": {"strategy": "default"}} + BINS_10 = {"binning": {"strategy": "num-bins", "num-bins": 10}} + BINS_50 = {"binning": {"strategy": "num-bins", "num-bins": 50}} + BINS_100 = {"binning": {"strategy": "num-bins", "num-bins": 100}} + NONE = None + + +class GroupBy(Field): + def __init__(self, field_id: int, option=None): + super(GroupBy, self).__init__(id=field_id, option=option) + + +class TemporalGroupBy(GroupBy): + def __init__(self, field_id: int, option: TemporalOption): + super(TemporalGroupBy, self).__init__(field_id=field_id, option=option.value) + + +class BinnedGroupBy(GroupBy): + def __init__(self, field_id: int, option: BinOption): + super(BinnedGroupBy, self).__init__(field_id=field_id, option=option.value) diff --git a/src/metabase/mbql/query.py b/src/metabase/mbql/query.py new file mode 100644 index 0000000..f72a577 --- /dev/null +++ b/src/metabase/mbql/query.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass, field +from typing import List, Union + +from metabase.mbql.aggregations import Aggregation +from metabase.mbql.groupby import GroupBy + + +@dataclass +class Query: + table_id: int + aggregations: List[Union[Aggregation, Metric]] + group_by: List[GroupBy] = field(default_factory=list) + filters: List[Filter] = field(default_factory=list) + + def compile(self): + return { + "source-table": self.table_id, + "aggregation": [agg.compile() for agg in self.aggregations], + "breakout": [], + "filter": [], + } From a144a782d33f618676f97de6420d083ce1fa05e0 Mon Sep 17 00:00:00 2001 From: Charles Larivier Date: Tue, 22 Feb 2022 21:50:52 -0500 Subject: [PATCH 02/16] feat: add Mbql Signed-off-by: Charles Larivier --- src/metabase/mbql/__init__.py | 1 - src/metabase/mbql/base.py | 15 +++++++-------- tests/mbql/__init__.py | 0 tests/mbql/test_base.py | 21 +++++++++++++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 tests/mbql/__init__.py create mode 100644 tests/mbql/test_base.py diff --git a/src/metabase/mbql/__init__.py b/src/metabase/mbql/__init__.py index 97206e2..e69de29 100644 --- a/src/metabase/mbql/__init__.py +++ b/src/metabase/mbql/__init__.py @@ -1 +0,0 @@ -from metabase.mbql.query import Query diff --git a/src/metabase/mbql/base.py b/src/metabase/mbql/base.py index da3aede..e5ffadd 100644 --- a/src/metabase/mbql/base.py +++ b/src/metabase/mbql/base.py @@ -1,18 +1,17 @@ from typing import List -class Mbql: - def compile(self): - raise NotImplementedError() - - def __repr__(self): - return str(self.compile()) +class Option: + pass -class Field(Mbql): - def __init__(self, id: int, option=None): +class Mbql: + def __init__(self, id: int, option: Option = None): self.id = id self.option = option def compile(self) -> List: return ["field", self.id, self.option] + + def __repr__(self): + return str(self.compile()) diff --git a/tests/mbql/__init__.py b/tests/mbql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/mbql/test_base.py b/tests/mbql/test_base.py new file mode 100644 index 0000000..3eb9215 --- /dev/null +++ b/tests/mbql/test_base.py @@ -0,0 +1,21 @@ +from unittest import TestCase + +from metabase.mbql.base import Mbql + + +class MbqlTests(TestCase): + def test_compile(self): + """Ensure Mbql.compile() returns a list formatted as ["field", self.id, self.option].""" + mbql = Mbql(id=5, option={"foo": "bar"}) + self.assertEqual(["field", 5, {"foo": "bar"}], mbql.compile()) + + mbql = Mbql(id=10, option=None) + self.assertEqual(["field", 10, None], mbql.compile()) + + def test_repr(self): + """Ensure Mbql.__repr__() returns the compiled Mbql as a string.""" + mbql = Mbql(id=5, option={"foo": "bar"}) + self.assertEqual("['field', 5, {'foo': 'bar'}]", mbql.__repr__()) + + mbql = Mbql(id=10, option=None) + self.assertEqual("['field', 10, None]", mbql.__repr__()) From 7ccd7e6842c0a4fc7d839bb59c95ba5c3f48817f Mon Sep 17 00:00:00 2001 From: Charles Larivier Date: Tue, 22 Feb 2022 21:55:55 -0500 Subject: [PATCH 03/16] feat: add aggregations.py Signed-off-by: Charles Larivier --- src/metabase/mbql/aggregations.py | 52 +++++++++++++++++++++---------- tests/mbql/test_aggregations.py | 25 +++++++++++++++ 2 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 tests/mbql/test_aggregations.py diff --git a/src/metabase/mbql/aggregations.py b/src/metabase/mbql/aggregations.py index bddfb71..e03f7bc 100644 --- a/src/metabase/mbql/aggregations.py +++ b/src/metabase/mbql/aggregations.py @@ -1,32 +1,52 @@ +from typing import List + from metabase.mbql.base import Mbql class Aggregation(Mbql): - mbql: str + function: str - def compile(self): - return [self.mbql] + def compile(self) -> List: + return [self.function, super(Aggregation, self).compile()] -class ColumnAggregation(Aggregation): - def __init__(self, field_id: int): - self.field_id = field_id +class Count(Aggregation): + function = "count" - def compile(self): - return [self.mbql, ["field", self.field_id, None]] + def __init__(self, id: int = None): + self.id = id + def compile(self) -> List: + return [self.function] -class Count(Aggregation): - mbql = "count" + +class Sum(Aggregation): + function = "sum" + + +class Average(Aggregation): + function = "avg" + + +class Distinct(Aggregation): + function = "distinct" + + +class CumulativeSum(Aggregation): + function = "cum-sum" + + +class CumulativeCount(Aggregation): + function = "cum-count" -class Sum(ColumnAggregation): - mbql = "sum" +class StandardDeviation(Aggregation): + function = "stddev" -class Max(ColumnAggregation): - mbql = "max" +class Min(Aggregation): + function = "min" -class Min(ColumnAggregation): - mbql = "min" +class Max(Aggregation): + function = "max" diff --git a/tests/mbql/test_aggregations.py b/tests/mbql/test_aggregations.py new file mode 100644 index 0000000..82f19a1 --- /dev/null +++ b/tests/mbql/test_aggregations.py @@ -0,0 +1,25 @@ +from unittest import TestCase + +from metabase.mbql.aggregations import Aggregation, Count + + +class AggregationTests(TestCase): + def test_aggregation(self): + """Ensure Aggregation.compile() returns [self.function, ['field', self.id, self.option']].""" + + class Mock(Aggregation): + function = "mock" + + aggregation = Mock(id=2) + self.assertEqual(["mock", ["field", 2, None]], aggregation.compile()) + + aggregation = Mock(id=2, option={"foo": "bar"}) + self.assertEqual(["mock", ["field", 2, {"foo": "bar"}]], aggregation.compile()) + + def test_count(self): + """Ensure Count optionally accepts an id attribute.""" + count = Count() + self.assertEqual(["count"], count.compile()) + + count = Count(id=5) + self.assertEqual(["count"], count.compile()) From 488a4aaa24e718b14c42bd43936785e5955880cc Mon Sep 17 00:00:00 2001 From: Charles Larivier Date: Tue, 22 Feb 2022 22:07:34 -0500 Subject: [PATCH 04/16] feat: add groupby.py Signed-off-by: Charles Larivier --- src/metabase/mbql/groupby.py | 23 +++++------------------ tests/mbql/test_groupby.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 tests/mbql/test_groupby.py diff --git a/src/metabase/mbql/groupby.py b/src/metabase/mbql/groupby.py index 0142bf2..8d2ea1b 100644 --- a/src/metabase/mbql/groupby.py +++ b/src/metabase/mbql/groupby.py @@ -1,9 +1,7 @@ -from enum import Enum +from metabase.mbql.base import Mbql, Option -from metabase.mbql.base import Field - -class TemporalOption(Enum): +class TemporalOption(Option): MINUTE = {"temporal-unit": "minute"} HOUR = {"temporal-unit": "hour"} DAY = {"temporal-unit": "day"} @@ -21,7 +19,7 @@ class TemporalOption(Enum): QUARTER_OF_YEAR = {"temporal-unit": "quarter-of-year"} -class BinOption(Enum): +class BinOption(Option): AUTO = {"binning": {"strategy": "default"}} BINS_10 = {"binning": {"strategy": "num-bins", "num-bins": 10}} BINS_50 = {"binning": {"strategy": "num-bins", "num-bins": 50}} @@ -29,16 +27,5 @@ class BinOption(Enum): NONE = None -class GroupBy(Field): - def __init__(self, field_id: int, option=None): - super(GroupBy, self).__init__(id=field_id, option=option) - - -class TemporalGroupBy(GroupBy): - def __init__(self, field_id: int, option: TemporalOption): - super(TemporalGroupBy, self).__init__(field_id=field_id, option=option.value) - - -class BinnedGroupBy(GroupBy): - def __init__(self, field_id: int, option: BinOption): - super(BinnedGroupBy, self).__init__(field_id=field_id, option=option.value) +class GroupBy(Mbql): + pass diff --git a/tests/mbql/test_groupby.py b/tests/mbql/test_groupby.py new file mode 100644 index 0000000..6a052aa --- /dev/null +++ b/tests/mbql/test_groupby.py @@ -0,0 +1,19 @@ +from unittest import TestCase + +from metabase.mbql.groupby import BinOption, GroupBy, TemporalOption + + +class GroupbyTests(TestCase): + def test_groupby_compile(self): + """Ensure GroupBy.compile() returns ["field", self.id, self.option]""" + groupby = GroupBy(id=4) + self.assertEqual(["field", 4, None], groupby.compile()) + + groupby = GroupBy(id=4, option=TemporalOption.DAY) + self.assertEqual(["field", 4, {"temporal-unit": "day"}], groupby.compile()) + + groupby = GroupBy(id=4, option=BinOption.BINS_10) + self.assertEqual( + ["field", 4, {"binning": {"strategy": "num-bins", "num-bins": 10}}], + groupby.compile(), + ) From 6c8de95e0d510604426831d9263e0c3ed557ed00 Mon Sep 17 00:00:00 2001 From: Charles Larivier Date: Tue, 22 Feb 2022 23:08:36 -0500 Subject: [PATCH 05/16] feat: add filter.py Signed-off-by: Charles Larivier --- src/metabase/mbql/filter.py | 141 ++++++++++++++++++++++++++++++++++++ tests/mbql/test_filter.py | 54 ++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 src/metabase/mbql/filter.py create mode 100644 tests/mbql/test_filter.py diff --git a/src/metabase/mbql/filter.py b/src/metabase/mbql/filter.py new file mode 100644 index 0000000..d695812 --- /dev/null +++ b/src/metabase/mbql/filter.py @@ -0,0 +1,141 @@ +from typing import Any, List + +from metabase.mbql.base import Mbql, Option + + +class CaseOption(Option): + CASE_SENSITIVE = {"case-sensitive": True} + CASE_INSENSITIVE = {"case-sensitive": False} + + +class TimeGrainOption(Option): + MINUTE = "minute" + HOUR = "hour" + DAY = "day" + WEEK = "week" + MONTH = "month" + QUARTER = "quarter" + YEAR = "year" + + +class Filter(Mbql): + function: str + + def __init__(self, id: int, option: Option = None): + self.id = id + self.option = None + self.filter_option = option + + def compile(self) -> List: + compiled = [self.function, super(Filter, self).compile()] + + if self.filter_option is not None: + compiled = compiled + [self.filter_option] + + return compiled + + +class ValueFilter(Filter): + def __init__(self, id: int, value: Any, option: Option = None): + self.id = id + self.value = value + self.option = None + self.filter_option = option + + def compile(self) -> List: + compiled = [self.function, super(Filter, self).compile(), self.value] + + if self.filter_option is not None: + compiled = compiled + [self.filter_option] + + return compiled + + +class Equal(ValueFilter): + function = "=" + + +class NotEqual(ValueFilter): + function = "!=" + + +class Greater(ValueFilter): + function = ">" + + +class Less(ValueFilter): + function = "<" + + +class Between(Filter): + function = "between" + + def __init__( + self, id: int, lower_bound: float, upper_bound: float, option: Option = None + ): + self.id = id + self.option = None + self.filter_option = option + self.lower_bound = lower_bound + self.upper_bound = upper_bound + + def compile(self) -> List: + return super(Between, self).compile() + [self.lower_bound, self.upper_bound] + + +class GreaterEqual(ValueFilter): + function = ">=" + + +class LessEqual(ValueFilter): + function = "<=" + + +class IsNull(Filter): + function = "is-null" + + +class IsNotNull(Filter): + function = "not-null" + + +class Contains(ValueFilter): + function = "contains" + + +class StartsWith(ValueFilter): + function = "starts-with" + + +class EndsWith(ValueFilter): + function = "ends-with" + + +class TimeInterval(Filter): + function = "time-interval" + + def __init__( + self, + id: int, + value: Any, + time_grain: TimeGrainOption, + include_current: bool = True, + ): + self.id = id + self.value = value + self.option = None + self.time_grain = time_grain + self.include_current = include_current + + def compile(self) -> List: + compiled = [ + self.function, + super(Filter, self).compile(), + self.value, + self.time_grain, + ] + + if self.include_current: + compiled = compiled + [{"include-current": True}] + + return compiled diff --git a/tests/mbql/test_filter.py b/tests/mbql/test_filter.py new file mode 100644 index 0000000..da217ad --- /dev/null +++ b/tests/mbql/test_filter.py @@ -0,0 +1,54 @@ +from unittest import TestCase + +from metabase.mbql.filter import ( + Between, + CaseOption, + TimeGrainOption, + TimeInterval, + ValueFilter, +) + + +class FilterTests(TestCase): + def test_value_filter_compile(self): + """ + Ensure ValueFilter.compile() returns + [self.function, ['field', self.id, None], self.value, self.filter_option]. + """ + + class Mock(ValueFilter): + function = "mock" + + filter = Mock(id=5, value="gmail", option=CaseOption.CASE_SENSITIVE) + self.assertEqual( + ["mock", ["field", 5, None], "gmail", {"case-sensitive": True}], + filter.compile(), + ) + + def test_between_compile(self): + """ + Ensure Between.compile() returns + ['between', ['field', self.id', None], self.lower_bound, self.upper_bound]. + """ + between = Between(id=2, lower_bound=2.5, upper_bound=6.7) + self.assertEqual(["between", ["field", 2, None], 2.5, 6.7], between.compile()) + + def test_time_interval(self): + interval = TimeInterval(id=4, value=30, time_grain=TimeGrainOption.YEAR) + self.assertEqual( + [ + "time-interval", + ["field", 4, None], + 30, + "year", + {"include-current": True}, + ], + interval.compile(), + ) + + interval = TimeInterval( + id=4, value=30, time_grain=TimeGrainOption.YEAR, include_current=False + ) + self.assertEqual( + ["time-interval", ["field", 4, None], 30, "year"], interval.compile() + ) From 36cb4863a97aff6d6c63ea991d5ad0f5d44b9d90 Mon Sep 17 00:00:00 2001 From: Charles Larivier Date: Tue, 22 Feb 2022 23:27:58 -0500 Subject: [PATCH 06/16] feat: add query.py Signed-off-by: Charles Larivier --- src/metabase/mbql/query.py | 29 ++++++++++++--- tests/mbql/test_query.py | 76 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 tests/mbql/test_query.py diff --git a/src/metabase/mbql/query.py b/src/metabase/mbql/query.py index f72a577..c05f730 100644 --- a/src/metabase/mbql/query.py +++ b/src/metabase/mbql/query.py @@ -1,21 +1,40 @@ from dataclasses import dataclass, field -from typing import List, Union +from typing import List from metabase.mbql.aggregations import Aggregation +from metabase.mbql.filter import Filter from metabase.mbql.groupby import GroupBy @dataclass class Query: table_id: int - aggregations: List[Union[Aggregation, Metric]] + aggregations: List[Aggregation] group_by: List[GroupBy] = field(default_factory=list) filters: List[Filter] = field(default_factory=list) def compile(self): return { "source-table": self.table_id, - "aggregation": [agg.compile() for agg in self.aggregations], - "breakout": [], - "filter": [], + "aggregation": self._aggregations, + "breakout": self._group_by, + "filter": self._filters, } + + @property + def _aggregations(self): + return [aggregation.compile() for aggregation in self.aggregations] + + @property + def _group_by(self): + return [group.compile() for group in self.group_by] + + @property + def _filters(self): + if len(self.filters) == 0: + return self.filters + + if len(self.filters) == 1: + return self.filters[0].compile() + + return ["and"] + [filt.compile() for filt in self.filters] diff --git a/tests/mbql/test_query.py b/tests/mbql/test_query.py new file mode 100644 index 0000000..87768d6 --- /dev/null +++ b/tests/mbql/test_query.py @@ -0,0 +1,76 @@ +from unittest import TestCase + +from metabase.mbql.aggregations import Count, Max +from metabase.mbql.filter import Equal +from metabase.mbql.groupby import GroupBy +from metabase.mbql.query import Query + + +class QueryTests(TestCase): + def test_compile(self): + """Ensure Query.compile() returns valid MBQL.""" + query = Query( + table_id=14, + aggregations=[Count(), Max(5)], + group_by=[GroupBy(14)], + filters=[Equal(2, 5), Equal(5, "foo")], + ) + + self.assertEqual( + { + "source-table": 14, + "aggregation": [["count"], ["max", ["field", 5, None]]], + "breakout": [["field", 14, None]], + "filter": [ + "and", + ["=", ["field", 2, None], 5], + ["=", ["field", 5, None], "foo"], + ], + }, + query.compile(), + ) + + def test__aggregations(self): + """Ensure Query._aggregations returns a list of compiled Aggregation.""" + query = Query(table_id=12, aggregations=[]) + self.assertEqual([], query._aggregations) + + query = Query( + table_id=12, + aggregations=[Count(), Max(5)], + ) + self.assertEqual([["count"], ["max", ["field", 5, None]]], query._aggregations) + + def test__group_by(self): + """Ensure Query._group_by returns a list of compiled GroupBy.""" + query = Query(table_id=12, aggregations=[Count()], group_by=[]) + self.assertEqual([], query._group_by) + + query = Query( + table_id=12, + aggregations=[Count()], + group_by=[GroupBy(5)], + ) + self.assertEqual([["field", 5, None]], query._group_by) + + def test__filters(self): + """Ensure Query._filters returns a list of compiled Filter.""" + query = Query(table_id=12, aggregations=[Count()], filters=[]) + self.assertEqual([], query._filters) + + query = Query( + table_id=12, + aggregations=[Count()], + filters=[Equal(5, 2)], + ) + self.assertListEqual(["=", ["field", 5, None], 2], query._filters) + + query = Query( + table_id=12, + aggregations=[Count()], + filters=[Equal(5, 2), Equal(6, 1)], + ) + self.assertEqual( + ["and", ["=", ["field", 5, None], 2], ["=", ["field", 6, None], 1]], + query._filters, + ) From 2a1540408d21ae11a0daf300ca644e38408cd9e0 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 20:52:25 -0500 Subject: [PATCH 07/16] fix: update default H2 database path in integration tests Signed-off-by: Charles Lariviere --- tests/resources/test_database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/resources/test_database.py b/tests/resources/test_database.py index db86415..e2203ac 100644 --- a/tests/resources/test_database.py +++ b/tests/resources/test_database.py @@ -34,7 +34,7 @@ def test_create(self): name="Test", engine="h2", details={ - "db": "zip:/app/metabase.jar!/sample-dataset.db;USER=GUEST;PASSWORD=guest" + "db": "zip:/app/metabase.jar!/sample-database.db;USER=GUEST;PASSWORD=guest" }, using=self.metabase, ) @@ -73,7 +73,7 @@ def test_delete(self): name="Test", engine="h2", details={ - "db": "zip:/app/metabase.jar!/sample-dataset.db;USER=GUEST;PASSWORD=guest" + "db": "zip:/app/metabase.jar!/sample-database.db;USER=GUEST;PASSWORD=guest" }, using=self.metabase, ) From 4bd011a49b0a5165a445a112f9df44070f68a331 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 21:36:26 -0500 Subject: [PATCH 08/16] feat(aggregation): add support for aggregation name Signed-off-by: Charles Lariviere --- src/metabase/mbql/aggregations.py | 31 +++++++++++++++++++++++++++---- tests/mbql/test_aggregations.py | 20 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/metabase/mbql/aggregations.py b/src/metabase/mbql/aggregations.py index e03f7bc..289aa82 100644 --- a/src/metabase/mbql/aggregations.py +++ b/src/metabase/mbql/aggregations.py @@ -1,23 +1,46 @@ from typing import List -from metabase.mbql.base import Mbql +from metabase.mbql.base import Mbql, Option class Aggregation(Mbql): function: str + def __init__(self, id: int, name: str = None, option: Option = None): + self.name = name + super(Aggregation, self).__init__(id=id, option=option) + def compile(self) -> List: - return [self.function, super(Aggregation, self).compile()] + compiled = [self.function, super(Aggregation, self).compile()] + + if self.name is not None: + compiled = self.compile_name(compiled, self.name) + + return compiled + + @staticmethod + def compile_name(compiled, name: str) -> str: + return ( + ["aggregation-options"] + + [compiled] + + [{"name": name, "display-name": name}] + ) class Count(Aggregation): function = "count" - def __init__(self, id: int = None): + def __init__(self, id: int = None, name: str = None, option: Option = None): self.id = id + self.name = name def compile(self) -> List: - return [self.function] + compiled = [self.function] + + if self.name is not None: + compiled = self.compile_name(compiled, self.name) + + return compiled class Sum(Aggregation): diff --git a/tests/mbql/test_aggregations.py b/tests/mbql/test_aggregations.py index 82f19a1..347a8df 100644 --- a/tests/mbql/test_aggregations.py +++ b/tests/mbql/test_aggregations.py @@ -16,6 +16,16 @@ class Mock(Aggregation): aggregation = Mock(id=2, option={"foo": "bar"}) self.assertEqual(["mock", ["field", 2, {"foo": "bar"}]], aggregation.compile()) + aggregation = Mock(id=2, name="My Aggregation", option={"foo": "bar"}) + self.assertEqual( + [ + "aggregation-options", + ["mock", ["field", 2, {"foo": "bar"}]], + {"name": "My Aggregation", "display-name": "My Aggregation"}, + ], + aggregation.compile(), + ) + def test_count(self): """Ensure Count optionally accepts an id attribute.""" count = Count() @@ -23,3 +33,13 @@ def test_count(self): count = Count(id=5) self.assertEqual(["count"], count.compile()) + + count = Count(id=5, name="My Count") + self.assertEqual( + [ + "aggregation-options", + ["count"], + {"name": "My Count", "display-name": "My Count"}, + ], + count.compile(), + ) From 631630ca0313cd4d5988509d8127a537194e4710 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 21:41:37 -0500 Subject: [PATCH 09/16] feat(mbql): import mbql classes in __init__ Signed-off-by: Charles Lariviere --- src/metabase/__init__.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/metabase/__init__.py b/src/metabase/__init__.py index 30dc4c7..d3de5f8 100644 --- a/src/metabase/__init__.py +++ b/src/metabase/__init__.py @@ -1,3 +1,30 @@ +from metabase.mbql.aggregations import ( + Average, + Count, + CumulativeCount, + CumulativeSum, + Distinct, + Max, + Min, + StandardDeviation, + Sum, +) +from metabase.mbql.filter import ( + Between, + CaseOption, + EndsWith, + Equal, + Greater, + GreaterEqual, + IsNotNull, + IsNull, + Less, + LessEqual, + NotEqual, + StartsWith, +) +from metabase.mbql.groupby import BinOption, GroupBy, TemporalOption +from metabase.mbql.query import Query from metabase.metabase import Metabase from metabase.resources.card import Card from metabase.resources.database import Database From be52fbc2bb5dbfd7fe6ead17d30a19d8f9a09c8b Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 22:13:00 -0500 Subject: [PATCH 10/16] docs(README): add example usage for new MBQL sub-package Signed-off-by: Charles Lariviere --- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fa662e6..dc01247 100644 --- a/README.md +++ b/README.md @@ -90,10 +90,11 @@ for user in User.list(): ) ``` -You can also execute queries and get results back as a Pandas DataFrame. Currently, you need to provide -the exact MBQL (i.e. Metabase Query Language) as the `query` argument. +You can also execute queries and get results back as a Pandas DataFrame. You can provide the exact MBQL, or use +the `Query` object to compile MBQL (i.e. Metabase Query Language) from Python classes included in this package. + ```python -from metabase import Dataset +from metabase import Dataset, Query, Count, GroupBy, TemporalOption dataset = Dataset.create( using.metabase, @@ -106,9 +107,76 @@ dataset = Dataset.create( }, ) +# compile the MBQL above using the Query object +dataset = Dataset.create( + database=1, + type="query", + query=Query( + table_id=2, + aggregations=[Count()], + group_by=[GroupBy(id=7, option=TemporalOption.YEAR)] + ).compile(), +) + df = dataset.to_pandas() ``` +As shown above, the `Query` object allows you to easily compile MBQL from Python objects. Here is a +more complete example: +```python +from metabase import Query, Sum, Average, Greater, GroupBy, BinOption, TemporalOption + +query = Query( + table_id=5, + aggregations=[ + Sum(id=5), # Provide the ID for the Metabase field + Average(id=5, name="Average of Price") # Optionally, you can provide a name + ], + filters=[ + Greater(id=1, value=5.5) # Filter for values of FieldID 1 greater than 5.5 + ], + group_by=[ + GroupBy(id=4), # Group by FieldID 4 + GroupBy(id=5, option=BinOption.AUTO), # You can use Metabase's binning feature for numeric fields + GroupBy(id=5, option=TemporalOption.YEAR) # Or it's temporal option for date fields + ] +) + +print(query.compile()) +{ + 'source-table': 5, + 'aggregation': [ + ['sum', ['field', 5, None]], + ['aggregation-options', ['avg', ['field', 5, None]], {'name': 'Average of Price', 'display-name': 'Average of Price'}] + ], + 'breakout': [ + ['field', 4, None], + ['field', 5, {'binning': {'strategy': 'default'}}], + ['field', 5, {'temporal-unit': 'year'}] + ], + 'filter': ['>', ['field', 1, None], 5.5] +} +``` + +This can also be used to more easily create `Metric` objects. + +```python +from metabase import Metric, Query, Count, EndsWith, CaseOption + + +metric = Metric.create( + name="Gmail Users", + description="Number of users with a @gmail.com email address.", + table_id=2, + definition=Query( + table_id=1, + aggregations=[Count()], + filters=[EndsWith(id=4, value="@gmail.com", option=CaseOption.CASE_INSENSITIVE)] + ).compile() +) +``` + + ## Endpoints From eba28bfa25df1129e08f35ee4b9be38fa0560d61 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 22:45:23 -0500 Subject: [PATCH 11/16] fix: circular import Signed-off-by: Charles Lariviere --- src/metabase/resource.py | 2 +- tests/helpers.py | 2 +- tests/mbql/test_query.py | 1 + tests/resources/test_card.py | 2 +- tests/resources/test_database.py | 4 +++- tests/resources/test_field.py | 2 +- tests/resources/test_metric.py | 2 +- tests/resources/test_permission_group.py | 2 +- tests/resources/test_permission_membership.py | 3 ++- tests/resources/test_segment.py | 2 +- tests/resources/test_table.py | 3 ++- tests/resources/test_user.py | 2 +- tests/test_metabase.py | 2 +- tests/test_resource.py | 1 + 14 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/metabase/resource.py b/src/metabase/resource.py index 409c35c..2d1813c 100644 --- a/src/metabase/resource.py +++ b/src/metabase/resource.py @@ -2,8 +2,8 @@ from requests import HTTPError -from metabase import Metabase from metabase.exceptions import NotFoundError +from metabase.metabase import Metabase from metabase.missing import MISSING diff --git a/tests/helpers.py b/tests/helpers.py index 231b0a4..4359da9 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,7 +2,7 @@ import requests -from metabase import Metabase +from metabase.metabase import Metabase class IntegrationTestCase(TestCase): diff --git a/tests/mbql/test_query.py b/tests/mbql/test_query.py index 87768d6..2869f62 100644 --- a/tests/mbql/test_query.py +++ b/tests/mbql/test_query.py @@ -4,6 +4,7 @@ from metabase.mbql.filter import Equal from metabase.mbql.groupby import GroupBy from metabase.mbql.query import Query +from metabase.resources.metric import Metric class QueryTests(TestCase): diff --git a/tests/resources/test_card.py b/tests/resources/test_card.py index 92e2be4..ca8d9bd 100644 --- a/tests/resources/test_card.py +++ b/tests/resources/test_card.py @@ -1,4 +1,4 @@ -from metabase import Card +from metabase.resources.card import Card from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_database.py b/tests/resources/test_database.py index e2203ac..874558b 100644 --- a/tests/resources/test_database.py +++ b/tests/resources/test_database.py @@ -1,5 +1,7 @@ -from metabase import Database, Field, Table from metabase.exceptions import NotFoundError +from metabase.resources.database import Database +from metabase.resources.field import Field +from metabase.resources.table import Table from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_field.py b/tests/resources/test_field.py index 36f4ae8..24838cd 100644 --- a/tests/resources/test_field.py +++ b/tests/resources/test_field.py @@ -1,4 +1,4 @@ -from metabase import Field +from metabase.resources.field import Field from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_metric.py b/tests/resources/test_metric.py index bf0da86..b211bff 100644 --- a/tests/resources/test_metric.py +++ b/tests/resources/test_metric.py @@ -1,5 +1,5 @@ -from metabase import Metric from metabase.exceptions import NotFoundError +from metabase.resources.metric import Metric from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_permission_group.py b/tests/resources/test_permission_group.py index 8f1e66b..0d07bd7 100644 --- a/tests/resources/test_permission_group.py +++ b/tests/resources/test_permission_group.py @@ -1,5 +1,5 @@ -from metabase import PermissionGroup from metabase.exceptions import NotFoundError +from metabase.resources.permission_group import PermissionGroup from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_permission_membership.py b/tests/resources/test_permission_membership.py index dce5af8..e97abe0 100644 --- a/tests/resources/test_permission_membership.py +++ b/tests/resources/test_permission_membership.py @@ -1,4 +1,5 @@ -from metabase import PermissionGroup, PermissionMembership +from metabase.resources.permission_group import PermissionGroup +from metabase.resources.permission_membership import PermissionMembership from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_segment.py b/tests/resources/test_segment.py index d159514..a51cac4 100644 --- a/tests/resources/test_segment.py +++ b/tests/resources/test_segment.py @@ -1,5 +1,5 @@ -from metabase import Segment from metabase.exceptions import NotFoundError +from metabase.resources.segment import Segment from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_table.py b/tests/resources/test_table.py index 230abc4..a76c150 100644 --- a/tests/resources/test_table.py +++ b/tests/resources/test_table.py @@ -1,4 +1,5 @@ -from metabase import Field, Metric +from metabase.resources.field import Field +from metabase.resources.metric import Metric from metabase.resources.segment import Segment from metabase.resources.table import Dimension, Table from tests.helpers import IntegrationTestCase diff --git a/tests/resources/test_user.py b/tests/resources/test_user.py index d818c38..ea15d1c 100644 --- a/tests/resources/test_user.py +++ b/tests/resources/test_user.py @@ -1,7 +1,7 @@ from random import randint -from metabase import User from metabase.exceptions import NotFoundError +from metabase.resources.user import User from tests.helpers import IntegrationTestCase diff --git a/tests/test_metabase.py b/tests/test_metabase.py index 57e0631..4fc7736 100644 --- a/tests/test_metabase.py +++ b/tests/test_metabase.py @@ -1,7 +1,7 @@ from unittest import TestCase -from metabase import Metabase from metabase.exceptions import AuthenticationError +from metabase.metabase import Metabase from tests.helpers import IntegrationTestCase diff --git a/tests/test_resource.py b/tests/test_resource.py index 7eab526..bd8ae19 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -3,6 +3,7 @@ from requests import HTTPError from metabase.exceptions import NotFoundError +from metabase.metabase import Metabase from metabase.missing import MISSING from metabase.resource import ( CreateResource, From 0c33d3f145aadf5c2c5172ddfc1e06acd964043f Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 22:45:44 -0500 Subject: [PATCH 12/16] feat(query): add support for Metric in aggregations Signed-off-by: Charles Lariviere --- src/metabase/mbql/query.py | 14 +++++++++++--- tests/mbql/test_query.py | 7 +++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/metabase/mbql/query.py b/src/metabase/mbql/query.py index c05f730..2498d8d 100644 --- a/src/metabase/mbql/query.py +++ b/src/metabase/mbql/query.py @@ -1,15 +1,16 @@ from dataclasses import dataclass, field -from typing import List +from typing import List, Union from metabase.mbql.aggregations import Aggregation from metabase.mbql.filter import Filter from metabase.mbql.groupby import GroupBy +from metabase.resources.metric import Metric @dataclass class Query: table_id: int - aggregations: List[Aggregation] + aggregations: List[Union[Aggregation, Metric]] group_by: List[GroupBy] = field(default_factory=list) filters: List[Filter] = field(default_factory=list) @@ -23,7 +24,14 @@ def compile(self): @property def _aggregations(self): - return [aggregation.compile() for aggregation in self.aggregations] + aggregations = [] + for aggregation in self.aggregations: + if isinstance(aggregation, Metric): + aggregations.append(["metric", aggregation.id]) + else: + aggregations.append(aggregation.compile()) + + return aggregations @property def _group_by(self): diff --git a/tests/mbql/test_query.py b/tests/mbql/test_query.py index 2869f62..c950f30 100644 --- a/tests/mbql/test_query.py +++ b/tests/mbql/test_query.py @@ -1,5 +1,6 @@ from unittest import TestCase +from metabase import Metric from metabase.mbql.aggregations import Count, Max from metabase.mbql.filter import Equal from metabase.mbql.groupby import GroupBy @@ -42,6 +43,12 @@ def test__aggregations(self): ) self.assertEqual([["count"], ["max", ["field", 5, None]]], query._aggregations) + query = Query( + table_id=12, + aggregations=[Count(), Metric(id=4)], + ) + self.assertEqual([["count"], ["metric", 4]], query._aggregations) + def test__group_by(self): """Ensure Query._group_by returns a list of compiled GroupBy.""" query = Query(table_id=12, aggregations=[Count()], group_by=[]) From ccd3dcc3f3065b690825dccb6f4fd5a718c06c42 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Wed, 23 Feb 2022 22:46:01 -0500 Subject: [PATCH 13/16] docs(README): add example of using a Metric in a Query aggregation Signed-off-by: Charles Lariviere --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc01247..dbd5c1c 100644 --- a/README.md +++ b/README.md @@ -124,13 +124,14 @@ df = dataset.to_pandas() As shown above, the `Query` object allows you to easily compile MBQL from Python objects. Here is a more complete example: ```python -from metabase import Query, Sum, Average, Greater, GroupBy, BinOption, TemporalOption +from metabase import Query, Sum, Average, Metric, Greater, GroupBy, BinOption, TemporalOption query = Query( table_id=5, aggregations=[ Sum(id=5), # Provide the ID for the Metabase field - Average(id=5, name="Average of Price") # Optionally, you can provide a name + Average(id=5, name="Average of Price"), # Optionally, you can provide a name + Metric.get(5) # You can also provide your Metabase Metrics ], filters=[ Greater(id=1, value=5.5) # Filter for values of FieldID 1 greater than 5.5 @@ -147,7 +148,8 @@ print(query.compile()) 'source-table': 5, 'aggregation': [ ['sum', ['field', 5, None]], - ['aggregation-options', ['avg', ['field', 5, None]], {'name': 'Average of Price', 'display-name': 'Average of Price'}] + ['aggregation-options', ['avg', ['field', 5, None]], {'name': 'Average of Price', 'display-name': 'Average of Price'}], + ["metric", 5] ], 'breakout': [ ['field', 4, None], From 67c65a7915163cc1180ba9f9e5aae8739b2e0367 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Thu, 24 Feb 2022 00:25:53 -0500 Subject: [PATCH 14/16] fix: circular import Signed-off-by: Charles Lariviere --- src/metabase/resources/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metabase/resources/metric.py b/src/metabase/resources/metric.py index 2bebab0..266ae49 100644 --- a/src/metabase/resources/metric.py +++ b/src/metabase/resources/metric.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import List -from metabase import Metabase +from metabase.metabase import Metabase from metabase.missing import MISSING from metabase.resource import CreateResource, GetResource, ListResource, UpdateResource From 4479947844b0280e7ba097e2ad9c3af9f295585b Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Thu, 24 Feb 2022 00:26:07 -0500 Subject: [PATCH 15/16] docs(README): add missing references to using=metabase Signed-off-by: Charles Lariviere --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dbd5c1c..afe3a64 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ pip install metabase-python ``` ## Usage -This API is still experimental and may change significantly between minor versions. +### Connection Start by creating an instance of Metabase with your credentials. ```python @@ -27,6 +27,7 @@ metabase = Metabase( ) ``` +### Interacting with Endpoints You can then interact with any of the supported endpoints through the classes included in this package. Methods that instantiate an object from the Metabase API require the `using` parameter which expects an instance of `Metabase` such as the one we just instantiated above. All changes are reflected in Metabase instantly. @@ -84,12 +85,14 @@ my_group = PermissionGroup.create(name="My Group", using=metabase) for user in User.list(): # add all users to my_group PermissionMembership.create( - using=metabase, group_id=my_group.id, - user_id=user.id + user_id=user.id, + using=metabase, ) ``` +### Querying & MBQL + You can also execute queries and get results back as a Pandas DataFrame. You can provide the exact MBQL, or use the `Query` object to compile MBQL (i.e. Metabase Query Language) from Python classes included in this package. @@ -97,7 +100,6 @@ the `Query` object to compile MBQL (i.e. Metabase Query Language) from Python cl from metabase import Dataset, Query, Count, GroupBy, TemporalOption dataset = Dataset.create( - using.metabase, database=1, type="query", query={ @@ -105,6 +107,7 @@ dataset = Dataset.create( "aggregation": [["count"]], "breakout": ["field", 7, {"temporal-unit": "year"},], }, + using=metabase, ) # compile the MBQL above using the Query object @@ -116,6 +119,7 @@ dataset = Dataset.create( aggregations=[Count()], group_by=[GroupBy(id=7, option=TemporalOption.YEAR)] ).compile(), + using=metabase ) df = dataset.to_pandas() @@ -174,7 +178,8 @@ metric = Metric.create( table_id=1, aggregations=[Count()], filters=[EndsWith(id=4, value="@gmail.com", option=CaseOption.CASE_INSENSITIVE)] - ).compile() + ).compile(), + using=metabase ) ``` From 7fafd8be2e877ae514113bd6e3ee1be9b7e7eb50 Mon Sep 17 00:00:00 2001 From: Charles Lariviere Date: Thu, 24 Feb 2022 00:26:20 -0500 Subject: [PATCH 16/16] test(mbql): add missing using=metabase Signed-off-by: Charles Lariviere --- tests/mbql/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mbql/test_query.py b/tests/mbql/test_query.py index c950f30..bdcd5d2 100644 --- a/tests/mbql/test_query.py +++ b/tests/mbql/test_query.py @@ -45,7 +45,7 @@ def test__aggregations(self): query = Query( table_id=12, - aggregations=[Count(), Metric(id=4)], + aggregations=[Count(), Metric(id=4, _using=None)], ) self.assertEqual([["count"], ["metric", 4]], query._aggregations)