From 532468b00249a815e2d1c731e0fb12698755d519 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Thu, 6 Oct 2022 08:09:15 -0500 Subject: [PATCH 01/15] flake8 cleanup removing E203, W503, F401 --- .flake8 | 2 +- pytabular/column.py | 4 +--- pytabular/partition.py | 1 - pytabular/pytabular.py | 17 ++++++----------- pytabular/table.py | 1 - 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.flake8 b/.flake8 index d24977e..16520fc 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,2 @@ [flake8] -ignore = E501, E203, W503, F401 \ No newline at end of file +ignore = E501 \ No newline at end of file diff --git a/pytabular/column.py b/pytabular/column.py index ddfd248..5f165b7 100644 --- a/pytabular/column.py +++ b/pytabular/column.py @@ -1,9 +1,7 @@ import logging from object import PyObject, PyObjects -from logic_utils import ticks_to_datetime -import pandas as pd -from typing import List + logger = logging.getLogger("PyTabular") diff --git a/pytabular/partition.py b/pytabular/partition.py index 1fdf204..1a91aec 100644 --- a/pytabular/partition.py +++ b/pytabular/partition.py @@ -3,7 +3,6 @@ from object import PyObject, PyObjects from logic_utils import ticks_to_datetime import pandas as pd -from typing import List logger = logging.getLogger("PyTabular") diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 4077afd..04dabaf 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -7,10 +7,9 @@ Table, DataColumn, Partition, - MPartitionSource, - Measure, + MPartitionSource ) -from Microsoft.AnalysisServices.AdomdClient import AdomdCommand, AdomdConnection + from Microsoft.AnalysisServices import UpdateOptions from typing import Any, Dict, List, Union from collections import namedtuple @@ -150,8 +149,7 @@ def _Refresh_Report(Property_Changes) -> pd.DataFrame: refresh_data = [] for property_change in Property_Changes: if ( - isinstance(property_change.Object, Partition) - and property_change.Property_Name == "RefreshedTime" + isinstance(property_change.Object, Partition) and property_change.Property_Name == "RefreshedTime" ): table, partition, refreshed_time = ( property_change.Object.Table.Name, @@ -331,8 +329,7 @@ def rename(items): relationships = [ relationship.Clone() for relationship in self.Model.Relationships.GetEnumerator() - if relationship.ToTable.Name == remove_suffix(table.Name, "_backup") - or relationship.FromTable.Name == remove_suffix(table.Name, "_backup") + if relationship.ToTable.Name == remove_suffix(table.Name, "_backup") or relationship.FromTable.Name == remove_suffix(table.Name, "_backup") ] logger.info("Renaming Relationships") rename(relationships) @@ -416,15 +413,13 @@ def Revert_Table(self, table_str: str) -> bool: main_relationships = [ relationship for relationship in self.Model.Relationships.GetEnumerator() - if relationship.ToTable.Name == main.Name - or relationship.FromTable.Name == main.Name + if relationship.ToTable.Name == main.Name or relationship.FromTable.Name == main.Name ] logger.debug("Finding backup relationships") backup_relationships = [ relationship for relationship in self.Model.Relationships.GetEnumerator() - if relationship.ToTable.Name == backup.Name - or relationship.FromTable.Name == backup.Name + if relationship.ToTable.Name == backup.Name or relationship.FromTable.Name == backup.Name ] def remove_role_permissions(): diff --git a/pytabular/table.py b/pytabular/table.py index cfcc091..7ded903 100644 --- a/pytabular/table.py +++ b/pytabular/table.py @@ -1,6 +1,5 @@ import logging -from Microsoft.AnalysisServices.Tabular import Table from object import PyObject import pandas as pd from partition import PyPartition, PyPartitions From 30a4410f30fea3e177680334ef2d270ac18ed029 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Thu, 6 Oct 2022 08:59:18 -0500 Subject: [PATCH 02/15] introduce measure class --- pytabular/measure.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pytabular/measure.py diff --git a/pytabular/measure.py b/pytabular/measure.py new file mode 100644 index 0000000..4e2455d --- /dev/null +++ b/pytabular/measure.py @@ -0,0 +1,25 @@ +import logging + +from object import PyObject, PyObjects +from logic_utils import ticks_to_datetime +import pandas as pd + +logger = logging.getLogger("PyTabular") + + +class PyMeasure(PyObject): + """Wrapper for [Microsoft.AnalysisServices.Measure](https://learn.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.measure?view=analysisservices-dotnet). + With a few other bells and whistles added to it. WIP + + Args: + Table: Parent Table to the Measure + """ + + def __init__(self, object, table) -> None: + super().__init__(object) + self.Table = table + + +class PyMeasures(PyObjects): + def __init__(self, objects) -> None: + super().__init__(objects) From a1a52c8a9265d5081acbc588e4219af60b29a78d Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Thu, 6 Oct 2022 08:59:34 -0500 Subject: [PATCH 03/15] add values --- pytabular/column.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pytabular/column.py b/pytabular/column.py index 5f165b7..3a648ff 100644 --- a/pytabular/column.py +++ b/pytabular/column.py @@ -1,5 +1,5 @@ import logging - +import pandas as pd from object import PyObject, PyObjects @@ -18,13 +18,30 @@ def __init__(self, object, table) -> None: super().__init__(object) self.Table = table - def Distinct_Count(self, No_Blank=False): + def Distinct_Count(self, No_Blank=False) -> int: + '''Get [DISTINCTCOUNT](https://learn.microsoft.com/en-us/dax/distinctcount-function-dax) of Column. + + Args: + No_Blank (bool, optional): Ability to call [DISTINCTCOUNTNOBLANK](https://learn.microsoft.com/en-us/dax/distinctcountnoblank-function-dax). Defaults to False. + + Returns: + int: Number of Distinct Count from column. If `No_Blank == True` then will return number of Distinct Count no blanks. + ''' func = "DISTINCTCOUNT" if No_Blank: func += "NOBLANK" return self.Table.Model.Adomd.Query( f"EVALUATE {{{func}('{self.Table.Name}'[{self.Name}])}}" ) + def Values(self) -> pd.DataFrame: + '''Get single column DataFrame of [VALUES](https://learn.microsoft.com/en-us/dax/values-function-dax) + + Returns: + pd.DataFrame: Single Column DataFrame of Values. + ''' + return self.Table.Model.Adomd.Query( + f"EVALUATE VALUES('{self.Table.Name}'[{self.Name}])" + ) class PyColumns(PyObjects): From 5692d59d37cc80638fdf860e7e1e1e8744e71cff Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Thu, 6 Oct 2022 08:59:50 -0500 Subject: [PATCH 04/15] add measures to table class --- pytabular/table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pytabular/table.py b/pytabular/table.py index 7ded903..3a02d35 100644 --- a/pytabular/table.py +++ b/pytabular/table.py @@ -1,9 +1,9 @@ import logging - from object import PyObject import pandas as pd from partition import PyPartition, PyPartitions from column import PyColumn, PyColumns +from measure import PyMeasure, PyMeasures from pytabular.object import PyObjects logger = logging.getLogger("PyTabular") @@ -31,6 +31,9 @@ def __init__(self, object, model) -> None: self.Columns = PyColumns( [PyColumn(column, self) for column in self._object.Columns.GetEnumerator()] ) + self.Measures = PyMeasures( + [PyMeasure(measure, self) for measure in self._object.Measures.GetEnumerator()] + ) def Row_Count(self) -> int: """Method to return count of rows. Simple Dax Query: From 0c101c1c5e722c6b4043eb43975a7c0a497e8219 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Thu, 6 Oct 2022 09:00:05 -0500 Subject: [PATCH 05/15] remove measures attribute --- pytabular/__init__.py | 3 +-- pytabular/basic_checks.py | 3 ++- pytabular/object.py | 5 +++++ pytabular/pytabular.py | 5 ----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pytabular/__init__.py b/pytabular/__init__.py index 64947ed..4d2ce3b 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -43,8 +43,7 @@ from .basic_checks import ( Return_Zero_Row_Tables, Table_Last_Refresh_Times, - BPA_Violations_To_DF, - Last_X_Interval, + BPA_Violations_To_DF ) from .logic_utils import ( pd_dataframe_to_m_expression, diff --git a/pytabular/basic_checks.py b/pytabular/basic_checks.py index 8a1986d..f1db838 100644 --- a/pytabular/basic_checks.py +++ b/pytabular/basic_checks.py @@ -83,7 +83,7 @@ def BPA_Violations_To_DF(model: pytabular.Tabular, te2: str, bpa: str) -> pd.Dat columns = ["Object", "Violation"] return pd.DataFrame(data, columns=columns) - +''' def Last_X_Interval( Model: pytabular.Tabular, Measure: Union[str, pytabular.pytabular.Measure], @@ -131,3 +131,4 @@ def Last_X_Interval( f"Running query for {Column_Name} in the last {Number_Of_Intervals} {Interval}s..." ) return Model.Query(Query_Str) +''' \ No newline at end of file diff --git a/pytabular/object.py b/pytabular/object.py index 0a00574..18ea09c 100644 --- a/pytabular/object.py +++ b/pytabular/object.py @@ -35,3 +35,8 @@ def __iter__(self): def __len__(self): return len(self._objects) + + def Find(self, object_str): + return [ + object for object in self._objects if object_str.lower() in object.Name.lower() + ] diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 04dabaf..2356dc1 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -88,11 +88,6 @@ def Reload_Model_Info(self) -> bool: self.Tables = PyTables( [PyTable(table, self) for table in self.Model.Tables.GetEnumerator()] ) - self.Measures = [ - measure - for table in self.Tables - for measure in table.Measures.GetEnumerator() - ] self.Database.Refresh() return True From f4c9c7aaf7ba162e6a4fe96b7ea4cf325b01054c Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Fri, 7 Oct 2022 14:50:31 -0500 Subject: [PATCH 06/15] refresh_handler improvement --- pytabular/tabular_tracing.py | 49 +++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/pytabular/tabular_tracing.py b/pytabular/tabular_tracing.py index 8943cd1..2470173 100644 --- a/pytabular/tabular_tracing.py +++ b/pytabular/tabular_tracing.py @@ -177,11 +177,48 @@ def _Query_DMV_For_Event_Categories(self): return Event_Categories -def refresh_handler(source, args): - if args.EventSubclass == TraceEventSubclass.ReadData: - logger.info(f"{args.ProgressTotal} - {args.ObjectPath}") +def _refresh_handler(source, args): + TextData = args.TextData.replace('','').replace('','') + + if args.EventClass == TraceEventClass.ProgressReportCurrent and args.EventSubclass == TraceEventSubclass.ReadData: + logger.info(f"{args.ProgressTotal}::{'::'.join(args.ObjectPath.split('.')[-2:])}") + + elif args.EventClass == TraceEventClass.ProgressReportEnd and args.EventSubclass == TraceEventSubclass.ReadData: + if args.ProgressTotal == 0: + logger.warning(f"{'::'.join(args.ObjectPath.split('.')[-2:])} QUERIED {args.ProgressTotal} ROWS!") + else: + logger.info(f"Finished Reading {'::'.join(args.ObjectPath.split('.')[-2:])} for {args.ProgressTotal} Rows!") + + elif args.EventSubclass == TraceEventSubclass.SwitchingDictionary: + logger.warning(f"{TextData}") + + elif args.EventClass == TraceEventClass.ProgressReportBegin and args.EventSubclass in [ + TraceEventSubclass.TabularSequencePoint, + TraceEventSubclass.TabularRefresh, + TraceEventSubclass.Process, + TraceEventSubclass.VertiPaq, + TraceEventSubclass.CompressSegment, + TraceEventSubclass.TabularCommit, + TraceEventSubclass.RelationshipBuildPrepare, + TraceEventSubclass.AnalyzeEncodeData, + TraceEventSubclass.ReadData + ]: + logger.info(f"{TextData}") + + elif args.EventClass == TraceEventClass.ProgressReportEnd and args.EventSubclass in [ + TraceEventSubclass.TabularSequencePoint, + TraceEventSubclass.TabularRefresh, + TraceEventSubclass.Process, + TraceEventSubclass.VertiPaq, + TraceEventSubclass.CompressSegment, + TraceEventSubclass.TabularCommit, + TraceEventSubclass.RelationshipBuildPrepare, + TraceEventSubclass.AnalyzeEncodeData + ]: + logger.info(f"{TextData}") + else: - logger.info(f"{args.EventClass} - {args.EventSubclass} - {args.ObjectName}") + logger.debug(f"{args.EventClass}::{args.EventSubclass}::{TextData}") class Refresh_Trace(Base_Trace): @@ -209,8 +246,8 @@ def __init__( TraceColumn.SessionID, TraceColumn.TextData, TraceColumn.EventClass, - TraceColumn.ProgressTotal, + TraceColumn.ProgressTotal ], - Handler: Callable = refresh_handler, + Handler: Callable = _refresh_handler, ) -> None: super().__init__(Tabular_Class, Trace_Events, Trace_Event_Columns, Handler) From ee553da42427b409fcf8a7f050f80dc68f86db89 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Fri, 7 Oct 2022 14:50:51 -0500 Subject: [PATCH 07/15] last x interval fix --- pytabular/__init__.py | 3 ++- pytabular/basic_checks.py | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pytabular/__init__.py b/pytabular/__init__.py index 4d2ce3b..de42def 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -43,7 +43,8 @@ from .basic_checks import ( Return_Zero_Row_Tables, Table_Last_Refresh_Times, - BPA_Violations_To_DF + BPA_Violations_To_DF, + Last_X_Interval ) from .logic_utils import ( pd_dataframe_to_m_expression, diff --git a/pytabular/basic_checks.py b/pytabular/basic_checks.py index f1db838..27b9832 100644 --- a/pytabular/basic_checks.py +++ b/pytabular/basic_checks.py @@ -4,7 +4,7 @@ from logic_utils import ticks_to_datetime import sys import pandas as pd - +from measure import PyMeasure logger = logging.getLogger("PyTabular") @@ -83,10 +83,10 @@ def BPA_Violations_To_DF(model: pytabular.Tabular, te2: str, bpa: str) -> pd.Dat columns = ["Object", "Violation"] return pd.DataFrame(data, columns=columns) -''' + def Last_X_Interval( Model: pytabular.Tabular, - Measure: Union[str, pytabular.pytabular.Measure], + Measure: Union[str, PyMeasure], Column_Name: Union[str, None] = None, Date_Column_Identifier: str = "'Date'[DATE_DTE_KEY]", Number_Of_Intervals: int = 90, @@ -105,7 +105,7 @@ def Last_X_Interval( Returns: pd.DataFrame: Pandas DataFrame of results. """ - if isinstance(Measure, str): + if isinstance(PyMeasure, str): try: Measure = [ measure for measure in Model.Measures if measure.Name == Measure @@ -131,4 +131,3 @@ def Last_X_Interval( f"Running query for {Column_Name} in the last {Number_Of_Intervals} {Interval}s..." ) return Model.Query(Query_Str) -''' \ No newline at end of file From 1f295a1a91a3a02aa68ba5dcbe3e56d6bba0bbb1 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Fri, 7 Oct 2022 14:51:16 -0500 Subject: [PATCH 08/15] partition, column, measure attribute addition --- pytabular/pytabular.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 2356dc1..5bb6c17 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -25,6 +25,9 @@ ) from query import Connection from table import PyTable, PyTables +from partition import PyPartitions +from column import PyColumns +from measure import PyMeasures from tabular_tracing import Refresh_Trace logger = logging.getLogger("PyTabular") @@ -88,6 +91,15 @@ def Reload_Model_Info(self) -> bool: self.Tables = PyTables( [PyTable(table, self) for table in self.Model.Tables.GetEnumerator()] ) + self.Partitions = PyPartitions( + [partition for table in self.Tables for partition in table.Partitions] + ) + self.Columns = PyColumns( + [column for table in self.Tables for column in table.Columns] + ) + self.Measures = PyMeasures( + [measure for table in self.Tables for measure in table.Measures] + ) self.Database.Refresh() return True From 3e31bc149cd440cd3ec39f4b45cc9769c64d8d3b Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Fri, 7 Oct 2022 14:52:18 -0500 Subject: [PATCH 09/15] project specific addition --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 187ab13..f4068c5 100644 --- a/.gitignore +++ b/.gitignore @@ -167,6 +167,7 @@ __pycache__ /TE2 setup.py notes.txt +adhoc.py *.bim *.csv docs/_config.yml# Byte-compiled / optimized / DLL files From aa11cba6eba090970762e73f279d6c547e21fb4c Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 8 Oct 2022 12:21:29 -0500 Subject: [PATCH 10/15] black styling --- pytabular/__init__.py | 7 +++---- pytabular/pytabular.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pytabular/__init__.py b/pytabular/__init__.py index de42def..02ddecf 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -3,11 +3,10 @@ import os import sys import platform +from rich.logging import RichHandler logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s::%(module)s::%(funcName)s::%(levelname)s::%(message)s", - datefmt="%y/%m/%d %H:%M:%S %z", + level=logging.DEBUG, format="%(message)s", datefmt="[%x]", handlers=[RichHandler()] ) logger = logging.getLogger("PyTabular") logger.setLevel(logging.INFO) @@ -44,7 +43,7 @@ Return_Zero_Row_Tables, Table_Last_Refresh_Times, BPA_Violations_To_DF, - Last_X_Interval + Last_X_Interval, ) from .logic_utils import ( pd_dataframe_to_m_expression, diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 5bb6c17..4e965a7 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -7,7 +7,7 @@ Table, DataColumn, Partition, - MPartitionSource + MPartitionSource, ) from Microsoft.AnalysisServices import UpdateOptions @@ -156,7 +156,8 @@ def _Refresh_Report(Property_Changes) -> pd.DataFrame: refresh_data = [] for property_change in Property_Changes: if ( - isinstance(property_change.Object, Partition) and property_change.Property_Name == "RefreshedTime" + isinstance(property_change.Object, Partition) + and property_change.Property_Name == "RefreshedTime" ): table, partition, refreshed_time = ( property_change.Object.Table.Name, @@ -336,7 +337,8 @@ def rename(items): relationships = [ relationship.Clone() for relationship in self.Model.Relationships.GetEnumerator() - if relationship.ToTable.Name == remove_suffix(table.Name, "_backup") or relationship.FromTable.Name == remove_suffix(table.Name, "_backup") + if relationship.ToTable.Name == remove_suffix(table.Name, "_backup") + or relationship.FromTable.Name == remove_suffix(table.Name, "_backup") ] logger.info("Renaming Relationships") rename(relationships) @@ -420,13 +422,15 @@ def Revert_Table(self, table_str: str) -> bool: main_relationships = [ relationship for relationship in self.Model.Relationships.GetEnumerator() - if relationship.ToTable.Name == main.Name or relationship.FromTable.Name == main.Name + if relationship.ToTable.Name == main.Name + or relationship.FromTable.Name == main.Name ] logger.debug("Finding backup relationships") backup_relationships = [ relationship for relationship in self.Model.Relationships.GetEnumerator() - if relationship.ToTable.Name == backup.Name or relationship.FromTable.Name == backup.Name + if relationship.ToTable.Name == backup.Name + or relationship.FromTable.Name == backup.Name ] def remove_role_permissions(): @@ -621,6 +625,6 @@ def Create_Table(self, df: pd.DataFrame, table_name: str) -> bool: f"Adding table: {new_table.Name} to {self.Server.Name}::{self.Database.Name}::{self.Model.Name}" ) self.Model.Tables.Add(new_table) - self.Refresh([new_table]) + self.Refresh([new_table], Tracing=True) self.SaveChanges() return True From 09c2eb8ab31d96ae7b2c1ac558bac0a65abe8209 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 8 Oct 2022 12:21:57 -0500 Subject: [PATCH 11/15] black styling --- pytabular/basic_checks.py | 1 + pytabular/column.py | 9 +++++---- pytabular/measure.py | 2 -- pytabular/object.py | 6 ++++-- pytabular/table.py | 5 ++++- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pytabular/basic_checks.py b/pytabular/basic_checks.py index 27b9832..9fe46c8 100644 --- a/pytabular/basic_checks.py +++ b/pytabular/basic_checks.py @@ -5,6 +5,7 @@ import sys import pandas as pd from measure import PyMeasure + logger = logging.getLogger("PyTabular") diff --git a/pytabular/column.py b/pytabular/column.py index 3a648ff..cdc9c22 100644 --- a/pytabular/column.py +++ b/pytabular/column.py @@ -19,26 +19,27 @@ def __init__(self, object, table) -> None: self.Table = table def Distinct_Count(self, No_Blank=False) -> int: - '''Get [DISTINCTCOUNT](https://learn.microsoft.com/en-us/dax/distinctcount-function-dax) of Column. + """Get [DISTINCTCOUNT](https://learn.microsoft.com/en-us/dax/distinctcount-function-dax) of Column. Args: No_Blank (bool, optional): Ability to call [DISTINCTCOUNTNOBLANK](https://learn.microsoft.com/en-us/dax/distinctcountnoblank-function-dax). Defaults to False. Returns: int: Number of Distinct Count from column. If `No_Blank == True` then will return number of Distinct Count no blanks. - ''' + """ func = "DISTINCTCOUNT" if No_Blank: func += "NOBLANK" return self.Table.Model.Adomd.Query( f"EVALUATE {{{func}('{self.Table.Name}'[{self.Name}])}}" ) + def Values(self) -> pd.DataFrame: - '''Get single column DataFrame of [VALUES](https://learn.microsoft.com/en-us/dax/values-function-dax) + """Get single column DataFrame of [VALUES](https://learn.microsoft.com/en-us/dax/values-function-dax) Returns: pd.DataFrame: Single Column DataFrame of Values. - ''' + """ return self.Table.Model.Adomd.Query( f"EVALUATE VALUES('{self.Table.Name}'[{self.Name}])" ) diff --git a/pytabular/measure.py b/pytabular/measure.py index 4e2455d..a1e22f4 100644 --- a/pytabular/measure.py +++ b/pytabular/measure.py @@ -1,8 +1,6 @@ import logging from object import PyObject, PyObjects -from logic_utils import ticks_to_datetime -import pandas as pd logger = logging.getLogger("PyTabular") diff --git a/pytabular/object.py b/pytabular/object.py index 18ea09c..bead097 100644 --- a/pytabular/object.py +++ b/pytabular/object.py @@ -35,8 +35,10 @@ def __iter__(self): def __len__(self): return len(self._objects) - + def Find(self, object_str): return [ - object for object in self._objects if object_str.lower() in object.Name.lower() + object + for object in self._objects + if object_str.lower() in object.Name.lower() ] diff --git a/pytabular/table.py b/pytabular/table.py index 3a02d35..eb9f924 100644 --- a/pytabular/table.py +++ b/pytabular/table.py @@ -32,7 +32,10 @@ def __init__(self, object, model) -> None: [PyColumn(column, self) for column in self._object.Columns.GetEnumerator()] ) self.Measures = PyMeasures( - [PyMeasure(measure, self) for measure in self._object.Measures.GetEnumerator()] + [ + PyMeasure(measure, self) + for measure in self._object.Measures.GetEnumerator() + ] ) def Row_Count(self) -> int: From 8f5cf5063b83a7a3578b289cceed8bb272bdfe11 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 8 Oct 2022 12:22:16 -0500 Subject: [PATCH 12/15] refresh handler refactor --- pytabular/tabular_tracing.py | 76 +++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/pytabular/tabular_tracing.py b/pytabular/tabular_tracing.py index 2470173..3632255 100644 --- a/pytabular/tabular_tracing.py +++ b/pytabular/tabular_tracing.py @@ -178,43 +178,63 @@ def _Query_DMV_For_Event_Categories(self): def _refresh_handler(source, args): - TextData = args.TextData.replace('','').replace('','') + TextData = args.TextData.replace("", "").replace("", "") - if args.EventClass == TraceEventClass.ProgressReportCurrent and args.EventSubclass == TraceEventSubclass.ReadData: - logger.info(f"{args.ProgressTotal}::{'::'.join(args.ObjectPath.split('.')[-2:])}") + if ( + args.EventClass == TraceEventClass.ProgressReportCurrent + and args.EventSubclass == TraceEventSubclass.ReadData + ): + logger.info( + f"Total Rows Read {args.ProgressTotal} From Table '{args.ObjectPath.split('.')[-2]}' Partition '{args.ObjectPath.split('.')[-1]}' " + ) - elif args.EventClass == TraceEventClass.ProgressReportEnd and args.EventSubclass == TraceEventSubclass.ReadData: + elif ( + args.EventClass == TraceEventClass.ProgressReportEnd + and args.EventSubclass == TraceEventSubclass.ReadData + ): if args.ProgressTotal == 0: - logger.warning(f"{'::'.join(args.ObjectPath.split('.')[-2:])} QUERIED {args.ProgressTotal} ROWS!") + logger.warning( + f"{'::'.join(args.ObjectPath.split('.')[-2:])} QUERIED {args.ProgressTotal} ROWS!" + ) else: - logger.info(f"Finished Reading {'::'.join(args.ObjectPath.split('.')[-2:])} for {args.ProgressTotal} Rows!") + logger.info( + f"Finished Reading {'::'.join(args.ObjectPath.split('.')[-2:])} for {args.ProgressTotal} Rows!" + ) elif args.EventSubclass == TraceEventSubclass.SwitchingDictionary: logger.warning(f"{TextData}") - elif args.EventClass == TraceEventClass.ProgressReportBegin and args.EventSubclass in [ - TraceEventSubclass.TabularSequencePoint, - TraceEventSubclass.TabularRefresh, - TraceEventSubclass.Process, - TraceEventSubclass.VertiPaq, - TraceEventSubclass.CompressSegment, - TraceEventSubclass.TabularCommit, - TraceEventSubclass.RelationshipBuildPrepare, - TraceEventSubclass.AnalyzeEncodeData, - TraceEventSubclass.ReadData - ]: + elif ( + args.EventClass == TraceEventClass.ProgressReportBegin + and args.EventSubclass + in [ + TraceEventSubclass.TabularSequencePoint, + TraceEventSubclass.TabularRefresh, + TraceEventSubclass.Process, + TraceEventSubclass.VertiPaq, + TraceEventSubclass.CompressSegment, + TraceEventSubclass.TabularCommit, + TraceEventSubclass.RelationshipBuildPrepare, + TraceEventSubclass.AnalyzeEncodeData, + TraceEventSubclass.ReadData, + ] + ): logger.info(f"{TextData}") - elif args.EventClass == TraceEventClass.ProgressReportEnd and args.EventSubclass in [ - TraceEventSubclass.TabularSequencePoint, - TraceEventSubclass.TabularRefresh, - TraceEventSubclass.Process, - TraceEventSubclass.VertiPaq, - TraceEventSubclass.CompressSegment, - TraceEventSubclass.TabularCommit, - TraceEventSubclass.RelationshipBuildPrepare, - TraceEventSubclass.AnalyzeEncodeData - ]: + elif ( + args.EventClass == TraceEventClass.ProgressReportEnd + and args.EventSubclass + in [ + TraceEventSubclass.TabularSequencePoint, + TraceEventSubclass.TabularRefresh, + TraceEventSubclass.Process, + TraceEventSubclass.VertiPaq, + TraceEventSubclass.CompressSegment, + TraceEventSubclass.TabularCommit, + TraceEventSubclass.RelationshipBuildPrepare, + TraceEventSubclass.AnalyzeEncodeData, + ] + ): logger.info(f"{TextData}") else: @@ -246,7 +266,7 @@ def __init__( TraceColumn.SessionID, TraceColumn.TextData, TraceColumn.EventClass, - TraceColumn.ProgressTotal + TraceColumn.ProgressTotal, ], Handler: Callable = _refresh_handler, ) -> None: From 801f854c538923336e2ede4c524d791ff8587683 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 8 Oct 2022 12:22:25 -0500 Subject: [PATCH 13/15] add W503 --- .flake8 | 2 +- pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index 16520fc..9214cb9 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,2 @@ [flake8] -ignore = E501 \ No newline at end of file +ignore = E501, W503 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9be8a69..f76d14b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,8 @@ dependencies = [ "clr-loader==0.1.7", "xmltodict==0.13.0", "pandas", - "requests" + "requests", + "rich" ] description = "Connect to your tabular model and perform operations programmatically" readme = "README.md" From f43900056136147382bd66720e819006979e6a0c Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 8 Oct 2022 12:22:56 -0500 Subject: [PATCH 14/15] rich inclusion and 0.1.7 build --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f76d14b..4e3aab6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.1.6" +version = "0.1.7" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] From 9bad7f6412c4b3388348b50f5201bbe30165c0e8 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 8 Oct 2022 12:34:13 -0500 Subject: [PATCH 15/15] logging to logger fix --- README.md | 27 +++++++++++++++++++++++++++ pytabular/pytabular.py | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 00d6753..6f51858 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ import pytabular model = pytabular.Tabular(CONNECTION_STR) ``` +I'm a big fan of logging, if you don't want any just get the logger and disable it. +```python +import pytabular +pytabular.logger.disabled = True +``` + You can query your models with the Query method from your tabular class. For Dax Queries, it will need the full Dax syntax. See [EVALUATE example](https://dax.guide/st/evaluate/). This will return a [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html). If you are looking to return a single value, see below. Simply wrap your query in the the curly brackets. The method will take that single cell table and just return the individual value. You can also query your DMV. See below for example. See [PyTabular Docs for Query](https://curts0.github.io/PyTabular/Tabular/#query). ```python #Run basic queries @@ -80,10 +86,31 @@ model.Refresh({'Table Name':'Partition Name'}) #or any kind of weird combination like model.Refresh([{:,'Table Name':['Partition1','Partition2']},'Table Name','Table Name2']) +#You can even run through the Tables & Partition Attributes +model.Tables['Table Name'].Refresh() + +#or +model.Tables['Table Name'].Partitions['Partition Name'].Refresh() + #Add Tracing=True for simple Traces tracking the refresh. model.Refresh(['Table1','Table2'], Tracing=True) ``` +It's not uncommon to need to run through some checks on specific Tables, Partitions, Columns, Etc... +```python +#Get Row Count from model +model.Tables['Table Name'].Row_Count() + +#Get Last Refresh time from a partition +model.Tables['Table Name'].Last_Refresh() + +#Get Distinct Count or Values from a Column +model.Tables['Table Name'].Columns['Column Name'].Distinct_Count() +#or +model.Tables['Table Name'].Columns['Column Name'].Values() +``` + + ### Use Cases #### If blank table, then refresh table. diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 4e965a7..07d05f9 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -200,7 +200,7 @@ def find_table(table_str: str) -> Table: result = self.Model.Tables.Find(table_str) if result is None: raise Exception(f"Unable to find table! from {table_str}") - logging.debug(f"Found table {result.Name}") + logger.debug(f"Found table {result.Name}") return result def find_partition(table: Table, partition_str: str) -> Partition: @@ -209,7 +209,7 @@ def find_partition(table: Table, partition_str: str) -> Partition: raise Exception( f"Unable to find partition! {table.Name}|{partition_str}" ) - logging.debug(f"Found partition {result.Table.Name}|{result.Name}") + logger.debug(f"Found partition {result.Table.Name}|{result.Name}") return result def refresh(Object):