diff --git a/dist/python_tabular-0.0.40-py3-none-any.whl b/dist/python_tabular-0.0.40-py3-none-any.whl new file mode 100644 index 0000000..bf198e4 Binary files /dev/null and b/dist/python_tabular-0.0.40-py3-none-any.whl differ diff --git a/dist/python_tabular-0.0.40.tar.gz b/dist/python_tabular-0.0.40.tar.gz new file mode 100644 index 0000000..5ec2254 Binary files /dev/null and b/dist/python_tabular-0.0.40.tar.gz differ diff --git a/docs/BPA.md b/docs/BPA.md index dabac1d..f624eeb 100644 --- a/docs/BPA.md +++ b/docs/BPA.md @@ -2,7 +2,7 @@ ## BPA -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L392) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L467) ```python BPA( rules_location: str = 'https: //raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json' diff --git a/docs/TE2.md b/docs/TE2.md index 6d70a8b..b8bf3c5 100644 --- a/docs/TE2.md +++ b/docs/TE2.md @@ -2,7 +2,7 @@ ## TE2 -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L416) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L491) ```python TE2( TE_Location = 'https: //github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip' diff --git a/docs/Tabular.md b/docs/Tabular.md index 8060b05..d074e3c 100644 --- a/docs/Tabular.md +++ b/docs/Tabular.md @@ -2,7 +2,7 @@ ## Tabular -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L21) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L23) ```python Tabular( CONNECTION_STR: str @@ -11,7 +11,7 @@ Tabular( --- -Tabular Class to perform operations:[Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet) +Tabular Class to perform operations: [Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet) **Args** @@ -24,7 +24,7 @@ Tabular Class to perform operations:[Microsoft.AnalysisServices.Tabular](https:/ ### .Reload_Model_Info -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L53) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L55) ```python .Reload_Model_Info() ``` @@ -39,7 +39,7 @@ Runs on __init__ iterates through details, can be called after any model changes ### .Disconnect -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L64) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L66) ```python .Disconnect() ``` @@ -54,10 +54,11 @@ Disconnects from Model ### .Refresh -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L79) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L81) ```python .Refresh( - Object: Union[str, Table, Partition, Iterable], RefreshType = RefreshType.Full + Object: Union[str, Table, Partition, Iterable], RefreshType = RefreshType.Full, + Run: bool = True ) ``` @@ -72,7 +73,8 @@ Input Object(s) to be refreshed in the tabular model. Combine with .SaveChanges( ### .Update -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L99) + +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L113) ```python .Update( UpdateOptions: UpdateOptions = UpdateOptions.ExpandFull @@ -94,7 +96,7 @@ Input Object(s) to be refreshed in the tabular model. Combine with .SaveChanges( ### .SaveChanges -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L110) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L124) ```python .SaveChanges() ``` @@ -109,7 +111,7 @@ Just a simple wrapper to call self.Model.SaveChanges() bool: ### .Backup_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L120) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L134) ```python .Backup_Table( table_str: str @@ -133,7 +135,7 @@ Refresh is performed from source during backup. ### .Revert_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L187) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L201) ```python .Revert_Table( table_str: str @@ -161,7 +163,7 @@ Example scenario -> ### .Query -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L255) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L269) ```python .Query( Query_Str: str @@ -183,7 +185,7 @@ Executes Query on Model and Returns Results in Pandas DataFrame ### .Query_Every_Column -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L285) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L299) ```python .Query_Every_Column( query_function: str = 'COUNTROWS(VALUES(_))' @@ -206,7 +208,7 @@ This will dynamically create a query to pull all columns from the model and run ### .Query_Every_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L307) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L321) ```python .Query_Every_Table( query_function: str = 'COUNTROWS(_)' @@ -229,7 +231,7 @@ It will replace the _ with the table to run. ### .Analyze_BPA -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L327) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L341) ```python .Analyze_BPA( Tabular_Editor_Exe: str, Best_Practice_Analyzer: str @@ -238,8 +240,8 @@ It will replace the _ with the table to run. --- Takes your Tabular Model and performs TE2s BPA. Runs through Command line. -https://docs.tabulareditor.com/te2/Best-Practice-Analyzer.html -https://docs.tabulareditor.com/te2/Command-line-Options.html +[Tabular Editor BPA](https://docs.tabulareditor.com/te2/Best-Practice-Analyzer.html) +[Tabular Editor Command Line Options](https://docs.tabulareditor.com/te2/Command-line-Options.html) **Args** @@ -254,7 +256,7 @@ https://docs.tabulareditor.com/te2/Command-line-Options.html ### .Create_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L351) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L365) ```python .Create_Table( df: pd.DataFrame, table_name: str diff --git a/pyproject.toml b/pyproject.toml index 729ac71..118548c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,13 +4,14 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.0.35" +version = "0.0.40" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] dependencies = [ "pythonnet==3.0.0a2", "clr-loader==0.1.7", + "xmltodict==0.13.0", "pandas", "requests" ] diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index cb90e71..d99c67a 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -1,14 +1,14 @@ import logging logging.debug(f'Importing Microsoft.AnalysisServices.Tabular') -from Microsoft.AnalysisServices.Tabular import Server, Database, RefreshType, DataType, ConnectionDetails, ColumnType, MetadataPermission, Table, DataColumn, Partition, MPartitionSource, PartitionSourceType +from Microsoft.AnalysisServices.Tabular import Server, Database, RefreshType, DataType, ConnectionDetails, ColumnType, MetadataPermission, Table, DataColumn, Partition, MPartitionSource, PartitionSourceType, Trace, TraceEvent, TraceEventHandler logging.debug(f'Importing Microsoft.AnalysisServices.AdomdClient') from Microsoft.AnalysisServices.AdomdClient import (AdomdCommand, AdomdConnection) logging.debug(f'Importing Microsoft.AnalysisServices') -from Microsoft.AnalysisServices import UpdateOptions +from Microsoft.AnalysisServices import UpdateOptions, TraceEventClass, TraceEventSubclass, TraceEventCollection, TraceColumn logging.debug('Importing Other Packages...') -from typing import List, Union +from typing import List, Union, Callable from collections.abc import Iterable import requests as r import pandas as pd @@ -16,10 +16,12 @@ import os import subprocess import atexit +import random +import xmltodict from logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype class Tabular: - '''Tabular Class to perform operations:[Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet) + '''Tabular Class to perform operations: [Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet) Args: CONNECTION_STR (str): [Connection String](https://docs.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services?view=asallproducts-allversions) @@ -76,7 +78,7 @@ def Disconnect(self) -> bool: else: logging.debug(f'Disconnect Successful') return True - def Refresh(self, Object:Union[str,Table,Partition,Iterable], RefreshType=RefreshType.Full) -> None: + def Refresh(self, Object:Union[str,Table,Partition,Iterable], RefreshType=RefreshType.Full, Run:bool = True) -> None: '''Input Object(s) to be refreshed in the tabular model. Combine with .SaveChanges() to actually run the refresh on the model. Args: @@ -84,6 +86,12 @@ def Refresh(self, Object:Union[str,Table,Partition,Iterable], RefreshType=Refres RefreshType (_type_, optional): [RefreshType](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.refreshtype?view=analysisservices-dotnet). Defaults to RefreshType.Full. ''' logging.debug(f'Beginning RequestRefresh cadence...') + + if Run: + refresh_trace = Tabular_Trace(self, [TraceEventClass.ProgressReportBegin,TraceEventClass.ProgressReportCurrent,TraceEventClass.ProgressReportEnd,TraceEventClass.ProgressReportError],[TraceColumn.EventSubclass,TraceColumn.CurrentTime, TraceColumn.ObjectName, TraceColumn.ObjectPath, TraceColumn.DatabaseName, TraceColumn.SessionID, TraceColumn.TextData, TraceColumn.EventClass, TraceColumn.ProgressTotal]) + refresh_trace.Add() + refresh_trace.Update() + def refresh(object): if isinstance(object,str): logging.info(f'Requesting refresh for {object}') @@ -96,6 +104,13 @@ def refresh(object): [refresh(object) for object in Object] else: refresh(Object) + + if Run: + refresh_trace.Start() + self.SaveChanges() + refresh_trace.Stop() + refresh_trace.Drop() + def Update(self, UpdateOptions:UpdateOptions =UpdateOptions.ExpandFull) -> None: '''[Update Model](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.majorobject.update?view=analysisservices-dotnet#microsoft-analysisservices-majorobject-update(microsoft-analysisservices-updateoptions)) @@ -326,8 +341,8 @@ def Query_Every_Table(self,query_function:str='COUNTROWS(_)') -> pd.DataFrame: return self.Query(query_str) def Analyze_BPA(self,Tabular_Editor_Exe:str,Best_Practice_Analyzer:str) -> List[str]: '''Takes your Tabular Model and performs TE2s BPA. Runs through Command line. - https://docs.tabulareditor.com/te2/Best-Practice-Analyzer.html - https://docs.tabulareditor.com/te2/Command-line-Options.html + [Tabular Editor BPA](https://docs.tabulareditor.com/te2/Best-Practice-Analyzer.html) + [Tabular Editor Command Line Options](https://docs.tabulareditor.com/te2/Command-line-Options.html) Args: Tabular_Editor_Exe (str): TE2 Exe File path. Feel free to use class TE2().EXE_Path or provide your own. @@ -387,13 +402,74 @@ def Create_Table(self,df:pd.DataFrame, table_name:str) -> bool: self.SaveChanges() return True +def main_handler(source, args): + if args.EventSubclass == TraceEventSubclass.ReadData: + logging.debug(f'{args.ProgressTotal} - {args.ObjectPath}') + else: + logging.debug(f'{args.EventClass} - {args.EventSubclass} - {args.ObjectName}') + +class Tabular_Trace: + def __init__(self, Tabular_Class:Tabular, TE:List[TraceEvent],TEC:List[TraceColumn],Handler:Callable=main_handler) -> None: + logging.debug(f'Request to Initialize Trace beginning...') + Name = 'PyTabular_'+''.join(random.SystemRandom().choices([str(x) for x in [0,1,2,3,4,5,6,7,8,9]],k=5)) + ID = Name.replace('PyTabular_','') + self.Tabular = Tabular_Class + logging.debug(f'Creating Trace Events...') + logging.debug(f'Creating Trace... {Name}') + self.Trace = Trace(Name,ID) + self.Get_Event_Categories() + TE = [TraceEvent(trace_event) for trace_event in TE] + logging.debug(f'Adding Events to... {self.Trace.Name}') + [self.Trace.get_Events().Add(te) for te in TE] + def add_column(trace_event,trace_event_column): + try: + trace_event.Columns.Add(trace_event_column) + except: + logging.warning(f'{trace_event} - {trace_event_column} Skipped') + logging.debug(f'Adding Trace Event Columns...') + #TODO Need to clarify if column gets skipped... + [add_column(trace_event,trace_event_column) for trace_event_column in TEC for trace_event in TE if str(trace_event_column.value__) in self.Event_Categories[str(trace_event.EventID.value__)] ] + logging.debug(f'Adding Handler to... {self.Trace.Name}') + self.Handler = TraceEventHandler(Handler) + self.Trace.OnEvent += self.Handler + pass + def Get_Event_Categories(self): + logging.info(f'Starting to retrieve Event Categories') + self.Event_Categories = {} + events = [] + logging.debug(f'Searching DMV...') + df = self.Tabular.Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES") + for index, row in df.iterrows(): + xml_data = xmltodict.parse(row.Data) + if type(xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT']) == list: + events += [event for event in xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT'] ] + else: + events += [xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT']] + for event in events: + self.Event_Categories[event['ID']] = [column['ID'] for column in event['EVENTCOLUMNLIST']['EVENTCOLUMN']] + def Add(self) -> bool: + logging.debug(f'Adding {self.Trace.Name} to {self.Tabular.Server.Name}') + self.item_number = self.Tabular.Server.Traces.Add(self.Trace) + return True + def Update(self) -> bool: + logging.debug(f'Running update for - {self.Trace.Name}') + self.Tabular.Server.Traces.get_Item(self.item_number).Update() + def Start(self) -> bool: + logging.debug(f'Starting Trace - {self.Trace.Name}') + self.Tabular.Server.Traces.get_Item(self.item_number).Start() + def Stop(self) -> bool: + logging.debug(f'Stopping Trace - {self.Trace.Name}') + self.Tabular.Server.Traces.get_Item(self.item_number).Stop() + def Drop(self) -> bool: + logging.debug(f'Dropping Trace - {self.Trace.Name}') + self.Tabular.Server.Traces.get_Item(self.item_number).Drop() class BPA: '''_summary_ ''' '''Best Practice Analyzer Class. Can provide Url, Json File Path, or Python List. If nothing is provided it will default to Microsofts Analysis Services report with BPA Rules. - [Default BPA](https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json) + [Default BPA File](https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json) ''' def __init__(self,rules_location:str='https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json') -> None: ''' @@ -412,7 +488,7 @@ def __init__(self,rules_location:str='https://raw.githubusercontent.com/microsof self.Rules = json.load(json_file) logging.debug(f'Rules from file path collected...') pass -#Todo... subclass with a namedtuple +#TODO... subclass with a namedtuple class TE2: '''TE2 Class, to use any built TabularEditor Command Line Scripts [TE2 Command Line Example](https://docs.tabulareditor.com/te2/Command-line-Options.html)