diff --git a/README.md b/README.md index 615809a..99ae859 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,7 @@ DAX Query tables_to_refresh = ['Table Name 1', 'Table Name 2', , ] #Queue up the tables and partitions that you want to refresh. model.Refresh(tables_to_refresh) - - #Once you are ready, update to execute the refresh - model.SaveChanges() + #NOTE if you monitor the logs you will notice a Trace is executed on the refreshes. ``` Built In Dax Query Helpers diff --git a/dist/python_tabular-0.0.50-py3-none-any.whl b/dist/python_tabular-0.0.50-py3-none-any.whl new file mode 100644 index 0000000..4af2412 Binary files /dev/null and b/dist/python_tabular-0.0.50-py3-none-any.whl differ diff --git a/dist/python_tabular-0.0.50.tar.gz b/dist/python_tabular-0.0.50.tar.gz new file mode 100644 index 0000000..f18b91c Binary files /dev/null and b/dist/python_tabular-0.0.50.tar.gz differ diff --git a/docs/BPA.md b/docs/BPA.md deleted file mode 100644 index f624eeb..0000000 --- a/docs/BPA.md +++ /dev/null @@ -1,15 +0,0 @@ -# - - -## BPA -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L467) -```python -BPA( - rules_location: str = 'https: //raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json' -) -``` - - ---- -_summary_ - diff --git a/docs/Best Practice Analyzer.md b/docs/Best Practice Analyzer.md new file mode 100644 index 0000000..352ecae --- /dev/null +++ b/docs/Best Practice Analyzer.md @@ -0,0 +1,43 @@ +# + + +## BPA +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/best_practice_analyzer.py\#L34) +```python +BPA( + File_Path: str = 'Default' +) +``` + + +--- +Setting BPA Class for future work... + + +---- + + +### Download_BPA_File +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/best_practice_analyzer.py\#L8) +```python +.Download_BPA_File( + Download_Location: str = 'https: //raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json', + Folder: str = 'Best_Practice_Analyzer', Auto_Remove = True +) +``` + +--- +Runs a request.get() to retrieve the json file from web. Will return and store in directory. Will also register the removal of the new directory and file when exiting program. + + +**Args** + +* **Download_Location** (_type_, optional) : F. Defaults to [Microsoft GitHub BPA]'https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json'. +* **Folder** (str, optional) : New Folder String. Defaults to 'Best_Practice_Analyzer'. +* **Auto_Remove** (bool, optional) : If you wish to Auto Remove when script exits. Defaults to True. + + +**Returns** + +* **str** : File Path for the newly downloaded BPA. + diff --git a/docs/Utils.md b/docs/Logic Utils.md similarity index 81% rename from docs/Utils.md rename to docs/Logic Utils.md index 1fdd606..6281c7e 100644 --- a/docs/Utils.md +++ b/docs/Logic Utils.md @@ -2,7 +2,7 @@ ### ticks_to_datetime -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L10) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L11) ```python .ticks_to_datetime( ticks: int @@ -27,7 +27,7 @@ Converts a C# System DateTime Tick into a Python DateTime ### pandas_datatype_to_tabular_datatype -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L21) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L22) ```python .pandas_datatype_to_tabular_datatype( df: pd.DataFrame @@ -52,7 +52,7 @@ WiP takes dataframe columns and gets respective tabular column datatype. ([NumP ### pd_dataframe_to_m_expression -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L75) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L76) ```python .pd_dataframe_to_m_expression( df: pd.DataFrame @@ -89,3 +89,23 @@ Source * **str** : Currently only returning string values in your tabular model. + +---- + + +### remove_folder_and_contents +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/logic_utils.py\#L120) +```python +.remove_folder_and_contents( + folder_location +) +``` + +--- +Internal used in tabular_editor.py and best_practice_analyzer.py. + + +**Args** + +* **folder_location** (str) : Folder path to remove directory and contents. + diff --git a/docs/TE2.md b/docs/TE2.md deleted file mode 100644 index b8bf3c5..0000000 --- a/docs/TE2.md +++ /dev/null @@ -1,16 +0,0 @@ -# - - -## TE2 -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L491) -```python -TE2( - TE_Location = 'https: //github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip' -) -``` - - ---- -TE2 Class, to use any built TabularEditor Command Line Scripts -[TE2 Command Line Example](https://docs.tabulareditor.com/te2/Command-line-Options.html) -[TE2 Download](https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip) diff --git a/docs/Tabular Editor 2.md b/docs/Tabular Editor 2.md new file mode 100644 index 0000000..7753d02 --- /dev/null +++ b/docs/Tabular Editor 2.md @@ -0,0 +1,43 @@ +# + + +## Tabular_Editor +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_editor.py\#L38) +```python +Tabular_Editor( + EXE_File_Path: str = 'Default' +) +``` + + +--- +Setting Tabular_Editor Class for future work. + + +---- + + +### Download_Tabular_Editor +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_editor.py\#L8) +```python +.Download_Tabular_Editor( + Download_Location: str = 'https: //github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip', + Folder: str = 'Tabular_Editor_2', Auto_Remove = True +) +``` + +--- +Runs a request.get() to retrieve the zip file from web. Will unzip response and store in directory. Will also register the removal of the new directory and files when exiting program. + + +**Args** + +* **Download_Location** (str, optional) : File path for zip of Tabular Editor 2. Defaults to [Tabular Editor 2 Github Zip Location]'https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip'. +* **Folder** (str, optional) : New Folder Location. Defaults to 'Tabular_Editor_2'. +* **Auto_Remove** (bool, optional) : Boolean to determine auto removal of files once script exits. Defaults to True. + + +**Returns** + +* **str** : _description_ + diff --git a/docs/Tabular.md b/docs/Tabular.md index d074e3c..42e6068 100644 --- a/docs/Tabular.md +++ b/docs/Tabular.md @@ -2,7 +2,7 @@ ## Tabular -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L23) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L22) ```python Tabular( CONNECTION_STR: str @@ -24,7 +24,7 @@ Tabular Class to perform operations: [Microsoft.AnalysisServices.Tabular](https: ### .Reload_Model_Info -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L55) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L54) ```python .Reload_Model_Info() ``` @@ -39,7 +39,7 @@ Runs on __init__ iterates through details, can be called after any model changes ### .Disconnect -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L66) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L65) ```python .Disconnect() ``` @@ -54,7 +54,7 @@ Disconnects from Model ### .Refresh -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L81) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L80) ```python .Refresh( Object: Union[str, Table, Partition, Iterable], RefreshType = RefreshType.Full, @@ -73,8 +73,7 @@ Input Object(s) to be refreshed in the tabular model. Combine with .SaveChanges( ### .Update - -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L113) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L110) ```python .Update( UpdateOptions: UpdateOptions = UpdateOptions.ExpandFull @@ -96,7 +95,7 @@ Input Object(s) to be refreshed in the tabular model. Combine with .SaveChanges( ### .SaveChanges -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L124) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L121) ```python .SaveChanges() ``` @@ -111,7 +110,7 @@ Just a simple wrapper to call self.Model.SaveChanges() bool: ### .Backup_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L134) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L131) ```python .Backup_Table( table_str: str @@ -135,7 +134,7 @@ Refresh is performed from source during backup. ### .Revert_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L201) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L198) ```python .Revert_Table( table_str: str @@ -163,7 +162,7 @@ Example scenario -> ### .Query -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L269) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L263) ```python .Query( Query_Str: str @@ -185,7 +184,7 @@ Executes Query on Model and Returns Results in Pandas DataFrame ### .Query_Every_Column -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L299) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L293) ```python .Query_Every_Column( query_function: str = 'COUNTROWS(VALUES(_))' @@ -208,7 +207,7 @@ This will dynamically create a query to pull all columns from the model and run ### .Query_Every_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L321) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L315) ```python .Query_Every_Table( query_function: str = 'COUNTROWS(_)' @@ -231,7 +230,7 @@ It will replace the _ with the table to run. ### .Analyze_BPA -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L341) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L335) ```python .Analyze_BPA( Tabular_Editor_Exe: str, Best_Practice_Analyzer: str @@ -256,7 +255,7 @@ Takes your Tabular Model and performs TE2s BPA. Runs through Command line. ### .Create_Table -[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L365) +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/pytabular.py\#L359) ```python .Create_Table( df: pd.DataFrame, table_name: str diff --git a/docs/Traces.md b/docs/Traces.md new file mode 100644 index 0000000..023dc35 --- /dev/null +++ b/docs/Traces.md @@ -0,0 +1,161 @@ +# + + +## Base_Trace +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L10) +```python +Base_Trace( + Tabular_Class, Trace_Events: List[TraceEvent], + Trace_Event_Columns: List[TraceColumn], Handler: Callable +) +``` + + + + +**Methods:** + + +### .Build +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L37) +```python +.Build() +``` + +--- +Run on initialization. This will take the inputed arguments for the class and attempt to build the Trace. + + +**Returns** + +* **bool** : True if successful + + +### .Arguments +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L62) +```python +.Arguments( + Trace_Events: List[TraceEvent], Trace_Event_Columns: List[TraceColumn], + Handler: Callable +) +``` + + +### .Add +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L65) +```python +.Add() +``` + +--- +Runs on initialization. Adds built Trace to the Server. + + +**Returns** + +* **int** : Return int of placement in Server.Traces.get_Item(int) + + +### .Update +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L74) +```python +.Update() +``` + +--- +Runs on initialization. Syncs with Server. + + +**Returns** + +* **None** : Returns None. Unless unsuccessful then it will return the error from Server. + + +### .Start +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L83) +```python +.Start() +``` + +--- +Call when you want to start the Trace + + +**Returns** + +* **None** : Returns None. Unless unsuccessful then it will return the error from Server. + + +### .Stop +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L92) +```python +.Stop() +``` + +--- +Call when you want to stop the Trace + + +**Returns** + +* **None** : Returns None. Unless unsuccessful then it will return the error from Server. + + +### .Drop +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L101) +```python +.Drop() +``` + +--- +Call when you want to drop the Trace + + +**Returns** + +* **None** : Returns None. Unless unsuccessful then it will return the error from Server. + + +### .Query_DMV_For_Event_Categories +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L110) +```python +.Query_DMV_For_Event_Categories() +``` + +--- +Internal use. Called during the building process to locate allowed columns for event categories. This is done by executing a Tabular().Query() on the DISCOVER_EVENT_CATEGORIES table in the DMV. Then the function will parse the results, as it is xml inside of rows. + + +**Returns** + +* **_type_** : _description_ + + +---- + + +## Refresh_Trace +[source](https://github.com/Curts0/PyTabular\blob\master\pytabular/tabular_tracing.py\#L138) +```python +Refresh_Trace( + Tabular_Class, + Trace_Events: List[TraceEvent] = [TraceEventClass.ProgressReportBegin, + TraceEventClass.ProgressReportCurrent, TraceEventClass.ProgressReportEnd, + TraceEventClass.ProgressReportError], + Trace_Event_Columns: List[TraceColumn] = [TraceColumn.EventSubclass, + TraceColumn.CurrentTime, TraceColumn.ObjectName, TraceColumn.ObjectPath, + TraceColumn.DatabaseName, TraceColumn.SessionID, TraceColumn.TextData, + TraceColumn.EventClass, TraceColumn.ProgressTotal], + Handler: Callable = default_refresh_handler +) +``` + + +--- +Subclass of Base_Trace. For built-in Refresh Tracing. + + +**Args** + +* **Base_Trace** (_type_) : _description_ + diff --git a/docs/index.md b/docs/index.md index 615809a..99ae859 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,9 +36,7 @@ DAX Query tables_to_refresh = ['Table Name 1', 'Table Name 2',
, ] #Queue up the tables and partitions that you want to refresh. model.Refresh(tables_to_refresh) - - #Once you are ready, update to execute the refresh - model.SaveChanges() + #NOTE if you monitor the logs you will notice a Trace is executed on the refreshes. ``` Built In Dax Query Helpers diff --git a/mkdocs.yml b/mkdocs.yml index e578a30..66ce801 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,8 +3,9 @@ site_url: https://curts0.github.io/PyTabular/ nav: - Home: index.md - Tabular: Tabular.md - - BPA: BPA.md - - TE2: TE2.md - - Utils: Utils.md + - Trace: Traces.md + - Best Practice Analyzer: Best Practice Analyzer.md + - Tabular Editor: Tabular Editor 2.md + - Logic Utils: Logic Utils.md - Examples: Examples.md theme: readthedocs diff --git a/mkgendocs.yml b/mkgendocs.yml index 1296e2d..b1d03cb 100644 --- a/mkgendocs.yml +++ b/mkgendocs.yml @@ -1,31 +1,41 @@ sources_dir: docs templates_dir: docs/templates -repo: https://github.com/Curts0/PyTabular #link to sources on github -version: master #link to sources on github -#gendocs --config mkgendocs.yml -#need to automate in workflow +repo: https://github.com/Curts0/PyTabular +version: master + + pages: - page: "Tabular.md" source: 'pytabular/pytabular.py' classes: - Tabular - - page: "BPA.md" - source: 'pytabular/pytabular.py' + - page: "Traces.md" + source: 'pytabular/tabular_tracing.py' + classes: + - Base_Trace + - Refresh_Trace + - page: "Best Practice Analyzer.md" + source: 'pytabular/best_practice_analyzer.py' + functions: + - Download_BPA_File classes: - BPA - - page: "TE2.md" - source: 'pytabular/pytabular.py' + - page: "Tabular Editor 2.md" + source: 'pytabular/tabular_editor.py' + functions: + - Download_Tabular_Editor classes: - - TE2 + - Tabular_Editor - page: "Examples.md" source: 'pytabular/basic_checks.py' functions: - Return_Zero_Row_Tables - Table_Last_Refresh_Times - BPA_Violations_To_DF - - page: "Utils.md" + - page: "Logic Utils.md" source: 'pytabular/logic_utils.py' functions: - ticks_to_datetime - pandas_datatype_to_tabular_datatype - - pd_dataframe_to_m_expression \ No newline at end of file + - pd_dataframe_to_m_expression + - remove_folder_and_contents \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 118548c..d59a40f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.0.40" +version = "0.0.50" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] diff --git a/pytabular/__init__.py b/pytabular/__init__.py index a193844..aa322cb 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -19,6 +19,9 @@ clr.AddReference('Microsoft.AnalysisServices') logging.debug(f"Importing from the rest...") -from . pytabular import Tabular, BPA, TE2 +from . pytabular import Tabular from . basic_checks import Return_Zero_Row_Tables, Table_Last_Refresh_Times, BPA_Violations_To_DF -from . logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype \ No newline at end of file +from . logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype +from . tabular_tracing import Base_Trace, Refresh_Trace +from . tabular_editor import Tabular_Editor +from . best_practice_analyzer import BPA \ No newline at end of file diff --git a/pytabular/best_practice_analyzer.py b/pytabular/best_practice_analyzer.py new file mode 100644 index 0000000..377ffb2 --- /dev/null +++ b/pytabular/best_practice_analyzer.py @@ -0,0 +1,43 @@ +import logging +import requests as r +import atexit +import json +import os +from logic_utils import remove_folder_and_contents + +def Download_BPA_File(Download_Location:str = 'https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json', +Folder:str = 'Best_Practice_Analyzer', +Auto_Remove = True) -> str: + '''Runs a request.get() to retrieve the json file from web. Will return and store in directory. Will also register the removal of the new directory and file when exiting program. + + Args: + Download_Location (_type_, optional): F. Defaults to [Microsoft GitHub BPA]'https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json'. + Folder (str, optional): New Folder String. Defaults to 'Best_Practice_Analyzer'. + Auto_Remove (bool, optional): If you wish to Auto Remove when script exits. Defaults to True. + + Returns: + str: File Path for the newly downloaded BPA. + ''' + logging.info(f'Downloading BPA from {Download_Location}') + folder_location = os.path.join(os.getcwd(),Folder) + if os.path.exists(folder_location) == False: + os.makedirs(folder_location) + response = r.get(Download_Location) + file_location = os.path.join(folder_location,Download_Location.split('/')[-1]) + with open(file_location, 'w', encoding='utf-8') as bpa: + json.dump(response.json(), bpa, ensure_ascii=False, indent= 4) + if Auto_Remove: + logging.debug(f'Registering removal on termination... For {folder_location}') + atexit.register(remove_folder_and_contents, folder_location) + return file_location + +class BPA: + '''Setting BPA Class for future work... + ''' + def __init__(self, File_Path:str = 'Default') -> None: + logging.debug(f'Initializing BPA Class:: {File_Path}') + if File_Path == 'Default': + self.Location: str = Download_BPA_File() + else: + self.Location: str = File_Path + pass \ No newline at end of file diff --git a/pytabular/logic_utils.py b/pytabular/logic_utils.py index ec89454..45a65a0 100644 --- a/pytabular/logic_utils.py +++ b/pytabular/logic_utils.py @@ -1,5 +1,6 @@ import logging import datetime +import os from typing import Dict, List import pandas as pd import clr @@ -114,4 +115,15 @@ def m_list_expression_generator(list_of_strings:List[str]) -> str: for index, row in df.iterrows(): expression_list_rows += [m_list_expression_generator(row.to_list())] expression_str += f"\u007b\n{','.join(expression_list_rows)}\n\u007d)\nin\nSource" - return expression_str \ No newline at end of file + return expression_str + +def remove_folder_and_contents(folder_location): + '''Internal used in tabular_editor.py and best_practice_analyzer.py. + + Args: + folder_location (str): Folder path to remove directory and contents. + ''' + import shutil + if os.path.exists(folder_location): + logging.info(f'Removing Dir and Contents -> {folder_location}') + shutil.rmtree(folder_location) \ No newline at end of file diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index d99c67a..e0c26d0 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -16,9 +16,8 @@ import os import subprocess import atexit -import random -import xmltodict from logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype +from tabular_tracing import Refresh_Trace class Tabular: '''Tabular Class to perform operations: [Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet) @@ -87,11 +86,6 @@ def Refresh(self, Object:Union[str,Table,Partition,Iterable], RefreshType=Refres ''' logging.debug(f'Beginning RequestRefresh cadence...') - if Run: - refresh_trace = Tabular_Trace(self, [TraceEventClass.ProgressReportBegin,TraceEventClass.ProgressReportCurrent,TraceEventClass.ProgressReportEnd,TraceEventClass.ProgressReportError],[TraceColumn.EventSubclass,TraceColumn.CurrentTime, TraceColumn.ObjectName, TraceColumn.ObjectPath, TraceColumn.DatabaseName, TraceColumn.SessionID, TraceColumn.TextData, TraceColumn.EventClass, TraceColumn.ProgressTotal]) - refresh_trace.Add() - refresh_trace.Update() - def refresh(object): if isinstance(object,str): logging.info(f'Requesting refresh for {object}') @@ -100,17 +94,19 @@ def refresh(object): else: logging.info(f'Requesting refresh for {object.Name}') object.RequestRefresh(RefreshType) + + if isinstance(Object,Iterable) and isinstance(Object,str) == False: [refresh(object) for object in Object] else: refresh(Object) if Run: - refresh_trace.Start() + rt = Refresh_Trace(self) + rt.Start() self.SaveChanges() - refresh_trace.Stop() - refresh_trace.Drop() - + rt.Stop() + rt.Drop() def Update(self, UpdateOptions:UpdateOptions =UpdateOptions.ExpandFull) -> None: '''[Update Model](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.majorobject.update?view=analysisservices-dotnet#microsoft-analysisservices-majorobject-update(microsoft-analysisservices-updateoptions)) @@ -213,9 +209,6 @@ def Revert_Table(self, table_str:str) -> bool: Returns: bool: Returns True if Successful, else will return error. - ''' - ''' - ''' logging.info(f'Beginning Revert for {table_str}') logging.debug(f'Finding original {table_str}') @@ -401,124 +394,3 @@ def Create_Table(self,df:pd.DataFrame, table_name:str) -> bool: self.Refresh([new_table]) self.SaveChanges() return True - -def main_handler(source, args): - if args.EventSubclass == TraceEventSubclass.ReadData: - logging.debug(f'{args.ProgressTotal} - {args.ObjectPath}') - else: - logging.debug(f'{args.EventClass} - {args.EventSubclass} - {args.ObjectName}') - -class Tabular_Trace: - def __init__(self, Tabular_Class:Tabular, TE:List[TraceEvent],TEC:List[TraceColumn],Handler:Callable=main_handler) -> None: - logging.debug(f'Request to Initialize Trace beginning...') - Name = 'PyTabular_'+''.join(random.SystemRandom().choices([str(x) for x in [0,1,2,3,4,5,6,7,8,9]],k=5)) - ID = Name.replace('PyTabular_','') - self.Tabular = Tabular_Class - logging.debug(f'Creating Trace Events...') - logging.debug(f'Creating Trace... {Name}') - self.Trace = Trace(Name,ID) - self.Get_Event_Categories() - TE = [TraceEvent(trace_event) for trace_event in TE] - logging.debug(f'Adding Events to... {self.Trace.Name}') - [self.Trace.get_Events().Add(te) for te in TE] - def add_column(trace_event,trace_event_column): - try: - trace_event.Columns.Add(trace_event_column) - except: - logging.warning(f'{trace_event} - {trace_event_column} Skipped') - logging.debug(f'Adding Trace Event Columns...') - #TODO Need to clarify if column gets skipped... - [add_column(trace_event,trace_event_column) for trace_event_column in TEC for trace_event in TE if str(trace_event_column.value__) in self.Event_Categories[str(trace_event.EventID.value__)] ] - logging.debug(f'Adding Handler to... {self.Trace.Name}') - self.Handler = TraceEventHandler(Handler) - self.Trace.OnEvent += self.Handler - pass - def Get_Event_Categories(self): - logging.info(f'Starting to retrieve Event Categories') - self.Event_Categories = {} - events = [] - logging.debug(f'Searching DMV...') - df = self.Tabular.Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES") - for index, row in df.iterrows(): - xml_data = xmltodict.parse(row.Data) - if type(xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT']) == list: - events += [event for event in xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT'] ] - else: - events += [xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT']] - for event in events: - self.Event_Categories[event['ID']] = [column['ID'] for column in event['EVENTCOLUMNLIST']['EVENTCOLUMN']] - def Add(self) -> bool: - logging.debug(f'Adding {self.Trace.Name} to {self.Tabular.Server.Name}') - self.item_number = self.Tabular.Server.Traces.Add(self.Trace) - return True - def Update(self) -> bool: - logging.debug(f'Running update for - {self.Trace.Name}') - self.Tabular.Server.Traces.get_Item(self.item_number).Update() - def Start(self) -> bool: - logging.debug(f'Starting Trace - {self.Trace.Name}') - self.Tabular.Server.Traces.get_Item(self.item_number).Start() - def Stop(self) -> bool: - logging.debug(f'Stopping Trace - {self.Trace.Name}') - self.Tabular.Server.Traces.get_Item(self.item_number).Stop() - def Drop(self) -> bool: - logging.debug(f'Dropping Trace - {self.Trace.Name}') - self.Tabular.Server.Traces.get_Item(self.item_number).Drop() - - -class BPA: - '''_summary_ - ''' - '''Best Practice Analyzer Class. Can provide Url, Json File Path, or Python List. If nothing is provided it will default to Microsofts Analysis Services report with BPA Rules. - [Default BPA File](https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json) - ''' - def __init__(self,rules_location:str='https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json') -> None: - ''' - ''' - self.Location = rules_location - self.Rules:list[dict()] = [] - logging.debug(f'Initializing BPA Class with: {rules_location}') - try: - logging.debug(f'Searching for Rules on the good ol\' internet...') - self.Rules = r.get(rules_location).json() - logging.debug(f'Rules recieved from: {rules_location}') - except: - logging.debug(f'Request to {rules_location} failed...') - logging.debug(f'Searching for Rules locally with file path...') - with open(rules_location,'r') as json_file: - self.Rules = json.load(json_file) - logging.debug(f'Rules from file path collected...') - pass -#TODO... subclass with a namedtuple -class TE2: - '''TE2 Class, to use any built TabularEditor Command Line Scripts - [TE2 Command Line Example](https://docs.tabulareditor.com/te2/Command-line-Options.html) - [TE2 Download](https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip) - ''' - def __init__(self,TE_Location='https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip') -> None: - logging.debug(f'Checking for TE2 in {os.getcwd()}') - te2_path = os.path.join(os.getcwd(),'TE2') - if os.path.exists(te2_path) == False: - logging.debug('Downloading Tabular Editor for BPA...') - self.TE_Location = TE_Location - response = r.get(self.TE_Location) - file_location = f"{os.getcwd()}\\{self.TE_Location.split('/')[-1]}" - with open(file_location,'wb') as te2: - te2.write(response.content) - logging.debug('TE2 Zip Download Complete!') - logging.debug('Import ZipFile') - import zipfile as Z - logging.debug('Unzipping file...') - with Z.ZipFile(file_location) as zip: - zip.extractall(path=te2_path) - logging.debug('All Unzipped!') - logging.debug('Removing Zip File') - os.remove(file_location) - else: - logging.debug(f'TE2 Directory Found with Files {os.listdir(path=te2_path)}') - self.EXE_Path = os.path.join(te2_path,'TabularEditor.exe') - if os.path.exists(self.EXE_Path): - logging.debug(f'TabularEditor.exe Located! {self.EXE_Path}') - else: - logging.error('TabularEditor.exe not found!') - pass - pass diff --git a/pytabular/tabular_editor.py b/pytabular/tabular_editor.py new file mode 100644 index 0000000..1f0ae63 --- /dev/null +++ b/pytabular/tabular_editor.py @@ -0,0 +1,47 @@ +import logging +import os +import requests as r +import zipfile as Z +import atexit +from logic_utils import remove_folder_and_contents + +def Download_Tabular_Editor(Download_Location:str = 'https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip', +Folder:str = 'Tabular_Editor_2', +Auto_Remove = True) -> str: + '''Runs a request.get() to retrieve the zip file from web. Will unzip response and store in directory. Will also register the removal of the new directory and files when exiting program. + + Args: + Download_Location (str, optional): File path for zip of Tabular Editor 2. Defaults to [Tabular Editor 2 Github Zip Location]'https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip'. + Folder (str, optional): New Folder Location. Defaults to 'Tabular_Editor_2'. + Auto_Remove (bool, optional): Boolean to determine auto removal of files once script exits. Defaults to True. + + Returns: + str: _description_ + ''' + logging.info(f'Downloading Tabular Editor 2...') + logging.info(f'From... {Download_Location}') + folder_location = os.path.join(os.getcwd(),Folder) + response = r.get(Download_Location) + file_location = f"{os.getcwd()}\\{Download_Location.split('/')[-1]}" + with open(file_location, 'wb') as te2_zip: + te2_zip.write(response.content) + with Z.ZipFile(file_location) as zipper: + zipper.extractall(path=folder_location) + logging.debug(f'Removing Zip File...') + os.remove(file_location) + logging.info(f'Tabular Editor Downloaded and Extracted to {folder_location}') + if Auto_Remove: + logging.debug(f'Registering removal on termination... For {folder_location}') + atexit.register(remove_folder_and_contents, folder_location) + return f'{folder_location}\\TabularEditor.exe' + +class Tabular_Editor: + '''Setting Tabular_Editor Class for future work. + ''' + def __init__(self, EXE_File_Path:str = 'Default') -> None: + logging.debug(f'Initializing Tabular Editor Class:: {EXE_File_Path}') + if EXE_File_Path == 'Default': + self.EXE: str = Download_Tabular_Editor() + else: + self.EXE: str = EXE_File_Path + pass \ No newline at end of file diff --git a/pytabular/tabular_tracing.py b/pytabular/tabular_tracing.py new file mode 100644 index 0000000..5113846 --- /dev/null +++ b/pytabular/tabular_tracing.py @@ -0,0 +1,147 @@ +import logging +import random +import xmltodict +from typing import List, Callable +from Microsoft.AnalysisServices.Tabular import Trace, TraceEvent, TraceEventHandler +from Microsoft.AnalysisServices import TraceColumn, TraceEventClass, TraceEventSubclass + + + +class Base_Trace: + def __init__(self, Tabular_Class, Trace_Events:List[TraceEvent], Trace_Event_Columns:List[TraceColumn], Handler:Callable) -> None: + '''Generates Trace to be run on Server. This is the base class to customize the type of Trace you are looking for. + [Server Traces](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.server.traces?view=analysisservices-dotnet#microsoft-analysisservices-tabular-server-traces) + + Args: + Tabular_Class (Tabular): Tabular Class to retrieve the connected Server and Model. + Trace_Events (List[TraceEvent]): List of Trace Events that you wish to track. [TraceEventClass](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.traceeventclass?view=analysisservices-dotnet) + Trace_Event_Columns (List[TraceColumn]): List of Trace Event Columns you with to track. [TraceEventColumn](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tracecolumn?view=analysisservices-dotnet) + Handler (Callable): Function to call when Trace returns response. Input needs to be two arguments. One is source (Which is currently None... Need to investigate why). Second is [TraceEventArgs](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.traceeventargs?view=analysisservices-dotnet) + ''' + logging.debug(f'Trace Base Class initializing...') + self.Name = 'PyTabular_'+''.join(random.SystemRandom().choices([str(x) for x in [y for y in range(0,10)]], k=10)) + self.ID = self.Name.replace('PyTabular_', '') + self.Trace = Trace(self.Name, self.ID) + logging.debug(f'Trace {self.Trace.Name} created...') + self.Tabular_Class = Tabular_Class + self.Event_Categories = self.Query_DMV_For_Event_Categories() + + self.Trace_Events = Trace_Events + self.Trace_Event_Columns = Trace_Event_Columns + self.Handler = Handler + + self.Build() + self.Add() + self.Update() + + def Build(self) -> bool: + '''Run on initialization. This will take the inputed arguments for the class and attempt to build the Trace. + + Returns: + bool: True if successful + ''' + logging.info(f'Building Trace {self.Name}') + TE = [TraceEvent(trace_event) for trace_event in self.Trace_Events] + logging.debug(f'Adding Events to... {self.Trace.Name}') + [self.Trace.get_Events().Add(te) for te in TE] + + def add_column(trace_event,trace_event_column): + try: + trace_event.Columns.Add(trace_event_column) + except: + logging.warning(f'{trace_event} - {trace_event_column} Skipped') + + logging.debug(f'Adding Trace Event Columns...') + [add_column(trace_event,trace_event_column) for trace_event_column in self.Trace_Event_Columns for trace_event in TE if str(trace_event_column.value__) in self.Event_Categories[str(trace_event.EventID.value__)] ] + + logging.debug(f'Adding Handler to Trace...') + self.Handler = TraceEventHandler(self.Handler) + self.Trace.OnEvent += self.Handler + return True + + def Arguments(Trace_Events: List[TraceEvent], Trace_Event_Columns: List[TraceColumn], Handler: Callable): + raise NotImplementedError + + def Add(self) -> int: + '''Runs on initialization. Adds built Trace to the Server. + + Returns: + int: Return int of placement in Server.Traces.get_Item(int) + ''' + logging.info(f'Adding {self.Name} to {self.Tabular_Class.Server.Name}') + return self.Tabular_Class.Server.Traces.Add(self.Trace) + + def Update(self) -> None: + '''Runs on initialization. Syncs with Server. + + Returns: + None: Returns None. Unless unsuccessful then it will return the error from Server. + ''' + logging.info(f'Updating {self.Name} in {self.Tabular_Class.Server.Name}') + return self.Trace.Update() + + def Start(self) -> None: + '''Call when you want to start the Trace + + Returns: + None: Returns None. Unless unsuccessful then it will return the error from Server. + ''' + logging.info(f'Starting {self.Name} in {self.Tabular_Class.Server.Name}') + return self.Trace.Start() + + def Stop(self) -> None: + '''Call when you want to stop the Trace + + Returns: + None: Returns None. Unless unsuccessful then it will return the error from Server. + ''' + logging.info(f'Stopping {self.Name} in {self.Tabular_Class.Server.Name}') + return self.Trace.Stop() + + def Drop(self) -> None: + '''Call when you want to drop the Trace + + Returns: + None: Returns None. Unless unsuccessful then it will return the error from Server. + ''' + logging.info(f'Dropping {self.Name} in {self.Tabular_Class.Server.Name}') + return self.Trace.Drop() + + def Query_DMV_For_Event_Categories(self): + '''Internal use. Called during the building process to locate allowed columns for event categories. This is done by executing a Tabular().Query() on the DISCOVER_EVENT_CATEGORIES table in the DMV. Then the function will parse the results, as it is xml inside of rows. + + Returns: + _type_: _description_ + ''' + Event_Categories = {} + events = [] + logging.debug(f'Querying DMV for columns rules...') + logging.debug(f'select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES') + df = self.Tabular_Class.Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES") + for index, row in df.iterrows(): + xml_data = xmltodict.parse(row.Data) + if type(xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT']) == list: + events += [event for event in xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT'] ] + else: + events += [xml_data['EVENTCATEGORY']['EVENTLIST']['EVENT']] + for event in events: + Event_Categories[event['ID']] = [column['ID'] for column in event['EVENTCOLUMNLIST']['EVENTCOLUMN']] + return Event_Categories + + +def default_refresh_handler(source, args): + if args.EventSubclass == TraceEventSubclass.ReadData: + logging.debug(f'{args.ProgressTotal} - {args.ObjectPath}') + else: + logging.debug(f'{args.EventClass} - {args.EventSubclass} - {args.ObjectName}') + +class Refresh_Trace(Base_Trace): + '''Subclass of Base_Trace. For built-in Refresh Tracing. + + Args: + Base_Trace (_type_): _description_ + ''' + def __init__(self, Tabular_Class, Trace_Events: List[TraceEvent] = [TraceEventClass.ProgressReportBegin,TraceEventClass.ProgressReportCurrent,TraceEventClass.ProgressReportEnd,TraceEventClass.ProgressReportError], + Trace_Event_Columns: List[TraceColumn] = [TraceColumn.EventSubclass,TraceColumn.CurrentTime, TraceColumn.ObjectName, TraceColumn.ObjectPath, TraceColumn.DatabaseName, TraceColumn.SessionID, TraceColumn.TextData, TraceColumn.EventClass, TraceColumn.ProgressTotal], + Handler: Callable = default_refresh_handler) -> None: + super().__init__(Tabular_Class, Trace_Events, Trace_Event_Columns, Handler) \ No newline at end of file diff --git a/test/test_tabular.py b/test/test_tabular.py index afd72a6..363b3c3 100644 --- a/test/test_tabular.py +++ b/test/test_tabular.py @@ -1,4 +1,4 @@ -from pytabular import pytabular +import pytabular from pytabular import localsecret import pytest import pandas as pd @@ -61,6 +61,6 @@ def test_table_removal(model): @pytest.mark.parametrize("model",testing_parameters) def test_bpa(model): - te2 = pytabular.TE2().EXE_Path + te2 = pytabular.Tabular_Editor().EXE bpa = pytabular.BPA().Location assert model.Analyze_BPA(te2,bpa)