diff --git a/dist/python_tabular-0.0.60-py3-none-any.whl b/dist/python_tabular-0.0.60-py3-none-any.whl new file mode 100644 index 0000000..94fba08 Binary files /dev/null and b/dist/python_tabular-0.0.60-py3-none-any.whl differ diff --git a/dist/python_tabular-0.0.60.tar.gz b/dist/python_tabular-0.0.60.tar.gz new file mode 100644 index 0000000..c6b809e Binary files /dev/null and b/dist/python_tabular-0.0.60.tar.gz differ diff --git a/pyproject.toml b/pyproject.toml index d59a40f..8d9bc20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.0.50" +version = "0.0.60" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] diff --git a/pytabular/__init__.py b/pytabular/__init__.py index aa322cb..a3a1f63 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -1,24 +1,30 @@ import logging logging.basicConfig(level=logging.DEBUG,format='%(asctime)s :: %(module)s :: %(levelname)s :: %(message)s') -logging.debug('Logging configured...') +logger = logging.getLogger('PyTabular') +logger.setLevel(logging.DEBUG) +logger.debug('Logging configured...') +logger.debug(f'To update PyTabular logger...') +logger.debug(f'>>> import logging') +logger.debug(f'>>> pytabular.logger.setLevel(level=logging.INFO)') +logger.debug(f'Visit https://docs.python.org/3/library/logging.html#logging-levels for info...') -logging.debug(f'Setting up file paths for {__file__}') +logger.debug(f'Setting up file paths for {__file__}') import os import sys dll = os.path.join(os.path.dirname(__file__),"dll") sys.path.append(dll) sys.path.append(os.path.dirname(__file__)) -logging.debug(f'Beginning CLR references...') +logger.debug(f'Beginning CLR references...') import clr -logging.debug('Adding Reference Microsoft.AnalysisServices.AdomdClient') +logger.debug('Adding Reference Microsoft.AnalysisServices.AdomdClient') clr.AddReference('Microsoft.AnalysisServices.AdomdClient') -logging.debug('Adding Reference Microsoft.AnalysisServices.Tabular') +logger.debug('Adding Reference Microsoft.AnalysisServices.Tabular') clr.AddReference('Microsoft.AnalysisServices.Tabular') -logging.debug('Adding Reference Microsoft.AnalysisServices') +logger.debug('Adding Reference Microsoft.AnalysisServices') clr.AddReference('Microsoft.AnalysisServices') -logging.debug(f"Importing from the rest...") +logger.debug(f"Importing specifics in module...") from . pytabular import Tabular from . basic_checks import Return_Zero_Row_Tables, Table_Last_Refresh_Times, BPA_Violations_To_DF from . logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype diff --git a/pytabular/basic_checks.py b/pytabular/basic_checks.py index e263045..9e62718 100644 --- a/pytabular/basic_checks.py +++ b/pytabular/basic_checks.py @@ -1,4 +1,5 @@ import logging +logger = logging.getLogger('PyTabular') from typing import List import pytabular from logic_utils import ticks_to_datetime @@ -14,7 +15,7 @@ def Return_Zero_Row_Tables(model:pytabular.Tabular) -> List[str]: Returns: List[str]: List of table names where DAX COUNTROWS('Table Name') is nan or 0. ''' - logging.info(f'Executing Basic Function {sys._getframe(0).f_code.co_name}') + logger.info(f'Executing Basic Function {sys._getframe(0).f_code.co_name}') query_function: str = 'COUNTROWS(_)' df: pd.DataFrame = model.Query_Every_Table(query_function) return df[df[f'[{query_function}]'].isna()]['[Table]'].to_list() @@ -33,7 +34,7 @@ def Table_Last_Refresh_Times(model:pytabular.Tabular, group_partition:bool = Tru pd.DataFrame: pd dataframe with the RefreshedTime property: https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.partition.refreshedtime?view=analysisservices-dotnet#microsoft-analysisservices-tabular-partition-refreshedtime If group_partition == True and the table has multiple partitions, then df.groupby(by["tables"]).max() ''' - logging.info(f'Executing Basic Function {sys._getframe(0).f_code.co_name}') + logger.info(f'Executing Basic Function {sys._getframe(0).f_code.co_name}') data = {\ "Tables":[partition.Table.Name for partition in model.Partitions],\ "Partitions":[partition.Name for partition in model.Partitions],\ @@ -41,10 +42,10 @@ def Table_Last_Refresh_Times(model:pytabular.Tabular, group_partition:bool = Tru } df = pd.DataFrame(data) if group_partition: - logging.debug('Grouping together to grain of Table') + logger.debug('Grouping together to grain of Table') return df[["Tables","RefreshedTime"]].groupby(by=["Tables"]).max().reset_index(drop=False) else: - logging.debug('Returning DF') + logger.debug('Returning DF') return df def BPA_Violations_To_DF(model:pytabular.Tabular,te2:str, bpa:str) -> pd.DataFrame: diff --git a/pytabular/best_practice_analyzer.py b/pytabular/best_practice_analyzer.py index 377ffb2..c63a277 100644 --- a/pytabular/best_practice_analyzer.py +++ b/pytabular/best_practice_analyzer.py @@ -1,4 +1,5 @@ import logging +logger = logging.getLogger('PyTabular') import requests as r import atexit import json @@ -18,7 +19,7 @@ def Download_BPA_File(Download_Location:str = 'https://raw.githubusercontent.com Returns: str: File Path for the newly downloaded BPA. ''' - logging.info(f'Downloading BPA from {Download_Location}') + logger.info(f'Downloading BPA from {Download_Location}') folder_location = os.path.join(os.getcwd(),Folder) if os.path.exists(folder_location) == False: os.makedirs(folder_location) @@ -27,7 +28,7 @@ def Download_BPA_File(Download_Location:str = 'https://raw.githubusercontent.com with open(file_location, 'w', encoding='utf-8') as bpa: json.dump(response.json(), bpa, ensure_ascii=False, indent= 4) if Auto_Remove: - logging.debug(f'Registering removal on termination... For {folder_location}') + logger.debug(f'Registering removal on termination... For {folder_location}') atexit.register(remove_folder_and_contents, folder_location) return file_location @@ -35,7 +36,7 @@ class BPA: '''Setting BPA Class for future work... ''' def __init__(self, File_Path:str = 'Default') -> None: - logging.debug(f'Initializing BPA Class:: {File_Path}') + logger.debug(f'Initializing BPA Class:: {File_Path}') if File_Path == 'Default': self.Location: str = Download_BPA_File() else: diff --git a/pytabular/logic_utils.py b/pytabular/logic_utils.py index 45a65a0..7c3ad24 100644 --- a/pytabular/logic_utils.py +++ b/pytabular/logic_utils.py @@ -1,4 +1,5 @@ import logging +logger = logging.getLogger('PyTabular') import datetime import os from typing import Dict, List @@ -28,7 +29,7 @@ def pandas_datatype_to_tabular_datatype(df:pd.DataFrame)-> Dict: Returns: Dict: EX {'col1': , 'col2': , 'col3': } ''' - logging.info(f'Getting DF Column Dtypes to Tabular Dtypes...') + logger.info(f'Getting DF Column Dtypes to Tabular Dtypes...') tabular_datatype_mapping_key = { 'b':DataType.Boolean, 'i':DataType.Int64, @@ -107,10 +108,10 @@ def m_list_expression_generator(list_of_strings:List[str]) -> str: ''' string_components = ','.join([f'\"{string_value}\"' for string_value in list_of_strings]) return f'\u007b{string_components}\u007d' - logging.debug(f'Executing m_list_generator()... for {df.columns}') + logger.debug(f'Executing m_list_generator()... for {df.columns}') columns = m_list_expression_generator(df.columns) expression_str = f"let\nSource=#table({columns},\n" - logging.debug(f'Iterating through rows to build expression... df has {len(df)} rows...') + logger.debug(f'Iterating through rows to build expression... df has {len(df)} rows...') expression_list_rows = [] for index, row in df.iterrows(): expression_list_rows += [m_list_expression_generator(row.to_list())] @@ -125,5 +126,5 @@ def remove_folder_and_contents(folder_location): ''' import shutil if os.path.exists(folder_location): - logging.info(f'Removing Dir and Contents -> {folder_location}') + logger.info(f'Removing Dir and Contents -> {folder_location}') shutil.rmtree(folder_location) \ No newline at end of file diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index e0c26d0..134d3ce 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -1,51 +1,54 @@ import logging +logger = logging.getLogger('PyTabular') -logging.debug(f'Importing Microsoft.AnalysisServices.Tabular') +logger.debug(f'Importing Microsoft.AnalysisServices.Tabular') from Microsoft.AnalysisServices.Tabular import Server, Database, RefreshType, DataType, ConnectionDetails, ColumnType, MetadataPermission, Table, DataColumn, Partition, MPartitionSource, PartitionSourceType, Trace, TraceEvent, TraceEventHandler -logging.debug(f'Importing Microsoft.AnalysisServices.AdomdClient') +logger.debug(f'Importing Microsoft.AnalysisServices.AdomdClient') from Microsoft.AnalysisServices.AdomdClient import (AdomdCommand, AdomdConnection) -logging.debug(f'Importing Microsoft.AnalysisServices') +logger.debug(f'Importing Microsoft.AnalysisServices') from Microsoft.AnalysisServices import UpdateOptions, TraceEventClass, TraceEventSubclass, TraceEventCollection, TraceColumn -logging.debug('Importing Other Packages...') +logger.debug('Importing Other Packages...') from typing import List, Union, Callable from collections.abc import Iterable +from collections import namedtuple import requests as r import pandas as pd import json import os import subprocess import atexit -from logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype +from logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype, ticks_to_datetime from tabular_tracing import Refresh_Trace + class Tabular: '''Tabular Class to perform operations: [Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet) Args: - CONNECTION_STR (str): [Connection String](https://docs.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services?view=asallproducts-allversions) + CONNECTION_STR (str): Valid [Connection String](https://docs.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services?view=asallproducts-allversions) for connecting to a Tabular Model. ''' def __init__(self,CONNECTION_STR:str): - logging.debug(f'Initializing Tabular Class') + logger.debug(f'Initializing Tabular Class') self.Server = Server() #[Server](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.server?view=analysisservices-dotnet) self.Server.Connect(CONNECTION_STR) - logging.debug(f'Connected to Server - {self.Server.Name}') + logger.info(f'Connected to Server - {self.Server.Name}') self.Catalog = self.Server.ConnectionInfo.Catalog - logging.debug(f'Received Catalog - {self.Catalog}') + logger.debug(f'Received Catalog - {self.Catalog}') try: self.Database = [database for database in self.Server.Databases.GetEnumerator() if database.Name == self.Catalog][0] except: - logging.error(f'Unable to find Database... {self.Catalog}') - logging.debug(f'Connected to Database - {self.Database.Name}') + logger.error(f'Unable to find Database... {self.Catalog}') + logger.info(f'Connected to Database - {self.Database.Name}') self.CompatibilityLevel: int = self.Database.CompatibilityLevel self.CompatibilityMode: int = self.Database.CompatibilityMode.value__ self.Model = self.Database.Model - logging.debug(f'Connected to Model - {self.Model.Name}') + logger.info(f'Connected to Model - {self.Model.Name}') self.DaxConnection = AdomdConnection() self.DaxConnection.ConnectionString = f"{self.Server.ConnectionString}Password='{self.Server.ConnectionInfo.Password}'" self.Reload_Model_Info() - logging.debug(f'Class Initialization Completed') - logging.debug(f'Registering Disconnect on Termination...') + logger.debug(f'Class Initialization Completed') + logger.debug(f'Registering Disconnect on Termination...') atexit.register(self.Disconnect) pass @@ -68,14 +71,14 @@ def Disconnect(self) -> bool: Returns: bool: True if successful ''' - logging.debug(f'Disconnecting from - {self.Server.Name}') + logger.debug(f'Disconnecting from - {self.Server.Name}') self.Server.Disconnect() if self.Server.Connected: - logging.error(f'Disconnect Unsuccessful') + logger.error(f'Disconnect Unsuccessful') return False else: - logging.debug(f'Disconnect Successful') + logger.debug(f'Disconnect Successful') return True def Refresh(self, Object:Union[str,Table,Partition,Iterable], RefreshType=RefreshType.Full, Run:bool = True) -> None: '''Input Object(s) to be refreshed in the tabular model. Combine with .SaveChanges() to actually run the refresh on the model. @@ -84,15 +87,20 @@ def Refresh(self, Object:Union[str,Table,Partition,Iterable], RefreshType=Refres Object (Union[str,Table,Partition,Iterable]): Can be str(table name only), Table object, Partition object, or an iterable combination of the three. RefreshType (_type_, optional): [RefreshType](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.refreshtype?view=analysisservices-dotnet). Defaults to RefreshType.Full. ''' - logging.debug(f'Beginning RequestRefresh cadence...') - + logger.debug(f'Beginning RequestRefresh cadence...') + def Refresh_Report(Property_Changes): + logger.debug(f'Running Refresh Report...') + for property_change in Property_Changes: + if isinstance(property_change.Object,Partition) and property_change.Property_Name == 'RefreshedTime': + logger.info(f'{property_change.Object.Table.Name} - {property_change.Object.Name} Refreshed! - {ticks_to_datetime(property_change.New_Value.Ticks).strftime("%m/%d/%Y, %H:%M:%S")}') + return True def refresh(object): if isinstance(object,str): - logging.info(f'Requesting refresh for {object}') + logger.info(f'Requesting refresh for {object}') table = [table for table in self.Tables if table.Name == object][0] table.RequestRefresh(RefreshType) else: - logging.info(f'Requesting refresh for {object.Name}') + logger.info(f'Requesting refresh for {object.Name}') object.RequestRefresh(RefreshType) @@ -104,9 +112,11 @@ def refresh(object): if Run: rt = Refresh_Trace(self) rt.Start() - self.SaveChanges() + m = self.SaveChanges() rt.Stop() rt.Drop() + Refresh_Report(m.Property_Changes) + return m.Property_Changes def Update(self, UpdateOptions:UpdateOptions =UpdateOptions.ExpandFull) -> None: '''[Update Model](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.majorobject.update?view=analysisservices-dotnet#microsoft-analysisservices-majorobject-update(microsoft-analysisservices-updateoptions)) @@ -116,18 +126,29 @@ def Update(self, UpdateOptions:UpdateOptions =UpdateOptions.ExpandFull) -> None: Returns: None: Placeholder to eventually change. ''' - logging.debug('Running Update Request') + logger.debug('Running Update Request') return self.Database.Update(UpdateOptions) def SaveChanges(self) -> bool: - '''TODO need to clean this up and add more flexibility. - Just a simple wrapper to call self.Model.SaveChanges() + def property_changes(Property_Changes): + Property_Change = namedtuple("Property_Change","New_Value Object Original_Value Property_Name Property_Type") + return [Property_Change(change.NewValue, change.Object, change.OriginalValue, change.PropertyName, change.PropertyType) for change in Property_Changes.GetEnumerator()] - Returns: - bool: - ''' - self.Reload_Model_Info() - self.Model.SaveChanges() - return True + + logger.info(f'Executing SaveChanges()...') + Model_Save_Results = self.Model.SaveChanges() + if isinstance(Model_Save_Results.Impact, type(None)): + logger.warning(f'No changes detected on save for {self.Model.Name}') + return None + else: + Property_Changes = Model_Save_Results.Impact.PropertyChanges + Added_Objects = Model_Save_Results.Impact.AddedObjects + Added_Subtree_Roots = Model_Save_Results.Impact.AddedSubtreeRoots + Removed_Objects = Model_Save_Results.Impact.RemovedObjects + Removed_Subtree_Roots = Model_Save_Results.Impact.RemovedSubtreeRoots + Changes = namedtuple("Changes","Property_Changes Added_Objects Added_Subtree_Roots Removed_Objects Removed_Subtree_Roots") + [property_changes(Property_Changes), Added_Objects, Added_Subtree_Roots, Removed_Objects, Removed_Subtree_Roots] + self.Reload_Model_Info() + return Changes(property_changes(Property_Changes), Added_Objects, Added_Subtree_Roots, Removed_Objects, Removed_Subtree_Roots) def Backup_Table(self,table_str:str) -> bool: '''USE WITH CAUTION, EXPERIMENTAL. Backs up table in memory, brings with it measures, columns, hierarchies, relationships, roles, etc. It will add suffix '_backup' to all objects. @@ -139,60 +160,60 @@ def Backup_Table(self,table_str:str) -> bool: Returns: bool: Returns True if Successful, else will return error. ''' - logging.info('Backup Beginning...') - logging.debug(f'Cloning {table_str}') + logger.info('Backup Beginning...') + logger.debug(f'Cloning {table_str}') table = self.Model.Tables.Find(table_str).Clone() - logging.info(f'Beginning Renames') + logger.info(f'Beginning Renames') def rename(items): for item in items: item.RequestRename(f'{item.Name}_backup') - logging.debug(f'Renamed - {item.Name}') - logging.info('Renaming Columns') + logger.debug(f'Renamed - {item.Name}') + logger.info('Renaming Columns') rename(table.Columns.GetEnumerator()) - logging.info('Renaming Partitions') + logger.info('Renaming Partitions') rename(table.Partitions.GetEnumerator()) - logging.info('Renaming Measures') + logger.info('Renaming Measures') rename(table.Measures.GetEnumerator()) - logging.info('Renaming Hierarchies') + logger.info('Renaming Hierarchies') rename(table.Hierarchies.GetEnumerator()) - logging.info('Renaming Table') + logger.info('Renaming Table') table.RequestRename(f'{table.Name}_backup') - logging.info('Adding Table to Model as backup') + logger.info('Adding Table to Model as backup') self.Model.Tables.Add(table) - logging.info('Finding Necessary Relationships... Cloning...') + logger.info('Finding Necessary Relationships... Cloning...') relationships = [relationship.Clone() for relationship in self.Model.Relationships.GetEnumerator() if relationship.ToTable.Name == table.Name.removesuffix('_backup') or relationship.FromTable.Name == table.Name.removesuffix('_backup')] - logging.info('Renaming Relationships') + logger.info('Renaming Relationships') rename(relationships) - logging.info('Switching Relationships to Clone Table & Column') + logger.info('Switching Relationships to Clone Table & Column') for relationship in relationships: - logging.debug(f'Renaming - {relationship.Name}') + logger.debug(f'Renaming - {relationship.Name}') if relationship.ToTable.Name == table.Name.removesuffix('_backup'): relationship.set_ToColumn(table.Columns.Find(f'{relationship.ToColumn.Name}_backup')) elif relationship.FromTable.Name == table.Name.removesuffix('_backup'): relationship.set_FromColumn(table.Columns.Find(f'{relationship.FromColumn.Name}_backup')) - logging.debug(f'Adding {relationship.Name} to {self.Model.Name}') + logger.debug(f'Adding {relationship.Name} to {self.Model.Name}') self.Model.Relationships.Add(relationship) def clone_role_permissions(): - logging.info(f'Beginning to handle roles and permissions for table...') - logging.debug(f'Finding Roles...') + logger.info(f'Beginning to handle roles and permissions for table...') + logger.debug(f'Finding Roles...') roles = [role for role in self.Model.Roles.GetEnumerator() for tablepermission in role.TablePermissions.GetEnumerator() if tablepermission.Name == table_str] for role in roles: - logging.debug(f'Role {role.Name} matched, looking into it...') - logging.debug(f'Searching for table specific permissions') + logger.debug(f'Role {role.Name} matched, looking into it...') + logger.debug(f'Searching for table specific permissions') tablepermissions = [table.Clone() for table in role.TablePermissions.GetEnumerator() if table.Name == table_str] for tablepermission in tablepermissions: - logging.debug(f'{tablepermission.Name} found... switching table to clone') + logger.debug(f'{tablepermission.Name} found... switching table to clone') tablepermission.set_Table(table) for column in tablepermission.ColumnPermissions.GetEnumerator(): - logging.debug(f'Column - {column.Name} copying permissions to clone...') + logger.debug(f'Column - {column.Name} copying permissions to clone...') column.set_Column(self.Model.Tables.Find(table.Name).Columns.Find(f'{column.Name}_backup')) - logging.debug(f'Adding {tablepermission.Name} to {role.Name}') + logger.debug(f'Adding {tablepermission.Name} to {role.Name}') role.TablePermissions.Add(tablepermission) return True clone_role_permissions() - logging.info(f'Refreshing Clone... {table.Name}') + logger.info(f'Refreshing Clone... {table.Name}') self.Refresh([table]) - logging.info(f'Updating Model {self.Model.Name}') + logger.info(f'Updating Model {self.Model.Name}') self.SaveChanges() return True def Revert_Table(self, table_str:str) -> bool: @@ -210,53 +231,53 @@ def Revert_Table(self, table_str:str) -> bool: Returns: bool: Returns True if Successful, else will return error. ''' - logging.info(f'Beginning Revert for {table_str}') - logging.debug(f'Finding original {table_str}') + logger.info(f'Beginning Revert for {table_str}') + logger.debug(f'Finding original {table_str}') main = self.Model.Tables.Find(table_str) - logging.debug(f'Finding backup {table_str}') + logger.debug(f'Finding backup {table_str}') backup = self.Model.Tables.Find(f'{table_str}_backup') - logging.debug(f'Finding original relationships') + logger.debug(f'Finding original relationships') main_relationships = [relationship for relationship in self.Model.Relationships.GetEnumerator() if relationship.ToTable.Name == main.Name or relationship.FromTable.Name == main.Name] - logging.debug(f'Finding backup relationships') + logger.debug(f'Finding backup relationships') backup_relationships = [relationship for relationship in self.Model.Relationships.GetEnumerator() if relationship.ToTable.Name == backup.Name or relationship.FromTable.Name == backup.Name] def remove_role_permissions(): - logging.debug(f'Finding table and column permission in roles to remove from {table_str}') + logger.debug(f'Finding table and column permission in roles to remove from {table_str}') roles = [role for role in self.Model.Roles.GetEnumerator() for tablepermission in role.TablePermissions.GetEnumerator() if tablepermission.Name == table_str] for role in roles: - logging.debug(f'Role {role.Name} Found') + logger.debug(f'Role {role.Name} Found') tablepermissions = [table for table in role.TablePermissions.GetEnumerator() if table.Name == table_str] for tablepermission in tablepermissions: - logging.debug(f'Removing {tablepermission.Name} from {role.Name}') + logger.debug(f'Removing {tablepermission.Name} from {role.Name}') role.TablePermissions.Remove(tablepermission) for relationship in main_relationships: - logging.debug(f'Cleaning relationships...') + logger.debug(f'Cleaning relationships...') if relationship.ToTable.Name == main.Name: - logging.debug(f'Removing {relationship.Name}') + logger.debug(f'Removing {relationship.Name}') self.Model.Relationships.Remove(relationship) elif relationship.FromTable.Name == main.Name: - logging.debug(f'Removing {relationship.Name}') + logger.debug(f'Removing {relationship.Name}') self.Model.Relationships.Remove(relationship) - logging.debug(f'Removing Original Table {main.Name}') + logger.debug(f'Removing Original Table {main.Name}') self.Model.Tables.Remove(main) remove_role_permissions() def dename(items): for item in items: - logging.debug(f'Removing Suffix for {item.Name}') + logger.debug(f'Removing Suffix for {item.Name}') item.RequestRename(f'{item.Name}'.removesuffix('_backup')) - logging.debug(f'Saving Changes... for {item.Name}') + logger.debug(f'Saving Changes... for {item.Name}') self.Model.SaveChanges() - logging.info(f'Name changes for Columns...') + logger.info(f'Name changes for Columns...') dename([column for column in backup.Columns.GetEnumerator() if column.Type != ColumnType.RowNumber]) - logging.info(f'Name changes for Partitions...') + logger.info(f'Name changes for Partitions...') dename(backup.Partitions.GetEnumerator()) - logging.info(f'Name changes for Measures...') + logger.info(f'Name changes for Measures...') dename(backup.Measures.GetEnumerator()) - logging.info(f'Name changes for Hierarchies...') + logger.info(f'Name changes for Hierarchies...') dename(backup.Hierarchies.GetEnumerator()) - logging.info(f'Name changes for Relationships...') + logger.info(f'Name changes for Relationships...') dename(backup_relationships) - logging.info(f'Name changes for Backup Table...') + logger.info(f'Name changes for Backup Table...') backup.RequestRename(backup.Name.removesuffix('_backup')) self.SaveChanges() return True @@ -269,25 +290,25 @@ def Query(self,Query_Str:str) -> pd.DataFrame: Returns: pd.DataFrame: Returns dataframe with results ''' - logging.info(f'Query Called...') + logger.debug(f'Beginning to Query...') try: - logging.debug(f'Attempting to Open Connection...') + logger.debug(f'Attempting to Open Adomd Connection...') self.DaxConnection.Open() - logging.debug(f'Connected!') + logger.debug(f'Connected!') except: - logging.debug(f'Connection skipped already connected...') + logger.debug(f'Connection skipped already connected...') pass - logging.debug(f'Querying Model with Query...') + logger.info(f'Querying Model with Query...') Query = AdomdCommand(Query_Str, self.DaxConnection).ExecuteReader() - logging.debug(f'Determining Field Count...') + logger.debug(f'Determining Field Count...') Column_Headers = [(index,Query.GetName(index)) for index in range(0,Query.FieldCount)] Results = list() - logging.debug(f'Converting Results into List...') + logger.debug(f'Converting Results into List...') while Query.Read(): Results.append([Query.GetValue(index) for index in range(0,len(Column_Headers))]) - logging.debug(f'Data retrieved and closing query...') + logger.info(f'Data retrieved and closing query...') Query.Close() - logging.debug(f'Converting to Pandas DataFrame...') + logger.debug(f'Converting to Pandas DataFrame...') df = pd.DataFrame(Results,columns=[value for _,value in Column_Headers]) return df def Query_Every_Column(self,query_function:str='COUNTROWS(VALUES(_))') -> pd.DataFrame: @@ -300,9 +321,9 @@ def Query_Every_Column(self,query_function:str='COUNTROWS(VALUES(_))') -> pd.Dat Returns: pd.DataFrame: Returns dataframe with results. ''' - logging.info(f'Beginning execution of querying every column...') - logging.debug(f'Function to be run: {query_function}') - logging.debug(f'Dynamically creating DAX query...') + logger.info(f'Beginning execution of querying every column...') + logger.debug(f'Function to be run: {query_function}') + logger.debug(f'Dynamically creating DAX query...') query_str = "EVALUATE UNION(\n" for column in self.Columns: if column.Type != ColumnType.RowNumber: @@ -322,9 +343,9 @@ def Query_Every_Table(self,query_function:str='COUNTROWS(_)') -> pd.DataFrame: Returns: pd.DataFrame: Returns dataframe with results ''' - logging.info(f'Beginning execution of querying every table...') - logging.debug(f'Function to be run: {query_function}') - logging.debug(f'Dynamically creating DAX query...') + logger.info(f'Beginning execution of querying every table...') + logger.debug(f'Function to be run: {query_function}') + logger.debug(f'Dynamically creating DAX query...') query_str = "EVALUATE UNION(\n" for table in self.Tables: table_name = table.get_Name() @@ -346,10 +367,10 @@ def Analyze_BPA(self,Tabular_Editor_Exe:str,Best_Practice_Analyzer:str) -> List[ ''' #Working TE2 Script in Python os.system(f"start /wait {te2.EXE_Path} \"Provider=MSOLAP;{model.DaxConnection.ConnectionString}\" FINANCE -B \"{os.getcwd()}\\Model.bim\" -A {l.BPA_LOCAL_FILE_PATH} -V/?") #start /wait - logging.debug(f'Beginning request to talk with TE2 & Find BPA...') + logger.debug(f'Beginning request to talk with TE2 & Find BPA...') cmd = f"{Tabular_Editor_Exe} \"Provider=MSOLAP;{self.DaxConnection.ConnectionString}\" {self.Database.Name} -B \"{os.getcwd()}\\Model.bim\" -A {Best_Practice_Analyzer} -V/?" - logging.debug(f'Command Generated') - logging.debug(f'Submitting Command...') + logger.debug(f'Command Generated') + logger.debug(f'Submitting Command...') sp = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True) raw_output,error = sp.communicate() if len(error) > 0: @@ -368,28 +389,28 @@ def Create_Table(self,df:pd.DataFrame, table_name:str) -> bool: Returns: bool: True if successful ''' - logging.debug(f'Beginning to create table for {table_name}...') + logger.debug(f'Beginning to create table for {table_name}...') new_table = Table() new_table.RequestRename(table_name) - logging.debug(f'Sorting through columns...') + logger.debug(f'Sorting through columns...') df_column_names = df.columns dtype_conversion = pandas_datatype_to_tabular_datatype(df) for df_column_name in df_column_names: - logging.debug(f'Adding {df_column_name} to Table...') + logger.debug(f'Adding {df_column_name} to Table...') column = DataColumn() column.RequestRename(df_column_name) column.set_SourceColumn(df_column_name) column.set_DataType(dtype_conversion[df_column_name]) new_table.Columns.Add(column) - logging.debug(f'Expression String Created...') - logging.debug(f'Creating MPartition...') + logger.debug(f'Expression String Created...') + logger.debug(f'Creating MPartition...') partition = Partition() partition.set_Source(MPartitionSource()) - logging.debug(f'Setting MPartition Expression...') + logger.debug(f'Setting MPartition Expression...') partition.Source.set_Expression(pd_dataframe_to_m_expression(df)) - logging.debug(f'Adding partition: {partition.Name} to {self.Server.Name}::{self.Database.Name}::{self.Model.Name}') + logger.debug(f'Adding partition: {partition.Name} to {self.Server.Name}::{self.Database.Name}::{self.Model.Name}') new_table.Partitions.Add(partition) - logging.debug(f'Adding table: {new_table.Name} to {self.Server.Name}::{self.Database.Name}::{self.Model.Name}') + logger.debug(f'Adding table: {new_table.Name} to {self.Server.Name}::{self.Database.Name}::{self.Model.Name}') self.Model.Tables.Add(new_table) self.Refresh([new_table]) self.SaveChanges() diff --git a/pytabular/tabular_editor.py b/pytabular/tabular_editor.py index 1f0ae63..11cb77c 100644 --- a/pytabular/tabular_editor.py +++ b/pytabular/tabular_editor.py @@ -1,4 +1,5 @@ import logging +logger = logging.getLogger('PyTabular') import os import requests as r import zipfile as Z @@ -18,8 +19,8 @@ def Download_Tabular_Editor(Download_Location:str = 'https://github.com/TabularE Returns: str: _description_ ''' - logging.info(f'Downloading Tabular Editor 2...') - logging.info(f'From... {Download_Location}') + logger.info(f'Downloading Tabular Editor 2...') + logger.info(f'From... {Download_Location}') folder_location = os.path.join(os.getcwd(),Folder) response = r.get(Download_Location) file_location = f"{os.getcwd()}\\{Download_Location.split('/')[-1]}" @@ -27,11 +28,11 @@ def Download_Tabular_Editor(Download_Location:str = 'https://github.com/TabularE te2_zip.write(response.content) with Z.ZipFile(file_location) as zipper: zipper.extractall(path=folder_location) - logging.debug(f'Removing Zip File...') + logger.debug(f'Removing Zip File...') os.remove(file_location) - logging.info(f'Tabular Editor Downloaded and Extracted to {folder_location}') + logger.info(f'Tabular Editor Downloaded and Extracted to {folder_location}') if Auto_Remove: - logging.debug(f'Registering removal on termination... For {folder_location}') + logger.debug(f'Registering removal on termination... For {folder_location}') atexit.register(remove_folder_and_contents, folder_location) return f'{folder_location}\\TabularEditor.exe' @@ -39,7 +40,7 @@ class Tabular_Editor: '''Setting Tabular_Editor Class for future work. ''' def __init__(self, EXE_File_Path:str = 'Default') -> None: - logging.debug(f'Initializing Tabular Editor Class:: {EXE_File_Path}') + logger.debug(f'Initializing Tabular Editor Class:: {EXE_File_Path}') if EXE_File_Path == 'Default': self.EXE: str = Download_Tabular_Editor() else: diff --git a/pytabular/tabular_tracing.py b/pytabular/tabular_tracing.py index 0d7b461..f068dd4 100644 --- a/pytabular/tabular_tracing.py +++ b/pytabular/tabular_tracing.py @@ -1,4 +1,5 @@ import logging +logger = logging.getLogger('PyTabular') import random import xmltodict from typing import List, Callable @@ -18,11 +19,11 @@ class Base_Trace: Handler (Callable): Function to call when Trace returns response. Input needs to be two arguments. One is source (Which is currently None... Need to investigate why). Second is [TraceEventArgs](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.traceeventargs?view=analysisservices-dotnet) ''' def __init__(self, Tabular_Class, Trace_Events:List[TraceEvent], Trace_Event_Columns:List[TraceColumn], Handler:Callable) -> None: - logging.debug(f'Trace Base Class initializing...') + logger.debug(f'Trace Base Class initializing...') self.Name = 'PyTabular_'+''.join(random.SystemRandom().choices([str(x) for x in [y for y in range(0,10)]], k=10)) self.ID = self.Name.replace('PyTabular_', '') self.Trace = Trace(self.Name, self.ID) - logging.debug(f'Trace {self.Trace.Name} created...') + logger.debug(f'Trace {self.Trace.Name} created...') self.Tabular_Class = Tabular_Class self.Event_Categories = self.Query_DMV_For_Event_Categories() @@ -40,21 +41,21 @@ def Build(self) -> bool: Returns: bool: True if successful ''' - logging.info(f'Building Trace {self.Name}') + logger.info(f'Building Trace {self.Name}') TE = [TraceEvent(trace_event) for trace_event in self.Trace_Events] - logging.debug(f'Adding Events to... {self.Trace.Name}') + logger.debug(f'Adding Events to... {self.Trace.Name}') [self.Trace.get_Events().Add(te) for te in TE] def add_column(trace_event,trace_event_column): try: trace_event.Columns.Add(trace_event_column) except: - logging.warning(f'{trace_event} - {trace_event_column} Skipped') + logger.warning(f'{trace_event} - {trace_event_column} Skipped') - logging.debug(f'Adding Trace Event Columns...') + logger.debug(f'Adding Trace Event Columns...') [add_column(trace_event,trace_event_column) for trace_event_column in self.Trace_Event_Columns for trace_event in TE if str(trace_event_column.value__) in self.Event_Categories[str(trace_event.EventID.value__)] ] - logging.debug(f'Adding Handler to Trace...') + logger.debug(f'Adding Handler to Trace...') self.Handler = TraceEventHandler(self.Handler) self.Trace.OnEvent += self.Handler return True @@ -68,7 +69,7 @@ def Add(self) -> int: Returns: int: Return int of placement in Server.Traces.get_Item(int) ''' - logging.info(f'Adding {self.Name} to {self.Tabular_Class.Server.Name}') + logger.info(f'Adding {self.Name} to {self.Tabular_Class.Server.Name}') return self.Tabular_Class.Server.Traces.Add(self.Trace) def Update(self) -> None: @@ -77,7 +78,7 @@ def Update(self) -> None: Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. ''' - logging.info(f'Updating {self.Name} in {self.Tabular_Class.Server.Name}') + logger.info(f'Updating {self.Name} in {self.Tabular_Class.Server.Name}') return self.Trace.Update() def Start(self) -> None: @@ -86,7 +87,7 @@ def Start(self) -> None: Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. ''' - logging.info(f'Starting {self.Name} in {self.Tabular_Class.Server.Name}') + logger.info(f'Starting {self.Name} in {self.Tabular_Class.Server.Name}') return self.Trace.Start() def Stop(self) -> None: @@ -95,7 +96,7 @@ def Stop(self) -> None: Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. ''' - logging.info(f'Stopping {self.Name} in {self.Tabular_Class.Server.Name}') + logger.info(f'Stopping {self.Name} in {self.Tabular_Class.Server.Name}') return self.Trace.Stop() def Drop(self) -> None: @@ -104,7 +105,7 @@ def Drop(self) -> None: Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. ''' - logging.info(f'Dropping {self.Name} in {self.Tabular_Class.Server.Name}') + logger.info(f'Dropping {self.Name} in {self.Tabular_Class.Server.Name}') return self.Trace.Drop() def Query_DMV_For_Event_Categories(self): @@ -115,8 +116,8 @@ def Query_DMV_For_Event_Categories(self): ''' Event_Categories = {} events = [] - logging.debug(f'Querying DMV for columns rules...') - logging.debug(f'select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES') + logger.debug(f'Querying DMV for columns rules...') + logger.debug(f'select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES') df = self.Tabular_Class.Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES") for index, row in df.iterrows(): xml_data = xmltodict.parse(row.Data) @@ -131,9 +132,9 @@ def Query_DMV_For_Event_Categories(self): def default_refresh_handler(source, args): if args.EventSubclass == TraceEventSubclass.ReadData: - logging.debug(f'{args.ProgressTotal} - {args.ObjectPath}') + logger.debug(f'{args.ProgressTotal} - {args.ObjectPath}') else: - logging.debug(f'{args.EventClass} - {args.EventSubclass} - {args.ObjectName}') + logger.debug(f'{args.EventClass} - {args.EventSubclass} - {args.ObjectName}') class Refresh_Trace(Base_Trace): '''Subclass of Base_Trace. For built-in Refresh Tracing.