From 00c3babb201f0b162af689a0831b1c792a67592f Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sat, 4 Feb 2023 14:45:40 -0600 Subject: [PATCH 01/10] initial flake8 pep8-naming pass --- pyproject.toml | 2 +- pytabular/__init__.py | 4 +- pytabular/best_practice_analyzer.py | 28 ++--- pytabular/column.py | 12 +-- pytabular/logic_utils.py | 10 +- pytabular/object.py | 2 +- pytabular/partition.py | 10 +- pytabular/pytabular.py | 158 ++++++++++++++-------------- pytabular/query.py | 48 ++++----- pytabular/refresh.py | 96 ++++++++--------- pytabular/relationship.py | 2 +- pytabular/table.py | 26 ++--- pytabular/tabular_editor.py | 42 ++++---- pytabular/tabular_tracing.py | 130 +++++++++++------------ test/conftest.py | 4 +- test/test_3tabular.py | 4 +- 16 files changed, 289 insertions(+), 289 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b502605..02d7907 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.3.4" +version = "0.3.5" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] diff --git a/pytabular/__init__.py b/pytabular/__init__.py index ca1a14e..ebe3c47 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -58,8 +58,8 @@ pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype, ) -from .tabular_tracing import Base_Trace, Refresh_Trace, Query_Monitor -from .tabular_editor import Tabular_Editor +from .tabular_tracing import BaseTrace, RefreshTrace, QueryMonitor +from .tabular_editor import TabularEditor from .best_practice_analyzer import BPA from .query import Connection from .pbi_helper import find_local_pbi_instances diff --git a/pytabular/best_practice_analyzer.py b/pytabular/best_practice_analyzer.py index 6b4103a..025b6ff 100644 --- a/pytabular/best_practice_analyzer.py +++ b/pytabular/best_practice_analyzer.py @@ -16,10 +16,10 @@ logger = logging.getLogger("PyTabular") -def Download_BPA_File( - Download_Location: str = "https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json", - Folder: str = "Best_Practice_Analyzer", - Auto_Remove=True, +def download_bpa_file( + download_location: str = "https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json", + folder: str = "Best_Practice_Analyzer", + auto_remove=True, ) -> str: """Runs a request.get() to retrieve the json file from web. Will return and store in directory. Will also register the removal of the new directory and file when exiting program. @@ -31,15 +31,15 @@ def Download_BPA_File( Returns: str: File Path for the newly downloaded BPA. """ - logger.info(f"Downloading BPA from {Download_Location}") - folder_location = os.path.join(os.getcwd(), Folder) + logger.info(f"Downloading BPA from {download_location}") + folder_location = os.path.join(os.getcwd(), folder) if os.path.exists(folder_location) is False: os.makedirs(folder_location) - response = r.get(Download_Location) - file_location = os.path.join(folder_location, Download_Location.split("/")[-1]) + response = r.get(download_location) + file_location = os.path.join(folder_location, download_location.split("/")[-1]) with open(file_location, "w", encoding="utf-8") as bpa: json.dump(response.json(), bpa, ensure_ascii=False, indent=4) - if Auto_Remove: + if auto_remove: logger.debug(f"Registering removal on termination... For {folder_location}") atexit.register(remove_folder_and_contents, folder_location) return file_location @@ -48,16 +48,16 @@ def Download_BPA_File( class BPA: """Setting BPA Class for future work...""" - def __init__(self, File_Path: str = "Default") -> None: + def __init__(self, file_path: str = "Default") -> None: """BPA class to be used with the TE2 class. Args: File_Path (str, optional): See `Download_BPA_File()`. Defaults to "Default". If "Default, then will run `Download_BPA_File()` without args. """ - logger.debug(f"Initializing BPA Class:: {File_Path}") - if File_Path == "Default": - self.Location: str = Download_BPA_File() + logger.debug(f"Initializing BPA Class:: {file_path}") + if file_path == "Default": + self.location: str = download_bpa_file() else: - self.Location: str = File_Path + self.location: str = file_path pass diff --git a/pytabular/column.py b/pytabular/column.py index 708d7be..aac5d9b 100644 --- a/pytabular/column.py +++ b/pytabular/column.py @@ -73,23 +73,23 @@ def get_sample_values(self, top_n: int = 3) -> pd.DataFrame: """ return self.Table.Model.Query(dax_query) - def Distinct_Count(self, No_Blank=False) -> int: + def distinct_count(self, no_blank=False) -> int: """Get [DISTINCTCOUNT](https://learn.microsoft.com/en-us/dax/distinctcount-function-dax) of Column. Args: - No_Blank (bool, optional): Ability to call [DISTINCTCOUNTNOBLANK](https://learn.microsoft.com/en-us/dax/distinctcountnoblank-function-dax). Defaults to False. + no_blank (bool, optional): Ability to call [DISTINCTCOUNTNOBLANK](https://learn.microsoft.com/en-us/dax/distinctcountnoblank-function-dax). Defaults to False. Returns: - int: Number of Distinct Count from column. If `No_Blank == True` then will return number of Distinct Count no blanks. + int: Number of Distinct Count from column. If `no_blank == True` then will return number of Distinct Count no blanks. """ func = "DISTINCTCOUNT" - if No_Blank: + if no_blank: func += "NOBLANK" return self.Table.Model.Adomd.Query( f"EVALUATE {{{func}('{self.Table.Name}'[{self.Name}])}}" ) - def Values(self) -> pd.DataFrame: + def values(self) -> pd.DataFrame: """Get single column DataFrame of [VALUES](https://learn.microsoft.com/en-us/dax/values-function-dax) Returns: @@ -111,7 +111,7 @@ class PyColumns(PyObjects): def __init__(self, objects) -> None: super().__init__(objects) - def Query_All(self, query_function: str = "COUNTROWS(VALUES(_))") -> pd.DataFrame: + def query_all(self, query_function: str = "COUNTROWS(VALUES(_))") -> pd.DataFrame: """This will dynamically create a query to pull all columns from the model and run the query function. It will replace the _ with the column to run. diff --git a/pytabular/logic_utils.py b/pytabular/logic_utils.py index 342007c..bd3de70 100644 --- a/pytabular/logic_utils.py +++ b/pytabular/logic_utils.py @@ -159,7 +159,7 @@ def get_sub_list(lst: list, n: int) -> list: return [lst[i : i + n] for i in range(0, len(lst), n)] -def get_value_to_df(Query: AdomdDataReader, index: int): +def get_value_to_df(query: AdomdDataReader, index: int): """Gets the values from the AdomdDataReader to convert the .Net Object into a tangible python value to work with in pandas. Lots of room for improvement on this one. @@ -169,12 +169,12 @@ def get_value_to_df(Query: AdomdDataReader, index: int): index (int): Index of the value to perform the logic on. """ if ( - Query.GetDataTypeName((index)) in ("Decimal") - and Query.GetValue(index) is not None + query.GetDataTypeName((index)) in ("Decimal") + and query.GetValue(index) is not None ): - return Query.GetValue(index).ToDouble(Query.GetValue(index)) + return query.GetValue(index).ToDouble(query.GetValue(index)) else: - return Query.GetValue(index) + return query.GetValue(index) def dataframe_to_dict(df: pd.DataFrame) -> List[dict]: diff --git a/pytabular/object.py b/pytabular/object.py index 1eae4dd..07ca11e 100644 --- a/pytabular/object.py +++ b/pytabular/object.py @@ -99,7 +99,7 @@ def __iadd__(self, obj): self.__init__(self._objects) return self - def Find(self, object_str: str): + def find(self, object_str: str): """Finds any or all `PyObject` inside of `PyObjects` that match the `object_str`. It is case insensitive. diff --git a/pytabular/partition.py b/pytabular/partition.py index 395d469..d58b3cd 100644 --- a/pytabular/partition.py +++ b/pytabular/partition.py @@ -29,10 +29,10 @@ def __init__(self, object, table) -> None: "SourceType", str(self._object.SourceType), end_section=True ) self._display.add_row( - "RefreshedTime", self.Last_Refresh().strftime("%m/%d/%Y, %H:%M:%S") + "RefreshedTime", self.last_refresh().strftime("%m/%d/%Y, %H:%M:%S") ) - def Last_Refresh(self) -> datetime: + def last_refresh(self) -> datetime: """Queries `RefreshedTime` attribute in the partition and converts from C# Ticks to Python datetime Returns: @@ -40,13 +40,13 @@ def Last_Refresh(self) -> datetime: """ return ticks_to_datetime(self.RefreshedTime.Ticks) - def Refresh(self, *args, **kwargs) -> pd.DataFrame: + def refresh(self, *args, **kwargs) -> pd.DataFrame: """Same method from Model Refresh, you can pass through any extra parameters. For example: - `Tabular().Tables['Table Name'].Partitions[0].Refresh(Tracing = True)` + `Tabular().Tables['Table Name'].Partitions[0].refresh(Tracing = True)` Returns: pd.DataFrame: Returns pandas dataframe with some refresh details """ - return self.Table.Model.Refresh(self, *args, **kwargs) + return self.Table.Model.refresh(self, *args, **kwargs) class PyPartitions(PyObjects): diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 553935d..12545f9 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -43,7 +43,7 @@ class Tabular(PyObject): """Tabular Class to perform operations: [Microsoft.AnalysisServices.Tabular](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular?view=analysisservices-dotnet). You can use this class as your main way to interact with your model. Args: - CONNECTION_STR (str): Valid [Connection String](https://docs.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services?view=asallproducts-allversions) for connecting to a Tabular Model. + connection_str (str): Valid [Connection String](https://docs.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services?view=asallproducts-allversions) for connecting to a Tabular Model. Attributes: Server (Server): See [Server MS Docs](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.server?view=analysisservices-dotnet). Catalog (str): Name of Database. See [Catalog MS Docs](https://learn.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.connectioninfo.catalog?view=analysisservices-dotnet#microsoft-analysisservices-connectioninfo-catalog). @@ -55,12 +55,12 @@ class Tabular(PyObject): Measures (List[Measure]): Easy access list of measures from model. See [Measure MS Docs](https://learn.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.table.measures?view=analysisservices-dotnet#microsoft-analysisservices-tabular-table-measures). """ - def __init__(self, CONNECTION_STR: str): + def __init__(self, connection_str: str): # Connecting to model... logger.debug("Initializing Tabular Class") self.Server = Server() - self.Server.Connect(CONNECTION_STR) + self.Server.Connect(connection_str) logger.info(f"Connected to Server - {self.Server.Name}") self.Catalog = self.Server.ConnectionInfo.Catalog logger.debug(f"Received Catalog - {self.Catalog}") @@ -84,7 +84,7 @@ def __init__(self, CONNECTION_STR: str): self.PyRefresh = PyRefresh # Build PyObjects - self.Reload_Model_Info() + self.reload_model_info() # Run subclass init super().__init__(self.Model) @@ -107,10 +107,10 @@ def __init__(self, CONNECTION_STR: str): # Finished and registering disconnect logger.debug("Class Initialization Completed") logger.debug("Registering Disconnect on Termination...") - atexit.register(self.Disconnect) + atexit.register(self.disconnect) - def Reload_Model_Info(self) -> bool: - """Runs on __init__ iterates through details, can be called after any model changes. Called in SaveChanges() + def reload_model_info(self) -> bool: + """Runs on __init__ iterates through details, can be called after any model changes. Called in save_changes() Returns: bool: True if successful @@ -144,7 +144,7 @@ def Reload_Model_Info(self) -> bool: ) return True - def Is_Process(self) -> bool: + def is_process(self) -> bool: """Run method to check if Processing is occurring. Will query DMV $SYSTEM.DISCOVER_JOBS to see if any processing is happening. Returns: @@ -153,18 +153,18 @@ def Is_Process(self) -> bool: _jobs_df = self.Query("select * from $SYSTEM.DISCOVER_JOBS") return len(_jobs_df[_jobs_df["JOB_DESCRIPTION"] == "Process"]) > 0 - def Disconnect(self) -> None: + def disconnect(self) -> None: """Disconnects from Model""" logger.info(f"Disconnecting from - {self.Server.Name}") - atexit.unregister(self.Disconnect) + atexit.unregister(self.disconnect) return self.Server.Disconnect() - def Reconnect(self) -> None: + def reconnect(self) -> None: """Reconnects to Model""" logger.info(f"Reconnecting to {self.Server.Name}") return self.Server.Reconnect() - def Refresh(self, *args, **kwargs) -> pd.DataFrame: + def refresh(self, *args, **kwargs) -> pd.DataFrame: """PyRefresh Class to handle refreshes of model. Args: @@ -178,69 +178,69 @@ def Refresh(self, *args, **kwargs) -> pd.DataFrame: Returns: pd.DataFrame """ - return self.PyRefresh(self, *args, **kwargs).Run() + return self.PyRefresh(self, *args, **kwargs).run() - def SaveChanges(self): + def save_changes(self): """Called after refreshes or any model changes. Currently will return a named tuple of all changes detected. However a ton of room for improvement here. """ if self.Server.Connected is False: - self.Reconnect() + self.reconnect() - def property_changes(Property_Changes): + def property_changes(property_changes_var): """ Returns any property changes. """ - Property_Change = namedtuple( - "Property_Change", - "New_Value Object Original_Value Property_Name Property_Type", + property_change = namedtuple( + "property_change", + "new_value object original_value property_name property_type", ) return [ - Property_Change( + property_change( change.NewValue, change.Object, change.OriginalValue, change.PropertyName, change.PropertyType, ) - for change in Property_Changes.GetEnumerator() + for change in property_changes_var.GetEnumerator() ] - logger.info("Executing SaveChanges()...") - Model_Save_Results = self.Model.SaveChanges() - if isinstance(Model_Save_Results.Impact, type(None)): + logger.info("Executing save_changes()...") + model_save_results = self.Model.SaveChanges() + if isinstance(model_save_results.Impact, type(None)): logger.warning(f"No changes detected on save for {self.Server.Name}") return None else: - Property_Changes = Model_Save_Results.Impact.PropertyChanges - Added_Objects = Model_Save_Results.Impact.AddedObjects - Added_Subtree_Roots = Model_Save_Results.Impact.AddedSubtreeRoots - Removed_Objects = Model_Save_Results.Impact.RemovedObjects - Removed_Subtree_Roots = Model_Save_Results.Impact.RemovedSubtreeRoots - Xmla_Results = Model_Save_Results.XmlaResults - Changes = namedtuple( - "Changes", - "Property_Changes Added_Objects Added_Subtree_Roots Removed_Objects Removed_Subtree_Roots Xmla_Results", + property_changes_var = model_save_results.Impact.PropertyChanges + added_objects = model_save_results.Impact.AddedObjects + added_subtree_roots = model_save_results.Impact.AddedSubtreeRoots + removed_objects = model_save_results.Impact.RemovedObjects + removed_subtree_roots = model_save_results.Impact.RemovedSubtreeRoots + xmla_results = model_save_results.XmlaResults + changes = namedtuple( + "changes", + "property_changes added_objects added_subtree_roots removed_objects removed_subtree_Roots xmla_results", ) [ - property_changes(Property_Changes), - Added_Objects, - Added_Subtree_Roots, - Removed_Objects, - Removed_Subtree_Roots, - Xmla_Results, + property_changes(property_changes_var), + added_objects, + added_subtree_roots, + removed_objects, + removed_subtree_roots, + xmla_results, ] - self.Reload_Model_Info() - return Changes( - property_changes(Property_Changes), - Added_Objects, - Added_Subtree_Roots, - Removed_Objects, - Removed_Subtree_Roots, - Xmla_Results, + self.reload_model_info() + return changes( + property_changes(property_changes_var), + added_objects, + added_subtree_roots, + removed_objects, + removed_subtree_roots, + xmla_results, ) - def Backup_Table(self, table_str: str) -> bool: + def backup_table(self, table_str: str) -> bool: """Will be removed. This is experimental with no written pytest for it. Backs up table in memory, brings with it measures, columns, hierarchies, relationships, roles, etc. It will add suffix '_backup' to all objects. @@ -254,7 +254,7 @@ def Backup_Table(self, table_str: str) -> bool: """ logger.info("Backup Beginning...") logger.debug(f"Cloning {table_str}") - table = self.Model.Tables.Find(table_str).Clone() + table = self.Model.Tables.find(table_str).Clone() logger.info("Beginning Renames") def rename(items): @@ -289,11 +289,11 @@ def rename(items): logger.debug(f"Renaming - {relationship.Name}") if relationship.ToTable.Name == remove_suffix(table.Name, "_backup"): relationship.set_ToColumn( - table.Columns.Find(f"{relationship.ToColumn.Name}_backup") + table.Columns.find(f"{relationship.ToColumn.Name}_backup") ) elif relationship.FromTable.Name == remove_suffix(table.Name, "_backup"): relationship.set_FromColumn( - table.Columns.Find(f"{relationship.FromColumn.Name}_backup") + table.Columns.find(f"{relationship.FromColumn.Name}_backup") ) logger.debug(f"Adding {relationship.Name} to {self.Model.Name}") self.Model.Relationships.Add(relationship) @@ -326,7 +326,7 @@ def clone_role_permissions(): f"Column - {column.Name} copying permissions to clone..." ) column.set_Column( - self.Model.Tables.Find(table.Name).Columns.Find( + self.Model.Tables.find(table.Name).Columns.find( f"{column.Name}_backup" ) ) @@ -336,13 +336,13 @@ def clone_role_permissions(): clone_role_permissions() logger.info(f"Refreshing Clone... {table.Name}") - self.Reload_Model_Info() - self.Refresh(table.Name, default_row_count_check=False) + self.reload_model_info() + self.refresh(table.Name, default_row_count_check=False) logger.info(f"Updating Model {self.Model.Name}") - self.SaveChanges() + self.save_changes() return True - def Revert_Table(self, table_str: str) -> bool: + def revert_table(self, table_str: str) -> bool: """Will be removed. This is experimental with no written pytest for it. This is used in conjunction with Backup_Table(). It will take the 'TableName_backup' and replace with the original. Example scenario -> @@ -359,9 +359,9 @@ def Revert_Table(self, table_str: str) -> bool: """ logger.info(f"Beginning Revert for {table_str}") logger.debug(f"Finding original {table_str}") - main = self.Model.Tables.Find(table_str) + main = self.Model.Tables.find(table_str) logger.debug(f"Finding backup {table_str}") - backup = self.Model.Tables.Find(f"{table_str}_backup") + backup = self.Model.Tables.find(f"{table_str}_backup") logger.debug("Finding original relationships") main_relationships = [ relationship @@ -421,7 +421,7 @@ def dename(items): logger.debug(f"Removing Suffix for {item.Name}") item.RequestRename(remove_suffix(item.Name, "_backup")) logger.debug(f"Saving Changes... for {item.Name}") - self.Model.SaveChanges() + self.Model.save_changes() logger.info("Name changes for Columns...") dename( @@ -441,11 +441,11 @@ def dename(items): dename(backup_relationships) logger.info("Name changes for Backup Table...") backup.RequestRename(remove_suffix(backup.Name, "_backup")) - self.SaveChanges() + self.save_changes() return True - def Query( - self, Query_Str: str, Effective_User: str = None + def query( + self, query_str: str, effective_user: str = None ) -> Union[pd.DataFrame, str, int]: """Executes Query on Model and Returns Results in Pandas DataFrame @@ -458,30 +458,30 @@ def Query( Returns: pd.DataFrame: Returns dataframe with results """ - if Effective_User is None: - return self.Adomd.Query(Query_Str) + if effective_user is None: + return self.Adomd.query(query_str) try: # This needs a public model with effective users to properly test - conn = self.Effective_Users[Effective_User] - logger.debug(f"Effective user found querying as... {Effective_User}") + conn = self.effective_users[effective_user] + logger.debug(f"Effective user found querying as... {effective_user}") except Exception: - logger.debug(f"Creating new connection with {Effective_User}") - conn = Connection(self.Server, Effective_User=Effective_User) - self.Effective_Users[Effective_User] = conn + logger.debug(f"Creating new connection with {effective_user}") + conn = Connection(self.Server, effective_user=effective_user) + self.effective_Users[effective_user] = conn - return conn.Query(Query_Str) + return conn.query(query_str) - def Analyze_BPA( - self, Tabular_Editor_Exe: str, Best_Practice_Analyzer: str + def analyze_bpa( + self, tabular_editor_exe: str, best_practice_analyzer: str ) -> List[str]: """Takes your Tabular Model and performs TE2s BPA. Runs through Command line. [Tabular Editor BPA](https://docs.tabulareditor.com/te2/Best-Practice-Analyzer.html) [Tabular Editor Command Line Options](https://docs.tabulareditor.com/te2/Command-line-Options.html) Args: - Tabular_Editor_Exe (str): TE2 Exe File path. Feel free to use class TE2().EXE_Path or provide your own. - Best_Practice_Analyzer (str): BPA json file path. Feel free to use class BPA().Location or provide your own. Defualts to https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json + tabular_editor_exe (str): TE2 Exe File path. Feel free to use class TE2().EXE_Path or provide your own. + best_practice_analyzer (str): BPA json file path. Feel free to use class BPA().Location or provide your own. Defualts to https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json Returns: List[str]: Assuming no failure, will return list of BPA violations. Else will return error from command line. @@ -489,7 +489,7 @@ def Analyze_BPA( logger.debug("Beginning request to talk with TE2 & Find BPA...") bim_file_location = f"{os.getcwd()}\\Model.bim" atexit.register(remove_file, bim_file_location) - cmd = f'{Tabular_Editor_Exe} "Provider=MSOLAP;{self.Adomd.ConnectionString}" {self.Database.Name} -B "{bim_file_location}" -A {Best_Practice_Analyzer} -V/?' + cmd = f'{tabular_editor_exe} "Provider=MSOLAP;{self.Adomd.ConnectionString}" {self.Database.Name} -B "{bim_file_location}" -A {best_practice_analyzer} -V/?' logger.debug("Command Generated") logger.debug("Submitting Command...") sp = subprocess.Popen( @@ -507,7 +507,7 @@ def Analyze_BPA( output for output in raw_output.split("\n") if "violates rule" in output ] - def Create_Table(self, df: pd.DataFrame, table_name: str) -> bool: + def create_table(self, df: pd.DataFrame, table_name: str) -> bool: """Creates tables from pd.DataFrame as an M-Partition. So will convert the dataframe to M-Partition logic via the M query table constructor. Runs refresh and will update model. @@ -546,7 +546,7 @@ def Create_Table(self, df: pd.DataFrame, table_name: str) -> bool: f"Adding table: {new_table.Name} to {self.Server.Name}::{self.Database.Name}::{self.Model.Name}" ) self.Model.Tables.Add(new_table) - self.SaveChanges() - self.Reload_Model_Info() - self.Refresh(new_table.Name) + self.save_changes() + self.reload_model_info() + self.refresh(new_table.Name) return True diff --git a/pytabular/query.py b/pytabular/query.py index c262c77..ea5a3cf 100644 --- a/pytabular/query.py +++ b/pytabular/query.py @@ -21,24 +21,24 @@ class Connection(AdomdConnection): AdomdConnection (_type_): _description_ """ - def __init__(self, Server, Effective_User=None) -> None: + def __init__(self, server, effective_user=None) -> None: super().__init__() - if Server.ConnectionInfo.Password is None: - connection_string = Server.ConnectionString + if server.ConnectionInfo.Password is None: + connection_string = server.ConnectionString else: connection_string = ( - f"{Server.ConnectionString}Password='{Server.ConnectionInfo.Password}'" + f"{server.ConnectionString}Password='{server.ConnectionInfo.Password}'" ) logger.debug(f"{connection_string}") - if Effective_User is not None: - connection_string += f";EffectiveUserName={Effective_User}" + if effective_user is not None: + connection_string += f";EffectiveUserName={effective_user}" self.ConnectionString = connection_string - def Query(self, Query_Str: str) -> Union[pd.DataFrame, str, int]: - """Executes Query on Model and Returns Results in Pandas DataFrame + def query(self, query_str: str) -> Union[pd.DataFrame, str, int]: + """Executes Query on Model and Returns results in Pandas DataFrame Args: - Query_Str (str): Dax Query. Note, needs full syntax (ex: EVALUATE). See (DAX Queries)[https://docs.microsoft.com/en-us/dax/dax-queries]. + query_str (str): Dax Query. Note, needs full syntax (ex: EVALUATE). See (DAX Queries)[https://docs.microsoft.com/en-us/dax/dax-queries]. Will check if query string is a file. If it is, then it will perform a query on whatever is read from the file. It is also possible to query DMV. For example. Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES"). See (DMVs)[https://docs.microsoft.com/en-us/analysis-services/instances/use-dynamic-management-views-dmvs-to-monitor-analysis-services?view=asallproducts-allversions] @@ -46,16 +46,16 @@ def Query(self, Query_Str: str) -> Union[pd.DataFrame, str, int]: pd.DataFrame: Returns dataframe with results """ try: - is_file = os.path.isfile(Query_Str) + is_file = os.path.isfile(query_str) except Exception: is_file = False if is_file: logger.debug( - f"File path detected, reading file... -> {Query_Str}", + f"File path detected, reading file... -> {query_str}", ) - with open(Query_Str, "r") as file: - Query_Str = str(file.read()) + with open(query_str, "r") as file: + query_str = str(file.read()) if str(self.get_State()) != "Open": # Works for now, need to update to handle different types of conneciton properties @@ -65,23 +65,23 @@ def Query(self, Query_Str: str) -> Union[pd.DataFrame, str, int]: logger.info(f"Connected! Session ID - {self.SessionID}") logger.debug("Querying Model...") - logger.debug(Query_Str) - Query = AdomdCommand(Query_Str, self).ExecuteReader() - Column_Headers = [ - (index, Query.GetName(index)) for index in range(0, Query.FieldCount) + logger.debug(query_str) + query = AdomdCommand(query_str, self).ExecuteReader() + column_headers = [ + (index, query.GetName(index)) for index in range(0, query.FieldCount) ] - Results = list() - while Query.Read(): - Results.append( + results = list() + while query.Read(): + results.append( [ - get_value_to_df(Query, index) - for index in range(0, len(Column_Headers)) + get_value_to_df(query, index) + for index in range(0, len(column_headers)) ] ) - Query.Close() + query.Close() logger.debug("Data retrieved... reading...") - df = pd.DataFrame(Results, columns=[value for _, value in Column_Headers]) + df = pd.DataFrame(results, columns=[value for _, value in column_headers]) if len(df) == 1 and len(df.columns) == 1: return df.iloc[0][df.columns[0]] return df diff --git a/pytabular/refresh.py b/pytabular/refresh.py index b46f51f..1da3a3b 100644 --- a/pytabular/refresh.py +++ b/pytabular/refresh.py @@ -1,7 +1,7 @@ """ `refresh.py` is the main file to handle all the components of refreshing your model. """ -from tabular_tracing import Refresh_Trace, Base_Trace +from tabular_tracing import RefreshTrace, BaseTrace import logging from Microsoft.AnalysisServices.Tabular import ( RefreshType, @@ -18,14 +18,14 @@ logger = logging.getLogger("PyTabular") -class Refresh_Check(ABC): - """`Refresh_Check` is an assertion you run after your refreshes. +class RefreshCheck(ABC): + """`RefreshCheck` is an assertion you run after your refreshes. It will run the given `function` before and after refreshes, then run the assertion of before and after. The default given in a refresh is to check row count. It will check row count before, and row count after. Then fail if row count after is zero. Args: - name (str): Name of Refresh_Check + name (str): Name of RefreshCheck function (function): Function to run before and after refresh. assertion (function): Functino that takes a `pre` and `post` argument to output True or False. """ @@ -40,7 +40,7 @@ def __init__(self, name: str, function, assertion=None) -> None: def __repr__(self) -> str: """ - `__repre__` that returns details on `Refresh_Check`. + `__repre__` that returns details on `RefreshCheck`. """ return f"{self.name} - {self.pre} - {self.post} - {str(self.function)}" @@ -128,25 +128,25 @@ def _check(self, stage: str): logger.info(f"{stage}-Check results for {self.name} - {results}") return results - def Pre_Check(self): + def pre_check(self): """Runs `self._check("Pre")`""" self._check("Pre") pass - def Post_Check(self): - """Runs `self._check("Post")` then `self.Assertion()`""" + def post_check(self): + """Runs `self._check("Post")` then `self.assertion_run()`""" self._check("Post") - self.Assertion() + self.assertion_run() pass - def Assertion(self): + def assertion_run(self): """Runs the given self.assertion function with `self.pre` and `self.post`. - So, `self.assertion(self.pre, self.post)`. + So, `self.assertion_run(self.pre, self.post)`. """ if self.assertion is None: logger.debug("Skipping assertion none given") else: - test = self.assertion(self.pre, self.post) + test = self.assertion_run(self.pre, self.post) assert_str = f"Test {self.name} - {test} - Pre Results - {self.pre} | Post Results {self.post}" if test: logger.info(assert_str) @@ -157,37 +157,37 @@ def Assertion(self): ), f"Test failed! Pre Results - {self.pre} | Post Results {self.post}" -class Refresh_Check_Collection: - """Groups together your `Refresh_Checks` to handle multiple types of checks in a single refresh.""" +class RefreshCheckCollection: + """Groups together your `RefreshChecks` to handle multiple types of checks in a single refresh.""" - def __init__(self, refresh_checks: Refresh_Check = []) -> None: - self._refresh_checks = refresh_checks + def __init__(self, refresh_checks: RefreshCheck = []) -> None: + self._refreshchecks = refresh_checks pass def __iter__(self): - """Basic iteration through the different `Refresh_Check`(s).""" - for refresh_check in self._refresh_checks: + """Basic iteration through the different `RefreshCheck`(s).""" + for refresh_check in self._refreshchecks: yield refresh_check - def add_refresh_check(self, refresh_check: Refresh_Check): - """Add a Refresh_Check + def add_refresh_check(self, refresh_check: RefreshCheck): + """Add a RefreshCheck Args: - refresh_check (Refresh_Check): `Refresh_Check` class. + RefreshCheck (RefreshCheck): `RefreshCheck` class. """ - self._refresh_checks.append(refresh_check) + self._refreshchecks.append(refresh_check) - def remove_refresh_check(self, refresh_check: Refresh_Check): - """Remove a Refresh_Check + def remove_refresh_check(self, refresh_check: RefreshCheck): + """Remove a RefreshCheck Args: - refresh_check (Refresh_Check): `Refresh_Check` class. + RefreshCheck (RefreshCheck): `RefreshCheck` class. """ - self._refresh_checks.remove(refresh_check) + self._refreshchecks.remove(refresh_check) def clear_refresh_checks(self): """Clear Refresh Checks.""" - self._refresh_checks.clear() + self._refreshchecks.clear() class PyRefresh: @@ -196,8 +196,8 @@ class PyRefresh: Args: model (Tabular): Main Tabular Class object (Union[str, PyTable, PyPartition, Dict[str, Any]]): Designed to handle a few different ways of selecting a refresh. Can be a string of 'Table Name' or dict of {'Table Name': 'Partition Name'} or even some combination with the actual PyTable and PyPartition classes. - trace (Base_Trace, optional): Set to `None` if no Tracing is desired, otherwise you can use default trace or create your own. Defaults to Refresh_Trace. - refresh_checks (Refresh_Check_Collection, optional): Add your `Refresh_Check`'s into a `Refresh_Check_Collection`. Defaults to Refresh_Check_Collection(). + trace (BaseTrace, optional): Set to `None` if no Tracing is desired, otherwise you can use default trace or create your own. Defaults to RefreshTrace. + RefreshChecks (RefreshCheckCollection, optional): Add your `RefreshCheck`'s into a `RefreshCheckCollection`. Defaults to RefreshCheckCollection(). default_row_count_check (bool, optional): Quick built in check will fail the refresh if post check row count is zero. Defaults to True. refresh_type (RefreshType, optional): Input RefreshType desired. Defaults to RefreshType.Full. """ @@ -206,8 +206,8 @@ def __init__( self, model, object: Union[str, PyTable, PyPartition, Dict[str, Any]], - trace: Base_Trace = Refresh_Trace, - refresh_checks: Refresh_Check_Collection = Refresh_Check_Collection(), + trace: BaseTrace = RefreshTrace, + refresh_checks: RefreshCheckCollection = RefreshCheckCollection(), default_row_count_check: bool = True, refresh_type: RefreshType = RefreshType.Full, ) -> None: @@ -224,8 +224,8 @@ def __init__( pass def _pre_checks(self): - """Checks if any `Base_Trace` classes are needed from `Tabular_Tracing.py`. - Then checks if any `Refresh_Checks` are needed, along with the default `Row_Count` check. + """Checks if any `BaseTrace` classes are needed from `Tabular_Tracing.py`. + Then checks if any `RefreshChecks` are needed, along with the default `Row_Count` check. """ logger.debug("Running Pre-checks") if self.trace is not None: @@ -247,27 +247,27 @@ def row_count_assertion(pre, post): return post > 0 for table in set(tables): - check = Refresh_Check( - f"{table.Name} Row Count", table.Row_Count, row_count_assertion + check = RefreshCheck( + f"{table.Name} Row Count", table.row_count, row_count_assertion ) self._checks.add_refresh_check(check) for check in self._checks: - check.Pre_Check() + check.pre_check() pass def _post_checks(self): """If traces are running it Stops and Drops it. - Runs through any `Post_Checks()` in `Refresh_Checks`. + Runs through any `post_checks()` in `RefreshChecks`. """ if self.trace is not None: - self.trace.Stop() - self.trace.Drop() + self.trace.stop() + self.trace.drop() for check in self._checks: - check.Post_Check() + check.post_check() self._checks.remove_refresh_check(check) pass - def _get_trace(self) -> Base_Trace: + def _get_trace(self) -> BaseTrace: """Creates Trace and creates it in model.""" return self.trace(self.model) @@ -338,7 +338,7 @@ def _request_refresh(self, object): else: [self._request_refresh(obj) for obj in object] - def _refresh_report(self, Property_Changes) -> pd.DataFrame: + def _refresh_report(self, property_changes) -> pd.DataFrame: """Builds a DataFrame that displays details on the refresh. Args: @@ -349,7 +349,7 @@ def _refresh_report(self, Property_Changes) -> pd.DataFrame: """ logger.debug("Running Refresh Report...") refresh_data = [] - for property_change in Property_Changes: + for property_change in property_changes: if ( isinstance(property_change.Object, Partition) and property_change.Property_Name == "RefreshedTime" @@ -367,19 +367,19 @@ def _refresh_report(self, Property_Changes) -> pd.DataFrame: refresh_data, columns=["Table", "Partition", "Refreshed Time"] ) - def Run(self) -> pd.DataFrame: + def run(self) -> pd.DataFrame: """Brings it all together. When ready, executes all the pre checks. Then refreshes. Then runs all the post checks. """ if self.model.Server.Connected is False: logger.info(f"{self.Server.Name} - Reconnecting...") - self.model.Server.Reconnect() + self.model.Server.reconnect() if self.trace is not None: - self.trace.Start() + self.trace.start() - save_changes = self.model.SaveChanges() + save_changes = self.model.save_changes() self._post_checks() - return self._refresh_report(save_changes.Property_Changes) + return self._refresh_report(save_changes.property_changes) diff --git a/pytabular/relationship.py b/pytabular/relationship.py index 718ebf6..435f36f 100644 --- a/pytabular/relationship.py +++ b/pytabular/relationship.py @@ -52,7 +52,7 @@ class PyRelationships(PyObjects): def __init__(self, objects) -> None: super().__init__(objects) - def Related(self, object: Union[PyTable, str]) -> PyTables: + def related(self, object: Union[PyTable, str]) -> PyTables: """Finds related tables of a given table. Args: diff --git a/pytabular/table.py b/pytabular/table.py index d273ba2..412d047 100644 --- a/pytabular/table.py +++ b/pytabular/table.py @@ -57,24 +57,24 @@ def __init__(self, object, model) -> None: ), ) - def Row_Count(self) -> int: + def row_count(self) -> int: """Method to return count of rows. Simple Dax Query: `EVALUATE {COUNTROWS('Table Name')}` Returns: int: Number of rows using [COUNTROWS](https://learn.microsoft.com/en-us/dax/countrows-function-dax). """ - return self.Model.Adomd.Query(f"EVALUATE {{COUNTROWS('{self.Name}')}}") + return self.Model.Adomd.query(f"EVALUATE {{COUNTROWS('{self.Name}')}}") - def Refresh(self, *args, **kwargs) -> pd.DataFrame: + def refresh(self, *args, **kwargs) -> pd.DataFrame: """Same method from Model Refresh, you can pass through any extra parameters. For example: `Tabular().Tables['Table Name'].Refresh(Tracing = True)` Returns: pd.DataFrame: Returns pandas dataframe with some refresh details """ - return self.Model.Refresh(self, *args, **kwargs) + return self.Model.refresh(self, *args, **kwargs) - def Last_Refresh(self) -> datetime: + def last_refresh(self) -> datetime: """Will query each partition for the last refresh time then select the max Returns: @@ -85,9 +85,9 @@ def Last_Refresh(self) -> datetime: ] return max(partition_refreshes) - def Related(self): + def related(self): """Returns tables with a relationship with the table in question.""" - return self.Model.Relationships.Related(self) + return self.Model.Relationships.related(self) class PyTables(PyObjects): @@ -101,12 +101,12 @@ class PyTables(PyObjects): def __init__(self, objects) -> None: super().__init__(objects) - def Refresh(self, *args, **kwargs): + def refresh(self, *args, **kwargs): """Refreshes all `PyTable`(s) in class.""" model = self._objects[0].Model - return model.Refresh(self, *args, **kwargs) + return model.refresh(self, *args, **kwargs) - def Query_All(self, query_function: str = "COUNTROWS(_)") -> pd.DataFrame: + def query_all(self, query_function: str = "COUNTROWS(_)") -> pd.DataFrame: """This will dynamically create a query to pull all tables from the model and run the query function. It will replace the _ with the table to run. @@ -125,9 +125,9 @@ def Query_All(self, query_function: str = "COUNTROWS(_)") -> pd.DataFrame: dax_table_identifier = f"'{table_name}'" query_str += f"ROW(\"Table\",\"{table_name}\",\"{query_function}\",{query_function.replace('_',dax_table_identifier)}),\n" query_str = f"{query_str[:-2]})" - return self[0].Model.Query(query_str) + return self[0].Model.query(query_str) - def Find_Zero_Rows(self): + def find_zero_rows(self): """Returns PyTables class of tables with zero rows queried.""" query_function: str = "COUNTROWS(_)" df = self.Query_All(query_function) @@ -137,7 +137,7 @@ def Find_Zero_Rows(self): tables = [self[name] for name in table_names] return self.__class__(tables) - def Last_Refresh(self, group_partition: bool = True) -> pd.DataFrame: + def last_refresh(self, group_partition: bool = True) -> pd.DataFrame: """Returns pd.DataFrame of tables with their latest refresh time. Optional 'group_partition' variable, default is True. If False an extra column will be include to have the last refresh time to the grain of the partition diff --git a/pytabular/tabular_editor.py b/pytabular/tabular_editor.py index 186d4df..0deb448 100644 --- a/pytabular/tabular_editor.py +++ b/pytabular/tabular_editor.py @@ -5,55 +5,55 @@ import logging import os import requests as r -import zipfile as Z +import zipfile as z import atexit from logic_utils import remove_folder_and_contents logger = logging.getLogger("PyTabular") -def Download_Tabular_Editor( - Download_Location: str = "https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip", - Folder: str = "Tabular_Editor_2", - Auto_Remove=True, +def download_tabular_editor( + download_location: str = "https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip", + folder: str = "Tabular_Editor_2", + auto_remove=True, ) -> str: """Runs a request.get() to retrieve the zip file from web. Will unzip response and store in directory. Will also register the removal of the new directory and files when exiting program. Args: - Download_Location (str, optional): File path for zip of Tabular Editor 2. Defaults to [Tabular Editor 2 Github Zip Location]'https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip'. - Folder (str, optional): New Folder Location. Defaults to 'Tabular_Editor_2'. - Auto_Remove (bool, optional): Boolean to determine auto removal of files once script exits. Defaults to True. + download_location (str, optional): File path for zip of Tabular Editor 2. Defaults to [Tabular Editor 2 Github Zip Location]'https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip'. + folder (str, optional): New Folder Location. Defaults to 'Tabular_Editor_2'. + auto_remove (bool, optional): Boolean to determine auto removal of files once script exits. Defaults to True. Returns: str: File path of TabularEditor.exe """ logger.info("Downloading Tabular Editor 2...") - logger.info(f"From... {Download_Location}") - folder_location = os.path.join(os.getcwd(), Folder) - response = r.get(Download_Location) - file_location = f"{os.getcwd()}\\{Download_Location.split('/')[-1]}" + logger.info(f"From... {download_location}") + folder_location = os.path.join(os.getcwd(), folder) + response = r.get(download_location) + file_location = f"{os.getcwd()}\\{download_location.split('/')[-1]}" with open(file_location, "wb") as te2_zip: te2_zip.write(response.content) - with Z.ZipFile(file_location) as zipper: + with z.ZipFile(file_location) as zipper: zipper.extractall(path=folder_location) logger.debug("Removing Zip File...") os.remove(file_location) logger.info(f"Tabular Editor Downloaded and Extracted to {folder_location}") - if Auto_Remove: + if auto_remove: logger.debug(f"Registering removal on termination... For {folder_location}") atexit.register(remove_folder_and_contents, folder_location) return f"{folder_location}\\TabularEditor.exe" -class Tabular_Editor: +class TabularEditor: """Setting Tabular_Editor Class for future work. - Mainly runs `Download_Tabular_Editor()` + Mainly runs `download_tabular_editor()` """ - def __init__(self, EXE_File_Path: str = "Default") -> None: - logger.debug(f"Initializing Tabular Editor Class:: {EXE_File_Path}") - if EXE_File_Path == "Default": - self.EXE: str = Download_Tabular_Editor() + def __init__(self, exe_file_path: str = "Default") -> None: + logger.debug(f"Initializing Tabular Editor Class:: {exe_file_path}") + if exe_file_path == "Default": + self.exe: str = download_tabular_editor() else: - self.EXE: str = EXE_File_Path + self.exe: str = exe_file_path pass diff --git a/pytabular/tabular_tracing.py b/pytabular/tabular_tracing.py index 0057294..985acd1 100644 --- a/pytabular/tabular_tracing.py +++ b/pytabular/tabular_tracing.py @@ -17,16 +17,16 @@ logger = logging.getLogger("PyTabular") -class Base_Trace: +class BaseTrace: """Generates Trace to be run on Server. This is the base class to customize the type of Trace you are looking for. [Server Traces](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.server.traces?view=analysisservices-dotnet#microsoft-analysisservices-tabular-server-traces) Args: - Tabular_Class (Tabular): Tabular Class. - Trace_Events (List[TraceEvent]): List of Trace Events. + tabular_class (Tabular): Tabular Class. + trace_events (List[TraceEvent]): List of Trace Events. [TraceEventClass](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.traceeventclass?view=analysisservices-dotnet) - Trace_Event_Columns (List[TraceColumn]): List of Trace Event Columns. + trace_event_columns (List[TraceColumn]): List of Trace Event Columns. [TraceEventColumn](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tracecolumn?view=analysisservices-dotnet) Handler (Callable): Function to call when Trace returns response. Input needs to be two arguments. @@ -37,10 +37,10 @@ class Base_Trace: def __init__( self, - Tabular_Class, - Trace_Events: List[TraceEvent], - Trace_Event_Columns: List[TraceColumn], - Handler: Callable, + tabular_class, + trace_events: List[TraceEvent], + trace_event_columns: List[TraceColumn], + handler: Callable, ) -> None: logger.debug("Trace Base Class initializing...") self.Name = "PyTabular_" + "".join( @@ -51,19 +51,19 @@ def __init__( self.ID = self.Name.replace("PyTabular_", "") self.Trace = Trace(self.Name, self.ID) logger.debug(f"Trace {self.Trace.Name} created...") - self.Tabular_Class = Tabular_Class - self.Event_Categories = self._Query_DMV_For_Event_Categories() + self.tabular_class = tabular_class + self.Event_Categories = self._query_dmv_for_event_categories() - self.Trace_Events = Trace_Events - self.Trace_Event_Columns = Trace_Event_Columns - self.Handler = Handler + self.trace_events = trace_events + self.trace_event_columns = trace_event_columns + self.handler = handler - self.Build() - self.Add() - self.Update() - atexit.register(self.Drop) + self.build() + self.add() + self.update() + atexit.register(self.drop) - def Build(self) -> bool: + def build(self) -> bool: """Run on initialization. This will take the inputed arguments for the class and attempt to build the Trace. @@ -72,9 +72,9 @@ def Build(self) -> bool: bool: True if successful """ logger.info(f"Building Trace {self.Name}") - TE = [TraceEvent(trace_event) for trace_event in self.Trace_Events] + te = [TraceEvent(trace_event) for trace_event in self.trace_events] logger.debug(f"Adding Events to... {self.Trace.Name}") - [self.Trace.get_Events().Add(te) for te in TE] + [self.Trace.get_Events().Add(t) for t in te] def add_column(trace_event, trace_event_column): """Adds the column to trace event.""" @@ -87,71 +87,71 @@ def add_column(trace_event, trace_event_column): logger.debug("Adding Trace Event Columns...") [ add_column(trace_event, trace_event_column) - for trace_event_column in self.Trace_Event_Columns - for trace_event in TE + for trace_event_column in self.trace_event_columns + for trace_event in te if str(trace_event_column.value__) in self.Event_Categories[str(trace_event.EventID.value__)] ] logger.debug("Adding Handler to Trace...") - self.Handler = TraceEventHandler(self.Handler) - self.Trace.OnEvent += self.Handler + self.handler = TraceEventHandler(self.handler) + self.Trace.OnEvent += self.handler return True - def Add(self) -> int: + def add(self) -> int: """Runs on initialization. Adds built Trace to the Server. Returns: int: Return int of placement in Server.Traces.get_Item(int) """ - logger.info(f"Adding {self.Name} to {self.Tabular_Class.Server.Name}") - return self.Tabular_Class.Server.Traces.Add(self.Trace) + logger.info(f"Adding {self.Name} to {self.tabular_class.Server.Name}") + return self.tabular_class.Server.Traces.Add(self.Trace) - def Update(self) -> None: + def update(self) -> None: """Runs on initialization. Syncs with Server. Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. """ - logger.info(f"Updating {self.Name} in {self.Tabular_Class.Server.Name}") - if self.Tabular_Class.Server.Connected is False: - self.Tabular_Class.Reconnect() + logger.info(f"Updating {self.Name} in {self.tabular_class.Server.Name}") + if self.tabular_class.Server.Connected is False: + self.tabular_class.reconnect() return self.Trace.Update() - def Start(self) -> None: + def start(self) -> None: """Call when you want to start the Trace Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. """ - logger.info(f"Starting {self.Name} in {self.Tabular_Class.Server.Name}") + logger.info(f"Starting {self.Name} in {self.tabular_class.Server.Name}") return self.Trace.Start() - def Stop(self) -> None: + def stop(self) -> None: """Call when you want to stop the Trace Returns: None: Returns None. Unless unsuccessful then it will return the error from Server. """ - logger.info(f"Stopping {self.Name} in {self.Tabular_Class.Server.Name}") + logger.info(f"Stopping {self.Name} in {self.tabular_class.Server.Name}") return self.Trace.Stop() - def Drop(self) -> None: + def drop(self) -> None: """Call when you want to drop the Trace Returns: None: Returns None. Unless unsuccessful, then it will return the error from Server. """ - logger.info(f"Dropping {self.Name} in {self.Tabular_Class.Server.Name}") - atexit.unregister(self.Drop) + logger.info(f"Dropping {self.Name} in {self.tabular_class.Server.Name}") + atexit.unregister(self.drop) return self.Trace.Drop() - def _Query_DMV_For_Event_Categories(self): + def _query_dmv_for_event_categories(self): """Internal use. Called during the building process to locate allowed columns for event categories. @@ -163,11 +163,11 @@ def _Query_DMV_For_Event_Categories(self): Returns: _type_: _description_ """ - Event_Categories = {} + event_categories = {} events = [] logger.debug("Querying DMV for columns rules...") logger.debug("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES") - df = self.Tabular_Class.Query( + df = self.tabular_class.query( "select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES" ) for index, row in df.iterrows(): @@ -179,17 +179,17 @@ def _Query_DMV_For_Event_Categories(self): else: events += [xml_data["EVENTCATEGORY"]["EVENTLIST"]["EVENT"]] for event in events: - Event_Categories[event["ID"]] = [ + event_categories[event["ID"]] = [ column["ID"] for column in event["EVENTCOLUMNLIST"]["EVENTCOLUMN"] ] - return Event_Categories + return event_categories def _refresh_handler(source, args): - """Default function called when `Refresh_Trace` is used. + """Default function called when `RefreshTrace` is used. It will log various steps of the refresh process. """ - TextData = args.TextData.replace("", "").replace("", "") + text_data = args.TextData.replace("", "").replace("", "") if ( args.EventClass == TraceEventClass.ProgressReportCurrent @@ -213,7 +213,7 @@ def _refresh_handler(source, args): ) elif args.EventSubclass == TraceEventSubclass.SwitchingDictionary: - logger.warning(f"{TextData}") + logger.warning(f"{text_data}") elif ( args.EventClass == TraceEventClass.ProgressReportBegin @@ -230,7 +230,7 @@ def _refresh_handler(source, args): TraceEventSubclass.ReadData, ] ): - logger.info(f"{TextData}") + logger.info(f"{text_data}") elif ( args.EventClass == TraceEventClass.ProgressReportEnd @@ -246,30 +246,30 @@ def _refresh_handler(source, args): TraceEventSubclass.AnalyzeEncodeData, ] ): - logger.info(f"{TextData}") + logger.info(f"{text_data}") else: - logger.debug(f"{args.EventClass}::{args.EventSubclass}::{TextData}") + logger.debug(f"{args.EventClass}::{args.EventSubclass}::{text_data}") -class Refresh_Trace(Base_Trace): - """Subclass of Base_Trace. For built-in Refresh Tracing. +class RefreshTrace(BaseTrace): + """Subclass of BaseTrace. For built-in Refresh Tracing. Run by default when refreshing tables or partitions. Args: - Base_Trace (Base_Trace): Base_Trace Class + BaseTrace (BaseTrace): BaseTrace Class """ def __init__( self, - Tabular_Class, - Trace_Events: List[TraceEvent] = [ + tabular_class, + trace_events: List[TraceEvent] = [ TraceEventClass.ProgressReportBegin, TraceEventClass.ProgressReportCurrent, TraceEventClass.ProgressReportEnd, TraceEventClass.ProgressReportError, ], - Trace_Event_Columns: List[TraceColumn] = [ + trace_event_columns: List[TraceColumn] = [ TraceColumn.EventSubclass, TraceColumn.CurrentTime, TraceColumn.ObjectName, @@ -280,9 +280,9 @@ def __init__( TraceColumn.EventClass, TraceColumn.ProgressTotal, ], - Handler: Callable = _refresh_handler, + handler: Callable = _refresh_handler, ) -> None: - super().__init__(Tabular_Class, Trace_Events, Trace_Event_Columns, Handler) + super().__init__(tabular_class, trace_events, trace_event_columns, handler) def _query_monitor_handler(source, args): @@ -305,19 +305,19 @@ def _query_monitor_handler(source, args): logger.debug(f"{args.TextData}") -class Query_Monitor(Base_Trace): - """Subclass of Base_Trace. For built-in Query Monitoring. +class QueryMonitor(BaseTrace): + """Subclass of BaseTrace. For built-in Query Monitoring. If you want to see full query text, set logger to debug. Args: - Base_Trace (Base_Trace): Base_Trace Class + BaseTrace (BaseTrace): BaseTrace Class """ def __init__( self, - Tabular_Class, - Trace_Events: List[TraceEvent] = [TraceEventClass.QueryEnd], - Trace_Event_Columns: List[TraceColumn] = [ + tabular_class, + trace_events: List[TraceEvent] = [TraceEventClass.QueryEnd], + trace_event_columns: List[TraceColumn] = [ TraceColumn.EventSubclass, TraceColumn.StartTime, TraceColumn.EndTime, @@ -329,7 +329,7 @@ def __init__( TraceColumn.ApplicationName, TraceColumn.TextData, ], - Handler: Callable = _query_monitor_handler, + handler: Callable = _query_monitor_handler, ) -> None: - super().__init__(Tabular_Class, Trace_Events, Trace_Event_Columns, Handler) + super().__init__(tabular_class, trace_events, trace_event_columns, handler) logger.info("Query text lives in DEBUG, adjust logging to see query text.") diff --git a/test/conftest.py b/test/conftest.py index 1654d5a..52c158b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,7 +6,7 @@ import pytabular as p -class testing_storage: +class TestStorage: query_trace = None documentation_class = None @@ -30,7 +30,7 @@ def remove_testing_table(model): def pytest_sessionstart(session): p.logger.info("Executing pytest setup...") remove_testing_table(local_pbix) - local_pbix.Create_Table(testingtabledf, testingtablename) + local_pbix.create_table(testingtabledf, testingtablename) return True diff --git a/test/test_3tabular.py b/test/test_3tabular.py index 3ed6de2..8a5df09 100644 --- a/test/test_3tabular.py +++ b/test/test_3tabular.py @@ -72,13 +72,13 @@ def test_nonetype_decimal_bug(model): @pytest.mark.parametrize("model", testing_parameters) -def test_Table_Last_Refresh_Times(model): +def test_table_last_refresh_times(model): """Really just testing the the function completes successfully and returns df""" assert isinstance(model.Tables.Last_Refresh(), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) -def test_Return_Zero_Row_Tables(model): +def test_return_zero_row_tables(model): """Testing that `Return_Zero_Row_Tables`""" assert isinstance(model.Tables.Find_Zero_Rows(), p.pytabular.PyTables) From ef7af2452fa8c3bf2fde04e95d71f03f506fd9dc Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sun, 5 Feb 2023 17:43:37 -0600 Subject: [PATCH 02/10] pep8-naming rework --- .github/workflows/flake8.yml | 1 + pytabular/column.py | 12 +++++----- pytabular/measure.py | 2 +- pytabular/pytabular.py | 10 ++++----- pytabular/refresh.py | 12 +++++----- pytabular/table.py | 6 ++--- test/conftest.py | 2 +- test/test_11document.py | 6 ++--- test/test_2object.py | 12 +++++----- test/test_3tabular.py | 42 +++++++++++++++++------------------ test/test_5column.py | 8 +++---- test/test_6table.py | 18 +++++++-------- test/test_7tabular_tracing.py | 32 +++++++++++++------------- test/test_8bpa.py | 8 +++---- test/test_9custom.py | 4 ++-- 15 files changed, 88 insertions(+), 87 deletions(-) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 428676d..658bedb 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -28,4 +28,5 @@ jobs: - uses: actions/setup-python@v2 - run: pip install --upgrade pip - run: pip install flake8 + - run: pip install pep8-naming - run: python3 -m flake8 --count \ No newline at end of file diff --git a/pytabular/column.py b/pytabular/column.py index aac5d9b..525755e 100644 --- a/pytabular/column.py +++ b/pytabular/column.py @@ -38,7 +38,7 @@ def __init__(self, object, table) -> None: def get_dependencies(self) -> pd.DataFrame: """Returns the dependant columns of a measure""" dmv_query = f"select * from $SYSTEM.DISCOVER_CALC_DEPENDENCY where [OBJECT] = '{self.Name}' and [TABLE] = '{self.Table.Name}'" - return self.Table.Model.Query(dmv_query) + return self.Table.Model.query(dmv_query) def get_sample_values(self, top_n: int = 3) -> pd.DataFrame: """Get sample values of column.""" @@ -58,7 +58,7 @@ def get_sample_values(self, top_n: int = 3) -> pd.DataFrame: ) ORDER BY {column_to_sample} """ - return self.Table.Model.Query(dax_query) + return self.Table.Model.query(dax_query) except Exception: # This is really tech debt anyways and should be replaced... dax_query = f""" @@ -71,7 +71,7 @@ def get_sample_values(self, top_n: int = 3) -> pd.DataFrame: ) ) """ - return self.Table.Model.Query(dax_query) + return self.Table.Model.query(dax_query) def distinct_count(self, no_blank=False) -> int: """Get [DISTINCTCOUNT](https://learn.microsoft.com/en-us/dax/distinctcount-function-dax) of Column. @@ -85,7 +85,7 @@ def distinct_count(self, no_blank=False) -> int: func = "DISTINCTCOUNT" if no_blank: func += "NOBLANK" - return self.Table.Model.Adomd.Query( + return self.Table.Model.Adomd.query( f"EVALUATE {{{func}('{self.Table.Name}'[{self.Name}])}}" ) @@ -95,7 +95,7 @@ def values(self) -> pd.DataFrame: Returns: pd.DataFrame: Single Column DataFrame of Values. """ - return self.Table.Model.Adomd.Query( + return self.Table.Model.Adomd.query( f"EVALUATE VALUES('{self.Table.Name}'[{self.Name}])" ) @@ -133,4 +133,4 @@ def query_all(self, query_function: str = "COUNTROWS(VALUES(_))") -> pd.DataFram dax_identifier = f"'{table_name}'[{column_name}]" query_str += f"ROW(\"Table\",\"{table_name}\",\"Column\",\"{column_name}\",\"{query_function}\",{query_function.replace('_',dax_identifier)}),\n" query_str = f"{query_str[:-2]})" - return self[0].Table.Model.Query(query_str) + return self[0].Table.Model.query(query_str) diff --git a/pytabular/measure.py b/pytabular/measure.py index 3999547..b5a0621 100644 --- a/pytabular/measure.py +++ b/pytabular/measure.py @@ -38,7 +38,7 @@ def get_dependencies(self) -> pd.DataFrame: """ dmv_query = f"select * from $SYSTEM.DISCOVER_CALC_DEPENDENCY where [OBJECT] = '{self.Name}' and [TABLE] = '{self.Table.Name}'" - return self.Table.Model.Query(dmv_query) + return self.Table.Model.query(dmv_query) class PyMeasures(PyObjects): diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 12545f9..5b87191 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -150,7 +150,7 @@ def is_process(self) -> bool: Returns: bool: True if DMV shows Process, False if not. """ - _jobs_df = self.Query("select * from $SYSTEM.DISCOVER_JOBS") + _jobs_df = self.query("select * from $SYSTEM.DISCOVER_JOBS") return len(_jobs_df[_jobs_df["JOB_DESCRIPTION"] == "Process"]) > 0 def disconnect(self) -> None: @@ -254,7 +254,7 @@ def backup_table(self, table_str: str) -> bool: """ logger.info("Backup Beginning...") logger.debug(f"Cloning {table_str}") - table = self.Model.Tables.find(table_str).Clone() + table = self.Model.Tables.Find(table_str).Clone() logger.info("Beginning Renames") def rename(items): @@ -359,9 +359,9 @@ def revert_table(self, table_str: str) -> bool: """ logger.info(f"Beginning Revert for {table_str}") logger.debug(f"Finding original {table_str}") - main = self.Model.Tables.find(table_str) + main = self.Tables.find(table_str)[0]._object logger.debug(f"Finding backup {table_str}") - backup = self.Model.Tables.find(f"{table_str}_backup") + backup = self.Tables.find(f"{table_str}_backup")[0]._object logger.debug("Finding original relationships") main_relationships = [ relationship @@ -421,7 +421,7 @@ def dename(items): logger.debug(f"Removing Suffix for {item.Name}") item.RequestRename(remove_suffix(item.Name, "_backup")) logger.debug(f"Saving Changes... for {item.Name}") - self.Model.save_changes() + self.save_changes() logger.info("Name changes for Columns...") dename( diff --git a/pytabular/refresh.py b/pytabular/refresh.py index 1da3a3b..78bc56d 100644 --- a/pytabular/refresh.py +++ b/pytabular/refresh.py @@ -146,7 +146,7 @@ def assertion_run(self): if self.assertion is None: logger.debug("Skipping assertion none given") else: - test = self.assertion_run(self.pre, self.post) + test = self.assertion(self.pre, self.post) assert_str = f"Test {self.name} - {test} - Pre Results - {self.pre} | Post Results {self.post}" if test: logger.info(assert_str) @@ -351,13 +351,13 @@ def _refresh_report(self, property_changes) -> pd.DataFrame: refresh_data = [] for property_change in property_changes: if ( - isinstance(property_change.Object, Partition) - and property_change.Property_Name == "RefreshedTime" + isinstance(property_change.object, Partition) + and property_change.property_name == "RefreshedTime" ): table, partition, refreshed_time = ( - property_change.Object.Table.Name, - property_change.Object.Name, - ticks_to_datetime(property_change.New_Value.Ticks), + property_change.object.Table.Name, + property_change.object.Name, + ticks_to_datetime(property_change.new_value.Ticks), ) logger.info( f'{table} - {partition} Refreshed! - {refreshed_time.strftime("%m/%d/%Y, %H:%M:%S")}' diff --git a/pytabular/table.py b/pytabular/table.py index 412d047..7ed51c9 100644 --- a/pytabular/table.py +++ b/pytabular/table.py @@ -81,7 +81,7 @@ def last_refresh(self) -> datetime: datetime: Last refresh time in datetime format """ partition_refreshes = [ - partition.Last_Refresh() for partition in self.Partitions + partition.last_refresh() for partition in self.Partitions ] return max(partition_refreshes) @@ -130,7 +130,7 @@ def query_all(self, query_function: str = "COUNTROWS(_)") -> pd.DataFrame: def find_zero_rows(self): """Returns PyTables class of tables with zero rows queried.""" query_function: str = "COUNTROWS(_)" - df = self.Query_All(query_function) + df = self.query_all(query_function) table_names = df[df[f"[{query_function}]"].isna()]["[Table]"].to_list() logger.debug(f"Found {table_names}") @@ -159,7 +159,7 @@ def last_refresh(self, group_partition: bool = True) -> pd.DataFrame: partition.Name for table in self for partition in table.Partitions ], "RefreshedTime": [ - partition.Last_Refresh() + partition.last_refresh() for table in self for partition in table.Partitions ], diff --git a/test/conftest.py b/test/conftest.py index 52c158b..79267b6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -37,6 +37,6 @@ def pytest_sessionstart(session): def pytest_sessionfinish(session, exitstatus): p.logger.info("Executing pytest cleanup...") remove_testing_table(local_pbix) - p.logger.info("Finding and closing PBIX file...") + # p.logger.info("Finding and closing PBIX file...") # subprocess.run(["powershell", "Stop-Process -Name PBIDesktop"]) return True diff --git a/test/test_11document.py b/test/test_11document.py index 8e40bf3..434140a 100644 --- a/test/test_11document.py +++ b/test/test_11document.py @@ -5,7 +5,7 @@ import pytabular as p import os from pytabular import logic_utils -from test.conftest import testing_storage +from test.conftest import TestStorage @pytest.mark.parametrize("model", testing_parameters) @@ -14,13 +14,13 @@ def test_basic_document_funcionality(model): docs = p.ModelDocumenter(model=model) docs.generate_documentation_pages() docs.save_documentation() - testing_storage.documentation_class = docs + TestStorage.documentation_class = docs except Exception: pytest.fail("Unsuccessful documentation generation..") def test_basic_documentation_removed(): - docs_class = testing_storage.documentation_class + docs_class = TestStorage.documentation_class remove = f"{docs_class.save_location}/{docs_class.friendly_name}" logic_utils.remove_folder_and_contents(remove) assert os.path.exists(remove) is False diff --git a/test/test_2object.py b/test/test_2object.py index a35fe69..79ac214 100644 --- a/test/test_2object.py +++ b/test/test_2object.py @@ -85,15 +85,15 @@ def test_get_attr(model): @pytest.mark.parametrize("model", testing_parameters) def test_iadd_tables(model): - a = model.Tables.Find("Sales") - b = model.Tables.Find("Date") + a = model.Tables.find("Sales") + b = model.Tables.find("Date") a += b - assert len(a.Find("Date")) > 0 + assert len(a.find("Date")) > 0 @pytest.mark.parametrize("model", testing_parameters) def test_iadd_table(model): - a = model.Tables.Find("Sales") - b = model.Tables.Find("Date")[0] + a = model.Tables.find("Sales") + b = model.Tables.find("Date")[0] a += b - assert len(a.Find("Date")) > 0 + assert len(a.find("Date")) > 0 diff --git a/test/test_3tabular.py b/test/test_3tabular.py index 8a5df09..ab9ed4d 100644 --- a/test/test_3tabular.py +++ b/test/test_3tabular.py @@ -6,8 +6,8 @@ @pytest.mark.parametrize("model", testing_parameters) def test_basic_query(model): - int_result = model.Query("EVALUATE {1}") - text_result = model.Query('EVALUATE {"Hello World"}') + int_result = model.query("EVALUATE {1}") + text_result = model.query('EVALUATE {"Hello World"}') assert int_result == 1 and text_result == "Hello World" @@ -21,7 +21,7 @@ def test_basic_query(model): @pytest.mark.parametrize("model", testing_parameters) def test_datatype_query(model): for query in datatype_queries: - result = model.Query(f"EVALUATE {{{query[1]}}}") + result = model.query(f"EVALUATE {{{query[1]}}}") assert result == query[0] @@ -30,7 +30,7 @@ def test_file_query(model): singlevaltest = get_test_path() + "\\singlevaltest.dax" dfvaltest = get_test_path() + "\\dfvaltest.dax" dfdupe = pd.DataFrame({"[Value1]": (1, 3), "[Value2]": (2, 4)}) - assert model.Query(singlevaltest) == 1 and model.Query(dfvaltest).equals(dfdupe) + assert model.query(singlevaltest) == 1 and model.query(dfvaltest).equals(dfdupe) @pytest.mark.parametrize("model", testing_parameters) @@ -40,22 +40,22 @@ def test_repr_str(model): @pytest.mark.parametrize("model", testing_parameters) def test_pytables_count(model): - assert model.Tables[testingtablename].Row_Count() > 0 + assert model.Tables[testingtablename].row_count() > 0 @pytest.mark.parametrize("model", testing_parameters) def test_pytables_refresh(model): - assert len(model.Tables[testingtablename].Refresh()) > 0 + assert len(model.Tables[testingtablename].refresh()) > 0 @pytest.mark.parametrize("model", testing_parameters) def test_pypartitions_refresh(model): - assert len(model.Tables[testingtablename].Partitions[0].Refresh()) > 0 + assert len(model.Tables[testingtablename].Partitions[0].refresh()) > 0 @pytest.mark.parametrize("model", testing_parameters) def test_pyobjects_adding(model): - table = model.Tables.Find(testingtablename) + table = model.Tables.find(testingtablename) table += table assert len(table) == 2 @@ -68,19 +68,19 @@ def test_nonetype_decimal_bug(model): (1, CONVERT( 1.24, CURRENCY ), "Hello"), (2, CONVERT( 87661, CURRENCY ), "World"), (3,,"Test") } """ - assert len(model.Query(query_str)) == 3 + assert len(model.query(query_str)) == 3 @pytest.mark.parametrize("model", testing_parameters) def test_table_last_refresh_times(model): """Really just testing the the function completes successfully and returns df""" - assert isinstance(model.Tables.Last_Refresh(), pd.DataFrame) + assert isinstance(model.Tables.last_refresh(), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) def test_return_zero_row_tables(model): """Testing that `Return_Zero_Row_Tables`""" - assert isinstance(model.Tables.Find_Zero_Rows(), p.pytabular.PyTables) + assert isinstance(model.Tables.find_zero_rows(), p.pytabular.PyTables) @pytest.mark.parametrize("model", testing_parameters) @@ -92,43 +92,43 @@ def test_get_dependencies(model): @pytest.mark.parametrize("model", testing_parameters) def test_disconnect(model): """Tests `Disconnect()` from `Tabular` class.""" - model.Disconnect() + model.disconnect() assert model.Server.Connected is False @pytest.mark.parametrize("model", testing_parameters) def test_reconnect(model): """Tests `Reconnect()` from `Tabular` class.""" - model.Reconnect() + model.reconnect() assert model.Server.Connected is True @pytest.mark.parametrize("model", testing_parameters) def test_reconnect_savechanges(model): - """This will test the `Reconnect()` gets called in `SaveChanges()`""" - model.Disconnect() - model.SaveChanges() + """This will test the `reconnect()` gets called in `save_changes()`""" + model.disconnect() + model.save_changes() assert model.Server.Connected is True @pytest.mark.parametrize("model", testing_parameters) def test_is_process(model): """Checks that `Is_Process()` from `Tabular` class returns bool""" - assert isinstance(model.Is_Process(), bool) + assert isinstance(model.is_process(), bool) @pytest.mark.parametrize("model", testing_parameters) def test_bad_table(model): """Checks for unable to find table exception""" with pytest.raises(Exception): - model.Refresh("badtablename") + model.refresh("badtablename") @pytest.mark.parametrize("model", testing_parameters) def test_refresh_dict(model): """Checks for refreshing dictionary""" table = model.Tables[testingtablename] - refresh = model.Refresh({table.Name: table.Partitions[0].Name}) + refresh = model.refresh({table.Name: table.Partitions[0].Name}) assert isinstance(refresh, pd.DataFrame) @@ -136,7 +136,7 @@ def test_refresh_dict(model): def test_refresh_dict_pypartition(model): """Checks for refreshing dictionary""" table = model.Tables[testingtablename] - refresh = model.Refresh({table.Name: table.Partitions[0]}) + refresh = model.refresh({table.Name: table.Partitions[0]}) assert isinstance(refresh, pd.DataFrame) @@ -145,4 +145,4 @@ def test_bad_partition(model): """Checks for refreshing dictionary""" table = model.Tables[testingtablename] with pytest.raises(Exception): - model.Refresh({table.Name: table.Partitions[0].Name + "fail"}) + model.refresh({table.Name: table.Partitions[0].Name + "fail"}) diff --git a/test/test_5column.py b/test/test_5column.py index d66fba2..df6571c 100644 --- a/test/test_5column.py +++ b/test/test_5column.py @@ -9,21 +9,21 @@ @pytest.mark.parametrize("model", testing_parameters) def test_values(model): """Tests for `Values()` of PyColumn class.""" - vals = model.Tables[testingtablename].Columns[1].Values() + vals = model.Tables[testingtablename].Columns[1].values() assert isinstance(vals, pd.DataFrame) or isinstance(vals, int) @pytest.mark.parametrize("model", testing_parameters) def test_distinct_count_no_blank(model): """Tests No_Blank=True for `Distinct_Count()` of PyColumn class.""" - vals = model.Tables[testingtablename].Columns[1].Distinct_Count(No_Blank=True) + vals = model.Tables[testingtablename].Columns[1].distinct_count(no_blank=True) assert isinstance(vals, int64) @pytest.mark.parametrize("model", testing_parameters) def test_distinct_count_blank(model): """Tests No_Blank=False for `Distinct_Count()` of PyColumn class.""" - vals = model.Tables[testingtablename].Columns[1].Distinct_Count(No_Blank=False) + vals = model.Tables[testingtablename].Columns[1].distinct_count(no_blank=False) assert isinstance(vals, int64) @@ -37,7 +37,7 @@ def test_get_sample_values(model): @pytest.mark.parametrize("model", testing_parameters) def test_query_every_column(model): """Tests `Query_All()` of PyColumns class.""" - assert isinstance(model.Tables[testingtablename].Columns.Query_All(), pd.DataFrame) + assert isinstance(model.Tables[testingtablename].Columns.query_all(), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) diff --git a/test/test_6table.py b/test/test_6table.py index 4b49f06..0c5a578 100644 --- a/test/test_6table.py +++ b/test/test_6table.py @@ -9,52 +9,52 @@ @pytest.mark.parametrize("model", testing_parameters) def test_row_count(model): """Tests for `Row_Count()` of PyTable class.""" - assert model.Tables[testingtablename].Row_Count() > 0 + assert model.Tables[testingtablename].row_count() > 0 @pytest.mark.parametrize("model", testing_parameters) def test_refresh(model): """Tests for `Refresh()` of PyTable class.""" - assert isinstance(model.Tables[testingtablename].Refresh(), pd.DataFrame) + assert isinstance(model.Tables[testingtablename].refresh(), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) def test_last_refresh(model): """Tests for `Last_Refresh()` of PyTable class.""" - assert isinstance(model.Tables[testingtablename].Last_Refresh(), datetime) + assert isinstance(model.Tables[testingtablename].last_refresh(), datetime) @pytest.mark.parametrize("model", testing_parameters) def test_related(model): """Tests for `Related()` of PyTable class.""" - assert isinstance(model.Tables[testingtablename].Related(), type(model.Tables)) + assert isinstance(model.Tables[testingtablename].related(), type(model.Tables)) @pytest.mark.parametrize("model", testing_parameters) def test_refresh_pytables(model): """Tests for `Refresh()` of PyTables class.""" - assert isinstance(model.Tables.Find(testingtablename).Refresh(), pd.DataFrame) + assert isinstance(model.Tables.find(testingtablename).refresh(), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) def test_query_all_pytables(model): """Tests for `Query_All()` of PyTables class.""" - assert isinstance(model.Tables.Query_All(), pd.DataFrame) + assert isinstance(model.Tables.query_all(), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) def test_find_zero_rows_pytables(model): """Tests for `Find_Zero_Rows()` of PyTables class.""" - assert isinstance(model.Tables.Find_Zero_Rows(), type(model.Tables)) + assert isinstance(model.Tables.find_zero_rows(), type(model.Tables)) @pytest.mark.parametrize("model", testing_parameters) def test_last_refresh_pytables_true(model): """Tests group_partition=True for `Last_Refresh()` of PyTables class.""" - assert isinstance(model.Tables.Last_Refresh(group_partition=True), pd.DataFrame) + assert isinstance(model.Tables.last_refresh(group_partition=True), pd.DataFrame) @pytest.mark.parametrize("model", testing_parameters) def test_last_refresh_pytables_false(model): """Tests group_partition=False for `Last_Refresh()` of PyTables class.""" - assert isinstance(model.Tables.Last_Refresh(group_partition=False), pd.DataFrame) + assert isinstance(model.Tables.last_refresh(group_partition=False), pd.DataFrame) diff --git a/test/test_7tabular_tracing.py b/test/test_7tabular_tracing.py index a091b21..b07bc74 100644 --- a/test/test_7tabular_tracing.py +++ b/test/test_7tabular_tracing.py @@ -3,43 +3,43 @@ from test.config import testing_parameters, testingtablename import pytest import pytabular as p -from test.conftest import testing_storage +from test.conftest import TestStorage @pytest.mark.parametrize("model", testing_parameters) def test_disconnect_for_trace(model): - """Tests `Disconnect()` from `Tabular` class.""" - model.Disconnect() + """Tests `disconnect()` from `Tabular` class.""" + model.disconnect() assert model.Server.Connected is False @pytest.mark.parametrize("model", testing_parameters) def test_reconnect_update(model): - """This will test the `Reconnect()` gets called in `Update()` + """This will test the `reconnect()` gets called in `update()` of the `Base_Trace` class. """ - model.Disconnect() - model.Tables[testingtablename].Refresh() + model.disconnect() + model.Tables[testingtablename].refresh() assert model.Server.Connected is True @pytest.mark.parametrize("model", testing_parameters) def test_query_monitor_start(model): - """This will test the `Query_Monitor` trace and `Start()` it.""" - query_trace = p.Query_Monitor(model) - query_trace.Start() - testing_storage.query_trace = query_trace - assert testing_storage.query_trace.Trace.IsStarted + """This will test the `QueryMonitor` trace and `start()` it.""" + query_trace = p.QueryMonitor(model) + query_trace.start() + TestStorage.query_trace = query_trace + assert TestStorage.query_trace.Trace.IsStarted @pytest.mark.parametrize("model", testing_parameters) def test_query_monitor_stop(model): - """Tests `Stop()` of `Query_Monitor` trace.""" - testing_storage.query_trace.Stop() - assert testing_storage.query_trace.Trace.IsStarted is False + """Tests `stop()` of `QueryMonitor` trace.""" + TestStorage.query_trace.stop() + assert TestStorage.query_trace.Trace.IsStarted is False @pytest.mark.parametrize("model", testing_parameters) def test_query_monitor_drop(model): - """Tests `Drop()` of `Query_Monitor` trace.""" - assert testing_storage.query_trace.Drop() is None + """Tests `drop()` of `QueryMonitor` trace.""" + assert TestStorage.query_trace.drop() is None diff --git a/test/test_8bpa.py b/test/test_8bpa.py index 6c77625..a9b8128 100644 --- a/test/test_8bpa.py +++ b/test/test_8bpa.py @@ -6,14 +6,14 @@ @pytest.mark.parametrize("model", testing_parameters) def test_bpa(model): - te2 = p.Tabular_Editor().EXE - bpa = p.BPA().Location - assert isinstance(model.Analyze_BPA(te2, bpa), list) + te2 = p.TabularEditor().exe + bpa = p.BPA().location + assert isinstance(model.analyze_bpa(te2, bpa), list) @pytest.mark.parametrize("model", testing_parameters) def test_te2_custom_file_path(model): - assert isinstance(p.Tabular_Editor(getcwd()), p.Tabular_Editor) + assert isinstance(p.TabularEditor(getcwd()), p.TabularEditor) @pytest.mark.parametrize("model", testing_parameters) diff --git a/test/test_9custom.py b/test/test_9custom.py index 80cc233..9e195cc 100644 --- a/test/test_9custom.py +++ b/test/test_9custom.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize("model", testing_parameters) def test_backingup_table(model): - model.Backup_Table(testingtablename) + model.backup_table(testingtablename) assert ( len( [ @@ -23,7 +23,7 @@ def test_backingup_table(model): @pytest.mark.parametrize("model", testing_parameters) def test_revert_table2(model): - model.Revert_Table(testingtablename) + model.revert_table(testingtablename) assert ( len( [ From 1da34242d9b1bba67c21183deb8d692e1b2c729c Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Sun, 5 Feb 2023 17:46:12 -0600 Subject: [PATCH 03/10] docs update --- mkgendocs.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mkgendocs.yml b/mkgendocs.yml index 99d8de4..7bfdd8e 100644 --- a/mkgendocs.yml +++ b/mkgendocs.yml @@ -17,8 +17,8 @@ pages: source: 'pytabular/refresh.py' classes: - PyRefresh - - Refresh_Check - - Refresh_Check_Collection + - RefreshCheck + - RefreshCheckCollection - page: "Table.md" source: 'pytabular/table.py' classes: @@ -37,21 +37,21 @@ pages: - page: "Traces.md" source: 'pytabular/tabular_tracing.py' classes: - - Base_Trace - - Refresh_Trace - - Query_Monitor + - BaseTrace + - RefreshTrace + - QueryMonitor - page: "Best Practice Analyzer.md" source: 'pytabular/best_practice_analyzer.py' functions: - - Download_BPA_File + - download_bpa_file classes: - BPA - page: "Tabular Editor 2.md" source: 'pytabular/tabular_editor.py' functions: - - Download_Tabular_Editor + - download_tabular_editor classes: - - Tabular_Editor + - TabularEditor - page: "PBI Helper.md" source: 'pytabular/pbi_helper.py' functions: From 93bed3c767864763396dd016e1307fb4cb9fcdff Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Mon, 6 Feb 2023 14:21:39 -0600 Subject: [PATCH 04/10] readme update for new naming convention --- README.md | 93 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 63d0a2e..d6abc1b 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,12 @@ [PyTabular](https://github.com/Curts0/PyTabular) (python-tabular in [pypi](https://pypi.org/project/python-tabular/)) is a python package that allows for programmatic execution on your tabular models! This is possible thanks to [Pythonnet](https://pythonnet.github.io/) and Microsoft's [.Net APIs on Azure Analysis Services](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices?view=analysisservices-dotnet). Currently, this build is tested and working on Windows Operating System only. Help is needed to expand this for other operating systems. The package should have the dll files included when you import it. See [Documentation Here](https://curts0.github.io/PyTabular/). PyTabular is still considered alpha while I'm working on building out the proper tests and testing environments, so I can ensure some kind of stability in features. Please send bugs my way! Preferably in the issues section in Github. I want to harden this project so many can use it easily. I currently have local pytest for python 3.6 to 3.10 and run those tests through a local AAS and Gen2 model. ### Getting Started -See the [Pypi project](https://pypi.org/project/python-tabular/) for available version. +See the [Pypi project](https://pypi.org/project/python-tabular/) for available versions. **To become PEP8 compliant with naming conventions, serious name changes were made in 0.3.5.** Instal v. 0.3.4 or lower to get the older naming conventions. ```powershell python3 -m pip install python-tabular + +#install specific version +python3 -m pip install python-tabular==0.3.4 ``` In your python environment, import pytabular and call the main Tabular Class. Only parameter needed is a solid connection string. @@ -33,19 +36,19 @@ You can query your models with the Query method from your tabular class. For Dax ```python #Run basic queries DAX_QUERY = "EVALUATE TOPN(100, 'Table1')" -model.Query(DAX_QUERY) #returns pd.DataFrame() +model.query(DAX_QUERY) #returns pd.DataFrame() #or... DMV_QUERY = "select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES" -model.Query(DMV_QUERY) #returns pd.DataFrame() +model.query(DMV_QUERY) #returns pd.DataFrame() #or... SINGLE_VALUE_QUERY_EX = "EVALUATE {1}" -model.Query(SINGLE_VALUE_QUERY_EX) #returns 1 +model.query(SINGLE_VALUE_QUERY_EX) #returns 1 #or... FILE_PATH = 'C:\\FILEPATHEXAMPLE\\file.dax' #or file.txt -model.Query(FILE_PATH) #Will return same logic as above, single values if possible else will return pd.DataFrame() +model.query(FILE_PATH) #Will return same logic as above, single values if possible else will return pd.DataFrame() ``` You can also explore your tables, partitions, and columns. Via the Attributes from your Tabular class. @@ -57,59 +60,59 @@ dir(model.Tables['Table Name']) dir(model.Tables['Table Name'].Partitions['Partition Name']) #Only a few features right now, but check out the built in methods. -model.Tables['Table Name'].Refresh(Tracing = True) +model.Tables['Table Name'].refresh() #or -model.Tables['Table Name'].Partitions['Partition Name'].Refresh(Tracing = True) +model.Tables['Table Name'].Partitions['Partition Name'].refresh() #or -model.Tables['Table Name'].Partitions['Partition Name'].Last_Refresh() +model.Tables['Table Name'].Partitions['Partition Name'].last_refresh() #or -model.Tables['Table Name'].Row_Count() +model.Tables['Table Name'].row_count() #or -model.Tables['Table Name'].Columns['Column Name'].Distinct_Count() +model.Tables['Table Name'].Columns['Column Name'].distinct_count() ``` -Refresh method to handle refreshes on your model. This is synchronous. Should be flexible enough to handle a variety of inputs. See [PyTabular Docs for Refreshing Tables and Partitions](https://curts0.github.io/PyTabular/Tabular/#refresh). Most basic way to refresh is input the table name string. The method will search for table and output exeption if unable to find it. For partitions you will need a key, value combination. Example, {'Table1':'Partition1'}. You can also take the key value pair and iterate through a group of partitions. Example, {'Table1':['Partition1','Partition2']}. Rather than providing a string, you can also input the actual class. See below for those examples, and you can acess them from the built in attributes self.Tables, self.Partitions or explore through the .Net classes yourself in self.Model.Tables. +Refresh method to handle refreshes on your model. This is synchronous. Should be flexible enough to handle a variety of inputs. See [PyTabular Docs for Refreshing Tables and Partitions](https://curts0.github.io/PyTabular/Tabular/#refresh). Most basic way to refresh is input the table name string. The method will search for table and output exeption if unable to find it. For partitions you will need a key, value combination. Example, `{'Table1':'Partition1'}`. You can also take the key value pair and iterate through a group of partitions. Example, `{'Table1':['Partition1','Partition2']}`. Rather than providing a string, you can also input the actual class. See below for those examples, and you can acess them from the built in attributes `self.Tables`, `self.Partitions` or explore through the .Net classes yourself in `self.Model.Tables`. ```python #You have a few options when refreshing. -model.Refresh('Table Name') +model.refresh('Table Name') #or... -model.Refresh(['Table1','Table2','Table3']) +model.refresh(['Table1','Table2','Table3']) #or... -model.Refresh() +model.refresh(
) #or... -model.Refresh() +model.refresh() #or... -model.Refresh({'Table Name':'Partition Name'}) +model.refresh({'Table Name':'Partition Name'}) #or any kind of weird combination like -model.Refresh([{
:,'Table Name':['Partition1','Partition2']},'Table Name','Table Name2']) +model.refresh([{
:,'Table Name':['Partition1','Partition2']},'Table Name','Table Name2']) #You can even run through the Tables & Partition Attributes -model.Tables['Table Name'].Refresh() +model.Tables['Table Name'].refresh() #or -model.Tables['Table Name'].Partitions['Partition Name'].Refresh() +model.Tables['Table Name'].Partitions['Partition Name'].refresh() -#Default Tracing happens automatically, but can be removed by -- -model.Refresh(['Table1','Table2'], Tracing = None) +#Default Tracing happens automatically, but can be removed by... +model.refresh(['Table1','Table2'], trace = None) ``` It's not uncommon to need to run through some checks on specific Tables, Partitions, Columns, Etc... ```python #Get Row Count from model -model.Tables['Table Name'].Row_Count() +model.Tables['Table Name'].row_count() #Get Last Refresh time from a partition -model.Tables['Table Name'].Last_Refresh() +model.Tables['Table Name'].last_refresh() #Get Distinct Count or Values from a Column -model.Tables['Table Name'].Columns['Column Name'].Distinct_Count() +model.Tables['Table Name'].Columns['Column Name'].distinct_count() #or -model.Tables['Table Name'].Columns['Column Name'].Values() +model.Tables['Table Name'].Columns['Column Name'].values() ``` @@ -120,9 +123,9 @@ This will use the function [Return_Zero_Row_Tables](https://curts0.github.io/PyT ```python import pytabular model = pytabular.Tabular(CONNECTION_STR) -tables = model.Tables.Find_Zero_Rows() +tables = model.Tables.find_zero_rows() if len(tables) > 0: - model.Refresh(tables) + model.refresh(tables) ``` #### Sneak in a refresh. @@ -130,10 +133,10 @@ This will use the method [Is_Process](https://curts0.github.io/PyTabular/Tabular ```python import pytabular model = pytabular.Tabular(CONNECTION_STR) -if model.Is_Process(): +if model.is_process(): #do what you want if there is a refresh happening else: - model.Refresh(TABLES_OR_PARTITIONS_TO_REFRESH) + model.refresh(TABLES_OR_PARTITIONS_TO_REFRESH) ``` #### Show refresh times in model. @@ -141,8 +144,8 @@ This will use the function [Table_Last_Refresh_Times](https://curts0.github.io/P ```python import pytabular model = pytabular.Tabular(CONNECTION_STR) -df = model.Tables.Last_Refresh() -model.Create_Table(df, 'Refresh Times') +df = model.Tables.last_refresh() +model.create_table(df, 'Refresh Times') ``` @@ -151,9 +154,9 @@ Uses a few things. First the [BPA Class](https://curts0.github.io/PyTabular/Best ```python import pytabular model = pytabular.Tabular(CONNECTION_STR) -TE2 = pytabular.Tabular_Editor() #Feel free to input your TE2 File path or this will download for you. -BPA = pytabular.BPA() #Fee free to input your own BPA file or this will download for you from: https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json -results = model.Analyze_BPA(TE2.EXE,BPA.Location) +te2 = pytabular.TabularEditor() #Feel free to input your TE2 File path or this will download for you. +bpa = pytabular.BPA() #Fee free to input your own BPA file or this will download for you from: https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json +results = model.analyze_bpa(te2.exe,bpa.location) if len(results) > 0: #Revert deployment here! @@ -166,14 +169,14 @@ import pytabular model = pytabular.Tabular(CONNECTION_STR) LIST_OF_FILE_PATHS = ['C:\\FilePath\\file1.dax','C:\\FilePath\\file1.txt','C:\\FilePath\\file2.dax','C:\\FilePath\\file2.txt'] for file_path in LIST_OF_FILE_PATHS: - model.Query(file_path) + model.query(file_path) ``` #### Advanced Refreshing with Pre and Post Checks Maybe you are introducing new logic to a fact table, and you need to ensure that a measure checking last month values never changes. To do that you can take advantage of the `Refresh_Check` and `Refresh_Check_Collection` classes (Sorry, I know the documentation stinks right now). But using those you can build out something that would first check the results of the measure, then refresh, then check the results of the measure after refresh, and lastly perform your desired check. In this case the `pre` value matches the `post` value. When refreshing and your pre does not equal post, it would fail and give an assertion error in your logging. ```python from pytabular import Tabular -from pytabular.refresh import Refresh_Check, Refresh_Check_Collection +from pytabular.refresh import RefreshCheck, RefreshCheckCollection model = Tabular(CONNECTION_STR) @@ -183,18 +186,18 @@ def sum_of_sales_assertion(pre, post): return pre == post # This is where we put it all together into the `Refresh_Check` class. Give it a name, give it a query to run, and give it the assertion you want to make. -sum_of_last_month_sales = Refresh_Check( +sum_of_last_month_sales = RefreshCheck( 'Last Month Sales', - lambda: model.Query("EVALUATE {[Last Month Sales]}") + lambda: model.query("EVALUATE {[Last Month Sales]}") ,sum_of_sales_assertion ) # Here we are adding it to a `Refresh_Check_Collection` because you can have more than on `Refresh_Check` to run. -all_refresh_check = Refresh_Check_Collection([sum_of_last_month_sales]) +all_refresh_check = RefreshCheckCollection([sum_of_last_month_sales]) model.Refresh( 'Fact Table Name', - refresh_checks = Refresh_Check_Collection([sum_of_last_month_sales]) + refresh_checks = RefreshCheckCollection([sum_of_last_month_sales]) ) ``` @@ -220,14 +223,14 @@ SUMMARIZE( user_email = 'user1@company.com' #Base line, to query as the user connecting to the model. -model.Query(query_str) +model.query(query_str) #Option 1, Connect via connection class... user1 = p.Connection(model.Server, Effective_User = user_email) -user1.Query(query_str) +user1.query(query_str) #Option 2, Just add Effective_User -model.Query(query_str, Effective_User = user_email) +model.query(query_str, Effective_User = user_email) #PyTabular will do it's best to handle multiple accounts... #So you won't have to reconnect on every query @@ -242,10 +245,10 @@ import pytabular as p model = p.Tabular(CONNECTION_STR) #Get related tables -tables = model.Tables[TABLE_NAME].Related() +tables = model.Tables[TABLE_NAME].related() #Now just refresh like usual... -tables.Refresh() +tables.refresh() ``` ### Contributing From 76484f2a5733eacf01b8211e7bc01507dfc2252a Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Mon, 6 Feb 2023 14:22:45 -0600 Subject: [PATCH 05/10] bug fix not clearing RefreshCheckCollection --- pytabular/refresh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytabular/refresh.py b/pytabular/refresh.py index 78bc56d..468f8b1 100644 --- a/pytabular/refresh.py +++ b/pytabular/refresh.py @@ -264,7 +264,8 @@ def _post_checks(self): self.trace.drop() for check in self._checks: check.post_check() - self._checks.remove_refresh_check(check) + #self._checks.remove_refresh_check(check) + self._checks.clear_refresh_checks() pass def _get_trace(self) -> BaseTrace: From 5eeb5563983fc0c8743264cc86d982b114401024 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Mon, 6 Feb 2023 14:24:31 -0600 Subject: [PATCH 06/10] flake8 comment issue --- pytabular/refresh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytabular/refresh.py b/pytabular/refresh.py index 468f8b1..99acb5d 100644 --- a/pytabular/refresh.py +++ b/pytabular/refresh.py @@ -264,7 +264,7 @@ def _post_checks(self): self.trace.drop() for check in self._checks: check.post_check() - #self._checks.remove_refresh_check(check) + # self._checks.remove_refresh_check(check) self._checks.clear_refresh_checks() pass From c8de5202eea9718785470ae12d67d678628d3178 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Mon, 6 Feb 2023 15:03:00 -0600 Subject: [PATCH 07/10] docstring tweak --- pytabular/partition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytabular/partition.py b/pytabular/partition.py index d58b3cd..db1e7b1 100644 --- a/pytabular/partition.py +++ b/pytabular/partition.py @@ -42,7 +42,7 @@ def last_refresh(self) -> datetime: def refresh(self, *args, **kwargs) -> pd.DataFrame: """Same method from Model Refresh, you can pass through any extra parameters. For example: - `Tabular().Tables['Table Name'].Partitions[0].refresh(Tracing = True)` + `Tabular().Tables['Table Name'].Partitions[0].refresh()` Returns: pd.DataFrame: Returns pandas dataframe with some refresh details """ From c3ebd1e9bf2c10532a4026d7f153635325cd5886 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Mon, 6 Feb 2023 15:03:24 -0600 Subject: [PATCH 08/10] effective user query fix --- pytabular/pytabular.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 5b87191..dbb36f9 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -80,7 +80,7 @@ def __init__(self, connection_str: str): self.Model = self.Database.Model logger.info(f"Connected to Model - {self.Model.Name}") self.Adomd: Connection = Connection(self.Server) - self.Effective_Users: dict = {} + self.effective_users: dict = {} self.PyRefresh = PyRefresh # Build PyObjects @@ -450,10 +450,10 @@ def query( """Executes Query on Model and Returns Results in Pandas DataFrame Args: - Query_Str (str): Dax Query. Note, needs full syntax (ex: EVALUATE). See (DAX Queries)[https://docs.microsoft.com/en-us/dax/dax-queries]. + query_Str (str): Dax Query. Note, needs full syntax (ex: EVALUATE). See (DAX Queries)[https://docs.microsoft.com/en-us/dax/dax-queries]. Will check if query string is a file. If it is, then it will perform a query on whatever is read from the file. It is also possible to query DMV. For example. Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES"). See (DMVs)[https://docs.microsoft.com/en-us/analysis-services/instances/use-dynamic-management-views-dmvs-to-monitor-analysis-services?view=asallproducts-allversions] - Effective_User (str): User you wish to query as. + effective_user (str): User you wish to query as. Returns: pd.DataFrame: Returns dataframe with results @@ -466,9 +466,9 @@ def query( conn = self.effective_users[effective_user] logger.debug(f"Effective user found querying as... {effective_user}") except Exception: - logger.debug(f"Creating new connection with {effective_user}") + logger.info(f"Creating new connection with {effective_user}") conn = Connection(self.Server, effective_user=effective_user) - self.effective_Users[effective_user] = conn + self.effective_users[effective_user] = conn return conn.query(query_str) From ee625297e84a41cd8937ae0e93ab0bb6aeb1c03e Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Mon, 6 Feb 2023 15:09:33 -0600 Subject: [PATCH 09/10] refresh reconnect fix --- pytabular/refresh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytabular/refresh.py b/pytabular/refresh.py index 99acb5d..cbc9790 100644 --- a/pytabular/refresh.py +++ b/pytabular/refresh.py @@ -374,7 +374,7 @@ def run(self) -> pd.DataFrame: """ if self.model.Server.Connected is False: logger.info(f"{self.Server.Name} - Reconnecting...") - self.model.Server.reconnect() + self.model.recconect() if self.trace is not None: self.trace.start() From b2f4aa214480f0c5855cb31c98a50bd52aab3b91 Mon Sep 17 00:00:00 2001 From: Curtis Stallings Date: Wed, 8 Feb 2023 07:30:52 -0600 Subject: [PATCH 10/10] documenting a model readme update --- README.md | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/README.md b/README.md index d6abc1b..7d0924b 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,84 @@ model.Tables['Table Name'].Columns['Column Name'].distinct_count() #or model.Tables['Table Name'].Columns['Column Name'].values() ``` +### Documenting a Model +The Tabular model contains a lot of information that can be used to generation documentation if filled in. Currently the markdown files are generated with the Docusaurs heading in place, but this will be changed in future to support multiple documentation platforms. +**Tip**: With Tabular Editor 2 (Free) or 3 (Paid) you can easily add Descriptioms, Translations (Cultures) and other additonal information that can later be used for generating the documentation. + +#### Args: +- **model**: Tabular +- **friendly_name**: Default > No Value + +To specify the location of the docs, just supply the save location with a new folder name argument. +- **save_location**: Default > docs + +Each page in the generation process has it's own specific name, with these arguments you can rename them to your liking. +- **general_page_url**: Default > 1-general-information.md +- **measure_page_url**: Default > 2-measures.md +- **table_page_url**: Default > 3-tables.md +- **column_page_url**: Default > 4-columns.md +- **roles_page_url**: Default > 5-roles.md + +#### Documenting a Model +The simpelst way to document a tabular model is to connect to the model, and initialize the documentation and execute `save_documentation()`. + +```python +import pytabular + +# Connect to a Tabular Model Model +model = pytabular.Tabular(CONNECTION_STR) + +# Initiate the Docs +docs = pytabular.ModelDocumenter(model) + +# Save docs to the default location +docs.save_documentation() +``` + +#### Documenting a Model with Cultures +Some model creators choose to add cultures to a tabular model for different kinds of reasons. We can leverage those cultures to use the translation names instead of the original object names. In order to this you can set translations to `True` and specify the culture you want to use (e.g. `'en-US'). + +```python +import pytabular + +# Connect to a Tabular Model Model +model = pytabular.Tabular(CONNECTION_STR) + +# Initiate the Docs +docs = pytabular.ModelDocumenter(model) + +# Set the translation for documentation to an available culture. +docs = pytabular.ModelDocumenter(model) + +# By setting the Tranlsations to `True` it will check if it exists and if it does, +# it will start using the translations for the docs +docs.set_transalation( + enable_translations = True, + culture = 'en-US' + ) + +# Save docs to the default location +docs.save_documentation() +``` +#### Documenting a Power BI Desktop Model +The Local model doesn't have a "name", only an Id. So we need to Supply a "Friendly Name", which will be used to store the markdown files. The result of this example with be a folder `my-docs-folder` with a subfolder `Adventure Works` where all the files are stored. +```python +import pytabular + +# Connect to a Tabular Model Model +model = pytabular.Tabular(CONNECTION_STR) + +# Initiate the Docs, set a friendly name to store the markdown files and overwrite the default location. +docs = pytabular.ModelDocumenter( + model = model, + friendly_name = "Adventure Works", + save_location = "my-docs-folder" +) + +# Save docs to the default location +docs.save_documentation() +``` ### Use Cases