diff --git a/README.md b/README.md index 3fab71d..fa477f2 100644 --- a/README.md +++ b/README.md @@ -21,16 +21,20 @@ In your python environment, import pytabular and call the main Tabular Class. On model = pytabular.Tabular(CONNECTION_STR) ``` -DAX Query +Query Model ```python - model.Query(DAX_QUERY) - - #Example Dax Query - #EVALUATE - #TOPN(100,'Table1') + #Run basic queries + DAX_QUERY = "EVALUATE TOPN(100, 'Table1')" + model.Query(DAX_QUERY) #returns pd.DataFrame() + + #or... + DMV_QUERY = "select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES" + model.Query(DMV_QUERY) #returns pd.DataFrame() - # Returns a Pandas DataFrame + #or... + SINGLE_VALUE_QUERY_EX = "EVALUATE {1}" + model.Query(SINGLE_VALUE_QUERY_EX) #returns 1 ``` See [Refresh Tables and Partitions](https://curts0.github.io/PyTabular/Tabular/#refresh). diff --git a/dist/python_tabular-0.0.80-py3-none-any.whl b/dist/python_tabular-0.0.80-py3-none-any.whl new file mode 100644 index 0000000..b4319dd Binary files /dev/null and b/dist/python_tabular-0.0.80-py3-none-any.whl differ diff --git a/dist/python_tabular-0.0.80.tar.gz b/dist/python_tabular-0.0.80.tar.gz new file mode 100644 index 0000000..24b9dda Binary files /dev/null and b/dist/python_tabular-0.0.80.tar.gz differ diff --git a/mkgendocs.yml b/mkgendocs.yml index b1d03cb..231fa72 100644 --- a/mkgendocs.yml +++ b/mkgendocs.yml @@ -32,10 +32,12 @@ pages: - Return_Zero_Row_Tables - Table_Last_Refresh_Times - BPA_Violations_To_DF + - Last_X_Interval - page: "Logic Utils.md" source: 'pytabular/logic_utils.py' functions: - ticks_to_datetime - pandas_datatype_to_tabular_datatype - pd_dataframe_to_m_expression - - remove_folder_and_contents \ No newline at end of file + - remove_folder_and_contents + - remove_suffix \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2483fd2..b12b0b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.0.70" +version = "0.0.80" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] diff --git a/pytabular/__init__.py b/pytabular/__init__.py index fc696ad..3200ec4 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -26,7 +26,7 @@ logger.debug(f"Importing specifics in module...") from . pytabular import Tabular -from . basic_checks import Return_Zero_Row_Tables, Table_Last_Refresh_Times, BPA_Violations_To_DF +from . basic_checks import Return_Zero_Row_Tables, Table_Last_Refresh_Times, BPA_Violations_To_DF, Last_X_Interval from . logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype from . tabular_tracing import Base_Trace, Refresh_Trace from . tabular_editor import Tabular_Editor diff --git a/pytabular/basic_checks.py b/pytabular/basic_checks.py index 9e62718..bc5414b 100644 --- a/pytabular/basic_checks.py +++ b/pytabular/basic_checks.py @@ -1,6 +1,6 @@ import logging logger = logging.getLogger('PyTabular') -from typing import List +from typing import List, Union import pytabular from logic_utils import ticks_to_datetime import sys @@ -62,4 +62,47 @@ def BPA_Violations_To_DF(model:pytabular.Tabular,te2:str, bpa:str) -> pd.DataFra results = model.Analyze_BPA(te2,bpa) data = [rule.replace(' violates rule ','^').replace('\"','').split('^') for rule in results] columns = ["Object","Violation"] - return pd.DataFrame(data,columns=columns) \ No newline at end of file + return pd.DataFrame(data,columns=columns) + +def Last_X_Interval( + Model:pytabular.Tabular, + Measure:Union[str,pytabular.pytabular.Measure], + Column_Name:Union[str,None] = None, + Date_Column_Identifier:str = "'Date'[DATE_DTE_KEY]", + Number_Of_Intervals:int = 90, + Interval:str = "DAY") -> pd.DataFrame: + '''Pulls the Last X Interval (Ex Last 90 Days) of a specific measure. + + Args: + Model (pytabular.Tabular): Tabular Model to perform query on. + Measure (Union[str,pytabular.pytabular.Measure]): Measure to query. If string, will first check for a measure in the model with that name, otherwise will assume it is a DAX Expression (Ex SUM(FactTable[ColumnValue]) ) and perform that as expression + Column_Name (Union[str,None], optional): Column Name to be outputted in DataFrame. You can provide your own otherwise will take from the Measure Name. Defaults to "Result". + Date_Column_Identifier (str, optional): Date column dax identifier. Defaults to "'Date'[DATE_DTE_KEY]". + Number_Of_Intervals (int, optional): This is used to plug in the variables for [DATESINPERIOD](https://docs.microsoft.com/en-us/dax/datesinperiod-function-dax). Defaults to 90. + Interval (str, optional): Sames as Number_Of_Intervals. Used to plug in parameters of DAX function [DATESINPERIOD](https://docs.microsoft.com/en-us/dax/datesinperiod-function-dax). Defaults to "DAY". Possible options are "DAY", "MONTH", "QUARTER", and "YEAR" + + Returns: + pd.DataFrame: Pandas DataFrame of results. + ''' + if isinstance(Measure,str): + try: + Measure = [measure for measure in Model.Measures if measure.Name == Measure][-1] + Column_Name = Measure.Name if Column_Name is None else Column_Name + Expression = f"[{Measure.Name}]" + except: + logging.debug(f'Measure is string but unable to find Measure...') + Column_Name = "Result" if Column_Name is None else Column_Name + Expression = Measure + else: + Column_Name = Measure.Name if Column_Name is None else Column_Name + Expression = f'[{Column_Name}]' + Query_Str = f''' + EVALUATE + SUMMARIZECOLUMNS( + {Date_Column_Identifier}, + KEEPFILTERS( DATESINPERIOD ( {Date_Column_Identifier}, UTCTODAY(), -{Number_Of_Intervals}, {Interval} ) ), + "{Column_Name}", {Expression} + ) + ''' + logging.info(f'Running query for {Column_Name} in the last {Number_Of_Intervals} {Interval}s...') + return Model.Query(Query_Str) \ No newline at end of file diff --git a/pytabular/logic_utils.py b/pytabular/logic_utils.py index 7c3ad24..2bccca6 100644 --- a/pytabular/logic_utils.py +++ b/pytabular/logic_utils.py @@ -127,4 +127,18 @@ def remove_folder_and_contents(folder_location): import shutil if os.path.exists(folder_location): logger.info(f'Removing Dir and Contents -> {folder_location}') - shutil.rmtree(folder_location) \ No newline at end of file + shutil.rmtree(folder_location) + +def remove_suffix(input_string, suffix): + '''Adding for >3.9 compatiblity. (Stackoverflow Answer)[https://stackoverflow.com/questions/66683630/removesuffix-returns-error-str-object-has-no-attribute-removesuffix] + + Args: + input_string (str): input string to remove suffix from + suffix (str): suffix to be removed + + Returns: + str: input_str with suffix removed + ''' + if suffix and input_string.endswith(suffix): + return input_string[:-len(suffix)] + return input_string \ No newline at end of file diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index b2796e9..485fe85 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -1,22 +1,23 @@ import logging +from pathlib import Path logger = logging.getLogger('PyTabular') logger.debug(f'Importing Microsoft.AnalysisServices.Tabular') -from Microsoft.AnalysisServices.Tabular import Server, RefreshType, ColumnType, Table, DataColumn, Partition, MPartitionSource +from Microsoft.AnalysisServices.Tabular import Server, RefreshType, ColumnType, Table, DataColumn, Partition, MPartitionSource, Measure logger.debug(f'Importing Microsoft.AnalysisServices.AdomdClient') from Microsoft.AnalysisServices.AdomdClient import (AdomdCommand, AdomdConnection) logger.debug(f'Importing Microsoft.AnalysisServices') from Microsoft.AnalysisServices import UpdateOptions logger.debug('Importing Other Packages...') -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Union, TYPE_CHECKING from collections.abc import Iterable from collections import namedtuple import pandas as pd import os import subprocess import atexit -from logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype, ticks_to_datetime +from logic_utils import pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype, ticks_to_datetime, remove_suffix from tabular_tracing import Refresh_Trace @@ -51,7 +52,7 @@ def __init__(self,CONNECTION_STR:str): pass def __repr__(self) -> str: - return f'{self.Server.Name}::{self.Database.Name}::{self.Model.Name}\n{self.Database.EstimatedSize} Estimated Size\n{len(self.Tables)} Tables\n{len(self.Columns)} Columns\n{len(self.Partitions)} Partitions\n{len(self.Measures)} Measures' + return f'{self.Server.Name}::{self.Database.Name}::{self.Model.Name}\nEstimated Size::{self.Database.EstimatedSize}\nTables::{len(self.Tables)}\nColumns::{len(self.Columns)}\nPartitions::{len(self.Partitions)}\nMeasures::{len(self.Measures)}' def Reload_Model_Info(self) -> bool: '''Runs on __init__ iterates through details, can be called after any model changes. Called in SaveChanges() @@ -72,11 +73,9 @@ def Disconnect(self) -> bool: ''' logger.debug(f'Disconnecting from - {self.Server.Name}') return self.Server.Disconnect() - def Refresh(self, Object:Union[ - str, Table, Partition, Dict[str, Any], - Iterable[str, Table, Partition, Dict[str, Any]] + str, Table, Partition, Dict[str, Any] ], RefreshType:RefreshType = RefreshType.Full, Tracing = False) -> None: @@ -167,7 +166,7 @@ def refresh(Object): Refresh_Report(m.Property_Changes) return m - def Update(self, UpdateOptions:UpdateOptions =UpdateOptions.ExpandFull) -> None: + def Update(self, UpdateOptions:UpdateOptions = UpdateOptions.ExpandFull) -> None: '''[Update Model](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.majorobject.update?view=analysisservices-dotnet#microsoft-analysisservices-majorobject-update(microsoft-analysisservices-updateoptions)) Args: @@ -232,15 +231,15 @@ def rename(items): logger.info('Adding Table to Model as backup') self.Model.Tables.Add(table) logger.info('Finding Necessary Relationships... Cloning...') - relationships = [relationship.Clone() for relationship in self.Model.Relationships.GetEnumerator() if relationship.ToTable.Name == table.Name.removesuffix('_backup') or relationship.FromTable.Name == table.Name.removesuffix('_backup')] + relationships = [relationship.Clone() for relationship in self.Model.Relationships.GetEnumerator() if relationship.ToTable.Name == remove_suffix(table.Name,'_backup') or relationship.FromTable.Name == remove_suffix(table.Name,'_backup')] logger.info('Renaming Relationships') rename(relationships) logger.info('Switching Relationships to Clone Table & Column') for relationship in relationships: logger.debug(f'Renaming - {relationship.Name}') - if relationship.ToTable.Name == table.Name.removesuffix('_backup'): + if relationship.ToTable.Name == remove_suffix(table.Name,'_backup'): relationship.set_ToColumn(table.Columns.Find(f'{relationship.ToColumn.Name}_backup')) - elif relationship.FromTable.Name == table.Name.removesuffix('_backup'): + elif relationship.FromTable.Name == remove_suffix(table.Name,'_backup'): relationship.set_FromColumn(table.Columns.Find(f'{relationship.FromColumn.Name}_backup')) logger.debug(f'Adding {relationship.Name} to {self.Model.Name}') self.Model.Relationships.Add(relationship) @@ -315,7 +314,7 @@ def remove_role_permissions(): def dename(items): for item in items: logger.debug(f'Removing Suffix for {item.Name}') - item.RequestRename(f'{item.Name}'.removesuffix('_backup')) + item.RequestRename(remove_suffix(item.Name,'_backup')) logger.debug(f'Saving Changes... for {item.Name}') self.Model.SaveChanges() logger.info(f'Name changes for Columns...') @@ -329,18 +328,27 @@ def dename(items): logger.info(f'Name changes for Relationships...') dename(backup_relationships) logger.info(f'Name changes for Backup Table...') - backup.RequestRename(backup.Name.removesuffix('_backup')) + backup.RequestRename(remove_suffix(backup.Name,'_backup')) self.SaveChanges() return True - def Query(self,Query_Str:str) -> pd.DataFrame: + def Query(self,Query_Str:str) -> Union[pd.DataFrame,str,int]: ''' Executes Query on Model and Returns Results in Pandas DataFrame Args: - Query_Str (str): Dax Query. Note, needs full syntax (ex: EVALUATE). See https://docs.microsoft.com/en-us/dax/dax-queries + Query_Str (str): Dax Query. Note, needs full syntax (ex: EVALUATE). See (DAX Queries)[https://docs.microsoft.com/en-us/dax/dax-queries]. + Will check if query string is a file. If it is, then it will perform a query on whatever is read from the file. + It is also possible to query DMV. For example. Query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES"). See (DMVs)[https://docs.microsoft.com/en-us/analysis-services/instances/use-dynamic-management-views-dmvs-to-monitor-analysis-services?view=asallproducts-allversions] Returns: pd.DataFrame: Returns dataframe with results ''' + + if os.path.isfile(Query_Str): + logging.debug(f'File path detected, reading file... -> {Query_Str}',) + with open(Query_Str,'r') as file: + Query_Str = str(file.read()) + + logger.debug(f'Beginning to Query...') try: logger.debug(f'Attempting to Open Adomd Connection...') @@ -361,6 +369,9 @@ def Query(self,Query_Str:str) -> pd.DataFrame: Query.Close() logger.debug(f'Converting to Pandas DataFrame...') df = pd.DataFrame(Results,columns=[value for _,value in Column_Headers]) + if len(df) == 1 and len(df.columns) == 1: + logging.debug(f'Returning single value...') + return df.iloc[0][df.columns[0]] return df def Query_Every_Column(self,query_function:str='COUNTROWS(VALUES(_))') -> pd.DataFrame: '''This will dynamically create a query to pull all columns from the model and run the query function. diff --git a/pytabular/tabular_editor.py b/pytabular/tabular_editor.py index 11cb77c..95a2109 100644 --- a/pytabular/tabular_editor.py +++ b/pytabular/tabular_editor.py @@ -17,7 +17,7 @@ def Download_Tabular_Editor(Download_Location:str = 'https://github.com/TabularE Auto_Remove (bool, optional): Boolean to determine auto removal of files once script exits. Defaults to True. Returns: - str: _description_ + str: File path of TabularEditor.exe ''' logger.info(f'Downloading Tabular Editor 2...') logger.info(f'From... {Download_Location}') diff --git a/test/test_tabular.py b/test/test_tabular.py index ac0bfaa..f252f2c 100644 --- a/test/test_tabular.py +++ b/test/test_tabular.py @@ -23,8 +23,8 @@ def test_database(model): @pytest.mark.parametrize("model",testing_parameters) def test_query(model): - df = model.Query('EVALUATE {1}') - assert df.iloc[0]['[Value]'] == 1 + result = model.Query('EVALUATE {1}') + assert result == 1 def remove_testing_table(model):