diff --git a/pyproject.toml b/pyproject.toml index bc384af..393bdd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.2.6" +version = "0.2.7" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] diff --git a/pytabular/__init__.py b/pytabular/__init__.py index db09d57..c431fb5 100644 --- a/pytabular/__init__.py +++ b/pytabular/__init__.py @@ -35,18 +35,19 @@ sys.path.append(dll) sys.path.append(os.path.dirname(__file__)) -logger.debug(f"Beginning CLR references...") +logger.info(f"Beginning CLR references...") import clr -logger.debug("Adding Reference Microsoft.AnalysisServices.AdomdClient") +logger.info("Adding Reference Microsoft.AnalysisServices.AdomdClient") clr.AddReference("Microsoft.AnalysisServices.AdomdClient") -logger.debug("Adding Reference Microsoft.AnalysisServices.Tabular") +logger.info("Adding Reference Microsoft.AnalysisServices.Tabular") clr.AddReference("Microsoft.AnalysisServices.Tabular") -logger.debug("Adding Reference Microsoft.AnalysisServices") +logger.info("Adding Reference Microsoft.AnalysisServices") clr.AddReference("Microsoft.AnalysisServices") -logger.debug(f"Importing specifics in module...") +logger.info(f"Importing specifics in module...") from .pytabular import Tabular + from .basic_checks import ( Return_Zero_Row_Tables, Table_Last_Refresh_Times, @@ -57,10 +58,10 @@ pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype, ) -from .tabular_tracing import Base_Trace, Refresh_Trace +from .tabular_tracing import Base_Trace, Refresh_Trace, Query_Monitor from .tabular_editor import Tabular_Editor from .best_practice_analyzer import BPA from .query import Connection from .pbi_helper import find_local_pbi_instances -logger.debug(f"Import successful...") +logger.info(f"Import successful...") diff --git a/pytabular/logic_utils.py b/pytabular/logic_utils.py index b53a2f5..a5c02b3 100644 --- a/pytabular/logic_utils.py +++ b/pytabular/logic_utils.py @@ -151,6 +151,17 @@ def remove_folder_and_contents(folder_location): shutil.rmtree(folder_location) +def remove_file(file_path): + """Just `os.remove()` but wanted a `logger.info()` with it. + + Args: + file_path: [See os.remove](https://docs.python.org/3/library/os.html) + """ + logger.info(f"Removing file - {file_path}") + os.remove(file_path) + pass + + def remove_suffix(input_string, suffix): """Adding for >3.9 compatiblity. [Stackoverflow Answer](https://stackoverflow.com/questions/66683630/removesuffix-returns-error-str-object-has-no-attribute-removesuffix) diff --git a/pytabular/pbi_helper.py b/pytabular/pbi_helper.py index 88fd090..94ed09a 100644 --- a/pytabular/pbi_helper.py +++ b/pytabular/pbi_helper.py @@ -3,6 +3,11 @@ def get_msmdsrv() -> list: + """Runs powershel command to retrieve the ProcessId from Get-CimInstance where Name == 'msmdsrv.exe'. + + Returns: + list: returns ProcessId(s) in list format to account for multiple PBIX files open at the same time. + """ p.logger.debug("Retrieving msmdsrv.exe(s)") msmdsrv = subprocess.check_output( [ @@ -16,6 +21,14 @@ def get_msmdsrv() -> list: def get_port_number(msmdsrv: str) -> str: + """Gets the local port number of given msmdsrv ProcessId. Via PowerShell. + + Args: + msmdsrv (str): A ProcessId returned from `get_msmdsrv()`. + + Returns: + str: `LocalPort` returned for specific ProcessId. + """ port = subprocess.check_output( [ "powershell", @@ -28,6 +41,14 @@ def get_port_number(msmdsrv: str) -> str: def get_parent_id(msmdsrv: str) -> str: + """Gets ParentProcessId via PowerShell from the msmdsrv ProcessId. + + Args: + msmdsrv (str): A ProcessId returned from `get_msmdsrv()`. + + Returns: + str: Returns ParentProcessId in `str` format. + """ parent = subprocess.check_output( [ "powershell", @@ -40,6 +61,14 @@ def get_parent_id(msmdsrv: str) -> str: def get_parent_title(parent_id: str) -> str: + """Takes the ParentProcessId and gets the name of the PBIX file. + + Args: + parent_id (str): Takes ParentProcessId which can be retrieved from `get_parent_id(msmdsrv)` + + Returns: + str: Returns str of title of PBIX file. + """ pbi_title_suffixes: list = [ " \u002D Power BI Desktop", # Dash Punctuation - minus hyphen " \u2212 Power BI Desktop", # Math Symbol - minus sign @@ -59,6 +88,14 @@ def get_parent_title(parent_id: str) -> str: def create_connection_str(port_number: str) -> str: + """This takes the port number and adds to connection string. This is pretty bland right now, may improve later. + + Args: + port_number (str): Port Number retrieved from `get_port_number(msmdsrv)`. + + Returns: + str: _description_ + """ connection_str = f"Data Source=localhost:{port_number}" p.logger.debug(f"Local Connection Str - {connection_str}") return connection_str diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 24bf614..6af4db4 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -20,6 +20,7 @@ pd_dataframe_to_m_expression, pandas_datatype_to_tabular_datatype, remove_suffix, + remove_file, ) from table import PyTable, PyTables @@ -517,7 +518,9 @@ def Analyze_BPA( List[str]: Assuming no failure, will return list of BPA violations. Else will return error from command line. """ logger.debug("Beginning request to talk with TE2 & Find BPA...") - cmd = f'{Tabular_Editor_Exe} "Provider=MSOLAP;{self.Adomd.ConnectionString}" {self.Database.Name} -B "{os.getcwd()}\\Model.bim" -A {Best_Practice_Analyzer} -V/?' + bim_file_location = f"{os.getcwd()}\\Model.bim" + atexit.register(remove_file, bim_file_location) + cmd = f'{Tabular_Editor_Exe} "Provider=MSOLAP;{self.Adomd.ConnectionString}" {self.Database.Name} -B "{bim_file_location}" -A {Best_Practice_Analyzer} -V/?' logger.debug("Command Generated") logger.debug("Submitting Command...") sp = subprocess.Popen( diff --git a/pytabular/refresh.py b/pytabular/refresh.py index bb63f47..4e4f2c2 100644 --- a/pytabular/refresh.py +++ b/pytabular/refresh.py @@ -11,6 +11,7 @@ from table import PyTable, PyTables from partition import PyPartition from abc import ABC +import atexit logger = logging.getLogger("PyTabular") @@ -221,6 +222,7 @@ def _post_checks(self): if self.trace is not None: self.trace.Stop() self.trace.Drop() + atexit.unregister(self.trace.Drop) for check in self._checks: check.Post_Check() self._checks.remove_refresh_check(check) diff --git a/pytabular/tabular_tracing.py b/pytabular/tabular_tracing.py index 3632255..d24bb50 100644 --- a/pytabular/tabular_tracing.py +++ b/pytabular/tabular_tracing.py @@ -3,7 +3,12 @@ import xmltodict from typing import List, Callable from Microsoft.AnalysisServices.Tabular import Trace, TraceEvent, TraceEventHandler -from Microsoft.AnalysisServices import TraceColumn, TraceEventClass, TraceEventSubclass +from Microsoft.AnalysisServices import ( + TraceColumn, + TraceEventClass, + TraceEventSubclass, +) +import atexit logger = logging.getLogger("PyTabular") @@ -52,6 +57,7 @@ def __init__( self.Build() self.Add() self.Update() + atexit.register(self.Drop) def Build(self) -> bool: """Run on initialization. @@ -271,3 +277,43 @@ def __init__( Handler: Callable = _refresh_handler, ) -> None: super().__init__(Tabular_Class, Trace_Events, Trace_Event_Columns, Handler) + + +def _query_monitor_handler(source, args): + total_secs = args.Duration / 1000 + domain_site = args.NTUserName.find("\\") + if domain_site > 0: + user = args.NTUserName[domain_site + 1 :] + else: + user = args.NTUserName + logger.info(f"{args.EventSubclass} by {user} in {args.ApplicationName}") + logger.info( + f"From {args.StartTime} to {args.EndTime} for {str(total_secs)} seconds" + ) + if args.Severity == 3: + logger.error(f"Query failure... {str(args.Error)}") + logger.error(f"{args.TextData}") + logger.debug(f"{args.TextData}") + + +class Query_Monitor(Base_Trace): + def __init__( + self, + Tabular_Class, + Trace_Events: List[TraceEvent] = [TraceEventClass.QueryEnd], + Trace_Event_Columns: List[TraceColumn] = [ + TraceColumn.EventSubclass, + TraceColumn.StartTime, + TraceColumn.EndTime, + TraceColumn.Duration, + TraceColumn.Severity, + TraceColumn.Error, + TraceColumn.NTUserName, + TraceColumn.DatabaseName, + TraceColumn.ApplicationName, + TraceColumn.TextData, + ], + Handler: Callable = _query_monitor_handler, + ) -> None: + super().__init__(Tabular_Class, Trace_Events, Trace_Event_Columns, Handler) + logger.info("Query text lives in DEBUG, adjust logging to see query text.") diff --git a/test/adventureworks/AdventureWorks Sales.pbix b/test/adventureworks/AdventureWorks Sales.pbix new file mode 100644 index 0000000..d4c7877 Binary files /dev/null and b/test/adventureworks/AdventureWorks Sales.pbix differ diff --git a/test/adventureworks/AdventureWorks Sales.xlsx b/test/adventureworks/AdventureWorks Sales.xlsx new file mode 100644 index 0000000..6de29bc Binary files /dev/null and b/test/adventureworks/AdventureWorks Sales.xlsx differ diff --git a/test/config.py b/test/config.py index 4487ebe..7708040 100644 --- a/test/config.py +++ b/test/config.py @@ -1,12 +1,42 @@ import pytabular as p -import local +import os import pandas as pd import pytest +import subprocess +from time import sleep -PLACEHOLDER_AAS = local.AAS -PLACEHOLDER_GEN2 = local.GEN2 -aas = p.Tabular(PLACEHOLDER_AAS) -gen2 = p.Tabular(PLACEHOLDER_GEN2) + +def get_test_path(): + cwd = os.getcwd() + if os.path.basename(cwd) == "test": + return cwd + elif "test" in os.listdir(): + return cwd + "\\test" + else: + raise BaseException("Unable to find test path...") + + +adventureworks_path = f'"{get_test_path()}\\adventureworks\\AdventureWorks Sales.pbix"' + +p.logger.info(f"Opening {adventureworks_path}") +subprocess.run(["powershell", f"Start-Process {adventureworks_path}"]) + +# Got to be a better way to wait and ensure the PBIX file is open? +p.logger.info("sleep(30)... Need a better way to wait until PBIX is loaded...") +sleep(30) + +LOCAL_FILE = p.find_local_pbi_instances()[0] +p.logger.info(f"Connecting to... {LOCAL_FILE[0]} - {LOCAL_FILE[1]}") +local_pbix = p.Tabular(LOCAL_FILE[1]) + +# subprocess.Popen(["powershell","Start-Process \"AdventureWorks Sales.pbix\""]) + +p.logger.info("Generating test data...") testingtablename = "PyTestTable" testingtabledf = pd.DataFrame(data={"col1": [1, 2, 3], "col2": ["four", "five", "six"]}) -testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] +testing_parameters = [ + pytest.param(local_pbix, id="LOCAL"), +] + + +# os.path.basename(os.getcwd()) diff --git a/test/conftest.py b/test/conftest.py index d160f60..7b7ecf1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,10 @@ -from test.config import aas, gen2, testingtablename, testingtabledf +from test.config import ( + local_pbix, + testingtablename, + testingtabledf, +) import pytabular as p +import subprocess def pytest_report_header(config): @@ -13,21 +18,21 @@ def remove_testing_table(model): if testingtablename in table.Name ] for table in table_check: + p.logger.info(f"Removing table {table.Name} from {model.Server.Name}") model.Model.Tables.Remove(table) model.SaveChanges() def pytest_sessionstart(session): p.logger.info("Executing pytest setup...") - remove_testing_table(aas) - remove_testing_table(gen2) - aas.Create_Table(testingtabledf, testingtablename) - gen2.Create_Table(testingtabledf, testingtablename) + remove_testing_table(local_pbix) + local_pbix.Create_Table(testingtabledf, testingtablename) return True def pytest_sessionfinish(session, exitstatus): p.logger.info("Executing pytest cleanup...") - remove_testing_table(aas) - remove_testing_table(gen2) + remove_testing_table(local_pbix) + p.logger.info("Finding and closing PBIX file...") + subprocess.run(["powershell", "Stop-Process -Name PBIDesktop"]) return True diff --git a/test/run_versions.bat b/test/run_versions.bat index 62be57b..3eb4727 100644 --- a/test/run_versions.bat +++ b/test/run_versions.bat @@ -1,2 +1,2 @@ @echo on -pyenv shell 3.6.8 & python3 -m pytest test_2tabular.py & pyenv shell 3.7.9 & python3 -m pytest test_2tabular.py & pyenv shell 3.8.9 & python3 -m pytest test_2tabular.py & pyenv shell 3.9.13 & python3 -m pytest test_2tabular.py & pyenv shell 3.10.6 & python3 -m pytest test_2tabular.py & pause & pause \ No newline at end of file +pyenv shell 3.6.8 & python3 -m pytest & pyenv shell 3.7.9 & python3 -m pytest & pyenv shell 3.8.9 & python3 -m pytest & pyenv shell 3.9.13 & python3 -m pytest & pyenv shell 3.10.6 & python3 -m pytest & pause & pause \ No newline at end of file diff --git a/test/test_1sanity.py b/test/test_1sanity.py index 82d0c47..5acf587 100644 --- a/test/test_1sanity.py +++ b/test/test_1sanity.py @@ -1,8 +1,6 @@ -from test.config import aas, gen2 import pytest from Microsoft.AnalysisServices.Tabular import Database - -testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] +from test.config import testing_parameters @pytest.mark.parametrize("model", testing_parameters) diff --git a/test/test_2tabular.py b/test/test_2tabular.py index e605473..dc96df7 100644 --- a/test/test_2tabular.py +++ b/test/test_2tabular.py @@ -1,8 +1,7 @@ -import local import pytest import pandas as pd import pytabular as p -from test.config import testingtablename, testing_parameters +from test.config import testingtablename, testing_parameters, get_test_path @pytest.mark.parametrize("model", testing_parameters) @@ -28,8 +27,8 @@ def test_datatype_query(model): @pytest.mark.parametrize("model", testing_parameters) def test_file_query(model): - singlevaltest = local.SINGLEVALTESTPATH - dfvaltest = local.DFVALTESTPATH + singlevaltest = get_test_path() + "\\singlevaltest.dax" + dfvaltest = get_test_path() + "\\dfvaltest.dax" dfdupe = pd.DataFrame({"[Value1]": (1, 3), "[Value2]": (2, 4)}) assert model.Query(singlevaltest) == 1 and model.Query(dfvaltest).equals(dfdupe) diff --git a/test/test_3custom.py b/test/test_3custom.py index d836d70..265ebda 100644 --- a/test/test_3custom.py +++ b/test/test_3custom.py @@ -2,11 +2,9 @@ These were designed selfishly for my own uses. So seperating out, to one day sunset and remove. """ -from test.config import aas, gen2, testingtablename +from test.config import testing_parameters, testingtablename import pytest -testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] - @pytest.mark.parametrize("model", testing_parameters) def test_query_every_table(model): diff --git a/test/test_4bpa.py b/test/test_4bpa.py index 9961914..8e33c62 100644 --- a/test/test_4bpa.py +++ b/test/test_4bpa.py @@ -1,8 +1,6 @@ -from test.config import aas, gen2 import pytest import pytabular as p - -testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] +from test.config import testing_parameters @pytest.mark.parametrize("model", testing_parameters)