diff --git a/.flake8 b/.flake8 index 9214cb9..1518320 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,2 @@ [flake8] -ignore = E501, W503 \ No newline at end of file +ignore = E501, W503, E203 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 48d79c9..e80c6ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python_tabular" -version = "0.2.4" +version = "0.2.5" authors = [ { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, ] diff --git a/pytabular/logic_utils.py b/pytabular/logic_utils.py index 0d1b103..b53a2f5 100644 --- a/pytabular/logic_utils.py +++ b/pytabular/logic_utils.py @@ -4,6 +4,7 @@ from typing import Dict, List import pandas as pd from Microsoft.AnalysisServices.Tabular import DataType +from Microsoft.AnalysisServices.AdomdClient import AdomdDataReader logger = logging.getLogger("PyTabular") @@ -188,4 +189,22 @@ def get_sub_list(lst: list, n: int) -> list: Returns: list: Nested list. """ - return [lst[i: i + n] for i in range(0, len(lst), n)] + return [lst[i : i + n] for i in range(0, len(lst), n)] + + +def get_value_to_df(Query: AdomdDataReader, index: int): + """Gets the values from the AdomdDataReader to convert the .Net Object + into a tangible python value to work with in pandas. + Lots of room for improvement on this one. + + Args: + Query (AdomdDataReader): [AdomdDataReader](https://learn.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.adomdclient.adomddatareader?view=analysisservices-dotnet) + index (int): Index of the value to perform the logic on. + """ + if ( + Query.GetDataTypeName((index)) in ("Decimal") + and Query.GetValue(index) is not None + ): + return Query.GetValue(index).ToDouble(Query.GetValue(index)) + else: + return Query.GetValue(index) diff --git a/pytabular/object.py b/pytabular/object.py index deff0a2..2d1a7eb 100644 --- a/pytabular/object.py +++ b/pytabular/object.py @@ -1,6 +1,7 @@ from abc import ABC from rich.console import Console from rich.table import Table +from collections.abc import Iterable class PyObject(ABC): @@ -55,6 +56,15 @@ def __iter__(self): def __len__(self): return len(self._objects) + def __iadd__(self, obj): + if isinstance(obj, Iterable): + self._objects.__iadd__(obj._objects) + else: + self._objects.__iadd__([obj]) + + self.__init__(self._objects) + return self + def Find(self, object_str): items = [ object diff --git a/pytabular/partition.py b/pytabular/partition.py index 1b6dd9d..fff32db 100644 --- a/pytabular/partition.py +++ b/pytabular/partition.py @@ -3,6 +3,7 @@ from object import PyObject, PyObjects from logic_utils import ticks_to_datetime import pandas as pd +from datetime import datetime logger = logging.getLogger("PyTabular") @@ -27,7 +28,7 @@ def __init__(self, object, table) -> None: "RefreshedTime", self.Last_Refresh().strftime("%m/%d/%Y, %H:%M:%S") ) - def Last_Refresh(self): + def Last_Refresh(self) -> datetime: """Queries `RefreshedTime` attribute in the partition and converts from C# Ticks to Python datetime Returns: diff --git a/pytabular/pytabular.py b/pytabular/pytabular.py index 2d5178e..dfc81ad 100644 --- a/pytabular/pytabular.py +++ b/pytabular/pytabular.py @@ -152,7 +152,7 @@ def Disconnect(self) -> bool: Returns: bool: True if successful """ - logger.debug(f"Disconnecting from - {self.Server.Name}") + logger.info(f"Disconnecting from - {self.Server.Name}") return self.Server.Disconnect() def Refresh(self, *args, **kwargs) -> pd.DataFrame: diff --git a/pytabular/query.py b/pytabular/query.py index f61faf3..b6c79d7 100644 --- a/pytabular/query.py +++ b/pytabular/query.py @@ -1,6 +1,7 @@ import logging import os from typing import Union +from logic_utils import get_value_to_df import pandas as pd from Microsoft.AnalysisServices.AdomdClient import AdomdCommand, AdomdConnection @@ -54,18 +55,16 @@ def Query(self, Query_Str: str) -> Union[pd.DataFrame, str, int]: logger.info(f"Connected! Session ID - {self.SessionID}") logger.debug("Querying Model...") + logger.debug(Query_Str) Query = AdomdCommand(Query_Str, self).ExecuteReader() Column_Headers = [ (index, Query.GetName(index)) for index in range(0, Query.FieldCount) ] Results = list() while Query.Read(): - """This is a bit garbage will need to refactor later but fixing issue with System.Decimal conversion""" Results.append( [ - Query.GetValue(index).ToDouble(Query.GetValue(index)) - if Query.GetDataTypeName((index)) in ("Decimal") - else Query.GetValue(index) + get_value_to_df(Query, index) for index in range(0, len(Column_Headers)) ] ) diff --git a/pytabular/table.py b/pytabular/table.py index a6f4afc..6a7be3d 100644 --- a/pytabular/table.py +++ b/pytabular/table.py @@ -6,6 +6,7 @@ from measure import PyMeasure, PyMeasures from pytabular.object import PyObjects from logic_utils import ticks_to_datetime +from datetime import datetime logger = logging.getLogger("PyTabular") @@ -71,6 +72,17 @@ def Refresh(self, *args, **kwargs) -> pd.DataFrame: """ return self.Model.Refresh(self, *args, **kwargs) + def Last_Refresh(self) -> datetime: + """Will query each partition for the last refresh time then select the max + + Returns: + datetime: Last refresh time in datetime format + """ + partition_refreshes = [ + partition.Last_Refresh() for partition in self.Partitions + ] + return max(partition_refreshes) + def Related(self): return self.Model.Relationships.Related(self) diff --git a/test/config.py b/test/config.py new file mode 100644 index 0000000..4487ebe --- /dev/null +++ b/test/config.py @@ -0,0 +1,12 @@ +import pytabular as p +import local +import pandas as pd +import pytest + +PLACEHOLDER_AAS = local.AAS +PLACEHOLDER_GEN2 = local.GEN2 +aas = p.Tabular(PLACEHOLDER_AAS) +gen2 = p.Tabular(PLACEHOLDER_GEN2) +testingtablename = "PyTestTable" +testingtabledf = pd.DataFrame(data={"col1": [1, 2, 3], "col2": ["four", "five", "six"]}) +testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..d160f60 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,33 @@ +from test.config import aas, gen2, testingtablename, testingtabledf +import pytabular as p + + +def pytest_report_header(config): + return "PyTabular Local Testing" + + +def remove_testing_table(model): + table_check = [ + table + for table in model.Model.Tables.GetEnumerator() + if testingtablename in table.Name + ] + for table in table_check: + model.Model.Tables.Remove(table) + model.SaveChanges() + + +def pytest_sessionstart(session): + p.logger.info("Executing pytest setup...") + remove_testing_table(aas) + remove_testing_table(gen2) + aas.Create_Table(testingtabledf, testingtablename) + gen2.Create_Table(testingtabledf, testingtablename) + return True + + +def pytest_sessionfinish(session, exitstatus): + p.logger.info("Executing pytest cleanup...") + remove_testing_table(aas) + remove_testing_table(gen2) + return True diff --git a/test/run_versions.bat b/test/run_versions.bat index 66eda8b..62be57b 100644 --- a/test/run_versions.bat +++ b/test/run_versions.bat @@ -1,2 +1,2 @@ @echo on -pyenv shell 3.6.8 & python3 -m pytest test_tabular.py & pyenv shell 3.7.9 & python3 -m pytest test_tabular.py & pyenv shell 3.8.9 & python3 -m pytest test_tabular.py & pyenv shell 3.9.13 & python3 -m pytest test_tabular.py & pyenv shell 3.10.6 & python3 -m pytest test_tabular.py & pause & pause \ No newline at end of file +pyenv shell 3.6.8 & python3 -m pytest test_2tabular.py & pyenv shell 3.7.9 & python3 -m pytest test_2tabular.py & pyenv shell 3.8.9 & python3 -m pytest test_2tabular.py & pyenv shell 3.9.13 & python3 -m pytest test_2tabular.py & pyenv shell 3.10.6 & python3 -m pytest test_2tabular.py & pause & pause \ No newline at end of file diff --git a/test/test_1sanity.py b/test/test_1sanity.py new file mode 100644 index 0000000..82d0c47 --- /dev/null +++ b/test/test_1sanity.py @@ -0,0 +1,25 @@ +from test.config import aas, gen2 +import pytest +from Microsoft.AnalysisServices.Tabular import Database + +testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] + + +@pytest.mark.parametrize("model", testing_parameters) +def test_sanity_check(model): + """Just in case... I might be crazy""" + assert 1 == 1 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_connection(model): + """ + Does a quick check to the Tabular Class + To ensure that it can connnect + """ + assert model.Server.Connected + + +@pytest.mark.parametrize("model", testing_parameters) +def test_database(model): + assert isinstance(model.Database, Database) diff --git a/test/test_2tabular.py b/test/test_2tabular.py new file mode 100644 index 0000000..e4fb8e7 --- /dev/null +++ b/test/test_2tabular.py @@ -0,0 +1,71 @@ +import local +import pytest +import pandas as pd +from test.config import testingtablename, testing_parameters + + +@pytest.mark.parametrize("model", testing_parameters) +def test_basic_query(model): + int_result = model.Query("EVALUATE {1}") + text_result = model.Query('EVALUATE {"Hello World"}') + assert int_result == 1 and text_result == "Hello World" + + +datatype_queries = [ + ["this is a string", '"this is a string"'], + [1, 1], + [1000.78, "CONVERT(1000.78,CURRENCY)"], +] + + +@pytest.mark.parametrize("model", testing_parameters) +def test_datatype_query(model): + for query in datatype_queries: + result = model.Query(f"EVALUATE {{{query[1]}}}") + assert result == query[0] + + +@pytest.mark.parametrize("model", testing_parameters) +def test_file_query(model): + singlevaltest = local.SINGLEVALTESTPATH + dfvaltest = local.DFVALTESTPATH + dfdupe = pd.DataFrame({"[Value1]": (1, 3), "[Value2]": (2, 4)}) + assert model.Query(singlevaltest) == 1 and model.Query(dfvaltest).equals(dfdupe) + + +@pytest.mark.parametrize("model", testing_parameters) +def test_repr_str(model): + assert isinstance(model.__repr__(), str) + + +@pytest.mark.parametrize("model", testing_parameters) +def test_pytables_count(model): + assert model.Tables[testingtablename].Row_Count() > 0 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_pytables_refresh(model): + assert len(model.Tables[testingtablename].Refresh()) > 0 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_pypartitions_refresh(model): + assert len(model.Tables[testingtablename].Partitions[0].Refresh()) > 0 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_pyobjects_adding(model): + table = model.Tables.Find(testingtablename) + table += table + assert len(table) == 2 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_nonetype_decimal_bug(model): + query_str = """ + EVALUATE + { + (1, CONVERT( 1.24, CURRENCY ), "Hello"), (2, CONVERT( 87661, CURRENCY ), "World"), (3,,"Test") + } + """ + assert len(model.Query(query_str)) == 3 diff --git a/test/test_3custom.py b/test/test_3custom.py new file mode 100644 index 0000000..d836d70 --- /dev/null +++ b/test/test_3custom.py @@ -0,0 +1,48 @@ +"""These are tests I have for pretty janky features of PyTabular. +These were designed selfishly for my own uses. +So seperating out, to one day sunset and remove. +""" +from test.config import aas, gen2, testingtablename +import pytest + +testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] + + +@pytest.mark.parametrize("model", testing_parameters) +def test_query_every_table(model): + assert len(model.Query_Every_Table()) > 0 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_query_every_column(model): + assert len(model.Query_Every_Column()) > 0 + + +@pytest.mark.parametrize("model", testing_parameters) +def test_backingup_table(model): + model.Backup_Table(testingtablename) + assert ( + len( + [ + table + for table in model.Model.Tables.GetEnumerator() + if f"{testingtablename}_backup" == table.Name + ] + ) + == 1 + ) + + +@pytest.mark.parametrize("model", testing_parameters) +def test_revert_table(model): + model.Revert_Table(testingtablename) + assert ( + len( + [ + table + for table in model.Model.Tables.GetEnumerator() + if f"{testingtablename}" == table.Name + ] + ) + == 1 + ) diff --git a/test/test_4bpa.py b/test/test_4bpa.py new file mode 100644 index 0000000..9961914 --- /dev/null +++ b/test/test_4bpa.py @@ -0,0 +1,12 @@ +from test.config import aas, gen2 +import pytest +import pytabular as p + +testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] + + +@pytest.mark.parametrize("model", testing_parameters) +def test_bpa(model): + te2 = p.Tabular_Editor().EXE + bpa = p.BPA().Location + assert isinstance(model.Analyze_BPA(te2, bpa), list) diff --git a/test/test_tabular.py b/test/test_tabular.py deleted file mode 100644 index 0ef4c2e..0000000 --- a/test/test_tabular.py +++ /dev/null @@ -1,174 +0,0 @@ -import pytabular -import local -import pytest -import pandas as pd -from Microsoft.AnalysisServices.Tabular import Database - -aas = pytabular.Tabular(local.AAS) -gen2 = pytabular.Tabular(local.GEN2) -testing_parameters = [pytest.param(aas, id="AAS"), pytest.param(gen2, id="GEN2")] -testingtablename = "PyTestTable" -testingtabledf = pd.DataFrame(data={"col1": [1, 2, 3], "col2": ["four", "five", "six"]}) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_sanity_check(model): - assert 1 == 1 - - -@pytest.mark.parametrize("model", testing_parameters) -def test_connection(model): - """ - Does a quick check to the Tabular Class - To ensure that it can connnect - """ - assert model.Server.Connected - - -@pytest.mark.parametrize("model", testing_parameters) -def test_database(model): - assert isinstance(model.Database, Database) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_basic_query(model): - int_result = model.Query("EVALUATE {1}") - text_result = model.Query('EVALUATE {"Hello World"}') - assert int_result == 1 and text_result == "Hello World" - - -datatype_queries = [ - ["this is a string", '"this is a string"'], - [1, 1], - [1000.78, "CONVERT(1000.78,CURRENCY)"], -] - - -@pytest.mark.parametrize("model", testing_parameters) -def test_datatype_query(model): - for query in datatype_queries: - result = model.Query(f"EVALUATE {{{query[1]}}}") - assert result == query[0] - - -@pytest.mark.parametrize("model", testing_parameters) -def test_file_query(model): - singlevaltest = local.SINGLEVALTESTPATH - dfvaltest = local.DFVALTESTPATH - dfdupe = pd.DataFrame({"[Value1]": (1, 3), "[Value2]": (2, 4)}) - assert model.Query(singlevaltest) == 1 and model.Query(dfvaltest).equals(dfdupe) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_repr_str(model): - assert isinstance(model.__repr__(), str) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_query_every_table(model): - assert len(model.Query_Every_Table()) > 0 - - -@pytest.mark.parametrize("model", testing_parameters) -def test_query_every_column(model): - assert len(model.Query_Every_Column()) > 0 - - -def remove_testing_table(model): - table_check = [ - table - for table in model.Model.Tables.GetEnumerator() - if testingtablename in table.Name - ] - for table in table_check: - model.Model.Tables.Remove(table) - model.SaveChanges() - - -@pytest.mark.parametrize("model", testing_parameters) -def test_pre_table_checks(model): - remove_testing_table(model) - assert ( - len( - [ - table - for table in model.Model.Tables.GetEnumerator() - if testingtablename in table.Name - ] - ) - == 0 - ) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_create_table(model): - df = pd.DataFrame(data={"col1": [1, 2, 3], "col2": ["four", "five", "six"]}) - model.Create_Table(df, testingtablename) - assert len(model.Query(f"EVALUATE {testingtablename}")) == 3 - - -@pytest.mark.parametrize("model", testing_parameters) -def test_pytables_count(model): - assert model.Tables[testingtablename].Row_Count() > 0 - - -@pytest.mark.parametrize("model", testing_parameters) -def test_pytables_refresh(model): - assert len(model.Tables[testingtablename].Refresh()) > 0 - - -@pytest.mark.parametrize("model", testing_parameters) -def test_pypartitions_refresh(model): - assert len(model.Tables[testingtablename].Partitions[0].Refresh()) > 0 - - -@pytest.mark.parametrize("model", testing_parameters) -def test_backingup_table(model): - model.Backup_Table(testingtablename) - assert ( - len( - [ - table - for table in model.Model.Tables.GetEnumerator() - if f"{testingtablename}_backup" == table.Name - ] - ) - == 1 - ) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_revert_table(model): - model.Revert_Table(testingtablename) - assert ( - len( - [ - table - for table in model.Model.Tables.GetEnumerator() - if f"{testingtablename}" == table.Name - ] - ) - == 1 - ) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_table_removal(model): - remove_testing_table(model) - assert ( - len( - [ - table - for table in model.Model.Tables.GetEnumerator() - if testingtablename in table.Name - ] - ) - == 0 - ) - - -@pytest.mark.parametrize("model", testing_parameters) -def test_bpa(model): - te2 = pytabular.Tabular_Editor().EXE - bpa = pytabular.BPA().Location - assert isinstance(model.Analyze_BPA(te2, bpa), list)