Update dependencies / fix tests (#280)

* Update installation

* Add debug_mode

* Update installation

* Update versions

* Fix tests and upgrade requirements

* Downgrade

* Unrestricted requirements

* Remove dep versions

* Add dockerfile

* Install git

* Skip mxnet tests

* Iterating on dockerfile

* Skip mxnet tests

Author: nlathia

CONTRIBUTING.md

@@ -55,8 +55,7 @@ Notes:
 
 ### Run the tests
 
-We're using `pytest` for the tests. You can type `pytest`, or use
-this `Makefile` command:
+This project has _a lot_ of dependencies. To run the tests, I'm using Docker:
 
 ```bash
 ❯ make test

Dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu
+WORKDIR /usr/src/app
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y build-essential && \
+    apt-get install -y git ninja-build ccache libopenblas-dev libopencv-dev cmake && \
+    apt-get install -y gcc mono-mcs g++ && \
+    apt-get install -y python3 python3-pip && \
+    apt-get install -y default-jdk && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --upgrade pip setuptools
+
+# Install & install requirements
+ADD requirements-dev0.txt ./requirements-dev0.txt
+RUN pip3 install -r requirements-dev0.txt
+
+ADD requirements-dev1.txt ./requirements-dev1.txt
+RUN pip3 install -r requirements-dev1.txt
+
+ADD requirements.txt ./requirements.txt
+RUN pip3 install -r requirements.txt
+
+# Copy library source
+COPY modelstore ./modelstore
+COPY tests ./tests
+
+# Run tests
+ENTRYPOINT ["python3", "-m", "pytest", "--exitfirst", "./tests"]

Makefile

@@ -1,30 +1,38 @@
 VIRTUALENV_NAME=$(shell pwd | rev | cut -d '/' -f 1 | rev)-dev
 
-.PHONY: library test setup install uninstall release-test release-prod clean update
-
+.PHONY: uninstall
 uninstall:
 	@./bin/_pyenv_uninstall $(VIRTUALENV_NAME)
 
+.PHONY: setup
 setup:
 	@./bin/_brew_install
 
+.PHONY: install
 install: uninstall
 	@./bin/_pyenv_install $(VIRTUALENV_NAME)
 
+.PHONY: update
 update:
 	@./bin/_pyenv_update
 
+.PHONY: build
 build : test
 	@./bin/_build_library
 
+.PHONY: test
 test:
-	@python -m pytest tests/
+	@docker build . -t modelstore-dev
+	@docker run -it --rm modelstore-dev
 
+.PHONY: release-test
 release-test: build
 	@./bin/_release_test
 
+.PHONY: release-prod
 release-prod:
 	@./bin/_release_prod
 
+.PHONY: cleanup
 clean:
 	@./bin/_cleanup

bin/_cleanup

@@ -1,10 +1,10 @@
 #!/bin/bash
-echo "\n 🧼  Removing pycache files"
+echo -e "\n 🧼  Removing pycache files"
 find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
 
-echo "\n 🧼  Removing build directories"
+echo -e "\n 🧼  Removing build directories"
 rm -rf *.egg-info
 rm -rf build
 rm -rf dist
 
-echo "\n 🎉  Done."
+echo -e "\n 🎉  Done."

bin/_pyenv_config

@@ -1,9 +1,12 @@
 #!/bin/bash
 # export PYTHON_VERSION=3.7.15
-# export PYTHON_VERSION=3.8.12
-export PYTHON_VERSION=3.9.16
+export PYTHON_VERSION=3.8.12
+# export PYTHON_VERSION=3.9.16
 
 export VIRTUALENV_NAME="$1-${PYTHON_VERSION//./-}"
 export REPO_ROOT=$(cd $(dirname $0)/.. && pwd)
 
 echo -e "\n 💬  Using a venv called: ${VIRTUALENV_NAME}"
+
+eval "$(pyenv init --path)"
+eval "$(pyenv virtualenv-init -)"

bin/_pyenv_install

@@ -19,7 +19,9 @@ env PYTHON_CONFIGURE_OPTS="--enable-framework CC=clang" \
 echo -e "\n 💬  Setting local: $VIRTUALENV_NAME"
 pyenv local $VIRTUALENV_NAME
 
+echo -e "\n 💬  Upgrading pip"
 pip install --upgrade pip setuptools wheel
+
 for i in ./requirements*txt; do
     echo -e "\n\n 💬  Installing requirements in: $i"
     pip install -r $i

modelstore/__init__.py

@@ -1,3 +1,16 @@
+#    Copyright 2024 Neal Lathia
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
 from pkg_resources import DistributionNotFound, get_distribution
 
 # pylint: disable=unused-import

modelstore/models/managers.py

@@ -95,6 +95,8 @@ def get_manager(name: str, storage: CloudStorage = None) -> ModelManager:
         manager = TensorflowManager(storage)
     else:
         manager = _LIBRARIES[name](storage)
-    if all(module_exists(x) for x in manager.required_dependencies()):
-        return manager
-    raise ValueError(f"could not create manager for {name}: dependencies not installed")
+    for x in manager.required_dependencies():
+        if not module_exists(x):
+            raise ValueError(f"could not create manager for {name}: {x} not installed")
+    return manager
+    

modelstore/utils/log.py

@@ -14,7 +14,6 @@
 import logging
 import sys
 
-
 def get_logger():
     """Builds the modelstore logger"""
     log = logging.getLogger(name="modelstore")
@@ -27,3 +26,13 @@ def get_logger():
 
 
 logger = get_logger()
+
+
+def debug_mode(on: bool):
+    global logger
+    logger = get_logger()
+    if not on:
+        return
+    logger.setLevel(logging.DEBUG)
+    for handler in logger.handlers:
+        handler.setLevel(logging.DEBUG)

requirements-dev0.txt

@@ -10,9 +10,12 @@ rope>=1.6.0
 twine>=4.0.2
 
 # Data / dependencies for ML libraries
-numpy==1.23.5 # numba 0.58.0 requires numpy<1.26,>=1.21; numpy>1.23.5 currently breaks mxnet
-numba>=0.55.1
-Cython>=0.29.28
-python-Levenshtein>=0.12.2
+numba>=0.58.1
+numpy==1.23.5 # numpy>1.23.5 currently breaks mxnet
+Cython>=3.0.8
+python-Levenshtein>=0.24.0
 pandas>=1.3.5; python_version < '3.8'
 pandas>=1.4.1; python_version > '3.7'
+
+# ML Dependencies
+# pydoop<=2.0.0; sys_platform == 'darwin'