Release modelstore==0.0.80 (#270)

* Release 0.0.80

* Add URL

* Update requirements

* Disable hdfs tests in examples

* Refresh gcloud

* Only import storage when required

* Format code

* Downgrade requirememnts

* Check alternative tokenizer file name

* Remove f-strings

* Run isort

* Exclude yolov5 in azure

* Remove test (does not work with import inside function)

* Fix name

* Add free-disk-space action step

* Fix name change in latest shap library

Author: nlathia

.github/workflows/unit-tests.yml

@@ -18,6 +18,21 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+          
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: true
+        swap-storage: true
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
@@ -27,4 +42,4 @@ jobs:
         pip install -e .
     - name: Test with pytest
       run: |
-        pytest
+        pytest -x

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+multi_line_output = 3

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change log
 
-## Unreleased
+## modelstore 0.0.80 ([October 2023](https://github.com/operatorai/modelstore/pull/270))
 
 **🆕  New functionality**
 

examples/Makefile-example

@@ -1,16 +1,20 @@
 VIRTUALENV_NAME=modelstore.$(shell pwd | rev | cut -d '/' -f 1 | rev)
 REPO_ROOT=$(shell cd ../../ && pwd)
 
-.PHONY: name pyenv pyenv-local pyenv-prod pyenv-test pyenv-uninstall refresh
+.PHONY: name pyenv pyenv-local pyenv-prod pyenv-test pyenv-uninstall refresh gcloud
 
 name:
 	@echo $(VIRTUALENV_NAME)
 
 pyenv-uninstall:
 	@$(REPO_ROOT)/bin/_pyenv_uninstall $(VIRTUALENV_NAME)
 
+gcloud:
+	@gcloud components update
+	@gcloud auth application-default login
+
 pyenv: pyenv-uninstall
-	#@$(REPO_ROOT)/bin/_setup_brew
+	# @$(REPO_ROOT)/bin/_setup_brew
 	@$(REPO_ROOT)/bin/_pyenv_install $(VIRTUALENV_NAME)
 	pip install -r https://raw.githubusercontent.com/ultralytics/yolov5/master/requirements.txt
 

examples/examples-by-ml-library/.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+multi_line_output = 3

examples/examples-by-ml-library/libraries/huggingface/distilbert.py

@@ -12,7 +12,11 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-from transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer
+from transformers import (
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer
+)
 
 from modelstore.model_store import ModelStore
 

examples/examples-by-ml-library/libraries/huggingface/dpt.py

@@ -13,7 +13,11 @@
 #    limitations under the License.
 from typing import Tuple
 
-from transformers import DPTForDepthEstimation, DPTImageProcessor, DPTPreTrainedModel
+from transformers import (
+    DPTForDepthEstimation,
+    DPTImageProcessor,
+    DPTPreTrainedModel
+)
 
 from modelstore.model_store import ModelStore
 

examples/examples-by-ml-library/libraries/onnx_lightgbm_example.py

@@ -17,11 +17,15 @@
 from libraries.util.datasets import load_classification_dataset
 from libraries.util.domains import BREAST_CANCER_DOMAIN
 from lightgbm import LGBMClassifier
-from onnxmltools.convert.lightgbm.operator_converters.LightGbm import convert_lightgbm
+from onnxmltools.convert.lightgbm.operator_converters.LightGbm import (
+    convert_lightgbm
+)
 from onnxruntime import InferenceSession
 from skl2onnx import convert_sklearn, update_registered_converter
 from skl2onnx.common.data_types import FloatTensorType
-from skl2onnx.common.shape_calculator import calculate_linear_classifier_output_shapes
+from skl2onnx.common.shape_calculator import (
+    calculate_linear_classifier_output_shapes
+)
 from sklearn.metrics import mean_squared_error
 
 from modelstore.model_store import ModelStore
@@ -30,11 +34,11 @@
 def _train_example_model() -> onnx.ModelProto:
     X_train, X_test, y_train, y_test = load_classification_dataset()
 
-    print(f"🔍  Training a light gbm classifier")
+    print("🔍  Training a light gbm classifier")
     clf = LGBMClassifier(random_state=12)
     clf.fit(X_train, y_train)
 
-    print(f"🔍  Converting the model to onnx")
+    print("🔍  Converting the model to onnx")
     update_registered_converter(
         LGBMClassifier,
         "LightGbmLGBMClassifier",
@@ -50,7 +54,7 @@ def _train_example_model() -> onnx.ModelProto:
         target_opset={"": 12, "ai.onnx.ml": 2},
     )
 
-    print(f"🔍  Loading the onnx model as an inference session")
+    print("🔍  Loading the onnx model as an inference session")
     sess = InferenceSession(model.SerializeToString())
     y_pred = sess.run(None, {"X": X_test.astype(np.float32)})[0]
 

examples/examples-by-ml-library/libraries/util/datasets.py

@@ -13,7 +13,11 @@
 #    limitations under the License.
 import pandas as pd
 import torch
-from sklearn.datasets import fetch_20newsgroups, load_breast_cancer, load_diabetes
+from sklearn.datasets import (
+    fetch_20newsgroups,
+    load_breast_cancer,
+    load_diabetes
+)
 from sklearn.model_selection import train_test_split
 
 

examples/examples-by-ml-library/main.py

@@ -37,9 +37,15 @@
     tensorflow_example,
     xgboost_booster_example,
     xgboost_example,
-    yolo_example,
+    yolo_example
+)
+from libraries.huggingface import (
+    distilbert,
+    dpt,
+    gpt2_pytorch,
+    gpt2_tensorflow,
+    sam
 )
-from libraries.huggingface import distilbert, dpt, gpt2_pytorch, gpt2_tensorflow, sam
 from modelstores import MODELSTORES, create_model_store
 
 EXAMPLES = {
@@ -93,6 +99,10 @@ def main(modelstore_in, ml_framework):
     if sys.platform == "darwin" and ml_framework == "fastai":
         print(f"⏩  Skipping {ml_framework} on darwin.")
         return
+    if ml_framework == "yolov5" and modelstore_in == "azure-container":
+        # Upload time out bug
+        print(f"⏩  Skipping {ml_framework} in {modelstore_in}.")
+        return
     print(
         f"\n🆕  Running {ml_framework} modelstore example with {modelstore_in} backend."
     )