diff --git a/dataikuapi/dss/jupyternootebook.py b/dataikuapi/dss/jupyternootebook.py new file mode 100644 index 00000000..ee6f4e95 --- /dev/null +++ b/dataikuapi/dss/jupyternootebook.py @@ -0,0 +1,136 @@ +from .discussion import DSSObjectDiscussions + +class DSSJupyterNotebook(object): + def __init__(self, client, project_key, notebook_name): + self.client = client + self.project_key = project_key + self.notebook_name = notebook_name + + def unload(self, session_id=None): + """ + Stop this Jupyter notebook and release its resources + """ + sessions = self.get_sessions() + if sessions is None: + raise Exception("Notebook isn't running") + if len(sessions) == 0: + raise Exception("Notebook isn't running") + if session_id is None: + if len(sessions) > 1: + raise Exception("Several sessions of the notebook are running, choose one") + else: + session_id = sessions[0].get('sessionId', None) + return self.client._perform_json("DELETE", + "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, session_id)) + + def get_sessions(self, as_objects=False): + """ + Get the list of running sessions of this Jupyter notebook + + :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.notebook.DSSNotebookSession` + :rtype: list of :class:`dataikuapi.dss.notebook.DSSNotebookSession` or list of dict + """ + sessions = self.client._perform_json("GET", + "/projects/%s/jupyter-notebooks/%s/sessions" % (self.project_key, self.notebook_name)) + + if as_objects: + return [DSSNotebookSession(self.client, session) for session in sessions] + else: + return sessions + + def get_content(self): + """ + Get the content of this Jupyter notebook (metadata, cells, nbformat) + """ + raw_content = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) + return DSSNotebookContent(self.client, self.project_key, self.notebook_name, raw_content) + + def delete(self): + """ + Delete this Jupyter notebook and stop all of its active sessions. + """ + return self.client._perform_json("DELETE", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) + + ######################################################## + # Discussions + ######################################################## + def get_object_discussions(self): + """ + Get a handle to manage discussions on the notebook + + :returns: the handle to manage discussions + :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` + """ + return DSSObjectDiscussions(self.client, self.project_key, "JUPYTER_NOTEBOOK", self.notebook_name) + +class DSSNotebookContent(object): + """ + Content of a Jupyter Notebook. Do not create this directly, use :meth:`DSSJupyterNotebook.get_content` + """ + + """ + A Python/R/Scala notebook on the DSS instance + """ + def __init__(self, client, project_key, notebook_name, content): + self.client = client + self.project_key = project_key + self.notebook_name = notebook_name + self.content = content + + def get_raw(self): + """ + Get the content of this Jupyter notebook (metadata, cells, nbformat) + :rtype: a dict containing the full content of a notebook + """ + return self.content + + def get_metadata(self): + """ + Get the metadata associated to this Jupyter notebook + :rtype: dict with metadata + """ + return self.content["metadata"] + + def get_cells(self): + """ + Get the cells associated to this Jupyter notebook + :rtype: list of cells + """ + return self.content["cells"] + + def save(self): + """ + Save the content of this Jupyter notebook + """ + return self.client._perform_json("PUT", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name), + body=self.content) + +class DSSNotebookSession(object): + """ + Metadata associated to the session of a Jupyter Notebook. Do not create this directly, use :meth:`DSSJupyterNotebook.get_sessions()` + """ + + def __init__(self, client, session): + self.client = client + self.project_key = session.get("projectKey") + self.notebook_name = session.get("notebookName") + self.session_creator = session.get("sessionCreator") + self.session_creator_display_name = session.get("sessionCreatorDisplayName") + self.session_unix_owner = session.get("sessionUnixOwner") + self.session_id = session.get("sessionId") + self.kernel_id = session.get("kernelId") + self.kernel_pid = session.get("kernelPid") + self.kernel_connections = session.get("kernelConnections") + self.kernel_last_activity_time = session.get("kernelLastActivityTime") + self.kernel_execution_state = session.get("kernelExecutionState") + self.session_start_time = session.get("sessionStartTime") + + + def unload(self): + """ + Stop this Jupyter notebook and release its resources + """ + return self.client._perform_json("DELETE", + "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, self.session_id)) diff --git a/dataikuapi/dss/notebook.py b/dataikuapi/dss/notebook.py index 317531ba..ad435039 100644 --- a/dataikuapi/dss/notebook.py +++ b/dataikuapi/dss/notebook.py @@ -2,90 +2,63 @@ class DSSNotebook(object): """ + Deprecated. Use DSSJupyterNotebook A Python/R/Scala notebook on the DSS instance """ - def __init__(self, client, project_key, notebook_name, state=None, content=None): - self.client = client - self.project_key = project_key - self.notebook_name = notebook_name - self.state = state - self.content = content - self.state_is_peek = True + def __init__(self, client, project_key, notebook_name, state=None): + self.client = client + self.project_key = project_key + self.notebook_name = notebook_name + self.state = state + self.state_is_peek = True def unload(self, session_id=None): """ + Deprecated. Use DSSJupyterNotebook Stop this Jupyter notebook and release its resources """ - sessions = self.get_sessions() - if sessions is None: + state = self.get_state() + if state is None: raise Exception("Notebook isn't running") - if len(sessions) == 0: + if state.get('activeSessions', None) is None: + raise Exception("Notebook isn't running") + if len(state['activeSessions']) == 0: raise Exception("Notebook isn't running") if session_id is None: - if len(sessions) > 1: + if len(state['activeSessions']) > 1: raise Exception("Several sessions of the notebook are running, choose one") else: - session_id = sessions[0].get('sessionId', None) - return self.client._perform_json("DELETE", - "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, session_id)) + session_id = state['activeSessions'][0].get('sessionId', None) + return self.client._perform_json("DELETE", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name, 'sessionId' : session_id}) - def get_state(self, refresh=False): - """ - Get the status of this Jupyter notebook - :param bool refresh: if True, get the status of the notebook from the backend + def get_state(self): + """ + Deprecated. Use DSSJupyterNotebook + Get the metadata associated to this Jupyter notebook """ - notebook_states = self.client._perform_json("GET", - "/projects/%s/jupyter-notebooks/" % self.project_key, - params={"active": False}) - if self.state is None or refresh: - for notebook_state in notebook_states: - if notebook_state.get("name") == self.notebook_name: - self.state = notebook_state - break + if self.state is None: + self.state = self.client._perform_json("GET", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name}) return self.state def get_sessions(self): """ + Deprecated. Use DSSJupyterNotebook Get the list of running sessions of this Jupyter notebook """ - - if self.state is None: - self.state = {} - sessions = self.client._perform_json("GET", - "/projects/%s/jupyter-notebooks/%s/sessions" % (self.project_key, self.notebook_name)) - self.state["activeSessions"] = sessions - return sessions - - def get_content(self): - """ - Get the content of this Jupyter notebook (metadata, cells, nbformat) - """ - if self.content is None: - self.content = self.client._perform_json("GET", - "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) - return self.content - - def save(self): - """ - Save the content of this Jupyter notebook - """ - return self.client._perform_json("PUT", - "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name), - body=self.content) - - def delete(self): - """ - Delete this Jupyter notebook and stop all of its active sessions. - """ - return self.client._perform_json("DELETE", - "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) + state = self.get_state() + if state is None: + raise Exception("Notebook isn't running") + if state.get('activeSessions', None) is None: + raise Exception("Notebook isn't running") + return state['activeSessions'] ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ + Deprecated. Use DSSJupyterNotebook Get a handle to manage discussions on the notebook :returns: the handle to manage discussions diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 3c74d31b..efbcf3a6 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -1,5 +1,7 @@ import time, warnings, sys, os.path as osp from .dataset import DSSDataset, DSSDatasetListItem, DSSManagedDatasetCreationHelper +from .jupyternootebook import DSSJupyterNotebook, DSSNotebookContent +from .notebook import DSSNotebook from .streaming_endpoint import DSSStreamingEndpoint, DSSStreamingEndpointListItem, DSSManagedStreamingEndpointCreationHelper from .recipe import DSSRecipeListItem, DSSRecipe from . import recipe @@ -11,7 +13,6 @@ from .continuousactivity import DSSContinuousActivity from .apiservice import DSSAPIService from .future import DSSFuture -from .notebook import DSSNotebook from .macro import DSSMacro from .wiki import DSSWiki from .discussion import DSSObjectDiscussions @@ -19,7 +20,6 @@ from .analysis import DSSAnalysis from .flow import DSSProjectFlow from .app import DSSAppManifest -from ..utils import DataikuException class DSSProject(object): @@ -193,8 +193,9 @@ def duplicate(self, target_project_key, if target_project_folder is not None: obj["targetProjectFolderId"] = target_project_folder.project_folder_id - ref = self.client._perform_json("POST", "/projects/%s/duplicate/" % self.project_key, body = obj) - return ref + return self.client._perform_json( + "POST", "/projects/%s/duplicate/" % self.project_key, body=obj + ) ######################################################## # Project infos @@ -263,9 +264,9 @@ def list_datasets(self, as_type="listitems"): :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/datasets/" % self.project_key) - if as_type == "listitems" or as_type == "listitem": + if as_type in ["listitems", "listitem"]: return [DSSDatasetListItem(self.client, item) for item in items] - elif as_type == "objects" or as_type == "object": + elif as_type in ["objects", "object"]: return [DSSDataset(self.client, self.project_key, item["name"]) for item in items] else: raise ValueError("Unknown as_type") @@ -403,9 +404,9 @@ def list_streaming_endpoints(self, as_type="listitems"): :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/streamingendpoints/" % self.project_key) - if as_type == "listitems" or as_type == "listitem": + if as_type in ["listitems", "listitem"]: return [DSSStreamingEndpointListItem(self.client, item) for item in items] - elif as_type == "objects" or as_type == "object": + elif as_type in ["objects", "object"]: return [DSSStreamingEndpoint(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") @@ -715,7 +716,7 @@ def list_model_evaluation_stores(self, as_type=None): :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/" % self.project_key) - if as_type == "objects" or as_type == "object": + if as_type in ["objects", "object"]: return [DSSModelEvaluationStore(self.client, self.project_key, item["id"]) for item in items] else: return items @@ -841,12 +842,11 @@ def list_jupyter_notebooks(self, as_objects=True, active=False): :returns: The list of the notebooks - see as_objects for more information :rtype: list """ - notebooks = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, - params={"active": active}) + notebook_names = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": active}) if as_objects: - return [DSSNotebook(self.client, notebook_state['projectKey'], notebook_state['name'], state=notebook_state) for notebook_state in notebooks] + return [DSSJupyterNotebook(self.client, self.project_key, notebook_name) for notebook_name in notebook_names] else: - return notebooks + return notebook_names def get_jupyter_notebook(self, notebook_name): """ @@ -856,9 +856,7 @@ def get_jupyter_notebook(self, notebook_name): :returns: A handle to interact with this jupyter notebook :rtype: :class:`~dataikuapi.dss.notebook.DSSNotebook` jupyter notebook handle """ - notebook_content = self.client._perform_json("GET", - "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name)) - return DSSNotebook(self.client, self.project_key, notebook_name, content=notebook_content) + return DSSJupyterNotebook(self.client, self.project_key, notebook_name) def create_jupyter_notebook(self, notebook_name, notebook_content): """ @@ -871,10 +869,8 @@ def create_jupyter_notebook(self, notebook_name, notebook_content): :returns: A handle to interact with the newly created jupyter notebook :rtype: :class:`~dataikuapi.dss.notebook.DSSNotebook` jupyter notebook handle """ - created_notebook_content = self.client._perform_json("POST", - "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), - body=notebook_content) - return DSSNotebook(self.client, self.project_key, notebook_name, content=created_notebook_content) + self.client._perform_json("POST", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), body=notebook_content) + return self.get_jupyter_notebook(notebook_name) ######################################################## # Continuous activities @@ -926,9 +922,9 @@ def set_variables(self, obj): @param dict obj: must be a modified version of the object returned by get_variables """ - if not "standard" in obj: + if "standard" not in obj: raise ValueError("Missing 'standard' key in argument") - if not "local" in obj: + if "local" not in obj: raise ValueError("Missing 'local' key in argument") self.client._perform_empty( @@ -1145,9 +1141,9 @@ def list_recipes(self, as_type="listitems"): :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/recipes/" % self.project_key) - if as_type == "listitems" or as_type == "listitem": + if as_type in ["listitems", "listitem"]: return [DSSRecipeListItem(self.client, item) for item in items] - elif as_type == "objects" or as_type == "object": + elif as_type in ["objects", "object"]: return [DSSRecipe(self.client, self.project_key, item["name"]) for item in items] else: raise ValueError("Unknown as_type") @@ -1233,7 +1229,7 @@ def new_recipe(self, type, name=None): return recipe.SamplingRecipeCreator(name, self) elif type == "split": return recipe.SplitRecipeCreator(name, self) - elif type == "prepare" or type == "shaker": + elif type in ["prepare", "shaker"]: return recipe.PrepareRecipeCreator(name, self) elif type == "prediction_scoring": return recipe.PredictionScoringRecipeCreator(name, self) @@ -1273,19 +1269,20 @@ def sync_datasets_acls(self): ######################################################## # Notebooks ######################################################## - + def list_running_notebooks(self, as_objects=True): """ + Deprecated. Use :meth:`DSSProject.list_jupyter_notebooks` List the currently-running notebooks Returns: list of notebooks. Each object contains at least a 'name' field """ - list = self.client._perform_json("GET", "/projects/%s/notebooks/active" % self.project_key) + notebook_list = self.client._perform_json("GET", "/projects/%s/notebooks/active" % self.project_key) if as_objects: - return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], notebook) for notebook in list] + return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list] else: - return list + return notebook_list ######################################################## # Tags @@ -1591,11 +1588,9 @@ def add_exposed_object(self, object_type, object_id, target_project): found_eo = {"type" : object_type, "localName" : object_id, "rules" : []} self.settings["exposedObjects"]["objects"].append(found_eo) - already_exists = False - for rule in found_eo["rules"]: - if rule["targetProject"] == target_project: - already_exists = True - break + already_exists = any( + rule["targetProject"] == target_project for rule in found_eo["rules"] + ) if not already_exists: found_eo["rules"].append({"targetProject": target_project}) diff --git a/dataikuapi/dssclient.py b/dataikuapi/dssclient.py index 8a5cb2f8..8296ecf1 100644 --- a/dataikuapi/dssclient.py +++ b/dataikuapi/dssclient.py @@ -1,8 +1,12 @@ import json +import warnings + from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth +from dataikuapi.dss.jupyternootebook import DSSJupyterNotebook +from dataikuapi.dss.notebook import DSSNotebook from .dss.future import DSSFuture from .dss.projectfolder import DSSProjectFolder from .dss.project import DSSProject @@ -11,7 +15,6 @@ from .dss.admin import DSSUser, DSSOwnUser, DSSGroup, DSSConnection, DSSGeneralSettings, DSSCodeEnv, DSSGlobalApiKey, DSSCluster from .dss.meaning import DSSMeaning from .dss.sqlquery import DSSSQLQuery -from .dss.notebook import DSSNotebook from .dss.discussion import DSSObjectDiscussions from .dss.apideployer import DSSAPIDeployer from .dss.projectdeployer import DSSProjectDeployer @@ -90,9 +93,32 @@ def get_future(self, job_id): ######################################################## # Notebooks ######################################################## - + + + def list_jupyter_notebooks(self, active=False, as_objects=True): + """ + List the Jupyter notebooks for every projects + + :param boolean active: if True, only the active notebooks + :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.notebook.DSSJupyterNotebook` + + :return: list of notebooks. if as_objects is True, each entry in the list is a :class:`dataikuapi.dss.notebook.DSSJupyterNotebook`. Else, each item in the list is a dict which contains at least a "name" field. + :rtype: list of :class:`dataikuapi.dss.notebook.DSSJupyterNotebook` or list of dict + """ + project_keys = self.list_project_keys() + output = [] + for project_key in project_keys: + notebook_names = self._perform_json("GET", "/projects/%s/jupyter-notebooks/" % project_key, params={"active": active}) + if as_objects: + output += [DSSJupyterNotebook(self, project_key, notebook_name) for notebook_name in notebook_names] + else: + for notebook_name in notebook_names: + output.append({u"projectKey": project_key, u"name": notebook_name}) + return output + def list_running_notebooks(self, as_objects=True): """ + Deprecated. Use :meth:`DSSClient.list_jupyter_notebooks` List the currently-running Jupyter notebooks :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.notebook.DSSNotebook` @@ -100,11 +126,12 @@ def list_running_notebooks(self, as_objects=True): :return: list of notebooks. if as_objects is True, each entry in the list is a :class:`dataikuapi.dss.notebook.DSSNotebook`. Else, each item in the list is a dict which contains at least a "name" field. :rtype: list of :class:`dataikuapi.dss.notebook.DSSNotebook` or list of dict """ - list = self._perform_json("GET", "/admin/notebooks/") + warnings.warn("Use DSSClient.list_jupyter_notebooks", DeprecationWarning) + notebook_list = self._perform_json("GET", "/admin/notebooks/") if as_objects: - return [DSSNotebook(self, notebook['projectKey'], notebook['name'], notebook) for notebook in list] + return [DSSNotebook(self, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list] else: - return list + return notebook_list ######################################################## # Project folders @@ -564,10 +591,12 @@ def create_cluster(self, cluster_name, cluster_type='manual', params=None): :returns: A :class:`dataikuapi.dss.admin.DSSCluster` cluster handle """ - definition = {} - definition['name'] = cluster_name - definition['type'] = cluster_type - definition['params'] = params if params is not None else {} + definition = { + 'name': cluster_name, + 'type': cluster_type, + 'params': params if params is not None else {}, + } + resp = self._perform_json( "POST", "/admin/clusters/", body=definition) if resp is None: @@ -678,7 +707,7 @@ def create_meaning(self, id, label, type, description=None, :returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ def make_entry(v): - if isinstance(v, str) or isinstance(v, unicode): + if isinstance(v, (str, unicode)): return {'value':v} else: return v