From cf45b4c5a63e0ffd32bd2502b255d230ad8a5489 Mon Sep 17 00:00:00 2001 From: Ben Batha Date: Tue, 12 Apr 2022 11:18:27 -0400 Subject: [PATCH 1/3] feat: retire gradient v1 and v2 Retire Jobs, Experiments, Deployments V2 and Tensorboards. These features have been supplanted by workflows and deployments v3. BREAKING CHANGE: Removed clients for retired products --- Makefile | 1 - gradient/api_sdk/__init__.py | 2 - gradient/api_sdk/clients/__init__.py | 5 - gradient/api_sdk/clients/deployment_client.py | 369 --- gradient/api_sdk/clients/experiment_client.py | 894 ------- .../api_sdk/clients/hyperparameter_client.py | 258 -- gradient/api_sdk/clients/job_client.py | 438 ---- gradient/api_sdk/clients/sdk_client.py | 10 +- .../api_sdk/clients/tensorboards_client.py | 222 -- gradient/api_sdk/config.py | 37 +- gradient/api_sdk/graphql.py | 18 +- gradient/api_sdk/models/__init__.py | 6 - gradient/api_sdk/models/deployment.py | 107 - gradient/api_sdk/models/experiment.py | 216 -- gradient/api_sdk/models/experiment_dataset.py | 20 - gradient/api_sdk/models/hyperparameter.py | 62 - gradient/api_sdk/models/job.py | 95 - gradient/api_sdk/models/tensorboard.py | 21 - gradient/api_sdk/repositories/__init__.py | 55 - gradient/api_sdk/repositories/deployments.py | 211 -- gradient/api_sdk/repositories/experiments.py | 251 -- .../api_sdk/repositories/hyperparameter.py | 54 - gradient/api_sdk/repositories/jobs.py | 235 -- gradient/api_sdk/repositories/tensorboards.py | 101 - gradient/api_sdk/s3_downloader.py | 34 +- gradient/api_sdk/s3_uploader.py | 206 +- gradient/api_sdk/serializers/__init__.py | 6 - gradient/api_sdk/serializers/deployment.py | 71 - gradient/api_sdk/serializers/experiment.py | 147 -- .../api_sdk/serializers/experiment_dataset.py | 27 - .../api_sdk/serializers/hyperparameter.py | 42 - gradient/api_sdk/serializers/job.py | 120 - gradient/api_sdk/serializers/tensorboard.py | 59 - gradient/api_sdk/workspace.py | 115 - gradient/cli/__init__.py | 16 +- gradient/cli/deployments.py | 843 ------- gradient/cli/experiments.py | 969 -------- gradient/cli/hyperparameters.py | 254 -- gradient/cli/jobs.py | 639 ----- gradient/cli/tensorboards.py | 154 -- gradient/commands/deployments.py | 270 -- gradient/commands/experiments.py | 461 ---- gradient/commands/hyperparameters.py | 95 - gradient/commands/jobs.py | 305 --- gradient/commands/tensorboards.py | 157 -- gradient/examples/sdk_notebook_tutorial.ipynb | 276 +-- source/_static/gradient.css | 4 - source/_templates/.gitkeep | 0 source/cli_docs/gradient.cli.rst | 12 - source/conf.py | 194 -- source/gradient.api_sdk.clients.rst | 40 - source/gradient.api_sdk.models.rst | 48 - source/gradient.api_sdk.repositories.rst | 40 - source/gradient.api_sdk.rst | 8 - source/gradient.api_sdk.serializers.rst | 48 - source/gradient.cli.rst | 143 +- source/gradient.commands.rst | 40 - source/gradient.wizards.rst | 30 - source/index.rst | 22 - tests/config_files/deployments_create.yaml | 30 - tests/config_files/deployments_delete.yaml | 2 - tests/config_files/deployments_details.yaml | 2 - tests/config_files/deployments_list.yaml | 5 - tests/config_files/deployments_logs.yaml | 5 - .../config_files/deployments_metrics_get.yaml | 8 - .../deployments_metrics_stream.yaml | 6 - tests/config_files/deployments_start.yaml | 2 - tests/config_files/deployments_stop.yaml | 2 - tests/config_files/deployments_update.yaml | 39 - .../experiments_create_multi_node.yaml | 57 - ...experiments_create_multi_node_ds_objs.yaml | 50 - .../experiments_create_single_node.yaml | 25 - tests/config_files/experiments_delete.yaml | 2 - tests/config_files/experiments_details.yaml | 2 - tests/config_files/experiments_list.yaml | 9 - tests/config_files/experiments_logs.yaml | 5 - .../config_files/experiments_metrics_get.yaml | 8 - .../experiments_metrics_stream.yaml | 6 - tests/config_files/experiments_start.yaml | 3 - tests/config_files/experiments_stop.yaml | 2 - .../config_files/hyperparameters_create.yaml | 30 - .../config_files/hyperparameters_details.yaml | 2 - tests/config_files/hyperparameters_list.yaml | 1 - tests/config_files/hyperparameters_start.yaml | 2 - .../config_files/jobs_artifacts_destroy.yaml | 3 - tests/config_files/jobs_artifacts_get.yaml | 2 - tests/config_files/jobs_artifacts_list.yaml | 5 - tests/config_files/jobs_create.yaml | 29 - tests/config_files/jobs_list.yaml | 7 - tests/config_files/jobs_logs.yaml | 5 - tests/config_files/jobs_metrics_get.yaml | 8 - tests/config_files/jobs_metrics_stream.yaml | 6 - tests/config_files/run.yaml | 34 - tests/config_files/tensorboards_create.yaml | 7 - tests/config_files/tensorboards_details.yaml | 2 - tests/config_files/tensorboards_list.yaml | 1 - tests/functional/test_deployments.py | 255 -- tests/functional/test_experiments.py | 2167 ----------------- tests/functional/test_hyperparameters.py | 901 ------- tests/functional/test_jobs.py | 1210 --------- tests/functional/test_tags.py | 16 - tests/functional/test_tensorboards.py | 462 ---- tests/functional/test_uploaders.py | 155 -- tests/unit/test_archiver_class.py | 57 +- ...deployment_autoscaling_option_validator.py | 37 - tests/unit/test_experiments_commands.py | 79 - tests/unit/test_tensorboard_handler.py | 101 - tests/unit/test_workspace.py | 90 - 108 files changed, 208 insertions(+), 15319 deletions(-) delete mode 100644 gradient/api_sdk/clients/deployment_client.py delete mode 100644 gradient/api_sdk/clients/experiment_client.py delete mode 100644 gradient/api_sdk/clients/hyperparameter_client.py delete mode 100644 gradient/api_sdk/clients/job_client.py delete mode 100644 gradient/api_sdk/clients/tensorboards_client.py delete mode 100644 gradient/api_sdk/models/deployment.py delete mode 100644 gradient/api_sdk/models/experiment.py delete mode 100644 gradient/api_sdk/models/experiment_dataset.py delete mode 100644 gradient/api_sdk/models/hyperparameter.py delete mode 100644 gradient/api_sdk/models/job.py delete mode 100644 gradient/api_sdk/models/tensorboard.py delete mode 100644 gradient/api_sdk/repositories/deployments.py delete mode 100644 gradient/api_sdk/repositories/experiments.py delete mode 100644 gradient/api_sdk/repositories/hyperparameter.py delete mode 100644 gradient/api_sdk/repositories/jobs.py delete mode 100644 gradient/api_sdk/repositories/tensorboards.py delete mode 100644 gradient/api_sdk/serializers/deployment.py delete mode 100644 gradient/api_sdk/serializers/experiment.py delete mode 100644 gradient/api_sdk/serializers/experiment_dataset.py delete mode 100644 gradient/api_sdk/serializers/hyperparameter.py delete mode 100644 gradient/api_sdk/serializers/job.py delete mode 100644 gradient/api_sdk/serializers/tensorboard.py delete mode 100644 gradient/api_sdk/workspace.py delete mode 100644 gradient/cli/deployments.py delete mode 100644 gradient/cli/experiments.py delete mode 100644 gradient/cli/hyperparameters.py delete mode 100644 gradient/cli/jobs.py delete mode 100644 gradient/cli/tensorboards.py delete mode 100644 gradient/commands/deployments.py delete mode 100644 gradient/commands/experiments.py delete mode 100644 gradient/commands/hyperparameters.py delete mode 100644 gradient/commands/jobs.py delete mode 100644 gradient/commands/tensorboards.py delete mode 100644 source/_static/gradient.css delete mode 100644 source/_templates/.gitkeep delete mode 100644 source/cli_docs/gradient.cli.rst delete mode 100644 source/conf.py delete mode 100644 source/gradient.wizards.rst delete mode 100644 source/index.rst delete mode 100644 tests/config_files/deployments_create.yaml delete mode 100644 tests/config_files/deployments_delete.yaml delete mode 100644 tests/config_files/deployments_details.yaml delete mode 100644 tests/config_files/deployments_list.yaml delete mode 100644 tests/config_files/deployments_logs.yaml delete mode 100644 tests/config_files/deployments_metrics_get.yaml delete mode 100644 tests/config_files/deployments_metrics_stream.yaml delete mode 100644 tests/config_files/deployments_start.yaml delete mode 100644 tests/config_files/deployments_stop.yaml delete mode 100644 tests/config_files/deployments_update.yaml delete mode 100644 tests/config_files/experiments_create_multi_node.yaml delete mode 100644 tests/config_files/experiments_create_multi_node_ds_objs.yaml delete mode 100644 tests/config_files/experiments_create_single_node.yaml delete mode 100644 tests/config_files/experiments_delete.yaml delete mode 100644 tests/config_files/experiments_details.yaml delete mode 100644 tests/config_files/experiments_list.yaml delete mode 100644 tests/config_files/experiments_logs.yaml delete mode 100644 tests/config_files/experiments_metrics_get.yaml delete mode 100644 tests/config_files/experiments_metrics_stream.yaml delete mode 100644 tests/config_files/experiments_start.yaml delete mode 100644 tests/config_files/experiments_stop.yaml delete mode 100644 tests/config_files/hyperparameters_create.yaml delete mode 100644 tests/config_files/hyperparameters_details.yaml delete mode 100644 tests/config_files/hyperparameters_list.yaml delete mode 100644 tests/config_files/hyperparameters_start.yaml delete mode 100644 tests/config_files/jobs_artifacts_destroy.yaml delete mode 100644 tests/config_files/jobs_artifacts_get.yaml delete mode 100644 tests/config_files/jobs_artifacts_list.yaml delete mode 100644 tests/config_files/jobs_create.yaml delete mode 100644 tests/config_files/jobs_list.yaml delete mode 100644 tests/config_files/jobs_logs.yaml delete mode 100644 tests/config_files/jobs_metrics_get.yaml delete mode 100644 tests/config_files/jobs_metrics_stream.yaml delete mode 100644 tests/config_files/run.yaml delete mode 100644 tests/config_files/tensorboards_create.yaml delete mode 100644 tests/config_files/tensorboards_details.yaml delete mode 100644 tests/config_files/tensorboards_list.yaml delete mode 100644 tests/functional/test_deployments.py delete mode 100644 tests/functional/test_experiments.py delete mode 100644 tests/functional/test_hyperparameters.py delete mode 100644 tests/functional/test_jobs.py delete mode 100644 tests/functional/test_tensorboards.py delete mode 100644 tests/functional/test_uploaders.py delete mode 100644 tests/unit/test_deployment_autoscaling_option_validator.py delete mode 100644 tests/unit/test_experiments_commands.py delete mode 100644 tests/unit/test_tensorboard_handler.py delete mode 100644 tests/unit/test_workspace.py diff --git a/Makefile b/Makefile index 648b3eb0..b6b84f9b 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,6 @@ clean: prepare-docs-source: @sphinx-apidoc -f -o source gradient - @cp source/cli_docs/gradient.cli.rst source/gradient.cli.rst gh-pages: prepare-docs-source @make html diff --git a/gradient/api_sdk/__init__.py b/gradient/api_sdk/__init__.py index d4519119..e6ad62c2 100644 --- a/gradient/api_sdk/__init__.py +++ b/gradient/api_sdk/__init__.py @@ -2,7 +2,5 @@ from .constants import * from .models import * from .repositories import * -from .s3_downloader import JobArtifactsDownloader -from .s3_uploader import ExperimentFileUploader, ExperimentWorkspaceDirectoryUploader from .archivers import ZipArchiver from .sdk_exceptions import * diff --git a/gradient/api_sdk/clients/__init__.py b/gradient/api_sdk/clients/__init__.py index c7dd4be3..50761823 100644 --- a/gradient/api_sdk/clients/__init__.py +++ b/gradient/api_sdk/clients/__init__.py @@ -2,10 +2,6 @@ from .dataset_client import DatasetsClient from .dataset_tag_client import DatasetTagsClient from .dataset_version_client import DatasetVersionsClient -from .deployment_client import DeploymentsClient -from .experiment_client import ExperimentsClient -from .hyperparameter_client import HyperparameterJobsClient -from .job_client import JobsClient from .machine_types_client import MachineTypesClient from .machines_client import MachinesClient from .model_client import ModelsClient @@ -13,7 +9,6 @@ from .project_client import ProjectsClient from .secret_client import SecretsClient from .storage_provider_client import StorageProvidersClient -from .tensorboards_client import TensorboardClient from .workflow_client import WorkflowsClient from .sdk_client import SdkClient diff --git a/gradient/api_sdk/clients/deployment_client.py b/gradient/api_sdk/clients/deployment_client.py deleted file mode 100644 index 33250ea2..00000000 --- a/gradient/api_sdk/clients/deployment_client.py +++ /dev/null @@ -1,369 +0,0 @@ -""" -Deployment related client handler logic. - -Remember that in code snippets all highlighted lines are required other lines are optional. -""" -from .base_client import BaseClient, TagsSupportMixin -from .. import config, models, repositories - - -class DeploymentsClient(TagsSupportMixin, BaseClient): - """ - Client to handle deployment related actions. - - How to create instance of deployment client: - - .. code-block:: python - :linenos: - :emphasize-lines: 4 - - from gradient import DeploymentsClient - - deployment_client = DeploymentsClient( - api_key='your_api_key_here' - ) - """ - HOST_URL = config.config.CONFIG_HOST - entity = "deployment" - - def create( - self, - deployment_type, - name, - machine_type, - image_url, - instance_count=None, - model_id=None, - container_model_path=None, - image_username=None, - image_password=None, - image_server=None, - container_url_path=None, - endpoint_url_path=None, - method=None, - docker_args=None, - env=None, - api_type=None, - ports=None, - cluster_id=None, - auth_username=None, - auth_password=None, - tags=None, - command=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - project_id=None, - autoscaling=None, - ): - """ - Method to create a Deployment instance. - - To create a new Deployment, you must first create a Model. With a Model available, use the ``create`` subcommand - and specify all of the following parameters: deployment type, base image, name, machine type, and container - image for serving, as well as the instance count: - - .. code-block:: python - :linenos: - :emphasize-lines: 4 - - from gradient import DeploymentsClient - - deployment_client = DeploymentsClient( - api_key='your_api_key_here' - ) - - - To obtain your Model ID, you can run ``command gradient models list`` and copy the target Model ID from - your available Models. - - :param str deployment_type: Model deployment type. Only TensorFlow Model deployment type is currently supported [required] - :param str model_id: ID of a trained model [required] - :param str name: Human-friendly name for new model deployment [required] - :param str machine_type: [G1|G6|G12|K80|P100|GV100] Type of machine for new deployment [required] - :param str image_url: Docker image for model deployment [required] - :param int instance_count: Number of machine instances [required] - :param str container_model_path: Container model path - :param str image_username: Username used to access docker image - :param str image_password: Password used to access docker image - :param str image_server: Docker image server - :param str container_url_path: Container URL path - :param str endpoint_url_path: Endpoint URL path - :param str method: Method - :param list[str]|tuple[str] docker_args: List of docker args - :param dict[str,str] env: Environmental variables - :param str api_type: Type of API (REST/GRPC) - :param str ports: Ports - :param str cluster_id: cluster ID - :param str auth_username: Username - :param str auth_password: Password - :param list[str] tags: List of tags - :param str command: Deployment command - :param str workspace_url: Project git or s3repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param str project_id: Project ID - :param models.AutoscalingDefinition autoscaling: Deployment autoscaling definition - - :returns: Created deployment id - :rtype: str - """ - deployment = models.Deployment( - deployment_type=deployment_type, - model_id=model_id, - name=name, - machine_type=machine_type, - image_url=image_url, - instance_count=instance_count, - container_model_path=container_model_path, - image_username=image_username, - image_password=image_password, - image_server=image_server, - container_url_path=container_url_path, - endpoint_url_path=endpoint_url_path, - method=method, - docker_args=docker_args, - env=env, - api_type=api_type, - ports=ports, - cluster_id=cluster_id, - auth_username=auth_username, - auth_password=auth_password, - command=command, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - project_id=project_id, - autoscaling=autoscaling, - ) - - repository = self.build_repository(repositories.CreateDeployment) - deployment_id = repository.create(deployment) - if tags: - self.add_tags(entity_id=deployment_id, tags=tags) - return deployment_id - - def get(self, deployment_id): - """Get deployment instance - - :param str deployment_id: Deployment ID - - :return: Deployment instance - :rtype: models.Deployment - """ - repository = self.build_repository(repositories.GetDeployment) - deployment = repository.get(deployment_id=deployment_id) - return deployment - - def start(self, deployment_id): - """ - Start deployment - - *EXAMPLE*:: - - gradient deployments start --id - - :param str deployment_id: Deployment ID - """ - - repository = self.build_repository(repositories.StartDeployment) - repository.start(deployment_id) - - def stop(self, deployment_id): - """ - Stop deployment - - *EXAMPLE*:: - - gradient deployments stop --id - - :param deployment_id: Deployment ID - """ - - repository = self.build_repository(repositories.StopDeployment) - repository.stop(deployment_id) - - def list(self, state=None, project_id=None, model_id=None, tags=None): - """ - List deployments with optional filtering - - :param str state: state to filter deployments - :param str project_id: project ID to filter deployments - :param str model_id: model ID to filter deployments - :param list[str]|tuple[str] tags: tags to filter deployments with OR - - :returns: List of Deployment model instances - :rtype: list[models.Deployment] - """ - - repository = self.build_repository(repositories.ListDeployments) - deployments = repository.list(state=state, project_id=project_id, model_id=model_id, tags=tags) - return deployments - - def delete(self, deployment_id): - repository = self.build_repository(repositories.DeleteDeployment) - repository.delete(deployment_id) - - def update( - self, - deployment_id, - deployment_type=None, - model_id=None, - name=None, - machine_type=None, - image_url=None, - instance_count=None, - container_model_path=None, - image_username=None, - image_password=None, - image_server=None, - container_url_path=None, - endpoint_url_path=None, - method=None, - docker_args=None, - env=None, - api_type=None, - ports=None, - cluster_id=None, - auth_username=None, - auth_password=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - project_id=None, - command=None, - autoscaling=None, - ): - deployment = models.Deployment( - deployment_type=deployment_type, - model_id=model_id, - name=name, - machine_type=machine_type, - image_url=image_url, - instance_count=instance_count, - container_model_path=container_model_path, - image_username=image_username, - image_password=image_password, - image_server=image_server, - container_url_path=container_url_path, - endpoint_url_path=endpoint_url_path, - method=method, - docker_args=docker_args, - env=env, - api_type=api_type, - ports=ports, - cluster_id=cluster_id, - auth_username=auth_username, - auth_password=auth_password, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - project_id=project_id, - command=command, - autoscaling=autoscaling, - ) - - repository = self.build_repository(repositories.UpdateDeployment) - repository.update(deployment_id, deployment) - - def get_metrics(self, deployment_id, start=None, end=None, interval="30s", built_in_metrics=None): - """Get model deployment metrics - - :param str deployment_id: ID of deployment - :param datetime.datetime|str start: - :param datetime.datetime|str end: - :param str interval: - :param list[str] built_in_metrics: List of metrics to get if different than default - Available builtin metrics: cpuPercentage, memoryUsage, gpuMemoryFree, gpuMemoryUsed, gpuPowerDraw, - gpuTemp, gpuUtilization, gpuMemoryUtilization - - :returns: Metrics of a model deployment job - :rtype: dict[str,dict[str,list[dict]]] - """ - - repository = self.build_repository(repositories.GetDeploymentMetrics) - metrics = repository.get( - id=deployment_id, - start=start, - end=end, - interval=interval, - built_in_metrics=built_in_metrics, - ) - return metrics - - - def list_metrics(self, deployment_id, start=None, end=None, interval="30s"): - """List model deployment metrics - - :param str deployment_id: ID of deployment - :param datetime.datetime|str start: - :param datetime.datetime|str end: - :param str interval: - :returns: Metrics of a model deployment job - :rtype: dict[str,dict[str,list[dict]]] - """ - - repository = self.build_repository(repositories.ListDeploymentMetrics) - metrics = repository.get( - id=deployment_id, - start=start, - end=end, - interval=interval, - ) - return metrics - - def stream_metrics(self, deployment_id, interval="30s", built_in_metrics=None): - """Stream live model deployment metrics - - :param str deployment_id: ID of model deployment - :param str interval: - :param list[str] built_in_metrics: List of metrics to get if different than default - Available builtin metrics: cpuPercentage, memoryUsage, gpuMemoryFree, gpuMemoryUsed, gpuPowerDraw, - gpuTemp, gpuUtilization, gpuMemoryUtilization - - :returns: Generator object yielding live model deployment metrics - :rtype: Iterable[dict] - """ - - repository = self.build_repository(repositories.StreamDeploymentMetrics) - metrics = repository.stream( - id=deployment_id, - interval=interval, - built_in_metrics=built_in_metrics, - ) - return metrics - - def logs(self, deployment_id, line=1, limit=10000): - """Show list of latest logs from the specified deployment. - - :param str deployment_id: Deployment Id - :param int line: line number at which logs starts to display on screen - :param int limit: maximum lines displayed on screen, default set to 10 000 - - :returns: list of LogRows - :rtype: list[models.LogRow] - """ - - repository = self.build_repository(repositories.ListDeploymentLogs) - logs = repository.list(id=deployment_id, line=line, limit=limit) - return logs - - def yield_logs(self, deployment_id, line=1, limit=10000): - """Get log generator. Polls the API for new logs - - :param str deployment_id: Deployment Id - :param int line: line number at which logs starts to display on screen - :param int limit: maximum lines displayed on screen, default set to 10 000 - - :returns: generator yielding LogRow instances - :rtype: Iterator[models.LogRow] - """ - - repository = self.build_repository(repositories.ListDeploymentLogs) - logs = repository.yield_logs(id=deployment_id, line=line, limit=limit) - return logs diff --git a/gradient/api_sdk/clients/experiment_client.py b/gradient/api_sdk/clients/experiment_client.py deleted file mode 100644 index 29c6ff0a..00000000 --- a/gradient/api_sdk/clients/experiment_client.py +++ /dev/null @@ -1,894 +0,0 @@ -import datetime - -from .base_client import BaseClient, TagsSupportMixin -from .. import repositories, models, constants, utils -from ..sdk_exceptions import ResourceCreatingDataError, InvalidParametersError -from ..validation_messages import EXPERIMENT_MODEL_PATH_VALIDATION_ERROR - - -class ExperimentsClient(TagsSupportMixin, utils.ExperimentsClientHelpersMixin, BaseClient): - entity = "experiment" - - def create_single_node( - self, - project_id, - machine_type, - command, - ports=None, - name=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - datasets=None, - working_directory=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - model_type=None, - model_path=None, - is_preemptible=False, - container=None, - container_user=None, - registry_username=None, - registry_password=None, - registry_url=None, - tags=None, - ): - """ - Create single node experiment - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str machine_type: Machine type [required] - :param str command: Container entrypoint command [required] - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param dict|list[dict]|tuple[dict] datasets: Dict or list of dicts describing dataset(s) used in experiment. - Required keys: "id" or "url" - Optional keys: "tag" for S3 tag and "auth" for S3 token - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables in a JSON - :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str container: Container (dockerfile) [required] - :param str container_user: Container user for running the specified command in the container. If no containerUser is specified, the user will default to 'root' in the container. - :param str registry_username: Registry username for accessing private docker registry container if necessary - :param str registry_password: Registry password for accessing private docker registry container if necessary - :param str registry_url: Registry server URL for accessing private docker registry container if necessary - :param list[str] tags: List of tags - - :returns: experiment handle - :rtype: str - """ - self._validate_arguments(model_type=model_type, model_path=model_path) - - if not is_preemptible: - is_preemptible = None - - datasets = self._dataset_dicts_to_instances(datasets) - - experiment = models.SingleNodeExperiment( - experiment_type_id=constants.ExperimentType.SINGLE_NODE, - name=name, - project_id=project_id, - machine_type=machine_type, - ports=ports, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - datasets=datasets, - working_directory=working_directory, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - model_type=model_type, - model_path=model_path, - is_preemptible=is_preemptible, - container=container, - command=command, - container_user=container_user, - registry_username=registry_username, - registry_password=registry_password, - registry_url=registry_url, - ) - - repository = self.build_repository(repositories.CreateSingleNodeExperiment) - handle = repository.create(experiment) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - - return handle - - def create_multi_node( - self, - project_id, - worker_container, - worker_machine_type, - worker_command, - worker_count, - parameter_server_container, - parameter_server_machine_type, - parameter_server_command, - parameter_server_count, - name=None, - experiment_type_id=constants.ExperimentType.GRPC_MULTI_NODE, - ports=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - datasets=None, - working_directory=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - model_type=None, - model_path=None, - is_preemptible=False, - worker_container_user=None, - worker_registry_username=None, - worker_registry_password=None, - worker_registry_url=None, - parameter_server_container_user=None, - parameter_server_registry_username=None, - parameter_server_registry_password=None, - parameter_server_registry_url=None, - tags=None, - ): - """ - Create multinode experiment - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str worker_container: Worker container (dockerfile) [required] - :param str worker_machine_type: Worker machine type [required] - :param str worker_command: Worker command [required] - :param int worker_count: Worker count [required] - :param str parameter_server_container: Parameter server container [required] - :param str parameter_server_machine_type: Parameter server machine type [required] - :param str parameter_server_command: Parameter server command [required] - :param int parameter_server_count: Parameter server count [required] - :param int|str experiment_type_id: Experiment Type ID - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param dict|list[dict]|tuple[dict] datasets: Dict or list of dicts describing dataset(s) used in experiment. - Required keys: "url" - Optional keys: "tag" for S3 tag and "auth" for S3 token - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables in a JSON - :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str worker_container_user: Worker container user - :param str worker_registry_username: Registry username for accessing private docker registry container if necessary - :param str worker_registry_password: Registry password for accessing private docker registry container if necessary - :param str worker_registry_url: Registry server URL for accessing private docker registry container if necessary - :param str parameter_server_container_user: Parameter server container user - :param str parameter_server_registry_username: Registry username for accessing private docker registry container if necessary - :param str parameter_server_registry_password: Registry password for accessing private docker registry container if necessary - :param str parameter_server_registry_url: Registry server URL for accessing private docker registry container if necessary - :param list[str] tags: List of tags - - :returns: experiment handle - :rtype: str - """ - self._validate_arguments(model_type=model_type, model_path=model_path) - - experiment_type_id = self._get_experiment_type_id(experiment_type_id) - - if not is_preemptible: - is_preemptible = None - - datasets = self._dataset_dicts_to_instances(datasets) - - experiment = models.MultiNodeExperiment( - name=name, - project_id=project_id, - experiment_type_id=experiment_type_id, - worker_container=worker_container, - worker_machine_type=worker_machine_type, - worker_command=worker_command, - worker_count=worker_count, - parameter_server_container=parameter_server_container, - parameter_server_machine_type=parameter_server_machine_type, - parameter_server_command=parameter_server_command, - parameter_server_count=parameter_server_count, - ports=ports, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - datasets=datasets, - working_directory=working_directory, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - model_type=model_type, - model_path=model_path, - is_preemptible=is_preemptible, - worker_container_user=worker_container_user, - worker_registry_username=worker_registry_username, - worker_registry_password=worker_registry_password, - worker_registry_url=worker_registry_url, - parameter_server_container_user=parameter_server_container_user, - parameter_server_registry_username=parameter_server_registry_username, - parameter_server_registry_password=parameter_server_registry_password, - parameter_server_registry_url=parameter_server_registry_url, - ) - - repository = self.build_repository(repositories.CreateMultiNodeExperiment) - handle = repository.create(experiment) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - return handle - - def create_mpi_multi_node( - self, - project_id, - worker_container, - worker_machine_type, - worker_command, - worker_count, - name=None, - master_container=None, - master_machine_type=None, - master_command=None, - master_count=None, - ports=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - datasets=None, - working_directory=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - model_type=None, - model_path=None, - is_preemptible=False, - worker_container_user=None, - worker_registry_username=None, - worker_registry_password=None, - worker_registry_url=None, - master_container_user=None, - master_registry_username=None, - master_registry_password=None, - master_registry_url=None, - tags=None, - ): - """ - Create multinode experiment using MPI - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str worker_container: Worker container (dockerfile) [required] - :param str worker_machine_type: Worker machine type [required] - :param str worker_command: Worker command [required] - :param int worker_count: Worker count [required] - :param str master_container: Master container [required] - :param str master_machine_type: Master machine type [required] - :param str master_command: Master command [required] - :param int master_count: Master count [required] - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param dict|list[dict]|tuple[dict] datasets: Dict or list of dicts describing dataset(s) used in experiment. - Required keys: "url" - Optional keys: "tag" for S3 tag and "auth" for S3 token - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables in a JSON - :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str worker_container_user: Worker container user - :param str worker_registry_username: Registry username for accessing private docker registry container if necessary - :param str worker_registry_password: Registry password for accessing private docker registry container if necessary - :param str worker_registry_url: Registry server URL for accessing private docker registry container if necessary - :param str master_container_user: Master container user - :param str master_registry_username: Registry username for accessing private docker registry container if necessary - :param str master_registry_password: Registry password for accessing private docker registry container if necessary - :param str master_registry_url: Registry server URL for accessing private docker registry container if necessary - :param list[str] tags: List of tags - - :returns: experiment handle - :rtype: str - """ - self._validate_arguments(model_type=model_type, model_path=model_path) - - if not is_preemptible: - is_preemptible = None - - datasets = self._dataset_dicts_to_instances(datasets) - experiment_type_id = constants.ExperimentType.MPI_MULTI_NODE - - experiment = models.MpiMultiNodeExperiment( - name=name, - project_id=project_id, - experiment_type_id=experiment_type_id, - worker_container=worker_container, - worker_machine_type=worker_machine_type, - worker_command=worker_command, - worker_count=worker_count, - master_container=master_container, - master_machine_type=master_machine_type, - master_command=master_command, - master_count=master_count, - ports=ports, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - datasets=datasets, - working_directory=working_directory, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - model_type=model_type, - model_path=model_path, - is_preemptible=is_preemptible, - worker_container_user=worker_container_user, - worker_registry_username=worker_registry_username, - worker_registry_password=worker_registry_password, - worker_registry_url=worker_registry_url, - master_container_user=master_container_user, - master_registry_username=master_registry_username, - master_registry_password=master_registry_password, - master_registry_url=master_registry_url, - ) - - repository = self.build_repository(repositories.CreateMpiMultiNodeExperiment) - handle = repository.create(experiment) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - return handle - - def run_single_node( - self, - project_id, - machine_type, - command, - name=None, - ports=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - datasets=None, - working_directory=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - model_type=None, - model_path=None, - is_preemptible=False, - container=None, - container_user=None, - registry_username=None, - registry_password=None, - registry_url=None, - tags=None, - ): - """Create and start single node experiment - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str machine_type: Machine type [required] - :param str command: Container entrypoint command [required] - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param dict|list[dict]|tuple[dict] datasets: Dict or list of dicts describing dataset(s) used in experiment. - Required keys: "url" - Optional keys: "tag" for S3 tag and "auth" for S3 token - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables in a JSON - :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str container: Container (dockerfile) [required] - :param str container_user: Container user for running the specified command in the container. If no containerUser is specified, the user will default to 'root' in the container. - :param str registry_username: Registry username for accessing private docker registry container if necessary - :param str registry_password: Registry password for accessing private docker registry container if necessary - :param str registry_url: Registry server URL for accessing private docker registry container if necessary - :param list[str] tags: List of tags - - :returns: experiment handle - :rtype: str - """ - - self._validate_arguments(model_type=model_type, model_path=model_path) - - if not is_preemptible: - is_preemptible = None - - datasets = self._dataset_dicts_to_instances(datasets) - - experiment = models.SingleNodeExperiment( - experiment_type_id=constants.ExperimentType.SINGLE_NODE, - name=name, - project_id=project_id, - machine_type=machine_type, - ports=ports, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - datasets=datasets, - working_directory=working_directory, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - model_type=model_type, - model_path=model_path, - is_preemptible=is_preemptible, - container=container, - command=command, - container_user=container_user, - registry_username=registry_username, - registry_password=registry_password, - registry_url=registry_url, - ) - - repository = self.build_repository(repositories.RunSingleNodeExperiment) - handle = repository.create(experiment) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - return handle - - def run_multi_node( - self, - project_id, - worker_container, - worker_machine_type, - worker_command, - worker_count, - parameter_server_container, - parameter_server_machine_type, - parameter_server_command, - parameter_server_count, - name=None, - experiment_type_id=constants.ExperimentType.GRPC_MULTI_NODE, - ports=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - datasets=None, - working_directory=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - model_type=None, - model_path=None, - is_preemptible=False, - worker_container_user=None, - worker_registry_username=None, - worker_registry_password=None, - worker_registry_url=None, - parameter_server_container_user=None, - parameter_server_registry_username=None, - parameter_server_registry_password=None, - parameter_server_registry_url=None, - tags=None, - ): - """Create and start multinode experiment - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str worker_container: Worker container (dockerfile) [required] - :param str worker_machine_type: Worker machine type [required] - :param str worker_command: Worker command [required] - :param int worker_count: Worker count [required] - :param str parameter_server_container: Parameter server container [required] - :param str parameter_server_machine_type: Parameter server machine type [required] - :param str parameter_server_command: Parameter server command [required] - :param int parameter_server_count: Parameter server count [required] - :param int|str experiment_type_id: Experiment Type ID [required] - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param dict|list[dict]|tuple[dict] datasets: Dict or list of dicts describing dataset(s) used in experiment. - Required keys: "url" - Optional keys: "tag" for S3 tag and "auth" for S3 token - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables in a JSON - :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str worker_container_user: Worker container user - :param str worker_registry_username: Registry username for accessing private docker registry container if necessary - :param str worker_registry_password: Registry password for accessing private docker registry container if necessary - :param str worker_registry_url: Registry server URL for accessing private docker registry container if necessary - :param str parameter_server_container_user: Parameter server container user - :param str parameter_server_registry_username: Registry username for accessing private docker registry container if necessary - :param str parameter_server_registry_password: Registry password for accessing private docker registry container if necessary - :param str parameter_server_registry_url: Registry server URL for accessing private docker registry container if necessary - :param list[str] tags: List of tags - - :returns: experiment handle - :rtype: str - """ - self._validate_arguments(model_type=model_type, model_path=model_path) - - experiment_type_id = self._get_experiment_type_id(experiment_type_id) - - if not is_preemptible: - is_preemptible = None - - datasets = self._dataset_dicts_to_instances(datasets) - - experiment = models.MultiNodeExperiment( - name=name, - project_id=project_id, - experiment_type_id=experiment_type_id, - worker_container=worker_container, - worker_machine_type=worker_machine_type, - worker_command=worker_command, - worker_count=worker_count, - parameter_server_container=parameter_server_container, - parameter_server_machine_type=parameter_server_machine_type, - parameter_server_command=parameter_server_command, - parameter_server_count=parameter_server_count, - ports=ports, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - datasets=datasets, - working_directory=working_directory, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - model_type=model_type, - model_path=model_path, - is_preemptible=is_preemptible, - worker_container_user=worker_container_user, - worker_registry_username=worker_registry_username, - worker_registry_password=worker_registry_password, - worker_registry_url=worker_registry_url, - parameter_server_container_user=parameter_server_container_user, - parameter_server_registry_username=parameter_server_registry_username, - parameter_server_registry_password=parameter_server_registry_password, - parameter_server_registry_url=parameter_server_registry_url, - ) - - repository = self.build_repository(repositories.RunMultiNodeExperiment) - handle = repository.create(experiment) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - - return handle - - def run_mpi_multi_node( - self, - project_id, - worker_container, - worker_machine_type, - worker_command, - worker_count, - master_container, - master_machine_type, - master_command, - master_count, - name=None, - ports=None, - workspace_url=None, - workspace_ref=None, - workspace_username=None, - workspace_password=None, - datasets=None, - working_directory=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - model_type=None, - model_path=None, - is_preemptible=False, - worker_container_user=None, - worker_registry_username=None, - worker_registry_password=None, - worker_registry_url=None, - master_container_user=None, - master_registry_username=None, - master_registry_password=None, - master_registry_url=None, - tags=None, - ): - """Create and start multinode experiment using MPI - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str worker_container: Worker container (dockerfile) [required] - :param str worker_machine_type: Worker machine type [required] - :param str worker_command: Worker command [required] - :param int worker_count: Worker count [required] - :param str master_container: Master container [required] - :param str master_machine_type: Master machine type [required] - :param str master_command: Master command [required] - :param int master_count: Master count [required] - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param dict|list[dict]|tuple[dict] datasets: Dict or list of dicts describing dataset(s) used in experiment. - Required keys: "url" - Optional keys: "tag" for S3 tag and "auth" for S3 token - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables in a JSON - :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str worker_container_user: Worker container user - :param str worker_registry_username: Registry username for accessing private docker registry container if necessary - :param str worker_registry_password: Registry password for accessing private docker registry container if necessary - :param str worker_registry_url: Registry server URL for accessing private docker registry container if necessary - :param str master_container_user: Master container user - :param str master_registry_username: Registry username for accessing private docker registry container if necessary - :param str master_registry_password: Registry password for accessing private docker registry container if necessary - :param str master_registry_url: Registry server URL for accessing private docker registry container if necessary - :param list[str] tags: List of tags - - :returns: experiment handle - :rtype: str - """ - self._validate_arguments(model_type=model_type, model_path=model_path) - - if not is_preemptible: - is_preemptible = None - - datasets = self._dataset_dicts_to_instances(datasets) - - experiment_type_id = constants.ExperimentType.MPI_MULTI_NODE - - experiment = models.MpiMultiNodeExperiment( - name=name, - project_id=project_id, - experiment_type_id=experiment_type_id, - worker_container=worker_container, - worker_machine_type=worker_machine_type, - worker_command=worker_command, - worker_count=worker_count, - master_container=master_container, - master_machine_type=master_machine_type, - master_command=master_command, - master_count=master_count, - ports=ports, - workspace_url=workspace_url, - workspace_ref=workspace_ref, - workspace_username=workspace_username, - workspace_password=workspace_password, - datasets=datasets, - working_directory=working_directory, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - model_type=model_type, - model_path=model_path, - is_preemptible=is_preemptible, - worker_container_user=worker_container_user, - worker_registry_username=worker_registry_username, - worker_registry_password=worker_registry_password, - worker_registry_url=worker_registry_url, - master_container_user=master_container_user, - master_registry_username=master_registry_username, - master_registry_password=master_registry_password, - master_registry_url=master_registry_url, - ) - - repository = self.build_repository(repositories.RunMpiMultiNodeExperiment) - handle = repository.create(experiment) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - - return handle - - def start(self, experiment_id): - """Start existing experiment that has not run - - :param str experiment_id: Experiment ID - - :raises: exceptions.GradientSdkError - """ - - repository = self.build_repository(repositories.StartExperiment) - repository.start(experiment_id) - - def stop(self, experiment_id): - """Stop running experiment - - :param str experiment_id: Experiment ID - - :raises: exceptions.GradientSdkError - """ - - repository = self.build_repository(repositories.StopExperiment) - repository.stop(experiment_id) - - def list(self, project_id=None, offset=None, limit=None, get_meta=False, tags=None): - """Get a list of experiments. Optionally filter by project ID - - :param str|list|None project_id: - :param int offset: - :param int limit: - :param bool get_meta: get dict of metadata like number of total items, etc. Setting to True changes rtype - :param list[str]|tuple[str] tags: tags to filter with OR - - :return: experiments - :rtype: list[models.SingleNodeExperiment|models.MultiNodeExperiment]|tuple[list[models.SingleNodeExperiment|models.MultiNodeExperiment],dict] - """ - - repository = self.build_repository(repositories.ListExperiments) - experiments = repository.list(project_id=project_id, limit=limit, offset=offset, get_meta=get_meta, tags=tags) - return experiments - - def get(self, experiment_id): - """Get experiment instance - - :param str experiment_id: Experiment ID - :rtype: models.SingleNodeExperiment|models.MultiNodeExperiment|MpiMultiNodeExperiment - """ - repository = self.build_repository(repositories.GetExperiment) - experiment = repository.get(experiment_id=experiment_id) - return experiment - - def logs(self, experiment_id, line=1, limit=10000): - """Show list of latest logs from the specified experiment. - - :param str experiment_id: Experiment ID - :param int line: line number at which logs starts to display on screen - :param int limit: maximum lines displayed on screen, default set to 10 000 - - :returns: list of LogRows - :rtype: list[models.LogRow] - """ - - repository = self.build_repository(repositories.ListExperimentLogs) - logs = repository.list(id=experiment_id, line=line, limit=limit) - return logs - - def yield_logs(self, experiment_id, line=1, limit=10000): - """Get log generator. Polls the API for new logs - - :param str experiment_id: - :param int line: line number at which logs starts to display on screen - :param int limit: maximum lines displayed on screen, default set to 10 000 - - :returns: generator yielding LogRow instances - :rtype: Iterator[models.LogRow] - """ - - repository = self.build_repository(repositories.ListExperimentLogs) - logs_generator = repository.yield_logs(id=experiment_id, line=line, limit=limit) - return logs_generator - - def delete(self, experiment_id): - repository = self.build_repository(repositories.DeleteExperiment) - repository.delete(experiment_id) - - def _validate_arguments(self, **kwargs): - if kwargs.get("model_path") and not kwargs.get("model_type"): - raise InvalidParametersError( - EXPERIMENT_MODEL_PATH_VALIDATION_ERROR - ) - - def get_metrics(self, experiment_id, start=None, end=None, interval="30s", built_in_metrics=None): - """Get experiment metrics - - :param str experiment_id: ID of experiment - :param datetime.datetime|str start: - :param datetime.datetime|str end: - :param str interval: - :param list[str] built_in_metrics: List of metrics to get if different than default - Available builtin metrics: cpuPercentage, memoryUsage, gpuMemoryFree, gpuMemoryUsed, gpuPowerDraw, - gpuTemp, gpuUtilization, gpuMemoryUtilization - - :returns: Metrics of and experiment - :rtype: dict[str,dict[str,list[dict]]] - """ - - repository = self.build_repository(repositories.GetExperimentMetrics) - metrics = repository.get( - id=experiment_id, - start=start, - end=end, - interval=interval, - built_in_metrics=built_in_metrics, - ) - return metrics - - def list_metrics(self, experiment_id, start=None, end=None, interval="30s"): - """List experiment metrics - - :param str experiment_id: ID of experiment - :param datetime.datetime|str start: - :param datetime.datetime|str end: - :param str interval: - :returns: Metrics of and experiment - :rtype: dict[str,dict[str,list[dict]]] - """ - - repository = self.build_repository(repositories.ListExperimentMetrics) - metrics = repository.get( - id=experiment_id, - start=start, - end=end, - interval=interval, - ) - return metrics - - - def stream_metrics(self, experiment_id, interval="30s", built_in_metrics=None): - """Stream live experiment metrics - - :param str experiment_id: ID of experiment - :param str interval: - :param list[str] built_in_metrics: List of metrics to get if different than default - Available builtin metrics: cpuPercentage, memoryUsage, gpuMemoryFree, gpuMemoryUsed, gpuPowerDraw, - gpuTemp, gpuUtilization, gpuMemoryUtilization - - :returns: Generator object yielding live experiment metrics - :rtype: Iterable[dict] - """ - - repository = self.build_repository(repositories.StreamExperimentMetrics) - metrics = repository.stream( - id=experiment_id, - interval=interval, - built_in_metrics=built_in_metrics, - ) - return metrics - - def _dataset_dicts_to_instances(self, datasets): - if not datasets: - return None - - if isinstance(datasets, dict): - datasets = [datasets] - - for ds in datasets: - if not (ds.get("id") or ds.get("uri")): - raise ResourceCreatingDataError("Error while creating experiment with dataset: " - 'id or uri key is required') - - for ds in datasets: - volume_options = ds.setdefault("volume_options", {}) - volume_options.setdefault("kind", ds.pop("volume_kind", None)) - volume_options.setdefault("size", ds.pop("volume_size", None)) - - datasets = [models.ExperimentDataset(**ds) for ds in datasets] - return datasets diff --git a/gradient/api_sdk/clients/hyperparameter_client.py b/gradient/api_sdk/clients/hyperparameter_client.py deleted file mode 100644 index fbe563ee..00000000 --- a/gradient/api_sdk/clients/hyperparameter_client.py +++ /dev/null @@ -1,258 +0,0 @@ -from . import base_client -from .base_client import TagsSupportMixin -from .. import models, repositories - - -class HyperparameterJobsClient(TagsSupportMixin, base_client.BaseClient): - entity = "experiment" - - def create( - self, - name, - project_id, - tuning_command, - worker_container, - worker_machine_type, - worker_command, - worker_count, - worker_container_user=None, - worker_registry_username=None, - worker_registry_password=None, - is_preemptible=False, - ports=None, - workspace_url=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - trigger_event_id=None, - model_type=None, - model_path=None, - dockerfile_path=None, - hyperparameter_server_registry_username=None, - hyperparameter_server_registry_password=None, - hyperparameter_server_container=None, - hyperparameter_server_container_user=None, - hyperparameter_server_machine_type=None, - working_directory=None, - use_dockerfile=False, - tags=None, - ): - """Create hyperparameter tuning job - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str tuning_command: Tuning command [required] - :param str worker_container: Worker container [required] - :param str worker_machine_type: Worker machine type [required] - :param str worker_command: Worker command [required] - :param int worker_count: Worker count [required] - :param str worker_container_user: Worker Container user - :param str worker_registry_username: Worker registry username - :param str worker_registry_password: Worker registry password - :param bool is_preemptible: Flag: is preemptible - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables (in JSON) - :param str trigger_event_id: GradientCI trigger event id - :param str model_type: Model type - :param str model_path: Model path - :param str dockerfile_path: Path to dockerfile in project - :param str hyperparameter_server_registry_username: Hyperparameter server registry username - :param str hyperparameter_server_registry_password: Hyperparameter server registry password - :param str hyperparameter_server_container: Hyperparameter server container - :param str hyperparameter_server_container_user: Hyperparameter server container user - :param str hyperparameter_server_machine_type: Hyperparameter server machine type - :param str working_directory: Working directory for the experiment - :param bool use_dockerfile: Flag: use dockerfile - :param list[str] tags: List of tags - - :returns: ID of a new job - :rtype: str - """ - - if not is_preemptible: - is_preemptible = None - - if use_dockerfile is False: - use_dockerfile = None - - hyperparameter = models.Hyperparameter( - name=name, - project_id=project_id, - tuning_command=tuning_command, - worker_container=worker_container, - worker_container_user=worker_container_user, - worker_machine_type=worker_machine_type, - worker_command=worker_command, - worker_count=worker_count, - worker_registry_username=worker_registry_username, - worker_registry_password=worker_registry_password, - is_preemptible=is_preemptible, - ports=ports, - workspace_url=workspace_url, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - trigger_event_id=trigger_event_id, - model_type=model_type, - model_path=model_path, - dockerfile_path=dockerfile_path, - hyperparameter_server_machine_type=hyperparameter_server_machine_type, - hyperparameter_server_registry_username=hyperparameter_server_registry_username, - hyperparameter_server_registry_password=hyperparameter_server_registry_password, - hyperparameter_server_container=hyperparameter_server_container, - hyperparameter_server_container_user=hyperparameter_server_container_user, - working_directory=working_directory, - use_dockerfile=use_dockerfile, - ) - - repository = self.build_repository(repositories.CreateHyperparameterJob) - handle = repository.create(hyperparameter) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - - return handle - - def run( - self, - name, - project_id, - tuning_command, - worker_container, - worker_machine_type, - worker_command, - worker_count, - worker_registry_username=None, - worker_registry_password=None, - worker_container_user=None, - is_preemptible=False, - ports=None, - workspace_url=None, - artifact_directory=None, - cluster_id=None, - experiment_env=None, - trigger_event_id=None, - model_type=None, - model_path=None, - dockerfile_path=None, - hyperparameter_server_registry_username=None, - hyperparameter_server_registry_password=None, - hyperparameter_server_container_user=None, - hyperparameter_server_container=None, - hyperparameter_server_machine_type=None, - working_directory=None, - use_dockerfile=False, - tags=None, - ): - """Create and start hyperparameter tuning job - - :param str name: Name of new experiment [required] - :param str project_id: Project ID [required] - :param str tuning_command: Tuning command [required] - :param str worker_container: Worker container [required] - :param str worker_machine_type: Worker machine type [required] - :param str worker_command: Worker command [required] - :param int worker_count: Worker count [required] - :param str worker_container_user: Worker container user - :param worker_registry_password: Worker registry password - :param worker_registry_username: Worker registry username - :param bool is_preemptible: Flag: is preemptible - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID - :param dict experiment_env: Environment variables (in JSON) - :param str trigger_event_id: GradientCI trigger event id - :param str model_type: Model type - :param str model_path: Model path - :param str dockerfile_path: Path to dockerfile - :param str hyperparameter_server_registry_username: container registry username - :param str hyperparameter_server_registry_password: container registry password - :param str hyperparameter_server_container_user: hps container user - :param str hyperparameter_server_container: hps container - :param str hyperparameter_server_machine_type: hps machine type - :param str working_directory: Working directory for the experiment - :param bool use_dockerfile: Flag: use dockerfile - :param list[str] tags: List of tags - - :returns: ID of a new job - :rtype: str - """ - - if not is_preemptible: - is_preemptible = None - - if use_dockerfile is False: - use_dockerfile = None - - hyperparameter = models.Hyperparameter( - name=name, - project_id=project_id, - tuning_command=tuning_command, - worker_container=worker_container, - worker_machine_type=worker_machine_type, - worker_command=worker_command, - worker_count=worker_count, - worker_container_user=worker_container_user, - worker_registry_username=worker_registry_username, - worker_registry_password=worker_registry_password, - is_preemptible=is_preemptible, - ports=ports, - workspace_url=workspace_url, - artifact_directory=artifact_directory, - cluster_id=cluster_id, - experiment_env=experiment_env, - trigger_event_id=trigger_event_id, - model_type=model_type, - model_path=model_path, - dockerfile_path=dockerfile_path, - hyperparameter_server_registry_username=hyperparameter_server_registry_username, - hyperparameter_server_registry_password=hyperparameter_server_registry_password, - hyperparameter_server_container_user=hyperparameter_server_container_user, - hyperparameter_server_container=hyperparameter_server_container, - hyperparameter_server_machine_type=hyperparameter_server_machine_type, - working_directory=working_directory, - use_dockerfile=use_dockerfile, - ) - - repository = self.build_repository(repositories.CreateAndStartHyperparameterJob) - handle = repository.create(hyperparameter) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - - return handle - - def get(self, id): - """Get Hyperparameter tuning job's instance - - :param str id: Hyperparameter job id - - :returns: instance of Hyperparameter - :rtype: models.Hyperparameter - """ - - repository = self.build_repository(repositories.GetHyperparameterTuningJob) - job = repository.get(id=id) - return job - - def start(self, id): - """Start existing hyperparameter tuning job - - :param str id: Hyperparameter job id - :raises: exceptions.GradientSdkError - """ - - repository = self.build_repository(repositories.StartHyperparameterTuningJob) - repository.start(id_=id) - - def list(self): - """Get a list of hyperparameter tuning jobs - - :rtype: list[models.Hyperparameter] - """ - repository = self.build_repository(repositories.ListHyperparameterJobs) - experiments = repository.list() - return experiments diff --git a/gradient/api_sdk/clients/job_client.py b/gradient/api_sdk/clients/job_client.py deleted file mode 100644 index cbcc001b..00000000 --- a/gradient/api_sdk/clients/job_client.py +++ /dev/null @@ -1,438 +0,0 @@ -""" -Jobs related client handler logic. - -Remember that in code snippets all highlighted lines are required other lines are optional. -""" -from .base_client import BaseClient, TagsSupportMixin -from ..models import Artifact, Job -from ..repositories.jobs import ListJobs, ListJobLogs, ListJobArtifacts, CreateJob, DeleteJob, StopJob, \ - DeleteJobArtifacts, GetJobArtifacts, GetJobMetrics, ListJobMetrics, StreamJobMetrics - - -class JobsClient(TagsSupportMixin, BaseClient): - """ - Client to handle job related actions. - - How to create instance of job client: - - .. code-block:: python - :linenos: - :emphasize-lines: 4 - - from gradient import JobsClient - - job_client = JobClient( - api_key='your_api_key_here' - ) - - """ - entity = "job" - - def create( - self, - machine_type, - container, - project_id, - data=None, - name=None, - command=None, - ports=None, - is_public=None, - working_directory=None, - experiment_id=None, - env_vars=None, - use_dockerfile=None, - is_preemptible=None, - project=None, - started_by_user_id=None, - rel_dockerfile_path=None, - registry_username=None, - registry_password=None, - cluster=None, - cluster_id=None, - node_attrs=None, - workspace_file_name=None, - registry_target=None, - registry_target_username=None, - registry_target_password=None, - build_only=False, - tags=None, - datasets=None, - ): - """ - Method to create and start job in paperspace gradient. - - Example create job: - - .. code-block:: python - :linenos: - :emphasize-lines: 2,3,4 - - job = job_client.create( - machine_type='K80', - container='tensorflow/tensorflow:1.13.1-gpu-py3', - project_id='Som3ProjecTiD', - data=data, - name='Example job', - command='pip install -r requirements.txt && python mnist.py', - ports='5000:5000', - env_vars={'CUSTOM_ENV': 'Some value that will be set as system environment'} - ) - - :param str machine_type: Type of machine on which job should run. This field is **required**. - - We recommend to choose one of this: - - .. code-block:: - - K80 - P100 - TPU - GV100 - GV100x8 - G1 - G6 - G12 - - :param str container: name of docker container that should be used to run job. This field is **required**. - - Example value: ``tensorflow/tensorflow:1.13.1-gpu-py3`` - - :param str project_id: Identify to which project job should be connected. This field is **required**. - - :param None|MultipartEncoderMonitor data: None if there are no data to upload or - encoded multipart data information with files to upload. - :param str name: name for job that creator wish to have. If not provided it will be autogenerated. - :param str command: custom command that should be run against command from docker image - :param str ports: string with comma `,` separated mapped ports. - - Example value: ``5000:5000,8080:8080`` - - :param bool is_public: bool flag to select if job should be available by default None - :param str working_directory: location of code to run. By default ``/paperspace`` - :param str experiment_id: Id of experiment to which job should be connected. If not provided there will be - created new experiment for this job. - :param dict env_vars: key value collection of envs that are used in code - :param bool use_dockerfile: determines whether to build from Dockerfile (default false). - Do not include a --container argument when using this flag. - :param bool is_preemptible: flag if we you want to use spot instance. By default False - :param str project: name of project that job is linked to. - :param str started_by_user_id: id of user that started job. By default it take user id from access token - or api key. - :param str rel_dockerfile_path: relative location to your dockerfile. Default set to ``./Dockerfile`` - :param str registry_username: username for custom docker registry - :param str registry_password: password for custom docker registry - :param str cluster: name of cluster that job should be run on. - :param str cluster_id: id of cluster that job should be run on. If you use one of recommended machine type - cluster will be chosen so you do not need to provide it. - :param dict node_attrs: - :param str workspace_file_name: - :param str registry_target: custom docker registry - :param str registry_target_username: username for custom docker registry - :param str registry_target_password: password for custom docker registry - :param bool build_only: determines whether to only build and not run image - :param list[str] tags: List of tags - :param list[object] datasets: List of input/output datasets - - :returns: Job handle - :rtype: str - """ - - if not build_only: - build_only = None - - job = Job( - machine_type=machine_type, - container=container, - project_id=project_id, - name=name, - command=command, - ports=ports, - is_public=is_public, - working_directory=working_directory, - experiment_id=experiment_id, - env_vars=env_vars, - use_dockerfile=use_dockerfile, - is_preemptible=is_preemptible, - project=project, - started_by_user_id=started_by_user_id, - rel_dockerfile_path=rel_dockerfile_path, - registry_username=registry_username, - registry_password=registry_password, - cluster=cluster, - cluster_id=cluster_id, - target_node_attrs=node_attrs, - workspace_file_name=workspace_file_name, - registry_target=registry_target, - registry_target_username=registry_target_username, - registry_target_password=registry_target_password, - build_only=build_only, - datasets=datasets, - ) - repository = self.build_repository(CreateJob) - handle = repository.create(job, data=data) - - if tags: - self.add_tags(entity_id=handle, tags=tags) - - return handle - - def delete(self, job_id): - """ - Method to remove job. - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - job_client.delete( - job_id='Your_job_id_here' - ) - - :param str job_id: id of job that you want to remove - :raises: exceptions.GradientSdkError - """ - repository = self.build_repository(DeleteJob) - repository.delete(job_id) - - def stop(self, job_id): - """ - Method to stop working job - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - job_client.stop( - job_id='Your_job_id_here' - ) - - :param job_id: id of job that we want to stop - :raises: exceptions.GradientSdkError - """ - repository = self.build_repository(StopJob) - repository.stop(job_id) - - def list(self, project_id=None, project=None, experiment_id=None, tags=None): - """ - Method to list jobs. - - To retrieve all user jobs: - - .. code-block:: python - :linenos: - - jobs = job_client.list() - - To list jobs from project: - - .. code-block:: python - :linenos: - - job = job_client.list( - project_id="Your_project_id_here", - ) - - :param str project_id: id of project that you want to list jobs - :param str project: name of project that you want to list jobs - :param str experiment_id: id of experiment that you want to list jobs - :param list[str]|tuple[str] tags: tags to filter jobs with OR - - :returns: list of job models - :rtype: list[Job] - """ - repository = self.build_repository(ListJobs) - jobs = repository.list( - project_id=project_id, - project=project, - experiment_id=experiment_id, - tags=tags, - ) - return jobs - - def logs(self, job_id, line=1, limit=10000): - """ - Method to retrieve job logs. - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - job_logs = job_client.logs( - job_id='Your_job_id_here', - line=100, - limit=100 - ) - - :param str job_id: id of job that we want to retrieve logs - :param int line: from what line you want to retrieve logs. Default 0 - :param int limit: how much lines you want to retrieve logs. Default 10000 - - :returns: list of formatted logs lines - :rtype: list - """ - repository = self.build_repository(ListJobLogs) - logs = repository.list(id=job_id, line=line, limit=limit) - return logs - - def yield_logs(self, job_id, line=1, limit=10000): - """Get log generator. Polls the API for new logs - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - job_logs_generator = job_client.yield_logs( - job_id='Your_job_id_here', - line=100, - limit=100 - ) - - :param str job_id: - :param int line: line number at which logs starts to display on screen - :param int limit: maximum lines displayed on screen, default set to 10 000 - - :returns: generator yielding LogRow instances - :rtype: Iterator[models.LogRow] - """ - - repository = self.build_repository(ListJobLogs) - logs = repository.yield_logs(id=job_id, line=line, limit=limit) - return logs - - def artifacts_delete(self, job_id, files=None): - """ - Method to delete job artifact. - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - job_client.artifacts_delete( - job_id='Your_job_id_here', - files=files, - ) - - :param str job_id: Id of job which artifact you want to delete - :param str files: if you wish to remove only few files from artifact pass string with names of this files - separated by comma ``,`` - - :raises: exceptions.GradientSdkError - """ - repository = self.build_repository(DeleteJobArtifacts) - repository.delete(id_=job_id, files=files) - - def artifacts_get(self, job_id): - """ - Method to retrieve federated access information for job artifacts. - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - artifacts = job_client.artifacts_get( - job_id='your_job_id_here', - ) - - :param job_id: Id of job from which you want to retrieve artifacts information about location - - :returns: Information about artifact place - :rtype: dict - """ - repository = self.build_repository(GetJobArtifacts) - data = repository.get(jobId=job_id) - return data - - def artifacts_list(self, job_id, files=None, size=False, links=True, start_after=None): - """ - Method to retrieve all artifacts files. - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - artifacts = job_client.artifacts_list( - job_id='your_job_id_here', - files='your_files,here', - size=False, - links=True, - start_after='key', - ) - - :param str job_id: to limit artifact from this job. - :param str files: to limit result only to file names provided. You can use wildcard option ``*``. - :param bool size: flag to show file size. Default value is set to False. - :param bool links: flag to show file url. Default value is set to True. - :params str start_after: key to list after - - :returns: list of files with description if specified from job artifacts. - :rtype: Pagination - """ - repository = self.build_repository(ListJobArtifacts) - return repository.list(jobId=job_id, files=files, links=links, size=size, start_after=start_after) - - def get_metrics(self, job_id, start=None, end=None, interval="30s", built_in_metrics=None): - """Get job metrics - - :param str job_id: ID of a job - :param datetime.datetime|str start: - :param datetime.datetime|str end: - :param str interval: - :param list[str] built_in_metrics: List of metrics to get if different than default - Available builtin metrics: cpuPercentage, memoryUsage, gpuMemoryFree, gpuMemoryUsed, gpuPowerDraw, - gpuTemp, gpuUtilization, gpuMemoryUtilization - - :returns: Metrics of a job - :rtype: dict[str,dict[str,list[dict]]] - """ - - repository = self.build_repository(GetJobMetrics) - metrics = repository.get( - id=job_id, - start=start, - end=end, - interval=interval, - built_in_metrics=built_in_metrics, - ) - return metrics - - def list_metrics(self, job_id, start=None, end=None, interval="30s"): - """List job metrics - - :param str job_id: ID of a job - :param datetime.datetime|str start: - :param datetime.datetime|str end: - :param str interval: - :returns: Metrics of a job - :rtype: dict[str,dict[str,list[dict]]] - """ - - repository = self.build_repository(ListJobMetrics) - metrics = repository.get( - id=job_id, - start=start, - end=end, - interval=interval, - ) - return metrics - - def stream_metrics(self, job_id, interval="30s", built_in_metrics=None): - """Stream live job metrics - - :param str job_id: ID of a job - :param str interval: - :param list[str] built_in_metrics: List of metrics to get if different than default - Available builtin metrics: cpuPercentage, memoryUsage, gpuMemoryFree, gpuMemoryUsed, gpuPowerDraw, - gpuTemp, gpuUtilization, gpuMemoryUtilization - - :returns: Generator object yielding live job metrics - :rtype: Iterable[dict] - """ - - repository = self.build_repository(StreamJobMetrics) - metrics = repository.stream( - id=job_id, - interval=interval, - built_in_metrics=built_in_metrics, - ) - return metrics diff --git a/gradient/api_sdk/clients/sdk_client.py b/gradient/api_sdk/clients/sdk_client.py index e0da81f1..ce97a466 100644 --- a/gradient/api_sdk/clients/sdk_client.py +++ b/gradient/api_sdk/clients/sdk_client.py @@ -1,9 +1,7 @@ -from . import DeploymentsClient, ExperimentsClient, HyperparameterJobsClient, ModelsClient, ProjectsClient, \ +from . import ModelsClient, ProjectsClient, \ MachinesClient, NotebooksClient, SecretsClient, DatasetsClient, MachineTypesClient, DatasetVersionsClient, \ DatasetTagsClient, ClustersClient, StorageProvidersClient -from .job_client import JobsClient from .workflow_client import WorkflowsClient -from .tensorboards_client import TensorboardClient from .. import logger as sdk_logger @@ -18,11 +16,6 @@ def __init__(self, api_key, logger=sdk_logger.MuteLogger()): self.dataset_tags = DatasetTagsClient(api_key=api_key, logger=logger) self.dataset_versions = DatasetVersionsClient( api_key=api_key, logger=logger) - self.deployments = DeploymentsClient(api_key=api_key, logger=logger) - self.experiments = ExperimentsClient(api_key=api_key, logger=logger) - self.hyperparameters = HyperparameterJobsClient( - api_key=api_key, logger=logger) - self.jobs = JobsClient(api_key=api_key, logger=logger) self.machine_types = MachineTypesClient(api_key=api_key, logger=logger) self.machines = MachinesClient(api_key=api_key, logger=logger) self.models = ModelsClient(api_key=api_key, logger=logger) @@ -31,5 +24,4 @@ def __init__(self, api_key, logger=sdk_logger.MuteLogger()): self.secrets = SecretsClient(api_key=api_key, logger=logger) self.storage_providers = StorageProvidersClient( api_key=api_key, logger=logger) - self.tensorboards = TensorboardClient(api_key=api_key, logger=logger) self.workflows = WorkflowsClient(api_key=api_key, logger=logger) diff --git a/gradient/api_sdk/clients/tensorboards_client.py b/gradient/api_sdk/clients/tensorboards_client.py deleted file mode 100644 index 635f0944..00000000 --- a/gradient/api_sdk/clients/tensorboards_client.py +++ /dev/null @@ -1,222 +0,0 @@ -""" -Tensorboard logic related client handler. - -Remember that in code snippets all highlighted lines are required other lines are optional. -""" -from .base_client import BaseClient -from .. import repositories, models - - -class TensorboardClient(BaseClient): - """ - Client to handle tensorboard related actions. - - How to create instance of tensorboard client: - - .. code-block:: python - :linenos: - :emphasize-lines: 4 - - from gradient import TensorboardClient - - tb_client = TensorboardClient( - api_key='your_api_key_here' - ) - - """ - - def create( - self, - image=None, - username=None, - password=None, - instance_type=None, - instance_size=None, - instances_count=None, - experiments=None, - ): - """ - Method to create tensorboard in paperspace gradient. - - Example create tensorboard: - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - tb_id = tb_client.create( - experiments=['some_experiment_id'], - image='tensorflow/tensorflow:latest-py3', - username='your_username', - password='your_password', - instance_type='cpu', - instance_size='small', - instance_count=1 - ) - - :param str image: your tensorboard will run with this image. - By default it will be run with ``tensorflow/tensorflow:latest-py3`` - :param str username: if you wish to limit access to your tensorboard with base auth then provide username - :param str password: if you wish to limit access to your tensorboard with base auth then provide password - :param str instance_type: type of instance on which you want to run tensorboard. - Available choices: - - .. code-block:: - - cpu - gpu - - By default we use ``cpu`` instance type. - :param str instance_size: size of instance on which you want to run tensorboard. - Available choices: - - .. code-block:: - - small - medium - large - - By default we use ``small`` instance size. - :param int instances_count: on how many machines you want to run tensorboard. By default ``1`` is used. - :param list experiments: list of experiments that you wish to add to tensorboard. - To create tensorboard you need to provide at least one experiment id. This field is **required**. - - :return: Return tensorboard id - :rtype: str - - :raises: ResourceFetchingError: When there is problem with response from API - """ - tensorboard = models.Tensorboard( - image=image, - username=username, - password=password, - instance=models.Instance( - type=instance_type, - size=instance_size, - count=instances_count, - ), - experiments=experiments, - ) - - repository = self.build_repository(repositories.CreateTensorboard) - tensorboard_id = repository.create(tensorboard) - return tensorboard_id - - def get(self, id): - """ - Method to get tensorboard details. - - Example get tensorboard details: - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - tb = tb_client.get( - id='your_tb_id' - ) - - :param str id: Tensorboard id of which you want to get details - :return: Tensorbord object if found - :rtype: None|Tensorboard - - :raises: ResourceFetchingError: When there is problem with response from API - """ - repository = self.build_repository(repositories.GetTensorboard) - tensorboard = repository.get(id=id) - return tensorboard - - def list(self): - """ - Method to list your active tensorboards. - - Example usage: - - .. code-block:: python - :linenos: - - tb_list = tb_client.list() - - :return: list of active tensorboards - :rtype: list[models.Tensorboard] - - :raises: ResourceFetchingError: When there is problem with response from API - """ - repository = self.build_repository(repositories.ListTensorboards) - tensorboards = repository.list() - return tensorboards - - def add_experiments(self, id, added_experiments): - """ - Method to add experiments to existing tensorboard. - - Example usage: - - .. code-block:: python - :linenos: - :emphasize-lines: 2, 3 - - tb = tb_client.add_experiments( - id='your_tb_id', - added_experiments=['new_experiment_id', 'next_new_experiment_id'] - ) - - :param str id: tensorboard id to which you want to add experiments - :param list added_experiments: list of experiment ids which you want to add to tensroboard - - :return: updated tensorboard - :rtype: Tensorboard - - :raises: ResourceFetchingError: When there is problem with response from API - """ - repository = self.build_repository(repositories.UpdateTensorboard) - tensorboard = repository.update( - id=id, added_experiments=added_experiments) - return tensorboard - - def remove_experiments(self, id, removed_experiments): - """ - Method to remove experiments from existing tensorboard. - - Example usage: - - .. code-block:: python - :linenos: - :emphasize-lines: 2, 3 - - tb = tb_client.remove_experiments( - id='your_tb_id', - removed_experiments=['experiment_id', 'next_experiment_id'] - ) - - :param str id: tensorboard id from which you want to remove experiments - :param list removed_experiments: list of experiment ids which you want to remove from tensroboard - - :return: updated tensorboard - :rtype: Tensorboard - - :raises: ResourceFetchingError: When there is problem with response from API - """ - repository = self.build_repository(repositories.UpdateTensorboard) - tensorboard = repository.update( - id=id, removed_experiments=removed_experiments) - return tensorboard - - def delete(self, id): - """ - Method to delete tensorboard. - - Example usage: - - .. code-block:: python - :linenos: - :emphasize-lines: 2 - - tb_client.delete( - id='your_tb_id' - ) - - :param str id: Tensoboard id which you want to delete - """ - repository = self.build_repository(repositories.DeleteTensorboard) - repository.delete(id_=id) diff --git a/gradient/api_sdk/config.py b/gradient/api_sdk/config.py index ea9cb4b3..d5f99cc3 100644 --- a/gradient/api_sdk/config.py +++ b/gradient/api_sdk/config.py @@ -18,8 +18,6 @@ def get_api_key(config_dir_path, config_file_name): _DEFAULT_API_HOST = "https://api.paperspace.com/graphql" _DEFAULT_CONFIG_HOST = "https://api.paperspace.io" _DEFAULT_CONFIG_LOG_HOST = "https://logs.paperspace.io" -_DEFAULT_CONFIG_EXPERIMENTS_HOST = "https://services.paperspace.io/experiments/v1/" -_DEFAULT_CONFIG_EXPERIMENTS_HOST_V2 = "https://services.paperspace.io/experiments/v2/" _DEFAULT_CONFIG_SERVICE_HOST = "https://services.paperspace.io" _DEFAULT_CONFIG_DIR_PATH = "~/.paperspace" _DEFAULT_CONFIG_FILE_NAME = os.path.expanduser("config.json") @@ -45,19 +43,26 @@ class config(object): WEB_URL = os.environ.get("PAPERSPACE_WEB_URL", _DEFAULT_WEB_URL) API_HOST = os.environ.get("PAPERSPACE_API_HOST", _DEFAULT_API_HOST) - USE_LEGACY_DEPLOYMENTS = os.environ.get("USE_LEGACY_DEPLOYMENTS", _DEFAULT_USE_LEGACY_DEPLOYMENTS) - CONFIG_HOST = os.environ.get("PAPERSPACE_CONFIG_HOST", _DEFAULT_CONFIG_HOST) - CONFIG_LOG_HOST = os.environ.get("PAPERSPACE_CONFIG_LOG_HOST", _DEFAULT_CONFIG_LOG_HOST) - CONFIG_EXPERIMENTS_HOST = os.environ.get("PAPERSPACE_CONFIG_EXPERIMENTS_HOST", _DEFAULT_CONFIG_EXPERIMENTS_HOST) - CONFIG_EXPERIMENTS_HOST_V2 = os.environ.get("PAPERSPACE_CONFIG_EXPERIMENTS_HOST_V2", - _DEFAULT_CONFIG_EXPERIMENTS_HOST_V2) - CONFIG_SERVICE_HOST = os.environ.get("PAPERSPACE_CONFIG_SERVICE_HOST", _DEFAULT_CONFIG_SERVICE_HOST) - CONFIG_DIR_PATH = os.path.expanduser(os.environ.get("PAPERSPACE_CONFIG_PATH", _DEFAULT_CONFIG_DIR_PATH)) - CONFIG_FILE_NAME = os.environ.get("PAPERSPACE_CONFIG_FILE_NAME", _DEFAULT_CONFIG_FILE_NAME) - PAPERSPACE_API_KEY = os.environ.get("PAPERSPACE_API_KEY", get_api_key(CONFIG_DIR_PATH, CONFIG_FILE_NAME)) - - HELP_HEADERS_COLOR = os.environ.get("PAPERSPACE_HELP_HEADERS_COLOR", _DEFAULT_HELP_HEADERS_COLOR) - HELP_OPTIONS_COLOR = os.environ.get("PAPERSPACE_HELP_OPTIONS_COLOR", _DEFAULT_HELP_OPTIONS_COLOR) + USE_LEGACY_DEPLOYMENTS = os.environ.get( + "USE_LEGACY_DEPLOYMENTS", _DEFAULT_USE_LEGACY_DEPLOYMENTS) + CONFIG_HOST = os.environ.get( + "PAPERSPACE_CONFIG_HOST", _DEFAULT_CONFIG_HOST) + CONFIG_LOG_HOST = os.environ.get( + "PAPERSPACE_CONFIG_LOG_HOST", _DEFAULT_CONFIG_LOG_HOST) + CONFIG_SERVICE_HOST = os.environ.get( + "PAPERSPACE_CONFIG_SERVICE_HOST", _DEFAULT_CONFIG_SERVICE_HOST) + CONFIG_DIR_PATH = os.path.expanduser(os.environ.get( + "PAPERSPACE_CONFIG_PATH", _DEFAULT_CONFIG_DIR_PATH)) + CONFIG_FILE_NAME = os.environ.get( + "PAPERSPACE_CONFIG_FILE_NAME", _DEFAULT_CONFIG_FILE_NAME) + PAPERSPACE_API_KEY = os.environ.get( + "PAPERSPACE_API_KEY", get_api_key(CONFIG_DIR_PATH, CONFIG_FILE_NAME)) + + HELP_HEADERS_COLOR = os.environ.get( + "PAPERSPACE_HELP_HEADERS_COLOR", _DEFAULT_HELP_HEADERS_COLOR) + HELP_OPTIONS_COLOR = os.environ.get( + "PAPERSPACE_HELP_OPTIONS_COLOR", _DEFAULT_HELP_OPTIONS_COLOR) USE_CONSOLE_COLORS = os.environ.get("PAPERSPACE_USE_CONSOLE_COLORS", _DEFAULT_USE_CONSOLE_COLORS) in (True, "true", "1") - HELP_COLORS_DICT = get_help_colors_dict(USE_CONSOLE_COLORS, HELP_HEADERS_COLOR, HELP_OPTIONS_COLOR) + HELP_COLORS_DICT = get_help_colors_dict( + USE_CONSOLE_COLORS, HELP_HEADERS_COLOR, HELP_OPTIONS_COLOR) diff --git a/gradient/api_sdk/graphql.py b/gradient/api_sdk/graphql.py index 5493d9c7..6fbd0d1f 100644 --- a/gradient/api_sdk/graphql.py +++ b/gradient/api_sdk/graphql.py @@ -4,12 +4,14 @@ from .config import config + def graphql_client(api_key=None): - if api_key is None: - api_key = config.PAPERSPACE_API_KEY - headers = { - 'Authorization': f'Bearer {api_key}', - } - transport = RequestsHTTPTransport(headers=headers, url=config.API_HOST, verify=True, retries=3) - - return Client(transport=transport) \ No newline at end of file + if api_key is None: + api_key = config.PAPERSPACE_API_KEY + headers = { + 'Authorization': f'Bearer {api_key}', + } + transport = RequestsHTTPTransport( + headers=headers, url=config.API_HOST, verify=True, retries=3) + + return Client(transport=transport) diff --git a/gradient/api_sdk/models/__init__.py b/gradient/api_sdk/models/__init__.py index 123778a2..783fc521 100644 --- a/gradient/api_sdk/models/__init__.py +++ b/gradient/api_sdk/models/__init__.py @@ -4,11 +4,6 @@ from .dataset_tag import DatasetTag, DatasetVersionSummary from .dataset_version import DatasetVersion, DatasetVersionPreSignedS3Call, DatasetVersionPreSignedURL, \ DatasetVersionTagSummary -from .deployment import Deployment, AutoscalingDefinition, AutoscalingMetric -from .experiment import BaseExperiment, MultiNodeExperiment, SingleNodeExperiment, MpiMultiNodeExperiment -from .experiment_dataset import ExperimentDataset, VolumeOptions -from .hyperparameter import Hyperparameter -from .job import Job, JobDataset from .log import LogRow from .machine import Machine, MachineEvent, MachineUtilization from .model import Model, ModelFile @@ -18,6 +13,5 @@ from .secret import Secret from .storage_provider import StorageProvider from .tag import Tag -from .tensorboard import Instance, Tensorboard from .vm_type import VmType, VmTypeGpuModel from .workflows import Workflow, WorkflowRun, WorkflowSpec diff --git a/gradient/api_sdk/models/deployment.py b/gradient/api_sdk/models/deployment.py deleted file mode 100644 index 20e1031d..00000000 --- a/gradient/api_sdk/models/deployment.py +++ /dev/null @@ -1,107 +0,0 @@ -import datetime - -import attr - - -@attr.s -class AutoscalingMetric(object): - type = attr.ib(type=str) - name = attr.ib(type=str) - value_type = attr.ib(type=str) - value = attr.ib(type=float) - - -@attr.s -class AutoscalingDefinition(object): - min_instance_count = attr.ib(type=int, default=None) - max_instance_count = attr.ib(type=int, default=None) - scale_cooldown_period = attr.ib(type=int, default=None) - metrics = attr.ib(type=list, factory=list) # instances of AutoscalerMetric - - -@attr.s -class Deployment(object): - """ - Deployment class - - Deploy any model as a high-performance, low-latency micro-service with a RESTful API. Easily monitor, scale, and - version deployments. Deployments take a trained model and expose them as a persistent service at a known, secure - URL endpoint. - - :param str id_: deployment id - :param str name: Human-friendly name for new model deployment - :param str endpoint: url address to deployment, for example:: - - ``https://services.paperspace.io/model-serving/:predict`` - :param str api_type: api type of deployment - Options:: - - "GPRC" - "REST" - :param str state: state of Deployment - Options:: - - "BUILDING" - "PROVISIONING" - "STARTING" - "RUNNING" - "STOPPING" - "STOPPED" - "ERROR" - :param str model_id: model id - :param str project_id: project id - :param str image_url: Docker image for model deployment - :param str deployment_type: Model deployment type. - :param str machine_type: Type of machine for new deployment - Options:: - - "G1" - "G6" - "G12" - "K80" - "P100" - "GV100" - :param int instance_count: Number of machine instances - - """ - id = attr.ib(type=str, default=None) - name = attr.ib(type=str, default=None) - endpoint = attr.ib(type=str, default=None) - api_type = attr.ib(type=str, default=None) - - state = attr.ib(type=str, default=None) - - model_id = attr.ib(type=str, default=None) - project_id = attr.ib(type=str, default=None) - - image_url = attr.ib(type=str, default=None) - deployment_type = attr.ib(type=str, default=None) - machine_type = attr.ib(type=str, default=None) - instance_count = attr.ib(type=int, default=None) - container_model_path = attr.ib(type=str, default=None) - image_username = attr.ib(type=str, default=None) - image_password = attr.ib(type=str, default=None) - image_server = attr.ib(type=str, default=None) - container_url_path = attr.ib(type=str, default=None) - endpoint_url_path = attr.ib(type=str, default=None) - method = attr.ib(type=str, default=None) - docker_args = attr.ib(type=list, default=None) - env = attr.ib(type=dict, default=None) - ports = attr.ib(type=str, default=None) - auth_username = attr.ib(type=str, default=None) - auth_password = attr.ib(type=str, default=None) - cluster_id = attr.ib(type=int, default=None) - tags = attr.ib(type=list, factory=list) - command = attr.ib(type=str, default=None) - workspace_url = attr.ib(type=str, default=None) - workspace_ref = attr.ib(type=str, default=None) - workspace_username = attr.ib(type=str, default=None) - workspace_password = attr.ib(type=str, default=None) - metrics_url = attr.ib(type=str, default=None) - autoscaling = attr.ib(type=AutoscalingDefinition, default=None) - - dt_created = attr.ib(type=datetime.datetime, default=None) - dt_modified = attr.ib(type=datetime.datetime, default=None) - dt_started = attr.ib(type=datetime.datetime, default=None) - dt_stopped = attr.ib(type=datetime.datetime, default=None) - dt_deleted = attr.ib(type=datetime.datetime, default=None) diff --git a/gradient/api_sdk/models/experiment.py b/gradient/api_sdk/models/experiment.py deleted file mode 100644 index 59750e18..00000000 --- a/gradient/api_sdk/models/experiment.py +++ /dev/null @@ -1,216 +0,0 @@ -import datetime - -import attr - -from .. import constants - - -@attr.s -class BaseExperiment(object): - """ - Base experiment class. Single node and multi node experiments classes inherit from it. - - Experiments are intended to be used for intensive computational tasks like neural network training. Gradient - supports single-node experiments as well as distributed training through multinode experiments. - - Experiments can be run from the Experiment Builder web interface, the GradientCI bot, or the CLI. - - :param str name: Name of new experiment - :param str ports: Port to use in new experiment - :param str workspace_url: Project git repository url - :param str workspace_ref: Git commit hash, branch name or tag - :param str workspace_username: Project git repository username - :param str workspace_password: Project git repository password - :param Dataset dataset - :param str working_directory: Working directory for the experiment - :param str artifact_directory: Artifacts directory - :param str cluster_id: Cluster ID (handle) - :param dict experiment_env: Environment variables in a JSON - :param str project_id: Project ID - :param str model_type: defines the type of model that is being generated by the experiment. - :param str model_path: Model path - :param bool is_preemptible: Is preemptible - :param str id: - :param int state: state of experiment can be one of:: - - "created" - "provisioned" - "network setup" - "running" - "stopped" - "error" - "failed" - "canceled" - "network teardown" - "pending" - "provisioning" - "network setting up" - "network tearing down" - """ - name = attr.ib(type=str, default=None) - ports = attr.ib(type=str, default=None) - workspace_url = attr.ib(type=str, default=None) - workspace_ref = attr.ib(type=str, default=None) - workspace_username = attr.ib(type=str, default=None) - workspace_password = attr.ib(type=str, default=None) - datasets = attr.ib(type=list, default=None) - working_directory = attr.ib(type=str, default=None) - artifact_directory = attr.ib(type=str, default=None) - cluster_id = attr.ib(type=str, default=None) - experiment_env = attr.ib(type=dict, default=dict) - project_id = attr.ib(type=str, default=None) - model_type = attr.ib(type=str, default=None) - model_path = attr.ib(type=str, default=None) - is_preemptible = attr.ib(type=bool, default=None) - id = attr.ib(type=str, default=None) - state = attr.ib(type=int, default=None) - tags = attr.ib(type=list, factory=list) - - dt_created = attr.ib(type=datetime.datetime, default=None) - dt_modified = attr.ib(type=datetime.datetime, default=None) - dt_started = attr.ib(type=datetime.datetime, default=None) - dt_stopped = attr.ib(type=datetime.datetime, default=None) - dt_deleted = attr.ib(type=datetime.datetime, default=None) - - -@attr.s -class SingleNodeExperiment(BaseExperiment): - """ - Single node experiment class. Inherits from ``BaseExperiment`` class - - In your your CLI command or ``config.yaml``, specify the experiment type as ``singlenode`` - - :param str container: Container (dockerfile) - :param str machine_type: Machine type - Options:: - - "Air" - "Standard" - "Pro" - "Advanced" - "GPU+" - "P4000" - "P5000" - "P6000" - "V100" - "C1" - "C2" - "C3" - "C4" - "C5" - "C6" - "C7" - "C8" - "C9" - "C10" - :param str command: Container entrypoint command - :param str container_user: Container user for running the specified command in the container. If no containerUser is specified, the user will default to 'root' in the container. - :param str registry_username: Registry username for accessing private docker registry container if nessesary - :param str registry_password: Registry password for accessing private docker registry container if nessesary - :param int experiment_type_id: type of experiment - Options:: - - "single node" - "GRPC multi node" - "MPI multi node" - "Hyperparameter tuning" - """ - container = attr.ib(type=str, default=None) - machine_type = attr.ib(type=str, default=None) - command = attr.ib(type=str, default=None) - container_user = attr.ib(type=str, default=None) - registry_username = attr.ib(type=str, default=None) - registry_password = attr.ib(type=str, default=None) - registry_url = attr.ib(type=str, default=None) - experiment_type_id = attr.ib(type=int, default=constants.ExperimentType.SINGLE_NODE) - - @experiment_type_id.validator - def experiment_type_id_validator(self, attribute, value): - if value is not constants.ExperimentType.SINGLE_NODE: - raise ValueError("Single node experiment's type must equal {}". - format(constants.ExperimentType.SINGLE_NODE)) - - -@attr.s -class MultiNodeExperiment(BaseExperiment): - """ - Multi node experiment class. Inherits from ``BaseExperiment`` class - - Gradient supports both gRPC and MPI protocols for distributed TensorFlow model training. In your CLI command or - ``config.yaml``, specify the experiment type as either ``multinode``. - - The two types are:: - - type: "multi-grpc" - - or:: - - type: "multi-mpi" - - :param int experiment_type_id: type of experiment - Options:: - - "single node" - "GRPC multi node" - "MPI multi node" - "Hyperparameter tuning" - :param str worker_container: Worker container (dockerfile) - :param str worker_machine_type: Worker machine type - :param str worker_command: Worker command - :param int worker_count: Worker count - :param str parameter_server_container: Parameter server container - :param str parameter_server_command: Parameter server command - :param int parameter_server_count: Parameter server count - :param str worker_container_user: Worker container user - :param str worker_registry_username: Registry username for accessing private docker registry container if nessesary - :param str worker_registry_password: Registry password for accessing private docker registry container if nessesary - :param str parameter_server_container_user: Parameter server container user - :param str parameter_server_registry_username: Registry username for accessing private docker registry container if nessesary - :param str parameter_server_registry_password: Registry password for accessing private docker registry container if nessesary - """ - experiment_type_id = attr.ib(type=int, default=None) - worker_container = attr.ib(type=str, default=None) - worker_machine_type = attr.ib(type=str, default=None) - worker_command = attr.ib(type=str, default=None) - worker_count = attr.ib(type=int, default=None) - parameter_server_container = attr.ib(type=str, default=None) - parameter_server_machine_type = attr.ib(type=str, default=None) - parameter_server_command = attr.ib(type=str, default=None) - parameter_server_count = attr.ib(type=int, default=None) - worker_container_user = attr.ib(type=str, default=None) - worker_registry_username = attr.ib(type=str, default=None) - worker_registry_password = attr.ib(type=str, default=None) - worker_registry_url = attr.ib(type=str, default=None) - parameter_server_container_user = attr.ib(type=str, default=None) - parameter_server_registry_username = attr.ib(type=str, default=None) - parameter_server_registry_password = attr.ib(type=str, default=None) - parameter_server_registry_url = attr.ib(type=str, default=None) - - @experiment_type_id.validator - def experiment_type_id_validator(self, attribute, value): - if value not in (constants.ExperimentType.GRPC_MULTI_NODE, - constants.ExperimentType.MPI_MULTI_NODE): - raise ValueError("Multi node experiment's type must equal {} or {}". - format(constants.ExperimentType.GRPC_MULTI_NODE, - constants.ExperimentType.MPI_MULTI_NODE)) - - -@attr.s -class MpiMultiNodeExperiment(BaseExperiment): - experiment_type_id = attr.ib(type=int, default=None) - worker_container = attr.ib(type=str, default=None) - worker_machine_type = attr.ib(type=str, default=None) - worker_command = attr.ib(type=str, default=None) - worker_count = attr.ib(type=int, default=None) - master_container = attr.ib(type=str, default=None) - master_machine_type = attr.ib(type=str, default=None) - master_command = attr.ib(type=str, default=None) - master_count = attr.ib(type=str, default=None) - worker_container_user = attr.ib(type=str, default=None) - worker_registry_username = attr.ib(type=str, default=None) - worker_registry_password = attr.ib(type=str, default=None) - worker_registry_url = attr.ib(type=str, default=None) - master_container_user = attr.ib(type=str, default=None) - master_registry_username = attr.ib(type=str, default=None) - master_registry_password = attr.ib(type=str, default=None) - master_registry_url = attr.ib(type=str, default=None) diff --git a/gradient/api_sdk/models/experiment_dataset.py b/gradient/api_sdk/models/experiment_dataset.py deleted file mode 100644 index 3c821c45..00000000 --- a/gradient/api_sdk/models/experiment_dataset.py +++ /dev/null @@ -1,20 +0,0 @@ -import attr - - -@attr.s -class VolumeOptions(object): - kind = attr.ib(type=str, default=None) - size = attr.ib(type=str, default=None) - - -@attr.s -class ExperimentDataset(object): - id = attr.ib(type=str, default=None) - uri = attr.ib(type=str, default=None) - aws_access_key_id = attr.ib(type=str, default=None) - aws_secret_access_key = attr.ib(type=str, default=None) - aws_endpoint = attr.ib(type=str, default=None) - etag = attr.ib(type=str, default=None) - version_id = attr.ib(type=str, default=None) - name = attr.ib(type=str, default=None) - volume_options = attr.ib(type=VolumeOptions, factory=VolumeOptions) diff --git a/gradient/api_sdk/models/hyperparameter.py b/gradient/api_sdk/models/hyperparameter.py deleted file mode 100644 index 64c215b2..00000000 --- a/gradient/api_sdk/models/hyperparameter.py +++ /dev/null @@ -1,62 +0,0 @@ -import attr - -from .experiment import BaseExperiment -from .. import constants - - -@attr.s -class Hyperparameter(BaseExperiment): - """ - Hyperparameter job class. Inherits from ``BaseExperiment`` class - - :param int experiment_type_id: experiment type, for hyperparameter experiment set to 4 - :param str tuning_command: Tuning command - :param str worker_container: Worker container - :param str worker_machine_type: Worker machine type - :param str worker_command: Worker command - :param int worker_count: Worker count - :param bool worker_use_dockerfile: Flag: use dockerfile - :param str worker_dockerfile_path: path to dockerfile, if not set default is project root directory - :param str worker_container_user: Worker container user - :param str worker_registry_username: Worker registry username - :param str worker_registry_password: Worker registry password - - :param str hyperparameter_server_machine_type: Hyperparameter server machine type - :param str hyperparameter_server_container: Hyperparameter server container - :param str hyperparameter_server_container_user: Hyperparameter server container user - :param str hyperparameter_server_registry_username: Hyperparameter server registry username - :param str hyperparameter_server_registry_password: Hyperparameter server registry password - :param bool is_preemptible: Flag: is preemptible - :param str trigger_event_id: GradientCI trigger event id - :param str dockerfile_path: Path to dockerfile - :param bool use_dockerfile: Flag: use dockerfile - """ - experiment_type_id = attr.ib(type=int, default=constants.ExperimentType.HYPERPARAMETER_TUNING) - tuning_command = attr.ib(type=str, default=None) - worker_container = attr.ib(type=str, default=None) - worker_machine_type = attr.ib(type=str, default=None) - worker_command = attr.ib(type=str, default=None) - worker_count = attr.ib(type=int, default=None) - worker_use_dockerfile = attr.ib(type=bool, default=None) - worker_dockerfile_path = attr.ib(type=str, default=None) - worker_container_user = attr.ib(type=str, default=None) - worker_registry_username = attr.ib(type=str, default=None) - worker_registry_password = attr.ib(type=str, default=None) - - hyperparameter_server_machine_type = attr.ib(type=str, default=None) - hyperparameter_server_container = attr.ib(type=str, default=None) - hyperparameter_server_container_user = attr.ib(type=str, default=None) - hyperparameter_server_registry_username = attr.ib(type=str, default=None) - hyperparameter_server_registry_password = attr.ib(type=str, default=None) - - is_preemptible = attr.ib(type=bool, default=None) - trigger_event_id = attr.ib(type=str, default=None) - dockerfile_path = attr.ib(type=str, default=None) - use_dockerfile = attr.ib(type=bool, default=None) - tags = attr.ib(type=list, factory=list) - - @experiment_type_id.validator - def experiment_type_id_validator(self, attribute, value): - if value is not constants.ExperimentType.HYPERPARAMETER_TUNING: - raise ValueError("Hyperparameter tuning model's type must equal {}". - format(constants.ExperimentType.HYPERPARAMETER_TUNING)) diff --git a/gradient/api_sdk/models/job.py b/gradient/api_sdk/models/job.py deleted file mode 100644 index 8e72cc93..00000000 --- a/gradient/api_sdk/models/job.py +++ /dev/null @@ -1,95 +0,0 @@ -import attr - - -@attr.s -class JobDataset(object): - id = attr.ib(type=str, default=None) - name = attr.ib(type=str, default=None) - output = attr.ib(type=bool, default=None) - - -@attr.s -class Job(object): - id = attr.ib(type=str, default=None) - name = attr.ib(type=str, default=None) - state = attr.ib(type=str, default=None) - working_directory = attr.ib(type=str, default=None) - artifacts_directory = attr.ib(type=str, default=None) - entrypoint = attr.ib(type=str, default=None) - project_id = attr.ib(type=str, default=None) - project = attr.ib(type=str, default=None) - container = attr.ib(type=str, default=None) - container_url = attr.ib(type=str, default=None) - base_container = attr.ib(type=str, default=None) - base_container_url = attr.ib(type=str, default=None) - machine_type = attr.ib(type=str, default=None) - cluster = attr.ib(type=str, default=None) - cluster_id = attr.ib(type=str, default=None) - usage_rate = attr.ib(type=str, default=None) - started_by_user_id = attr.ib(type=str, default=None) - parent_job_id = attr.ib(type=str, default=None) - job_error = attr.ib(type=str, default=None) - dt_created = attr.ib(type=str, default=None) - dt_modified = attr.ib(type=str, default=None) - dt_provisioning_started = attr.ib(type=str, default=None) - dt_provisioning_finished = attr.ib(type=str, default=None) - dt_started = attr.ib(type=str, default=None) - dt_finished = attr.ib(type=str, default=None) - dt_teardown_started = attr.ib(type=str, default=None) - dt_teardown_finished = attr.ib(type=str, default=None) - dt_deleted = attr.ib(type=str, default=None) - exit_code = attr.ib(type=str, default=None) - queue_position = attr.ib(type=str, default=None) - seq_num = attr.ib(type=int, default=None) - storage_region = attr.ib(type=str, default=None) - cluster_machine = attr.ib(type=str, default=None) - fqdn = attr.ib(type=str, default=None) - ports = attr.ib(type=str, default=None) - is_public = attr.ib(type=bool, default=None) - container_user = attr.ib(type=str, default=None) - has_code = attr.ib(type=bool, default=None) - code_uploaded = attr.ib(type=bool, default=None) - code_commit = attr.ib(type=str, default=None) - run_till_cancelled = attr.ib(type=bool, default=None) - push_on_completion = attr.ib(type=bool, default=None) - new_image_name = attr.ib(type=str, default=None) - cpu_hostname = attr.ib(type=str, default=None) - cpu_count = attr.ib(type=int, default=None) - cpu_model = attr.ib(type=str, default=None) - cpu_flags = attr.ib(type=str, default=None) - cpu_mem = attr.ib(type=str, default=None) - gpu_name = attr.ib(type=str, default=None) - gpu_serial = attr.ib(type=str, default=None) - gpu_device = attr.ib(type=str, default=None) - gpu_driver = attr.ib(type=str, default=None) - gpu_count = attr.ib(type=int, default=None) - gpu_mem = attr.ib(type=str, default=None) - tpu_type = attr.ib(type=str, default=None) - tpu_name = attr.ib(type=str, default=None) - tpu_grpc_url = attr.ib(type=str, default=None) - tpu_tf_version = attr.ib(type=str, default=None) - tpu_dataset_dir = attr.ib(type=str, default=None) - tpu_model_dir = attr.ib(type=str, default=None) - target_node_attrs = attr.ib(type=dict, default=None) - job_env = attr.ib(type=dict, default=None) - env_vars = attr.ib(type=dict, default=None) - shared_mem_mbytes = attr.ib(type=int, default=None) - shutdown_timeout = attr.ib(type=int, default=None) - is_preemptible = attr.ib(type=bool, default=None) - metrics_url = attr.ib(type=str, default=None) - custom_metrics = attr.ib(type=str, default=None) - experiment_id = attr.ib(type=str, default=None) - - command = attr.ib(type=str, default=None) - workspace_file_name = attr.ib(type=str, default=None) - use_dockerfile = attr.ib(type=str, default=None) - rel_dockerfile_path = attr.ib(type=str, default=None) - registry_username = attr.ib(type=str, default=None) - registry_password = attr.ib(type=str, default=None) - build_only = attr.ib(type=bool, default=None) - - registry_target = attr.ib(type=str, default=None) - registry_target_username = attr.ib(type=str, default=None) - registry_target_password = attr.ib(type=str, default=None) - tags = attr.ib(type=list, factory=list) - datasets = attr.ib(type=list, factory=list) diff --git a/gradient/api_sdk/models/tensorboard.py b/gradient/api_sdk/models/tensorboard.py deleted file mode 100644 index 0f696416..00000000 --- a/gradient/api_sdk/models/tensorboard.py +++ /dev/null @@ -1,21 +0,0 @@ -import attr - - -@attr.s -class Instance(object): - type = attr.ib(type=str, default=None) - size = attr.ib(type=str, default=None) - count = attr.ib(type=int, default=None) - - -@attr.s -class Tensorboard(object): - id = attr.ib(type=str, default=None) - image = attr.ib(type=str, default=None) - username = attr.ib(type=str, default=None) - password = attr.ib(type=str, default=None) - instance = attr.ib(type=Instance, default=None) - experiments = attr.ib(type=list, default=None) - url = attr.ib(type=str, default=None) - state = attr.ib(type=int, default=None) - tags = attr.ib(type=list, factory=list) diff --git a/gradient/api_sdk/repositories/__init__.py b/gradient/api_sdk/repositories/__init__.py index e90bfed1..7cb26b1a 100644 --- a/gradient/api_sdk/repositories/__init__.py +++ b/gradient/api_sdk/repositories/__init__.py @@ -22,20 +22,6 @@ UpdateDatasetVersion ) -from .deployments import ( - ListDeployments, - CreateDeployment, - StartDeployment, - StopDeployment, - DeleteDeployment, - UpdateDeployment, - GetDeployment, - GetDeploymentMetrics, - ListDeploymentMetrics, - StreamDeploymentMetrics, - ListDeploymentLogs -) - from .gradient_deployments import ( create_deployment, list_deployments, @@ -45,40 +31,6 @@ get_deployment_logs, yield_deployment_logs ) -from .experiments import ( - ListExperiments, - GetExperiment, - ListExperimentLogs, - StartExperiment, - StopExperiment, - CreateSingleNodeExperiment, - CreateMultiNodeExperiment, - RunSingleNodeExperiment, - RunMultiNodeExperiment, - CreateMpiMultiNodeExperiment, - RunMpiMultiNodeExperiment, - DeleteExperiment, - GetExperimentMetrics, - ListExperimentMetrics, - StreamExperimentMetrics -) -from .hyperparameter import ( - CreateHyperparameterJob, - CreateAndStartHyperparameterJob, - ListHyperparameterJobs, - GetHyperparameterTuningJob, - StartHyperparameterTuningJob -) -from .jobs import ( - ListJobs, - ListResources, - ListJobArtifacts, - ListJobLogs, - GetJob, - GetJobMetrics, - ListJobMetrics, - StreamJobMetrics -) from .machine_types import ListMachineTypes from .machines import ( CheckMachineAvailability, @@ -133,13 +85,6 @@ GetStorageProvider, UpdateStorageProvider ) -from .tensorboards import ( - CreateTensorboard, - GetTensorboard, - ListTensorboards, - UpdateTensorboard, - DeleteTensorboard -) from .workflows import ( ListWorkflows, GetWorkflow, diff --git a/gradient/api_sdk/repositories/deployments.py b/gradient/api_sdk/repositories/deployments.py deleted file mode 100644 index 5395981e..00000000 --- a/gradient/api_sdk/repositories/deployments.py +++ /dev/null @@ -1,211 +0,0 @@ -from .common import ListResources, CreateResource, StartResource, StopResource, DeleteResource, AlterResource, \ - GetResource, GetMetrics, ListMetrics, StreamMetrics, ListLogs -from .. import serializers, config, sdk_exceptions -from ..sdk_exceptions import ResourceFetchingError, MalformedResponseError - - -class GetBaseDeploymentApiUrlMixin(object): - def _get_api_url(self, **_): - return config.config.CONFIG_HOST - - -class ListDeployments(GetBaseDeploymentApiUrlMixin, ListResources): - def get_request_url(self, **kwargs): - return "/deployments/getDeploymentList/" - - def _parse_objects(self, data, **kwargs): - deployment_dicts = self._get_deployments_dicts_from_json_data(data, kwargs) - deployments = [] - - for deployment_dict in deployment_dicts: - deployment = serializers.DeploymentSchema().get_instance(deployment_dict) - deployments.append(deployment) - - return deployments - - @staticmethod - def _get_deployments_dicts_from_json_data(data, kwargs): - return data["deploymentList"] - - def _get_request_json(self, kwargs): - filters = {} - if kwargs["model_id"]: - filters["modelId"] = kwargs["model_id"] - - if kwargs["state"]: - filters["state"] = kwargs["state"] - - if kwargs["project_id"]: - filters["projectId"] = kwargs["project_id"] - - if filters: - json_ = {"filter": {"where": {"and": [filters]}}} - else: - json_ = {} - - tags = kwargs.get("tags") - if tags: - json_["tagFilter"] = tags - - return json_ or None - - -class CreateDeployment(GetBaseDeploymentApiUrlMixin, CreateResource): - SERIALIZER_CLS = serializers.DeploymentCreateSchema - - def get_request_url(self, **kwargs): - if kwargs.get("cluster"): - return "/deployments/v2/createDeployment/" - - return "/deployments/createDeployment/" - - def _get_id_from_response(self, response): - handle = response.data["deployment"]["id"] - return handle - - -class StartDeployment(GetBaseDeploymentApiUrlMixin, StartResource): - def get_request_url(self, **kwargs): - return "/deployments/v2/updateDeployment/" - - def _get_request_json(self, kwargs): - data = { - "id": kwargs["id"], - "isRunning": True, - } - return data - - def _send_request(self, client, url, json_data=None): - response = client.post(url, json=json_data) - return response - - -class StopDeployment(GetBaseDeploymentApiUrlMixin, StopResource): - def get_request_url(self, **kwargs): - return "/deployments/v2/updateDeployment/" - - def _get_request_json(self, kwargs): - data = { - "id": kwargs["id"], - "isRunning": False, - } - return data - - def _send_request(self, client, url, json_data=None): - response = client.post(url, json=json_data) - return response - - -class DeleteDeployment(GetBaseDeploymentApiUrlMixin, DeleteResource): - def get_request_url(self, **kwargs): - return "/deployments/v2/deleteDeployment" - - def _get_request_json(self, kwargs): - data = { - "id": kwargs["id"], - "isRunning": False, - } - return data - - def _send_request(self, client, url, json_data=None): - response = client.post(url, json=json_data) - return response - - -class UpdateDeployment(GetBaseDeploymentApiUrlMixin, AlterResource): - SERIALIZER_CLS = serializers.DeploymentSchema - VALIDATION_ERROR_MESSAGE = "Failed to update resource" - - def update(self, id, instance): - instance_dict = self._get_instance_dict(instance) - self._run(id=id, **instance_dict) - - def get_request_url(self, **kwargs): - return "/deployments/v2/updateDeployment" - - def _get_request_json(self, kwargs): - # this temporary workaround is here because create and update - # endpoints have different names for docker args field - args = kwargs.pop("dockerArgs", None) - if args: - kwargs["args"] = args - - j = { - "id": kwargs.pop("id"), - "upd": kwargs, - } - return j - - -class GetDeployment(GetBaseDeploymentApiUrlMixin, GetResource): - SERIALIZER_CLS = serializers.DeploymentSchema - - def get_request_url(self, **kwargs): - return "/deployments/getDeploymentList/" - - def _get_request_json(self, kwargs): - deployment_id = kwargs["deployment_id"] - filter_ = {"where": {"and": [{"id": deployment_id}]}} - json_ = {"filter": filter_} - return json_ - - def _parse_object(self, instance_dict, **kwargs): - try: - instance_dict = instance_dict["deploymentList"][0] - except KeyError: - raise MalformedResponseError("Malformed response from API") - except IndexError: - raise ResourceFetchingError("Deployment not found") - - return super(GetDeployment, self)._parse_object(instance_dict, **kwargs) - - -class GetDeploymentMetrics(GetMetrics): - OBJECT_TYPE = "modelDeployment" - - def _get_instance_by_id(self, instance_id, **kwargs): - repository = GetDeployment(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(deployment_id=instance_id) - return instance - - def _get_start_date(self, instance, kwargs): - rv = super(GetDeploymentMetrics, self)._get_start_date(instance, kwargs) - if rv is None: - raise sdk_exceptions.GradientSdkError("Deployment job has not started yet") - - return rv - -class ListDeploymentMetrics(ListMetrics): - OBJECT_TYPE = "modelDeployment" - - def _get_instance_by_id(self, instance_id, **kwargs): - repository = GetDeployment(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(deployment_id=instance_id) - return instance - - def _get_start_date(self, instance, kwargs): - rv = super(ListDeploymentMetrics, self)._get_start_date(instance, kwargs) - if rv is None: - raise sdk_exceptions.GradientSdkError("Deployment job has not started yet") - - return rv - -class StreamDeploymentMetrics(StreamMetrics): - OBJECT_TYPE = "modelDeployment" - - def _get_metrics_api_url(self, instance_id, protocol="https"): - repository = GetDeployment(api_key=self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - deployment = repository.get(deployment_id=instance_id) - - metrics_api_url = super(StreamDeploymentMetrics, self)._get_metrics_api_url(deployment, protocol="wss") - return metrics_api_url - - -class ListDeploymentLogs(ListLogs): - def _get_request_params(self, kwargs): - params = { - "deploymentId": kwargs["id"], - "line": kwargs["line"], - "limit": kwargs["limit"] - } - return params diff --git a/gradient/api_sdk/repositories/experiments.py b/gradient/api_sdk/repositories/experiments.py deleted file mode 100644 index f8a3bfe6..00000000 --- a/gradient/api_sdk/repositories/experiments.py +++ /dev/null @@ -1,251 +0,0 @@ -import abc - -import six -import websocket - -from .common import ListResources, CreateResource, StartResource, StopResource, DeleteResource, GetResource, GetMetrics, ListMetrics, \ - StreamMetrics, ListLogs -from .. import config, serializers, sdk_exceptions -from ..repositories.jobs import ListJobs -from ..serializers import utils as serializers_utils -from ..utils import concatenate_urls - - -class GetBaseExperimentApiUrlMixin(object): - def _get_api_url(self, **_): - return config.config.CONFIG_EXPERIMENTS_HOST_V2 - - -class GetBaseExperimentApiUrlBasedOnClusterIdMixin(object): - def _get_api_url(self, **kwargs): - if kwargs.get("clusterId"): - return config.config.CONFIG_EXPERIMENTS_HOST_V2 - - return config.config.CONFIG_EXPERIMENTS_HOST - - -class ParseExperimentDictMixin(object): - def _parse_object(self, experiment_dict, **kwargs): - """ - :param dict experiment_dict: - :rtype BaseExperiment - """ - serializer = serializers_utils.get_serializer_for_experiment(experiment_dict) - experiment = serializer().get_instance(experiment_dict) - return experiment - - -class ListExperiments(ParseExperimentDictMixin, GetBaseExperimentApiUrlMixin, ListResources): - def get_request_url(self, **kwargs): - return "/experiments/" - - def _get_meta_data(self, resp): - return resp.data.get("meta") - - def _parse_objects(self, data, **kwargs): - experiments_dicts = self._get_experiments_dicts_from_json_data(data, kwargs) - experiments = [] - for experiment_dict in experiments_dicts: - experiment_dict.update(experiment_dict["templateHistory"].get("params", {})) - experiment = self._parse_object(experiment_dict) - experiments.append(experiment) - - return experiments - - @staticmethod - def _get_experiments_dicts_from_json_data(data, kwargs): - filtered = bool(kwargs.get("project_id")) - if not filtered: # If filtering by project ID response data has different format... - return data["data"] - - experiments = [] - for project_experiments in data["data"]: - for experiment in project_experiments["data"]: - experiments.append(experiment) - - return experiments - - def _get_request_params(self, kwargs): - params = { - "limit": kwargs.get("limit"), - "offset": kwargs.get("offset") - } - - project_id = kwargs.get("project_id") - if project_id: - if isinstance(project_id, six.string_types): - project_id = [project_id] - for i, experiment_id in enumerate(project_id): - key = "projectHandle[{}]".format(i) - params[key] = experiment_id - - tags = kwargs.get("tags") - if tags: - params["tag"] = tags - - return params - - -class GetExperiment(ParseExperimentDictMixin, GetBaseExperimentApiUrlMixin, GetResource): - def _parse_object(self, experiment_dict, **kwargs): - experiment_dict = experiment_dict["data"] - experiment_dict.update(experiment_dict["templateHistory"].get("params", {})) - return super(GetExperiment, self)._parse_object(experiment_dict, **kwargs) - - def get_request_url(self, **kwargs): - experiment_id = kwargs["experiment_id"] - url = "/experiments/{}/".format(experiment_id) - return url - - -class ListExperimentLogs(ListLogs): - def _get_request_params(self, kwargs): - params = { - "experimentId": kwargs["id"], - "line": kwargs["line"], - "limit": kwargs["limit"], - } - return params - - -@six.add_metaclass(abc.ABCMeta) -class BaseCreateExperiment(GetBaseExperimentApiUrlBasedOnClusterIdMixin, CreateResource): - def get_request_url(self, **_): - return "/experiments/" - - -class CreateSingleNodeExperiment(BaseCreateExperiment): - SERIALIZER_CLS = serializers.SingleNodeExperimentSchema - - -class CreateMultiNodeExperiment(BaseCreateExperiment): - SERIALIZER_CLS = serializers.MultiNodeExperimentSchema - - -class CreateMpiMultiNodeExperiment(BaseCreateExperiment): - SERIALIZER_CLS = serializers.MpiMultiNodeExperimentSchema - - -class RunSingleNodeExperiment(CreateSingleNodeExperiment): - def get_request_url(self, **_): - return "/experiments/run/" - - -class RunMultiNodeExperiment(CreateMultiNodeExperiment): - def get_request_url(self, **_): - return "/experiments/run/" - - -class RunMpiMultiNodeExperiment(CreateMpiMultiNodeExperiment): - def get_request_url(self, **_): - return "/experiments/run/" - - -class StartExperiment(GetBaseExperimentApiUrlMixin, StartResource): - VALIDATION_ERROR_MESSAGE = "Failed to start experiment" - - def get_request_url(self, **kwargs): - id_ = kwargs["id"] - url = "/experiments/{}/start/".format(id_) - return url - - -class StopExperiment(GetBaseExperimentApiUrlMixin, StopResource): - VALIDATION_ERROR_MESSAGE = "Failed to stop experiment" - - def get_request_url(self, **kwargs): - id_ = kwargs["id"] - url = "/experiments/{}/stop/".format(id_) - return url - - -class DeleteExperiment(GetBaseExperimentApiUrlMixin, DeleteResource): - def get_request_url(self, **kwargs): - experiment_id = kwargs["id"] - return "/experiments/{}/".format(experiment_id) - - -class GetExperimentMetricsApiUrlMixin(object): - def _get_metrics_api_url(self, instance, protocol="https"): - instance_id = getattr(instance, "id", instance) - repository = ListJobs(api_key=self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - try: - job = repository.list(experiment_id=instance_id)[0] - except IndexError: - raise sdk_exceptions.GradientSdkError("Experiment has not started yet") - - metrics_api_url = concatenate_urls(protocol + "://", job.metrics_url) - if not job.metrics_url: - raise sdk_exceptions.GradientSdkError("Metrics API url not found") - - return metrics_api_url - - -class GetExperimentMetrics(GetExperimentMetricsApiUrlMixin, GetMetrics): - OBJECT_TYPE = "experiment" - - def _get_instance_by_id(self, instance_id, **kwargs): - repository = GetExperiment(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(experiment_id=instance_id) - return instance - - def _get_start_date(self, instance, kwargs): - rv = super(GetExperimentMetrics, self)._get_start_date(instance, kwargs) - if rv is None: - raise sdk_exceptions.GradientSdkError("Experiment has not started yet") - - return rv - - def _get_instance(self, response, **kwargs): - try: - rv = super(GetExperimentMetrics, self)._get_instance(response, **kwargs) - except sdk_exceptions.ResourceFetchingError as e: - if '{"version":' in str(e): - # TODO: metrics are not working for v1 experiments at the moment - raise sdk_exceptions.GradientSdkError("Metrics are available for private clusters only") - else: - raise - - return rv - -class ListExperimentMetrics(GetExperimentMetricsApiUrlMixin, ListMetrics): - OBJECT_TYPE = "experiment" - - def _get_instance_by_id(self, instance_id, **kwargs): - repository = GetExperiment(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(experiment_id=instance_id) - return instance - - def _get_start_date(self, instance, kwargs): - rv = super(ListExperimentMetrics, self)._get_start_date(instance, kwargs) - if rv is None: - raise sdk_exceptions.GradientSdkError("Experiment has not started yet") - - return rv - - def _get_instance(self, response, **kwargs): - try: - rv = super(ListExperimentMetrics, self)._get_instance(response, **kwargs) - except sdk_exceptions.ResourceFetchingError as e: - if '{"version":' in str(e): - # TODO: metrics are not working for v1 experiments at the moment - raise sdk_exceptions.GradientSdkError("Custom metrics are available for private clusters only") - else: - raise - - return rv - -class StreamExperimentMetrics(GetExperimentMetricsApiUrlMixin, StreamMetrics): - OBJECT_TYPE = "experiment" - - def _get_connection(self, kwargs): - try: - ws = super(StreamExperimentMetrics, self)._get_connection(kwargs) - except websocket.WebSocketBadStatusException as e: - if "Handshake status 200 OK" in str(e): - # TODO: metrics are not working for v1 experiments at the moment - raise sdk_exceptions.GradientSdkError("Metrics are available for private clusters only") - else: - raise - - return ws diff --git a/gradient/api_sdk/repositories/hyperparameter.py b/gradient/api_sdk/repositories/hyperparameter.py deleted file mode 100644 index 186d9a23..00000000 --- a/gradient/api_sdk/repositories/hyperparameter.py +++ /dev/null @@ -1,54 +0,0 @@ -from .common import CreateResource, ListResources, GetResource, StartResource -from .. import serializers -from ..repositories.experiments import ParseExperimentDictMixin, GetBaseExperimentApiUrlMixin - - -class CreateHyperparameterJob(GetBaseExperimentApiUrlMixin, CreateResource): - SERIALIZER_CLS = serializers.HyperparameterSchema - - def get_request_url(self, **_): - return "/hyperopt/" - - -class CreateAndStartHyperparameterJob(CreateHyperparameterJob): - def get_request_url(self, **_): - return "/hyperopt/create_and_start/" - - -class ListHyperparameterJobs(GetBaseExperimentApiUrlMixin, ParseExperimentDictMixin, ListResources): - def get_request_url(self, **kwargs): - return "/hyperopt/" - - def _parse_objects(self, data, **kwargs): - experiments = [] - for experiment_dict in data["data"]: - experiment_dict.update(experiment_dict["templateHistory"].get("params", {})) - experiment = self._parse_object(experiment_dict) - experiments.append(experiment) - - return experiments - - def _get_request_params(self, kwargs): - return {"limit": -1} - - -class GetHyperparameterTuningJob(GetBaseExperimentApiUrlMixin, ParseExperimentDictMixin, GetResource): - def get_request_url(self, **kwargs): - id_ = kwargs["id"] - url = "/hyperopt/{}/".format(id_) - return url - - def _parse_object(self, job_dict, **kwargs): - data = job_dict["data"] - data.update(data["templateHistory"].get("params", {})) - instance = super(GetHyperparameterTuningJob, self)._parse_object(data) - return instance - - -class StartHyperparameterTuningJob(GetBaseExperimentApiUrlMixin, StartResource): - VALIDATION_ERROR_MESSAGE = "Failed to start hyperparameter tuning job" - - def get_request_url(self, **kwargs): - id_ = kwargs["id"] - url = "/hyperopt/{}/start/".format(id_) - return url diff --git a/gradient/api_sdk/repositories/jobs.py b/gradient/api_sdk/repositories/jobs.py deleted file mode 100644 index 76aa5348..00000000 --- a/gradient/api_sdk/repositories/jobs.py +++ /dev/null @@ -1,235 +0,0 @@ -import json - -import gradient.api_sdk.config -from .common import ListResources, CreateResource, GetResource, DeleteResource, StopResource, GetMetrics, ListMetrics, StreamMetrics, \ - ListLogs -from .. import serializers, sdk_exceptions -from ..clients import http_client -from ..serializers import JobSchema - - -class GetBaseJobApiUrlMixin(object): - def _get_api_url(self, **_): - return gradient.api_sdk.config.config.CONFIG_HOST - - -class ListJobs(GetBaseJobApiUrlMixin, ListResources): - - def get_request_url(self, **kwargs): - return "/jobs/getJobList/" - - def _parse_objects(self, data, **kwargs): - jobs = [] - - for job_dict in data["jobList"]: - job = self._parse_object(job_dict) - jobs.append(job) - - return jobs - - def _parse_object(self, job_dict): - job = serializers.JobSchema().get_instance(job_dict) - return job - - def _get_request_params(self, kwargs): - filters = {"filter": {"where": {}}} - if kwargs.get("project_id"): - filters["filter"]["where"]["projectId"] = kwargs.get("project_id") - - if kwargs.get("project"): - filters["filter"]["where"]["project"] = kwargs.get("project") - - if kwargs.get("experiment_id"): - filters["filter"]["where"]["experimentId"] = kwargs.get("experiment_id") - - params = {} - filter_string = json.dumps(filters) - params["filter"] = filter_string - - tags = kwargs.get("tags") - if tags: - params["modelName"] = "team" # TODO: filtering by tags won't work without this. Remove this when fixed. - for i, tag in enumerate(tags): - key = "tagFilter[{}]".format(i) - params[key] = tag - - return params or None - - -class ListJobLogs(ListLogs): - def _get_request_params(self, kwargs): - params = { - "jobId": kwargs["id"], - "line": kwargs["line"], - "limit": kwargs["limit"] - } - return params - - -class CreateJob(GetBaseJobApiUrlMixin, CreateResource): - SERIALIZER_CLS = JobSchema - HANDLE_FIELD = "id" - - def get_request_url(self, **kwargs): - return "/jobs/createJob/" - - def _get_id_from_response(self, response): - handle = response.data[self.HANDLE_FIELD] - return handle - - def _get_request_json(self, instance_dict): - return - - def _get_request_params(self, instance_dict): - if instance_dict.get('datasets'): - instance_dict['datasets'] = json.dumps(instance_dict['datasets']) - - return instance_dict - - -class DeleteJob(GetBaseJobApiUrlMixin, DeleteResource): - - def get_request_url(self, **kwargs): - return "/jobs/{}/destroy".format(kwargs.get("id")) - - def _send_request(self, client, url, json_data=None): - response = client.post(url, json=json_data) - return response - - -class StopJob(GetBaseJobApiUrlMixin, StopResource): - - def get_request_url(self, **kwargs): - return "/jobs/{}/stop".format(kwargs.get('id')) - - def _send_request(self, client, url, json_data=None): - response = client.post(url, json=json_data) - return response - - -class GetJob(GetBaseJobApiUrlMixin, GetResource): - def get_request_url(self, **kwargs): - return "/jobs/getPublicJob" - - def _get_request_json(self, kwargs): - json_ = { - "jobId": kwargs["job_id"] - } - return json_ - - def _send_request(self, client, url, json=None, params=None): - response = client.post(url, json=json, params=params) - return response - - def _parse_object(self, instance_dict, **kwargs): - instance_dict = instance_dict["job"] - job = serializers.JobSchema().get_instance(instance_dict) - return job - - -class ListJobArtifacts(GetBaseJobApiUrlMixin, ListResources): - def _parse_objects(self, data, **kwargs): - serializer = serializers.utils.paginate_schema(serializers.ArtifactSchema) - return serializer.get_instance(data) - - def get_request_url(self, **kwargs): - return "/jobs/artifactsListV2" - - def _get_request_params(self, kwargs): - params = { - "jobId": kwargs.get("jobId"), - } - - if kwargs.get("files"): - params["files"] = kwargs.get("files") - - if kwargs.get("size"): - params["size"] = kwargs.get("size") - - if kwargs.get("links"): - params["links"] = kwargs.get("links") - - if kwargs.get("start_after"): - params["startAfter"] = kwargs.get("start_after") - - return params - - -class DeleteJobArtifacts(GetBaseJobApiUrlMixin, DeleteResource): - VALIDATION_ERROR_MESSAGE = "Failed to delete resource" - - def get_request_url(self, **kwargs): - return "/jobs/{}/artifactsDestroy".format(kwargs.get("id")) - - def _send(self, url, **kwargs): - client = self._get_client(**kwargs) - params_data = self._get_request_params(kwargs) - response = self._send_request(client, url, params_data=params_data) - gradient_response = http_client.GradientResponse.interpret_response(response) - return gradient_response - - def _send_request(self, client, url, params_data=None): - response = client.post(url, params=params_data) - return response - - def _get_request_params(self, kwargs): - filters = dict() - - if kwargs.get("files"): - filters["files"] = kwargs.get("files") - - return filters or None - - -class GetJobArtifacts(GetBaseJobApiUrlMixin, GetResource): - def _parse_object(self, data, **kwargs): - return data - - def _get_request_params(self, kwargs): - return { - "jobId": kwargs.get("jobId") - } - - def get_request_url(self, **kwargs): - return "/jobs/artifactsGet" - - -class GetJobMetrics(GetMetrics): - OBJECT_TYPE = "mljob" - - def _get_instance_by_id(self, instance_id, **kwargs): - repository = GetJob(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(job_id=instance_id) - return instance - - def _get_start_date(self, instance, kwargs): - rv = super(GetJobMetrics, self)._get_start_date(instance, kwargs) - if rv is None: - raise sdk_exceptions.GradientSdkError("Job has not started yet") - - return rv - -class ListJobMetrics(ListMetrics): - OBJECT_TYPE = "mljob" - - def _get_instance_by_id(self, instance_id, **kwargs): - repository = GetJob(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(job_id=instance_id) - return instance - - def _get_start_date(self, instance, kwargs): - rv = super(ListJobMetrics, self)._get_start_date(instance, kwargs) - if rv is None: - raise sdk_exceptions.GradientSdkError("Job has not started yet") - - return rv - -class StreamJobMetrics(StreamMetrics): - OBJECT_TYPE = "mljob" - - def _get_metrics_api_url(self, instance_id, protocol="https"): - repository = GetJob(api_key=self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) - instance = repository.get(job_id=instance_id) - - metrics_api_url = super(StreamJobMetrics, self)._get_metrics_api_url(instance, protocol="wss") - return metrics_api_url diff --git a/gradient/api_sdk/repositories/tensorboards.py b/gradient/api_sdk/repositories/tensorboards.py deleted file mode 100644 index 046b1b56..00000000 --- a/gradient/api_sdk/repositories/tensorboards.py +++ /dev/null @@ -1,101 +0,0 @@ -from . import common -from .. import config -from .. import serializers, models - - -class GetTensorboardApiUrlMixin(object): - def _get_api_url(self, **kwargs): - return config.config.CONFIG_SERVICE_HOST - - -class ParseTensorboardMixin(object): - def _parse_object(self, instance_dict, **kwargs): - machine_instance_dict = instance_dict.get("instance") - if machine_instance_dict: - machine_instance = serializers.InstanceSchema().get_instance(machine_instance_dict) - else: - machine_instance = models.Instance() - - instance = super(ParseTensorboardMixin, self)._parse_object(instance_dict) - instance.instance = machine_instance - return instance - - -class CreateTensorboard(GetTensorboardApiUrlMixin, common.CreateResource): - SERIALIZER_CLS = serializers.TensorboardSchema - - def get_request_url(self, **kwargs): - return "/tensorboards/v1/" - - def _get_id_from_response(self, response): - handle = response.data["data"]["id"] - return handle - - -class GetTensorboard(ParseTensorboardMixin, GetTensorboardApiUrlMixin, common.GetResource): - SERIALIZER_CLS = serializers.TensorboardDetailSchema - - def get_request_url(self, **kwargs): - id_ = kwargs["id"] - return "/tensorboards/v1/{}/".format(id_) - - def _parse_object(self, instance_dict, **kwargs): - instance_dict = instance_dict["data"] - instance = super(GetTensorboard, self)._parse_object(instance_dict) - return instance - - -class ListTensorboards(ParseTensorboardMixin, GetTensorboardApiUrlMixin, common.ListResources): - SERIALIZER_CLS = serializers.TensorboardDetailSchema - - def _get_instance_dicts(self, data, **kwargs): - instance_dicts = data["data"] - return instance_dicts - - def get_request_url(self, **kwargs): - return "/tensorboards/v1/" - - -class UpdateTensorboard(ParseTensorboardMixin, GetTensorboardApiUrlMixin, common.AlterResource): - VALIDATION_ERROR_MESSAGE = "Failed to update resource" - - def update(self, id, **kwargs): - resp = self._run(id=id, **kwargs) - tensorboard = self._parse_object(resp.data) - return tensorboard - - def get_request_url(self, **kwargs): - return "/tensorboards/v1/{}".format(kwargs["id"]) - - def _send_request(self, client, url, json_data=None): - response = client.put(url, json=json_data) - return response - - def _get_request_json(self, kwargs): - return { - "added_experiments": kwargs.get("added_experiments", list()), - "removed_experiments": kwargs.get("removed_experiments", list()) - } - - def _parse_object(self, instance_dict, **kwargs): - instance_dict = instance_dict["data"] - instance = self._get_instance(instance_dict) - return instance - - def _get_instance(self, instance_dict): - machine_instance_dict = instance_dict.get("instance") - if machine_instance_dict: - machine_instance = serializers.InstanceSchema().get_instance(machine_instance_dict) - else: - machine_instance = models.Instance() - - instance = serializers.TensorboardDetailSchema().get_instance(instance_dict) - instance.instance = machine_instance - return instance - - -class DeleteTensorboard(GetTensorboardApiUrlMixin, common.DeleteResource): - VALIDATION_ERROR_MESSAGE = "Failed to delete resource" - - def get_request_url(self, **kwargs): - return "/tensorboards/v1/{}".format(kwargs["id"]) diff --git a/gradient/api_sdk/s3_downloader.py b/gradient/api_sdk/s3_downloader.py index 7a803538..521e093a 100644 --- a/gradient/api_sdk/s3_downloader.py +++ b/gradient/api_sdk/s3_downloader.py @@ -6,7 +6,7 @@ import six from . import sdk_exceptions -from .clients import JobsClient, ModelsClient +from .clients import ModelsClient from .clients.base_client import BaseClient from .logger import MuteLogger @@ -23,7 +23,8 @@ def download_list(self, sources, destination_dir): :param str destination_dir: """ for source in sources: - self.download_file(source, destination_dir, max_retries=self.file_download_retries) + self.download_file(source, destination_dir, + max_retries=self.file_download_retries) def download_file(self, source, destination_dir, max_retries=0): self._create_directory(destination_dir) @@ -38,10 +39,12 @@ def download_file(self, source, destination_dir, max_retries=0): response = requests.get(file_url) break except requests.exceptions.ConnectionError: - self.logger.debug("Downloading {} resulted in error. Trying again...".format(file_path)) + self.logger.debug( + "Downloading {} resulted in error. Trying again...".format(file_path)) time.sleep(0.1) else: # break statement not executed - ConnectionError `max_retries` times - raise sdk_exceptions.ResourceFetchingError("Downloading {} resulted in error".format(file_path)) + raise sdk_exceptions.ResourceFetchingError( + "Downloading {} resulted in error".format(file_path)) self._create_subdirectories(file_path, destination_dir) self._save_file(response, file_path, destination_dir) @@ -75,7 +78,8 @@ def __init__(self, api_key, logger=MuteLogger(), ps_client_name=None): self.api_key = api_key self.logger = logger self.ps_client_name = ps_client_name - self.client = self._build_client(self.CLIENT_CLASS, api_key, logger=logger) + self.client = self._build_client( + self.CLIENT_CLASS, api_key, logger=logger) def download(self, job_id, destination): files = self._get_files_list(job_id) @@ -103,26 +107,6 @@ def _build_client(self, client_class, *args, **kwargs): return client -class JobArtifactsDownloader(ResourceDownloader): - CLIENT_CLASS = JobsClient - - def _get_files_list(self, job_id): - start_after = None - files = [] - while True: - pagination_response = self.client.artifacts_list(job_id, start_after=start_after) - - if pagination_response.data: - files.extend(pagination_response.data) - start_after = pagination_response.start_after - - if start_after is None: - break - - files = tuple((f.file, f.url) for f in files) - return files - - class ModelFilesDownloader(ResourceDownloader): CLIENT_CLASS = ModelsClient diff --git a/gradient/api_sdk/s3_uploader.py b/gradient/api_sdk/s3_uploader.py index f45b633e..5c5d7392 100644 --- a/gradient/api_sdk/s3_uploader.py +++ b/gradient/api_sdk/s3_uploader.py @@ -37,8 +37,10 @@ def upload(self, file_path, url, s3_fields=None): ordered_s3_fields = collections.OrderedDict(s3_fields) with open(file_path, "rb") as file_handle: ordered_s3_fields["file"] = (file_path, file_handle) - multipart_encoder_monitor = self._get_multipart_encoder_monitor(ordered_s3_fields) - self.logger.debug("Uploading file: {} to url: {}...".format(file_path, url)) + multipart_encoder_monitor = self._get_multipart_encoder_monitor( + ordered_s3_fields) + self.logger.debug( + "Uploading file: {} to url: {}...".format(file_path, url)) self._upload(url, data=multipart_encoder_monitor) self.logger.debug("Uploading completed") @@ -55,7 +57,8 @@ def _upload(self, url, data): raise sdk_exceptions.S3UploadFailedError(response) def _get_client(self, url): - client = http_client.API(url, logger=self.logger, ps_client_name=self.ps_client_name) + client = http_client.API( + url, logger=self.logger, ps_client_name=self.ps_client_name) return client def _get_multipart_encoder_monitor(self, fields): @@ -88,90 +91,14 @@ def _upload(self, url, data): """ file_path = data.encoder.fields['file'][0] client = self._get_client(url) - client.headers = {"Content-Type": mimetypes.guess_type(file_path)[0] or ""} + client.headers = { + "Content-Type": mimetypes.guess_type(file_path)[0] or ""} response = client.put("", data=data) if not response.ok: raise sdk_exceptions.S3UploadFailedError(response) -class ExperimentFileUploader(object): - def __init__(self, api_key, uploader=None, logger=None, ps_client_name=None): - """ - :param str api_key: - :param S3FileUploader uploader: - :param Logger logger: - """ - self.logger = logger or MuteLogger() - self.experiments_api = http_client.API( - config.CONFIG_EXPERIMENTS_HOST, - api_key=api_key, - logger=self.logger, - ps_client_name=ps_client_name, - ) - self.uploader = uploader or S3FileUploader(logger=self.logger, ps_client_name=ps_client_name) - - def upload(self, file_path, project_id, cluster_id=None): - """Upload file to S3 bucket for a project - - :param str file_path: - :param str project_id: - :param str cluster_id: - - :rtype: str - :return: S3 bucket's URL - """ - url, bucket_name, fields = self._get_upload_data(file_path, project_id, cluster_id=cluster_id) - self.uploader.upload(file_path, url, fields) - bucket_url = self.uploader.get_bucket_url(bucket_name, fields) - return bucket_url - - def _get_upload_data(self, file_path, project_handle, cluster_id=None): - """Ask API for data required to upload a file to S3 - - :param str file_path: - :param str project_handle: - :param str cluster_id: - - :rtype: tuple[str,str,dict] - :return: URL to which send the file, name of the bucket and a dictionary required by S3 service - """ - file_name = os.path.basename(file_path) - params = {"workspaceName": file_name, "projectHandle": project_handle} - if cluster_id: - params['clusterHandle'] = cluster_id - - response = self.experiments_api.get("/workspace/get_presigned_url", params=params) - if response.status_code == 401: - raise sdk_exceptions.ProjectAccessDeniedError("Access to project denied") - if response.status_code == 403: - raise sdk_exceptions.PresignedUrlAccessDeniedError("Access denied") - if response.status_code == 404: - raise sdk_exceptions.PresignedUrlUnreachableError("URL not found") - if not response.ok: - raise sdk_exceptions.PresignedUrlConnectionError(response.reason) - - try: - response_data = response.json() - if response_data["message"] != "success": - raise sdk_exceptions.PresignedUrlError("Presigned url error: {}".format(response_data)) - - url = response_data["data"]["url"] - bucket_name = response_data["data"]["bucket_name"] - fields = response_data["data"]["fields"] - except (KeyError, ValueError): - raise sdk_exceptions.PresignedUrlMalformedResponseError("Response malformed") - - return url, bucket_name, fields - - -class S3ProjectFileUploader(ExperimentFileUploader): - """ - DEPRECATED: This class will be renamed to ExperimentFileUploader in release v0.8 - """ - pass - - class S3ModelFileUploader(object): DEFAULT_MULTIPART_ENCODER_CLS = MultipartEncoderWithProgressbar @@ -224,19 +151,22 @@ def _get_upload_data(self, file_path, model_id, cluster_id=None): if cluster_id: params["clusterId"] = cluster_id - response = self.ps_api_client.get("/mlModels/getPresignedModelUrl", params=params) + response = self.ps_api_client.get( + "/mlModels/getPresignedModelUrl", params=params) if not response.ok: raise sdk_exceptions.PresignedUrlConnectionError(response.reason) try: url = response.json() except (KeyError, ValueError): - raise sdk_exceptions.PresignedUrlMalformedResponseError("Response malformed") + raise sdk_exceptions.PresignedUrlMalformedResponseError( + "Response malformed") return url def _get_client(self, url, ps_client_name=None, api_key=None): - client = http_client.API(url, logger=self.logger, ps_client_name=ps_client_name, api_key=api_key) + client = http_client.API( + url, logger=self.logger, ps_client_name=ps_client_name, api_key=api_key) return client @@ -258,110 +188,6 @@ def _get_archiver(self): def _get_archive_path(self): archive_file_name = 'model.zip' - archive_file_path = os.path.join(tempfile.gettempdir(), archive_file_name) + archive_file_path = os.path.join( + tempfile.gettempdir(), archive_file_name) return archive_file_path - - -class ExperimentWorkspaceDirectoryUploader(object): - def __init__(self, api_key, temp_dir=None, archiver=None, project_uploader=None, ps_client_name=None): - """ - :param str api_key: - :param str temp_dir: - :param ZipArchiver archiver: - :param ExperimentFileUploader project_uploader: - """ - self.temp_dir = temp_dir or tempfile.gettempdir() - self.archiver = archiver or ZipArchiver() - self.ps_client_name = ps_client_name - self.project_uploader = project_uploader or ExperimentFileUploader(api_key, ps_client_name=ps_client_name) - - def upload(self, workspace_dir_path, project_id, exclude=None, temp_file_name="temp.zip"): - """Archive and upload a workspace directory - - :param str workspace_dir_path: - :param str project_id: - :param list|tuple|None exclude: - :param str temp_file_name: - - :rtype: str - :return: URL to the S3 bucket - """ - archive_path = self.get_archive_path(temp_file_name) - self.archiver.archive(workspace_dir_path, archive_path, exclude=exclude) - bucket_url = self.project_uploader.upload(archive_path, project_id) - return bucket_url - - def get_archive_path(self, temp_file_name): - archive_path = os.path.join(self.temp_dir, temp_file_name) - return archive_path - - -class S3WorkspaceDirectoryUploader(ExperimentWorkspaceDirectoryUploader): - """ - DEPRECATED: This class will be renamed to ExperimentWorkspaceDirectoryUploader in release v0.8 - """ - pass - - -class DeploymentWorkspaceDirectoryUploader(object): - def __init__(self, api_key, uploader=None, logger=None, ps_client_name=None): - """ - :param str api_key: - :param S3PutFileUploader uploader: - :param Logger logger: - """ - self.logger = logger or MuteLogger() - self.ps_api_client = http_client.API( - config.CONFIG_HOST, - api_key=api_key, - logger=self.logger, - ps_client_name=ps_client_name, - ) - self.uploader = S3PutFileUploader(logger=self.logger, ps_client_name=ps_client_name) - - def _get_upload_data(self, file_path, project_id, cluster_id=None): - """Ask API for data required to upload deployment workspace a file to S3 - - :param str file_path: - :param str project_id: - :param str cluster_id: - - :rtype: str - :return: URL to which send the file, name of the bucket and a dictionary required by S3 service - """ - file_name = os.path.basename(file_path) - params = { - "fileName": file_name, - "contentType": mimetypes.guess_type(file_path)[0] or "", - } - if project_id: - params['projectId'] = project_id - if cluster_id: - params['clusterHandle'] = cluster_id - response = self.ps_api_client.get("/deployments/getPresignedDeploymentUrl", params=params) - if not response.ok: - raise sdk_exceptions.PresignedUrlConnectionError(response.reason) - - try: - response_data = response.json() - presigned_url = response_data['presignedUrl'] - bucket_name = response_data['bucketName'] - workspace_url = response_data['workspaceUrl'] - except (KeyError, ValueError): - raise sdk_exceptions.PresignedUrlMalformedResponseError("Response malformed") - - return presigned_url, bucket_name, workspace_url - - def upload(self, file_path, project_id=None, cluster_id=None, **kwargs): - """Upload file to S3 bucket for a project - - :param str file_path: - :param str project_id: - :param str cluster_id: - - :rtype: str - :return: S3 bucket's URL - """ - url, bucket_name, workspace_url = self._get_upload_data(file_path, project_id, cluster_id) - self.uploader.upload(file_path, url) - return workspace_url diff --git a/gradient/api_sdk/serializers/__init__.py b/gradient/api_sdk/serializers/__init__.py index db059451..880d0b5e 100644 --- a/gradient/api_sdk/serializers/__init__.py +++ b/gradient/api_sdk/serializers/__init__.py @@ -2,11 +2,6 @@ from .dataset import DatasetSchema, DatasetRefSchema from .dataset_tag import DatasetTagSchema from .dataset_version import DatasetVersionSchema, DatasetVersionPreSignedURLSchema -from .deployment import DeploymentSchema, DeploymentCreateSchema -from .experiment import BaseExperimentSchema, MultiNodeExperimentSchema, SingleNodeExperimentSchema, \ - MpiMultiNodeExperimentSchema -from .hyperparameter import HyperparameterSchema -from .job import JobSchema from .log import LogRowSchema from .machine import MachineSchema, MachineSchemaForListing, MachineEventSchema from .model import Model, ModelFileSchema @@ -15,6 +10,5 @@ from .secret import SecretSchema from .storage_provider import StorageProviderSchema from .tag import TagSchema -from .tensorboard import InstanceSchema, TensorboardSchema, TensorboardDetailSchema from .vm_type import VmTypeSchema, VmTypeGpuModelSchema from .workflows import WorkflowSchema, WorkflowRunSchema, WorkflowSpecSchema diff --git a/gradient/api_sdk/serializers/deployment.py b/gradient/api_sdk/serializers/deployment.py deleted file mode 100644 index 173454ff..00000000 --- a/gradient/api_sdk/serializers/deployment.py +++ /dev/null @@ -1,71 +0,0 @@ -import marshmallow as ma - -from .base import BaseSchema -from .. import models - - -class AutoscalingMetricSchema(BaseSchema): - MODEL = models.AutoscalingMetric - - type = ma.fields.Str() - name = ma.fields.Str() - value_type = ma.fields.Str(dump_to="valueType", load_from="valueType") - value = ma.fields.Float() - - -class AutoscalingDefinitionSchema(BaseSchema): - MODEL = models.AutoscalingDefinition - - min_instance_count = ma.fields.Int(dump_to="minInstanceCount", load_from="minInstanceCount") - max_instance_count = ma.fields.Int(dump_to="maxInstanceCount", load_from="maxInstanceCount") - scale_cooldown_period = ma.fields.Int(dump_to="scaleCooldownPeriod", load_from="scaleCooldownPeriod") - metrics = ma.fields.Nested(AutoscalingMetricSchema, many=True, default=None) - - -class DeploymentSchema(BaseSchema): - MODEL = models.Deployment - - id = ma.fields.Str(dump_to="id", load_from="id") - name = ma.fields.Str(required=True) - endpoint = ma.fields.Str() - api_type = ma.fields.Str(dump_to="apiType", load_from="apiType") - - state = ma.fields.Str() - - model_id = ma.fields.Str(rquired=True, dump_to="modelId", load_from="modelId") - project_id = ma.fields.Str(dump_to="projectId", load_from="projectId") - - image_url = ma.fields.Str(required=True, dump_to="imageUrl", load_from="imageUrl") - deployment_type = ma.fields.Str(required=True, dump_to="deploymentType", load_from="deploymentType") - machine_type = ma.fields.Str(required=True, dump_to="machineType", load_from="machineType") - instance_count = ma.fields.Int(required=True, dump_to="instanceCount", load_from="instanceCount") - container_model_path = ma.fields.Str(dump_to="containerModelPath", load_from="containerModelPath") - image_username = ma.fields.Str(dump_to="imageUsername", load_from="imageUsername") - image_password = ma.fields.Str(dump_to="imagePassword", load_from="imagePassword") - image_server = ma.fields.Str(dump_to="imageServer", load_from="imageServer") - container_url_path = ma.fields.Str(dump_to="containerUrlPath", load_from="containerUrlPath") - endpoint_url_path = ma.fields.Str(dump_to="endpointUrlPath", load_from="endpointUrlPath") - method = ma.fields.Str(dump_to="method", load_from="method") - docker_args = ma.fields.List(ma.fields.Str(), dump_to="dockerArgs", load_from="dockerArgs") - env = ma.fields.Dict(dump_to="env", load_from="env") - ports = ma.fields.Str(dump_to="ports", load_from="ports") - auth_username = ma.fields.Str(dump_to="oauthKey", load_from="oauthKey") - auth_password = ma.fields.Str(dump_to="oauthSecret", load_from="oauthSecret") - cluster_id = ma.fields.Str(dump_to="clusterId", load_from="clusterId") - tags = ma.fields.List(ma.fields.Str(), load_only=True) - command = ma.fields.Str() - metrics_url = ma.fields.Str(dump_to="metricsURL", load_from="metricsURL") - workspace_url = ma.fields.Str(dump_to="workspaceUrl", load_from="workspaceUrl") - workspace_ref = ma.fields.Str(dump_to="workspaceRef", load_from="workspaceRef") - workspace_username = ma.fields.Str(dump_to="workspaceUsername", load_from="workspaceUsername") - workspace_password = ma.fields.Str(dump_to="workspacePassword", load_from="workspacePassword") - dt_created = ma.fields.DateTime(dump_to="dtCreated", load_from="dtCreated") - dt_modified = ma.fields.DateTime(dump_to="dtModified", load_from="dtModified") - dt_started = ma.fields.DateTime(dump_to="dtStarted", load_from="dtStarted") - dt_stopped = ma.fields.DateTime(dump_to="dtStopped", load_from="dtStopped") - dt_deleted = ma.fields.DateTime(dump_to="dtDeleted", load_from="dtDeleted") - autoscaling = ma.fields.Nested(AutoscalingDefinitionSchema) - - -class DeploymentCreateSchema(DeploymentSchema): - cluster_id = ma.fields.Str(dump_to="cluster", load_from="clusterId") diff --git a/gradient/api_sdk/serializers/experiment.py b/gradient/api_sdk/serializers/experiment.py deleted file mode 100644 index 2b5bd370..00000000 --- a/gradient/api_sdk/serializers/experiment.py +++ /dev/null @@ -1,147 +0,0 @@ -import copy - -import marshmallow - -from . import experiment_dataset -from .base import BaseSchema -from .tag import TagSchema -from .. import models, utils - - -class BaseExperimentSchema(BaseSchema): - experiment_type_id = marshmallow.fields.Int(required=True, dump_to="experimentTypeId", load_from="experimentTypeId") - name = marshmallow.fields.Str(required=True) - ports = marshmallow.fields.Str() - workspace_url = marshmallow.fields.Str(dump_to="workspaceUrl", load_from="workspaceUrl") - workspace_ref = marshmallow.fields.Str(dump_to="workspaceRef", load_from="workspaceRef") - workspace_username = marshmallow.fields.Str(dump_to="workspaceUsername", load_from="workspaceUsername") - workspace_password = marshmallow.fields.Str(dump_to="workspacePassword", load_from="workspacePassword") - datasets = marshmallow.fields.Nested(experiment_dataset.ExperimentDatasetSchema, many=True) - working_directory = marshmallow.fields.Str(dump_to="workingDirectory", load_from="workingDirectory") - artifact_directory = marshmallow.fields.Str(dump_to="artifactDirectory", load_from="artifactDirectory") - cluster_id = marshmallow.fields.String(dump_to="clusterId", load_from="clusterId") - experiment_env = marshmallow.fields.Dict(dump_to="experimentEnv", load_from="experimentEnv") - project_id = marshmallow.fields.Str(required=True, dump_to="projectHandle", load_from="project_handle") - model_type = marshmallow.fields.Str(dump_to="modelType", load_from="modelType") - model_path = marshmallow.fields.Str(dump_to="modelPath", load_from="modelPath") - is_preemptible = marshmallow.fields.Bool(dump_to="isPreemptible", load_from="isPreemptible") - id = marshmallow.fields.Str(load_from="handle") - state = marshmallow.fields.Int() - tags = marshmallow.fields.Nested(TagSchema, only="name", many=True, load_only=True) - - dt_created = marshmallow.fields.DateTime(dump_to="dtCreated", load_from="dtCreated") - dt_modified = marshmallow.fields.DateTime(dump_to="dtModified", load_from="dtModified") - dt_started = marshmallow.fields.DateTime(dump_to="dtStarted", load_from="dtStarted") - dt_stopped = marshmallow.fields.DateTime(dump_to="dtStopped", load_from="dtStopped") - dt_deleted = marshmallow.fields.DateTime(dump_to="dtDeleted", load_from="dtDeleted") - - def get_instance(self, obj_dict, many=False): - # without popping these marshmallow wouldn't use load_from - obj_dict.pop("id", None) - obj_dict.pop("project_id", None) - - ports = obj_dict.get("ports") - if isinstance(ports, int): - obj_dict["ports"] = str(ports) - - instance = super(BaseExperimentSchema, self).get_instance(obj_dict, many=many) - return instance - - -class SingleNodeExperimentSchema(BaseExperimentSchema): - MODEL = models.SingleNodeExperiment - - container = marshmallow.fields.Str(required=True, load_from="worker_container") - machine_type = marshmallow.fields.Str(required=True, dump_to="machineType", load_from="worker_machine_type") - command = marshmallow.fields.Str(required=True, load_from="worker_command") - container_user = marshmallow.fields.Str(dump_to="containerUser", load_from="containerUser") - registry_username = marshmallow.fields.Str(dump_to="registryUsername", load_from="registryUsername") - registry_password = marshmallow.fields.Str(dump_to="registryPassword", load_from="registryPassword") - registry_url = marshmallow.fields.Str(dump_to="registryUrl", load_from="registryUrl") - - @marshmallow.pre_dump - def preprocess(self, data, **kwargs): - data = copy.copy(data) - - utils.base64_encode_attribute(data, "command") - return data - - -class MultiNodeExperimentSchema(BaseExperimentSchema): - MODEL = models.MultiNodeExperiment - - worker_container = marshmallow.fields.Str(required=True, dump_to="workerContainer", load_from="workerContainer") - worker_machine_type = marshmallow.fields.Str(required=True, dump_to="workerMachineType", - load_from="workerMachineType") - worker_command = marshmallow.fields.Str(required=True, dump_to="workerCommand", load_from="workerCommand") - worker_count = marshmallow.fields.Int(required=True, dump_to="workerCount", load_from="workerCount") - parameter_server_container = marshmallow.fields.Str(required=True, dump_to="parameterServerContainer", - load_from="parameterServerContainer") - parameter_server_machine_type = marshmallow.fields.Str(required=True, dump_to="parameterServerMachineType", - load_from="parameterServerMachineType") - parameter_server_command = marshmallow.fields.Str(required=True, dump_to="parameterServerCommand", - load_from="parameterServerCommand") - parameter_server_count = marshmallow.fields.Int(required=True, dump_to="parameterServerCount", - load_from="parameterServerCount") - worker_container_user = marshmallow.fields.Str(dump_to="workerContainerUser", load_from="workerContainerUser") - worker_registry_username = marshmallow.fields.Str(dump_to="workerRegistryUsername", - load_from="workerRegistryUsername") - worker_registry_password = marshmallow.fields.Str(dump_to="workerRegistryPassword", - load_from="workerRegistryPassword") - worker_registry_url = marshmallow.fields.Str(dump_to="workerRegistryUrl", - load_from="workerRegistryUrl") - parameter_server_container_user = marshmallow.fields.Str(required=True, dump_to="parameterServerContainerUser", - load_from="parameterServerContainerUser") - parameter_server_registry_username = marshmallow.fields.Str(dump_to="parameterServerRegistryUsername", - load_from="parameterServerRegistryUsername") - parameter_server_registry_password = marshmallow.fields.Str(dump_to="parameterServerRegistryPassword", - load_from="parameterServerRegistryPassword") - parameter_server_registry_url = marshmallow.fields.Str(dump_to="parameterServerRegistryUrl", - load_from="parameterServerRegistryUrl") - - @marshmallow.pre_dump - def preprocess(self, data, **kwargs): - data = copy.copy(data) - - utils.base64_encode_attribute(data, "worker_command") - utils.base64_encode_attribute(data, "parameter_server_command") - return data - - -class MpiMultiNodeExperimentSchema(BaseExperimentSchema): - MODEL = models.MpiMultiNodeExperiment - - worker_container = marshmallow.fields.Str(required=True, dump_to="workerContainer", load_from="workerContainer") - worker_machine_type = marshmallow.fields.Str(required=True, dump_to="workerMachineType", - load_from="workerMachineType") - worker_command = marshmallow.fields.Str(required=True, dump_to="workerCommand", load_from="workerCommand") - worker_count = marshmallow.fields.Int(required=True, dump_to="workerCount", load_from="workerCount") - master_container = marshmallow.fields.Str(required=True, dump_to="masterContainer", - load_from="masterContainer") - master_machine_type = marshmallow.fields.Str(required=True, dump_to="masterMachineType", - load_from="masterMachineType") - master_command = marshmallow.fields.Str(required=True, dump_to="masterCommand", load_from="masterCommand") - master_count = marshmallow.fields.Int(required=True, dump_to="masterCount", load_from="masterCount") - worker_container_user = marshmallow.fields.Str(dump_to="workerContainerUser", load_from="workerContainerUser") - worker_registry_username = marshmallow.fields.Str(dump_to="workerRegistryUsername", - load_from="workerRegistryUsername") - worker_registry_password = marshmallow.fields.Str(dump_to="workerRegistryPassword", - load_from="workerRegistryPassword") - worker_registry_url = marshmallow.fields.Str(dump_to="workerRegistryUrl", - load_from="workerRegistryUrl") - master_container_user = marshmallow.fields.Str(required=True, dump_to="masterContainerUser", - load_from="masterContainerUser") - master_registry_username = marshmallow.fields.Str(dump_to="masterRegistryUsername", - load_from="masterRegistryUsername") - master_registry_password = marshmallow.fields.Str(dump_to="masterRegistryPassword", - load_from="masterRegistryPassword") - master_registry_url = marshmallow.fields.Str(dump_to="masterRegistryUrl", - load_from="masterRegistryUrl") - - @marshmallow.pre_dump - def preprocess(self, data, **kwargs): - data = copy.copy(data) - - utils.base64_encode_attribute(data, "worker_command") - utils.base64_encode_attribute(data, "master_command") - return data diff --git a/gradient/api_sdk/serializers/experiment_dataset.py b/gradient/api_sdk/serializers/experiment_dataset.py deleted file mode 100644 index 081868d3..00000000 --- a/gradient/api_sdk/serializers/experiment_dataset.py +++ /dev/null @@ -1,27 +0,0 @@ -import marshmallow as ma - -from .base import BaseSchema -from .tag import TagSchema -from .. import models - - -class VolumeOptionsSchema(BaseSchema): - MODEL = models.VolumeOptions - - kind = ma.fields.Str(required=True) - size = ma.fields.Str(required=True) - - -class ExperimentDatasetSchema(BaseSchema): - MODEL = models.ExperimentDataset - - id = ma.fields.String() - uri = ma.fields.String() - aws_access_key_id = ma.fields.String(dump_to="awsAccessKeyId", load_from="awsAccessKeyId") - aws_secret_access_key = ma.fields.String(dump_to="awsSecretAccessKey", load_from="awsSecretAccessKey") - aws_endpoint = ma.fields.String(dump_to="awsEndpoint", load_from="awsEndpoint") - etag = ma.fields.String() - version_id = ma.fields.String(dump_to="versionId", load_from="versionId") - name = ma.fields.String() - tags = ma.fields.Nested(TagSchema, only="name", many=True, load_only=True) - volume_options = ma.fields.Nested(VolumeOptionsSchema, dump_to="volumeOptions", load_from="volumeOptions") diff --git a/gradient/api_sdk/serializers/hyperparameter.py b/gradient/api_sdk/serializers/hyperparameter.py deleted file mode 100644 index fe55da32..00000000 --- a/gradient/api_sdk/serializers/hyperparameter.py +++ /dev/null @@ -1,42 +0,0 @@ -import copy - -import marshmallow - -from .experiment import BaseExperimentSchema -from .tag import TagSchema -from .. import models, utils - - -class HyperparameterSchema(BaseExperimentSchema): - MODEL = models.Hyperparameter - - tuning_command = marshmallow.fields.Str(dump_to="tuningCommand", load_from="tuningCommand") - worker_container = marshmallow.fields.Str(dump_to="workerContainer", load_from="workerContainer") - worker_container_user = marshmallow.fields.Str(dump_to="workerContainerUser", load_from="workerContainerUser") - worker_machine_type = marshmallow.fields.Str(dump_to="workerMachineType", load_from="workerMachineType") - worker_command = marshmallow.fields.Str(dump_to="workerCommand", load_from="workerCommand") - worker_count = marshmallow.fields.Int(dump_to="workerCount", load_from="workerCount") - use_dockerfile = marshmallow.fields.Bool(dump_to="useDockerfile", load_from="useDockerfile") - dockerfile_path = marshmallow.fields.Str(dump_to="dockerfilePath", load_from="dockerfilePath") - worker_registry_username = marshmallow.fields.Str(dump_to="workerRegistryUsername") - worker_registry_password = marshmallow.fields.Str(dump_to="workerRegistryPassword") - - hyperparameter_server_registry_username = marshmallow.fields.Str(dump_to="hyperparameterServerRegistryUsername", - load_from="hyperparameterServerRegistryUsername") - hyperparameter_server_registry_password = marshmallow.fields.Str(dump_to="hyperparameterServerRegistryPassword", - load_from="hyperparameterServerRegistryPassword") - hyperparameter_server_machine_type = marshmallow.fields.Str(dump_to="hyperparameterServerMachineType", - load_from="hyperparameterServerMachineType") - hyperparameter_server_container = marshmallow.fields.Str(dump_to="hyperparameterServerContainer", - load_from="hyperparameterServerContainer") - hyperparameter_server_container_user = marshmallow.fields.Str(dump_to="hyperparameterServerContainerUser", - load_from="hyperparameterServerContainerUser") - tags = marshmallow.fields.Nested(TagSchema, only="name", many=True, load_only=True) - - @marshmallow.pre_dump - def preprocess(self, data, **kwargs): - data = copy.copy(data) - - utils.base64_encode_attribute(data, "worker_command") - utils.base64_encode_attribute(data, "tuning_command") - return data diff --git a/gradient/api_sdk/serializers/job.py b/gradient/api_sdk/serializers/job.py deleted file mode 100644 index 9f5b2c37..00000000 --- a/gradient/api_sdk/serializers/job.py +++ /dev/null @@ -1,120 +0,0 @@ -import json - -import marshmallow - -from .base import BaseSchema -from .. import models - - -class JSONField(marshmallow.fields.Field): - """Field that serializes to json - """ - - def _serialize(self, value, attr, obj, **kwargs): - if value is None: - return None - - return json.dumps(value) - - def _deserialize(self, value, attr, data, **kwargs): - return value - - -class JobDatasetSchema(BaseSchema): - MODEL = models.JobDataset - - id = marshmallow.fields.Str(required=True) - name = marshmallow.fields.Str(required=True) - output = marshmallow.fields.Bool() - - -class JobSchema(BaseSchema): - MODEL = models.Job - - id = marshmallow.fields.Str(dump_to="id", load_from="id") - name = marshmallow.fields.Str(required=True) - state = marshmallow.fields.Str() - workspace_file_name = marshmallow.fields.Str(dump_to="workspaceFileName", load_from="workspaceFileName") - working_directory = marshmallow.fields.Str(dump_to="workingDirectory", load_from="workingDirectory") - artifacts_directory = marshmallow.fields.Str(dump_to="artifactsDirectory", load_from="artifactsDirectory") - entrypoint = marshmallow.fields.Str() - project_id = marshmallow.fields.Str(dump_to="projectId", load_from="projectId") - project = marshmallow.fields.Str() - container = marshmallow.fields.Str() - container_url = marshmallow.fields.Str(dump_to="containerUrl", load_from="containerUrl") - base_container = marshmallow.fields.Str(dump_to="baseContainer", load_from="baseContainer") - base_container_url = marshmallow.fields.Str(dump_to="baseContainerUrl", load_from="baseContainerUrl") - machine_type = marshmallow.fields.Str(dump_to="machineType", load_from="machineType") - cluster = marshmallow.fields.Str() - cluster_id = marshmallow.fields.Str(dump_to="clusterId", load_from="clusterId") - usage_rate = marshmallow.fields.Str(dump_to="usageRate", load_from="usageRate") - started_by_user_id = marshmallow.fields.Str(dump_to="startedByUserId", load_from="startedByUserId") - parent_job_id = marshmallow.fields.Str(dump_to="parentJobId", load_from="parentJobId") - job_error = marshmallow.fields.Str(dump_to="jobError", load_from="jobError") - dt_created = marshmallow.fields.Str(dump_to="dtCreated", load_from="dtCreated") - dt_modified = marshmallow.fields.Str(dump_to="dtModified", load_from="dtModified") - dt_provisioning_started = marshmallow.fields.Str(dump_to="dtProvisioningStarted", load_from="dtProvisioningStarted") - dt_provisioning_finished = marshmallow.fields.Str(dump_to="dtProvisioningFinished", - load_from="dtProvisioningFinished") - dt_started = marshmallow.fields.Str(dump_to="dtStarted", load_from="dtStarted") - dt_finished = marshmallow.fields.Str(dump_to="dtFinished", load_from="dtFinished") - dt_teardown_started = marshmallow.fields.Str(dump_to="dtTeardownStarted", load_from="dtTeardownStarted") - dt_teardown_finished = marshmallow.fields.Str(dump_to="dtTeardownFinished", load_from="dtTeardownFinished") - dt_deleted = marshmallow.fields.Str(dump_to="dtDeleted", load_from="dtDeleted") - exit_code = marshmallow.fields.Str(dump_to="exitCode", load_from="exitCode") - queue_position = marshmallow.fields.Str(dump_to="queuePosition", load_from="queuePosition") - seq_num = marshmallow.fields.Int(dump_to="seqNum", load_from="seqNum") - storage_region = marshmallow.fields.Str(dump_to="storageRegion", load_from="storageRegion") - cluster_machine = marshmallow.fields.Str(dump_to="clusterMachine", load_from="clusterMachine") - fqdn = marshmallow.fields.Str() - ports = marshmallow.fields.Str() - is_public = marshmallow.fields.Bool(dump_to="isPublic", load_from="isPublic") - container_user = marshmallow.fields.Str(dump_to="containerUser", load_from="containerUser") - has_code = marshmallow.fields.Bool(dump_to="hasCode", load_from="hasCode") - code_uploaded = marshmallow.fields.Bool(dump_to="codeUploaded", load_from="codeUploaded") - code_commit = marshmallow.fields.Str(dump_to="codeCommit", load_from="codeCommit") - run_till_cancelled = marshmallow.fields.Bool(dump_to="runTillCancelled", load_from="runTillCancelled") - push_on_completion = marshmallow.fields.Bool(dump_to="pushOnCompletion", load_from="pushOnCompletion") - new_image_name = marshmallow.fields.Str(dump_to="newImageName", load_from="newImageName") - cpu_hostname = marshmallow.fields.Str(dump_to="cpuHostname", load_from="cpuHostname") - cpu_count = marshmallow.fields.Int(dump_to="cpuCount", load_from="cpuCount") - cpu_model = marshmallow.fields.Str(dump_to="cpuModel", load_from="cpuModel") - cpu_flags = marshmallow.fields.Str(dump_to="cpuFlags", load_from="cpuFlags") - cpu_mem = marshmallow.fields.Str(dump_to="cpuMem", load_from="cpuMem") - gpu_name = marshmallow.fields.Str(dump_to="gpuName", load_from="gpuName") - gpu_serial = marshmallow.fields.Str(dump_to="gpuSerial", load_from="gpuSerial") - gpu_device = marshmallow.fields.Str(dump_to="gpuDevice", load_from="gpuDevice") - gpu_driver = marshmallow.fields.Str(dump_to="gpuDriver", load_from="gpuDriver") - gpu_count = marshmallow.fields.Int(dump_to="gpuCount", load_from="gpuCount") - gpu_mem = marshmallow.fields.Str(dump_to="gpuMem", load_from="gpuMem") - tpu_type = marshmallow.fields.Str(dump_to="tpuType", load_from="tpuType") - tpu_name = marshmallow.fields.Str(dump_to="tpuName", load_from="tpuName") - tpu_grpc_url = marshmallow.fields.Str(dump_to="tpuGrpcUrl", load_from="tpuGrpcUrl") - tpu_tf_version = marshmallow.fields.Str(dump_to="tpuTFVersion", load_from="tpuTFVersion") - tpu_dataset_dir = marshmallow.fields.Str(dump_to="tpuDatasetDir", load_from="tpuDatasetDir") - tpu_model_dir = marshmallow.fields.Str(dump_to="tpuModelDir", load_from="tpuModelDir") - target_node_attrs = marshmallow.fields.Dict(dump_to="targetNodeAttrs", load_from="targetNodeAttrs") - job_env = marshmallow.fields.Dict(dump_to="jobEnv", load_from="jobEnv") - env_vars = JSONField(dump_to="envVars", load_from="envVars") - shared_mem_mbytes = marshmallow.fields.Int(dump_to="sharedMemMBytes", load_from="sharedMemMBytes") - shutdown_timeout = marshmallow.fields.Int(dump_to="shutdownTimeout", load_from="shutdownTimeout") - is_preemptible = marshmallow.fields.Bool(dump_to="isPreemptible", load_from="isPreemptible") - metrics_url = marshmallow.fields.Str(dump_to="metricsURL", load_from="metricsURL") - custom_metrics = marshmallow.fields.Str(dump_to="customMetrics", load_from="customMetrics") - experiment_id = marshmallow.fields.Str(dump_to="experimentId", load_from="experimentId") - - command = marshmallow.fields.Str() - use_dockerfile = marshmallow.fields.Bool(dump_to="useDockerfile", load_from="useDockerfile") - rel_dockerfile_path = marshmallow.fields.Str(dump_to="relDockerfilePath", load_from="relDockerfilePath") - registry_username = marshmallow.fields.Str(dump_to="registryUsername", load_from="registryUsername") - registry_password = marshmallow.fields.Str(dump_to="registryPassword", load_from="registryPassword") - build_only = marshmallow.fields.Bool(dump_to="buildOnly", load_from="buildOnly") - - registry_target = marshmallow.fields.Str(dump_to="registryTarget", load_from="registryTarget") - registry_target_username = marshmallow.fields.Str( - dump_to="registryTargetUsername", load_from="registryTargetUsername") - registry_target_password = marshmallow.fields.Str( - dump_to="registryTargetPassword", load_from="registryTargetPassword") - - tags = marshmallow.fields.Str(many=True, load_only=True) - datasets = marshmallow.fields.Nested(JobDatasetSchema, dump_only=True, many=True) diff --git a/gradient/api_sdk/serializers/tensorboard.py b/gradient/api_sdk/serializers/tensorboard.py deleted file mode 100644 index 7d635d9f..00000000 --- a/gradient/api_sdk/serializers/tensorboard.py +++ /dev/null @@ -1,59 +0,0 @@ -import marshmallow -from enum import Enum -from marshmallow.validate import OneOf - -from .base import BaseSchema -from .tag import TagSchema -from .. import models - - -class InstanceType(Enum): - CPU = 'cpu' - GPU = 'gpu' - - -class InstanceSize(Enum): - SMALL = 'small' - MEDIUM = 'medium' - LARGE = 'large' - - -class InstanceSchema(BaseSchema): - MODEL = models.Instance - - type = marshmallow.fields.String(validate=marshmallow.validate.OneOf(choices=[e.value for e in InstanceType])) - size = marshmallow.fields.String(validate=marshmallow.validate.OneOf(choices=[e.value for e in InstanceSize])) - count = marshmallow.fields.Int() - - -class TBExperimentSchema(BaseSchema): - id = marshmallow.fields.String() - project_id = marshmallow.fields.String() - state = marshmallow.fields.String() - - -class TensorboardDetailSchema(BaseSchema): - MODEL = models.Tensorboard - - id = marshmallow.fields.Str() - image = marshmallow.fields.Str() - username = marshmallow.fields.Str() - password = marshmallow.fields.Str() - instance = marshmallow.fields.Nested(InstanceSchema, required=False, default=None) - experiments = marshmallow.fields.List(marshmallow.fields.Nested(TBExperimentSchema)) - url = marshmallow.fields.Str() - state = marshmallow.fields.String() - tags = marshmallow.fields.Nested(TagSchema, only="name", many=True, load_only=True) - - -class TensorboardSchema(BaseSchema): - MODEL = models.Tensorboard - - id = marshmallow.fields.Str() - image = marshmallow.fields.Str() - username = marshmallow.fields.Str() - password = marshmallow.fields.Str() - instance = marshmallow.fields.Nested(InstanceSchema, required=False, default=None) - experiments = marshmallow.fields.List(marshmallow.fields.String()) - url = marshmallow.fields.Str() - state = marshmallow.fields.String() diff --git a/gradient/api_sdk/workspace.py b/gradient/api_sdk/workspace.py deleted file mode 100644 index d5e14f74..00000000 --- a/gradient/api_sdk/workspace.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import tempfile - -from . import s3_uploader, archivers, utils, sdk_exceptions -from .logger import MuteLogger - - -class WorkspaceHandler(object): - WORKSPACE_ARCHIVER_CLS = archivers.ZipArchiver - - def __init__(self, logger_=None, archiver_cls=None): - """ - - :param logger_: gradient.logger - """ - self.logger = logger_ or MuteLogger() - self.archiver_cls = archiver_cls or self.WORKSPACE_ARCHIVER_CLS - - def _zip_workspace(self, workspace_path, ignore_files): - zip_file_name = 'workspace.zip' - zip_file_path = os.path.join(tempfile.gettempdir(), zip_file_name) - - if ignore_files: - ignore_files = ignore_files.split(",") - ignore_files = [f.strip() for f in ignore_files] - - zip_archiver = self._get_workspace_archiver() - zip_archiver.archive(workspace_path, zip_file_path, exclude=ignore_files) - - return zip_file_path - - def handle(self, input_data): - workspace_path = input_data.get('workspace') - if workspace_path is None: - return None - - ignore_files = input_data.get('ignore_files') - - if utils.PathParser.is_remote_path(workspace_path): - return workspace_path # nothing to do - - if utils.PathParser.is_local_path(workspace_path): - return self._handle_local_path(workspace_path, ignore_files) - - raise sdk_exceptions.WrongPathError("Invalid workspace path: {}".format(workspace_path)) - - def _handle_local_path(self, workspace_path, ignore_files): - workspace_path = os.path.abspath(workspace_path) - if utils.PathParser.parse_path(workspace_path) == utils.PathParser.LOCAL_DIR: - self.logger.log('Archiving your working directory for upload as your experiment workspace...' - '(See https://docs.paperspace.com/gradient/experiments/run-experiments for more ' - 'information.)') - workspace_path = self._zip_workspace(workspace_path, ignore_files) - - return workspace_path - - def _get_workspace_archiver(self): - workspace_archiver = self.archiver_cls(logger=self.logger) - return workspace_archiver - - -class S3WorkspaceHandler(WorkspaceHandler): - WORKSPACE_UPLOADER_CLS = s3_uploader.ExperimentFileUploader - - def __init__(self, api_key, client_name=None, uploader_cls=None, *args, **kwargs): - """ - :param str api_key: - :param str client_name: - :param object uploader_cls: - :param gradient.logger logger_: - """ - super(S3WorkspaceHandler, self).__init__(*args, **kwargs) - self.api_key = api_key - self.client_name = client_name - self.uploader_cls = uploader_cls or self.WORKSPACE_UPLOADER_CLS - - def handle(self, input_data): - workspace = super(S3WorkspaceHandler, self).handle(input_data) - if workspace is None: - return None - - if utils.PathParser.is_remote_path(workspace): - return workspace - - project_handle = input_data.get('projectHandle') or input_data.get("project_id") - cluster_id = input_data.get('clusterId') or input_data.get("cluster_id") - workspace = self._upload(workspace, project_handle, cluster_id=cluster_id) - return workspace - - def _upload(self, archive_path, project_id, cluster_id=None): - uploader = self._get_workspace_uploader(self.api_key) - workspace = uploader.upload(archive_path, project_id, cluster_id=cluster_id) - return workspace - - def _get_workspace_uploader(self, api_key): - workspace_uploader = self.WORKSPACE_UPLOADER_CLS(api_key, logger=self.logger, ps_client_name=self.client_name) - return workspace_uploader - - -class S3WorkspaceHandlerWithProgressbar(S3WorkspaceHandler): - WORKSPACE_ARCHIVER_CLS = archivers.ZipArchiverWithProgressbar - - def _get_workspace_uploader(self, api_key): - file_uploader = s3_uploader.S3FileUploader( - s3_uploader.MultipartEncoderWithProgressbar, - logger=self.logger, - ps_client_name=self.client_name, - ) - workspace_uploader = self.uploader_cls( - api_key, - uploader=file_uploader, - logger=self.logger, - ps_client_name=self.client_name - ) - return workspace_uploader diff --git a/gradient/cli/__init__.py b/gradient/cli/__init__.py index a21c5516..358c4a77 100644 --- a/gradient/cli/__init__.py +++ b/gradient/cli/__init__.py @@ -5,9 +5,6 @@ import gradient.cli.auth import gradient.cli.clusters import gradient.cli.datasets -import gradient.cli.experiments -import gradient.cli.hyperparameters -import gradient.cli.jobs import gradient.cli.machine_types import gradient.cli.machines import gradient.cli.models @@ -15,13 +12,10 @@ import gradient.cli.projects import gradient.cli.secrets import gradient.cli.storage_providers -import gradient.cli.tensorboards import gradient.cli.workflows from gradient.api_sdk.config import config -if config.USE_LEGACY_DEPLOYMENTS: - import gradient.cli.deployments -else: - import gradient.cli.gradient_deployments +import gradient.cli.gradient_deployments + def show(self, file=None): if file is None: @@ -34,8 +28,10 @@ def show(self, file=None): % (self.ctx.command_path, self.ctx.help_option_names[0])) if self.ctx is not None: color = self.ctx.color - click.echo(self.ctx.get_usage() + '\n%s' % hint, file=file, color=color) - msg = colorama.Fore.RED + 'Error: %s' % self.format_message() + colorama.Style.RESET_ALL + click.echo(self.ctx.get_usage() + '\n%s' % + hint, file=file, color=color) + msg = colorama.Fore.RED + 'Error: %s' % self.format_message() + \ + colorama.Style.RESET_ALL click.echo(msg, file=file, color=color) diff --git a/gradient/cli/deployments.py b/gradient/cli/deployments.py deleted file mode 100644 index cddbe549..00000000 --- a/gradient/cli/deployments.py +++ /dev/null @@ -1,843 +0,0 @@ -import collections - -import click - -from gradient import cliutils -from gradient import exceptions, clilogger, DEPLOYMENT_TYPES_MAP -from gradient.api_sdk import constants, workspace -from gradient.api_sdk.s3_uploader import DeploymentWorkspaceDirectoryUploader -from gradient.cli import common -from gradient.cli.cli import cli -from gradient.cli.cli_types import ChoiceType, json_string -from gradient.cli.common import api_key_option, del_if_value_is_none, ClickGroup, validate_comma_split_option -from gradient.cli_constants import CLI_PS_CLIENT_NAME -from gradient.commands import deployments as deployments_commands -from gradient.commands.deployments import DeploymentRemoveTagsCommand, DeploymentAddTagsCommand, \ - GetDeploymentMetricsCommand, ListDeploymentMetricsCommand, StreamDeploymentMetricsCommand, DeploymentLogsCommand - - -def get_workspace_handler(api_key): - logger_ = clilogger.CliLogger() - workspace_handler = workspace.S3WorkspaceHandlerWithProgressbar(api_key=api_key, - logger_=logger_, - uploader_cls=DeploymentWorkspaceDirectoryUploader, - client_name=CLI_PS_CLIENT_NAME) - return workspace_handler - - -def validate_autoscaling_metric_or_resource(ctx, param, value, metric_type): - """ - value in = ("cpu/targetAverage:10") - value out = ({"type": instance, - "name": "cpu", - "value_type": "targetAverage", - "value": 10}) - """ - - if value is None: - return None - - old_values = value - new_values = [] - - for old_value in old_values: - try: - name, values = old_value.split("/", 1) - value_type, value = values.split(":", 1) - value = float(value) - except Exception as e: - debug_msg = "Error occurred while validating autoscaling {} with value {}: {}" \ - .format(metric_type, old_value, e) - clilogger.CliLogger().debug(debug_msg) - - msg = "value need to be in format resource_name/value_type:value for example cpu/targetAverage:60" \ - .format(old_value) - raise click.BadParameter(msg) - - new_value = {"type": metric_type, - "name": name, - "value_type": value_type, - "value": value} - new_values.append(new_value) - - return tuple(new_values) - - -def validate_autoscaling_metric(ctx, param, value): - return validate_autoscaling_metric_or_resource(ctx, param, value, "Metric") - - -def validate_autoscaling_resource(ctx, param, value): - return validate_autoscaling_metric_or_resource(ctx, param, value, "Resource") - - -@cli.group("deployments", help="Manage deployments", cls=ClickGroup) -def deployments_group(): - pass - - -@deployments_group.group("tags", help="Manage deployments tags", cls=ClickGroup) -def deployments_tags(): - pass - - -@deployments_group.group(name="metrics", help="Read model deployment metrics", cls=ClickGroup) -def deployments_metrics(): - pass - - -@deployments_group.command("create", help="Create new deployment") -@click.option( - "--deploymentType", - "deployment_type", - type=ChoiceType(DEPLOYMENT_TYPES_MAP, case_sensitive=False), - required=True, - help="Model deployment type", - cls=common.GradientOption, -) -@click.option( - "--projectId", - "project_id", - help="Project ID", - cls=common.GradientOption, -) -@click.option( - "--modelId", - "model_id", - help="ID of a trained model", - cls=common.GradientOption, -) -@click.option( - "--name", - "name", - required=True, - help="Human-friendly name for new model deployment", - cls=common.GradientOption, -) -@click.option( - "--machineType", - "machine_type", - required=True, - help="Type of machine for new deployment", - cls=common.GradientOption, -) -@click.option( - "--imageUrl", - "image_url", - required=True, - help="Docker image for model serving", - cls=common.GradientOption, -) -@click.option( - "--instanceCount", - "instance_count", - type=int, - help="Number of machine instances", - cls=common.GradientOption, -) -@click.option( - "--command", - "command", - help="Deployment command", - cls=common.GradientOption, -) -@click.option( - "--containerModelPath", - "container_model_path", - help="Container model path", - cls=common.GradientOption, -) -@click.option( - "--imageUsername", - "image_username", - help="Username used to access docker image", - cls=common.GradientOption, -) -@click.option( - "--imagePassword", - "image_password", - help="Password used to access docker image", - cls=common.GradientOption, -) -@click.option( - "--imageServer", - "image_server", - help="Docker image server", - cls=common.GradientOption, -) -@click.option( - "--containerUrlPath", - "container_url_path", - help="Container URL path", - cls=common.GradientOption, -) -@click.option( - "--method", - "method", - help="Method", - cls=common.GradientOption, -) -@click.option( - "--dockerArgs", - "docker_args", - type=json_string, - help="JSON-style list of docker args", - cls=common.GradientOption, -) -@click.option( - "--env", - "env", - type=json_string, - help="JSON-style environmental variables map", - cls=common.GradientOption, -) -@click.option( - "--apiType", - "api_type", - help="Type of API", - cls=common.GradientOption, -) -@click.option( - "--ports", - "ports", - help="Ports", - cls=common.GradientOption, -) -@click.option( - "--clusterId", - "cluster_id", - required=True, - help="Cluster ID", - cls=common.GradientOption, -) -@click.option( - "--authUsername", - "auth_username", - help="Username", - cls=common.GradientOption, -) -@click.option( - "--authPassword", - "auth_password", - help="Password", - cls=common.GradientOption, -) -@click.option( - "--auth", - "generate_auth", - is_flag=True, - help="Generate username and password. Mutually exclusive with --authUsername and --authPassword", - cls=common.GradientOption, -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to model deployment job", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to model deployment job", - cls=common.GradientOption -) -@click.option( - "--workspace", - "workspace", - help="Path to workspace directory, archive, S3 or git repository", - cls=common.GradientOption, -) -@click.option( - "--workspaceRef", - "workspace_ref", - help="Git commit hash, branch name or tag", - cls=common.GradientOption, -) -@click.option( - "--workspaceUsername", - "workspace_username", - help="Workspace username", - cls=common.GradientOption, -) -@click.option( - "--workspacePassword", - "workspace_password", - help="Workspace password", - cls=common.GradientOption, -) -@click.option( - "--minInstanceCount", - "min_instance_count", - help="Minimal instance count", - cls=common.GradientOption, -) -@click.option( - "--maxInstanceCount", - "max_instance_count", - help="Maximal instance count", - cls=common.GradientOption, -) -@click.option( - "--scaleCooldownPeriod", - "scale_cooldown_period", - help="Scale cooldown period", - cls=common.GradientOption, -) -@click.option( - "--metric", - "metrics", - multiple=True, - callback=validate_autoscaling_metric, - help="Autoscaling metrics. Example: my_metric/targetAverage:21.37", - cls=common.GradientOption, -) -@click.option( - "--resource", - "resources", - multiple=True, - callback=validate_autoscaling_resource, - help="Autoscaling resources. Example: cpu/target:60", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def create_deployment(api_key, options_file, **kwargs): - cliutils.validate_auth_options(kwargs) - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) - del_if_value_is_none(kwargs) - command = deployments_commands.CreateDeploymentCommand(api_key=api_key, - workspace_handler=get_workspace_handler(api_key)) - command.execute(**kwargs) - - -DEPLOYMENT_STATES_MAP = collections.OrderedDict( - ( - ("BUILDING", "Building"), - ("PROVISIONING", "Provisioning"), - ("STARTING", "Starting"), - ("RUNNING", "Running"), - ("STOPPING", "Stopping"), - ("STOPPED", "Stopped"), - ("ERROR", "Error"), - ) -) - - -@deployments_group.command("list", help="List deployments with optional filtering") -@click.option( - "--state", - "state", - type=ChoiceType(DEPLOYMENT_STATES_MAP, case_sensitive=False), - help="Filter by deployment state", - cls=common.GradientOption, -) -@click.option( - "--projectId", - "project_id", - help="Use to filter by project ID", - cls=common.GradientOption, -) -@click.option( - "--modelId", - "model_id", - help="Use to filter by model ID", - cls=common.GradientOption, -) -@click.option( - "--tag", - "tags", - multiple=True, - cls=common.GradientOption, - help="Filter by tags. Multiple use" -) -@api_key_option -@common.options_file -def get_deployments_list(api_key, options_file, **filters): - del_if_value_is_none(filters) - command = deployments_commands.ListDeploymentsCommand(api_key=api_key) - try: - command.execute(**filters) - except exceptions.ApplicationError as e: - clilogger.CliLogger().error(e) - - -@deployments_group.command("start", help="Start deployment") -@click.option( - "--id", - "id_", - required=True, - help="Deployment ID", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def start_deployment(id_, options_file, api_key=None): - command = deployments_commands.StartDeploymentCommand(api_key=api_key) - command.execute(deployment_id=id_) - - -@deployments_group.command("stop", help="Stop deployment") -@click.option( - "--id", - "id_", - required=True, - help="Deployment ID", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def stop_deployment(id_, options_file, api_key=None): - command = deployments_commands.StopDeploymentCommand(api_key=api_key) - command.execute(deployment_id=id_) - - -@deployments_group.command("delete", help="Delete deployment") -@click.option( - "--id", - "id_", - required=True, - help="Deployment ID", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def delete_deployment(id_, options_file, api_key): - command = deployments_commands.DeleteDeploymentCommand(api_key=api_key) - command.execute(deployment_id=id_) - - -@deployments_group.command("update", help="Modify existing deployment") -@click.option( - "--id", - "deployment_id", - required=True, - help="ID of existing deployment", - cls=common.GradientOption, -) -@click.option( - "--deploymentType", - "deployment_type", - type=ChoiceType(DEPLOYMENT_TYPES_MAP, case_sensitive=False), - help="Model deployment type", - cls=common.GradientOption, -) -@click.option( - "--projectId", - "project_id", - help="Project ID", - cls=common.GradientOption, -) -@click.option( - "--modelId", - "model_id", - help="ID of a trained model", - cls=common.GradientOption, -) -@click.option( - "--name", - "name", - help="Human-friendly name for new model deployment", - cls=common.GradientOption, -) -@click.option( - "--machineType", - "machine_type", - help="Type of machine for new deployment", - cls=common.GradientOption, -) -@click.option( - "--imageUrl", - "image_url", - help="Docker image for model serving", - cls=common.GradientOption, -) -@click.option( - "--instanceCount", - "instance_count", - type=int, - help="Number of machine instances", - cls=common.GradientOption, -) -@click.option( - "--command", - "command", - help="Deployment command", - cls=common.GradientOption, -) -@click.option( - "--containerModelPath", - "container_model_path", - help="Container model path", - cls=common.GradientOption, -) -@click.option( - "--imageUsername", - "image_username", - help="Username used to access docker image", - cls=common.GradientOption, -) -@click.option( - "--imagePassword", - "image_password", - help="Password used to access docker image", - cls=common.GradientOption, -) -@click.option( - "--imageServer", - "image_server", - help="Docker image server", - cls=common.GradientOption, -) -@click.option( - "--containerUrlPath", - "container_url_path", - help="Container URL path", - cls=common.GradientOption, -) -@click.option( - "--method", - "method", - help="Method", - cls=common.GradientOption, -) -@click.option( - "--dockerArgs", - "docker_args", - type=json_string, - help="JSON-style list of docker args", - cls=common.GradientOption, -) -@click.option( - "--env", - "env", - type=json_string, - help="JSON-style environmental variables map", - cls=common.GradientOption, -) -@click.option( - "--apiType", - "api_type", - help="Type of API", - cls=common.GradientOption, -) -@click.option( - "--ports", - "ports", - help="Ports", - cls=common.GradientOption, -) -@click.option( - "--authUsername", - "auth_username", - help="Username", - cls=common.GradientOption, -) -@click.option( - "--authPassword", - "auth_password", - help="Password", - cls=common.GradientOption, -) -@click.option( - "--clusterId", - "cluster_id", - help="Cluster ID", - cls=common.GradientOption, -) -@click.option( - "--workspace", - "workspace", - help="Path to workspace directory, archive, S3 or git repository", - cls=common.GradientOption, -) -@click.option( - "--workspaceRef", - "workspace_ref", - help="Git commit hash, branch name or tag", - cls=common.GradientOption, -) -@click.option( - "--workspaceUsername", - "workspace_username", - metavar="", - help="Workspace username", - cls=common.GradientOption, -) -@click.option( - "--workspacePassword", - "workspace_password", - help="Workspace password", - cls=common.GradientOption, -) -@click.option( - "--minInstanceCount", - "min_instance_count", - help="Minimal instance count", - cls=common.GradientOption, -) -@click.option( - "--maxInstanceCount", - "max_instance_count", - help="Maximal instance count", - cls=common.GradientOption, -) -@click.option( - "--scaleCooldownPeriod", - "scale_cooldown_period", - help="Scale cooldown period", - cls=common.GradientOption, -) -@click.option( - "--metric", - "metrics", - multiple=True, - callback=validate_autoscaling_metric, - help="Autoscaling metrics. Example: my_metric/targetAverage:21.37", - cls=common.GradientOption, -) -@click.option( - "--resource", - "resources", - multiple=True, - callback=validate_autoscaling_resource, - help="Autoscaling resources. Example: cpu/target:60", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def update_deployment(deployment_id, api_key, options_file, **kwargs): - del_if_value_is_none(kwargs) - command = deployments_commands.UpdateDeploymentCommand(api_key=api_key, - workspace_handler=get_workspace_handler(api_key)) - command.execute(deployment_id, **kwargs) - - -@deployments_group.command("details", help="Get details of model deployment") -@click.option( - "--id", - "deployment_id", - required=True, - help="Deployment ID", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def get_deployment(deployment_id, api_key, options_file): - command = deployments_commands.GetDeploymentDetails(api_key=api_key) - command.execute(deployment_id) - - -@deployments_tags.command("add", help="Add tags to deployment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the deployment", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to deployment", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to deployment", - cls=common.GradientOption -) -@api_key_option -@common.options_file -def deployment_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = DeploymentAddTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - -@deployments_tags.command("remove", help="Remove tags from deployment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the deployment", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to remove from deployment", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want to remove from deployment", - cls=common.GradientOption -) -@api_key_option -@common.options_file -def deployment_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = DeploymentRemoveTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - -@deployments_metrics.command( - "get", - short_help="Get model deployment metrics", - help="Get model deployment metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "deployment_id", - required=True, - cls=common.GradientOption, - help="ID of the model deployment", -) -@click.option( - "--metric", - "metrics_list", - multiple=True, - type=str, - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help=("One or more metrics that you want to read: {}. Defaults to cpuPercentage and memoryUsage. To view available custom metrics, use command: `gradient deployments metrics list`".format(', '.join(map(str, constants.METRICS_MAP)))), - cls=common.GradientOption, -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@click.option( - "--start", - "start", - type=click.DateTime(), - help="Timestamp of first time series metric to collect", - cls=common.GradientOption, -) -@click.option( - "--end", - "end", - type=click.DateTime(), - help="Timestamp of last time series metric to collect", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def get_deployment_metrics(deployment_id, metrics_list, interval, start, end, options_file, api_key): - command = GetDeploymentMetricsCommand(api_key=api_key) - command.execute(deployment_id, start, end, interval, built_in_metrics=metrics_list) - -@deployments_metrics.command( - "list", - short_help="List model deployment metrics", - help="List model deployment metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "deployment_id", - required=True, - cls=common.GradientOption, - help="ID of the model deployment", -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@click.option( - "--start", - "start", - type=click.DateTime(), - help="Timestamp of first time series metric to collect", - cls=common.GradientOption, -) -@click.option( - "--end", - "end", - type=click.DateTime(), - help="Timestamp of last time series metric to collect", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_deployment_metrics(deployment_id, interval, start, end, options_file, api_key): - command = ListDeploymentMetricsCommand(api_key=api_key) - command.execute(deployment_id, start, end, interval) - - -@deployments_metrics.command( - "stream", - short_help="Watch live model deployment metrics", - help="Watch live model deployment metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "deployment_id", - required=True, - cls=common.GradientOption, - help="ID of the model deployment", -) -@click.option( - "--metric", - "metrics_list", - multiple=True, - type=ChoiceType(constants.METRICS_MAP, case_sensitive=False), - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help="One or more metrics that you want to read. Defaults to cpuPercentage and memoryUsage", - cls=common.GradientOption, -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def stream_model_deployment_metrics(deployment_id, metrics_list, interval, options_file, api_key): - command = StreamDeploymentMetricsCommand(api_key=api_key) - command.execute(deployment_id=deployment_id, interval=interval, built_in_metrics=metrics_list) - - -@deployments_group.command("logs", help="List deployment logs") -@click.option( - "--id", - "deployment_id", - required=True, - cls=common.GradientOption, -) -@click.option( - "--line", - "line", - default=0, - cls=common.GradientOption, -) -@click.option( - "--limit", - "limit", - default=10000, - cls=common.GradientOption, -) -@click.option( - "--follow", - "follow", - default=False, - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_logs(deployment_id, line, limit, follow, options_file, api_key=None): - command = DeploymentLogsCommand(api_key=api_key) - command.execute(deployment_id, line, limit, follow) diff --git a/gradient/cli/experiments.py b/gradient/cli/experiments.py deleted file mode 100644 index 2a4a150b..00000000 --- a/gradient/cli/experiments.py +++ /dev/null @@ -1,969 +0,0 @@ -import functools - -import click - -from gradient import clilogger -from gradient.api_sdk import constants, workspace -from gradient.cli import common, validators -from gradient.cli.cli import cli -from gradient.cli.cli_types import json_string, ChoiceType -from gradient.cli.common import api_key_option, ClickGroup, validate_comma_split_option -from gradient.cli.utils.flag_with_value import GradientRegisterReaderOption, GradientRegisterWriterOption, \ - GradientRegisterWriterCommand -from gradient.cli_constants import CLI_PS_CLIENT_NAME -from gradient.commands import experiments as experiments_commands -from gradient.commands.experiments import ExperimentAddTagsCommand, ExperimentRemoveTagsCommand, \ - GetExperimentMetricsCommand, ListExperimentMetricsCommand, StreamExperimentMetricsCommand - -MULTI_NODE_CREATE_EXPERIMENT_COMMANDS = { - constants.ExperimentType.GRPC_MULTI_NODE: experiments_commands.CreateMultiNodeExperimentCommand, - constants.ExperimentType.MPI_MULTI_NODE: experiments_commands.CreateMpiMultiNodeExperimentCommand, -} - -MULTI_NODE_RUN_EXPERIMENT_COMMANDS = { - constants.ExperimentType.GRPC_MULTI_NODE: experiments_commands.CreateAndStartMultiNodeExperimentCommand, - constants.ExperimentType.MPI_MULTI_NODE: experiments_commands.CreateAndStartMpiMultiNodeExperimentCommand, -} - - -def get_workspace_handler(api_key): - logger_ = clilogger.CliLogger() - workspace_handler = workspace.S3WorkspaceHandlerWithProgressbar(api_key=api_key, logger_=logger_, - client_name=CLI_PS_CLIENT_NAME) - return workspace_handler - - -@cli.group("experiments", help="Manage experiments", cls=ClickGroup) -def experiments_group(): - pass - - -@experiments_group.group("create", help="Create new experiment", cls=ClickGroup) -def create_experiment(): - pass - - -@experiments_group.group(name="run", help="Create and start new experiment", cls=ClickGroup) -def create_and_start_experiment(): - pass - - -@experiments_group.group(name="tags", help="Manage tags for experiment", cls=ClickGroup) -def experiments_tags(): - pass - - -@experiments_group.group(name="metrics", help="Read experiment metrics", cls=ClickGroup) -def experiments_metrics(): - pass - - -def common_experiments_create_options(f): - options = [ - click.option( - "--name", - metavar="", - help="Name of new experiment", - cls=common.GradientOption, - ), - click.option( - "--ports", - help="Port to use in new experiment", - cls=common.GradientOption, - ), - click.option( - "--workspace", - "workspace", - help="Path to workspace directory, archive, S3 or git repository", - cls=common.GradientOption, - ), - click.option( - "--workspaceRef", - "workspace_ref", - help="Git commit hash, branch name or tag", - cls=common.GradientOption, - ), - click.option( - "--workspaceUsername", - "workspace_username", - metavar="", - help="Workspace username", - cls=common.GradientOption, - ), - click.option( - "--workspacePassword", - "workspace_password", - help="Workspace password", - cls=common.GradientOption, - ), - click.option( - "--ignoreFiles", - "ignore_files", - help="Ignore certain files from uploading", - cls=common.GradientOption, - ), - click.option( - "--workingDirectory", - "working_directory", - help="Working directory for the experiment", - cls=common.GradientOption, - ), - click.option( - "--artifactDirectory", - "artifact_directory", - help="Artifacts directory", - cls=common.GradientOption, - ), - click.option( - "--clusterId", - "cluster_id", - metavar="", - help="Cluster ID", - cls=common.GradientOption, - ), - click.option( - "--experimentEnv", - "experiment_env", - type=json_string, - help="Environment variables in a JSON", - cls=common.GradientOption, - ), - click.option( - "--projectId", - "project_id", - metavar="", - required=True, - help="Project ID", - cls=common.GradientOption, - ), - click.option( - "--modelType", - "model_type", - metavar="", - help="Model type", - cls=common.GradientOption, - ), - click.option( - "--modelPath", - "model_path", - metavar="", - help="Model path", - cls=common.GradientOption, - ), - click.option( - "--isPreemptible", - "is_preemptible", - type=bool, - is_flag=True, - help="Flag: is preemptible", - cls=common.GradientOption, - ), - click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to experiment", - cls=common.GradientOption - ), - click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to experiment", - cls=common.GradientOption - ) - ] - return functools.reduce(lambda x, opt: opt(x), reversed(options), f) - - -def dataset_options(f): - options = [ - click.option( - "--datasetId", - "dataset_id_list", - metavar="", - multiple=True, - help="Dataset ID", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetUri", - "dataset_uri_list", - metavar="", - multiple=True, - help="Url to S3 bucket with dataset", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetName", - "dataset_name_list", - multiple=True, - metavar="", - help="Name of dataset", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetAwsAccessKeyId", - "dataset_access_key_id_list", - multiple=True, - metavar="", - help="S3 bucket's Access Key ID", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetAwsSecretAccessKey", - "dataset_secret_access_key_list", - multiple=True, - help="S3 bucket's Secret Access Key", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetAwsEndpoint", - "dataset_endpoint_list", - multiple=True, - help="S3 endpoint URL", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetVersionId", - "dataset_version_id_list", - metavar="", - multiple=True, - help="S3 dataset's version ID", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetEtag", - "dataset_etag_list", - metavar="", - multiple=True, - help="S3 dataset's ETag", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetVolumeKind", - "dataset_volume_kind_list", - multiple=True, - type=ChoiceType(constants.DATASET_VOLUME_KINDS, case_sensitive=False), - help="S3 dataset's volume kind. If used, --datasetVolumeSize has to be set as well", - cls=common.GradientDatasetOption, - ), - click.option( - "--datasetVolumeSize", - "dataset_volume_size_list", - multiple=True, - help="S3 dataset's volume size", - cls=common.GradientDatasetOption, - ), - ] - return functools.reduce(lambda x, opt: opt(x), reversed(options), f) - - -def common_experiment_create_multi_node_options(f): - options = [ - click.option( - "--experimentType", - "experiment_type_id", - type=ChoiceType(constants.MULTI_NODE_EXPERIMENT_TYPES_MAP, case_sensitive=False), - required=True, - help="Experiment Type", - cls=common.GradientOption, - ), - click.option( - "--workerContainer", - "worker_container", - metavar="", - required=True, - help="Worker container", - cls=common.GradientOption, - ), - click.option( - "--workerMachineType", - "worker_machine_type", - metavar="", - required=True, - help="Worker machine type", - cls=common.GradientOption, - ), - click.option( - "--workerCommand", - "worker_command", - metavar="", - required=True, - help="Worker command", - cls=common.GradientOption, - ), - click.option( - "--workerCount", - "worker_count", - type=int, - required=True, - help="Worker count", - cls=common.GradientOption, - ), - click.option( - "--parameterServerContainer", - "parameter_server_container", - metavar="", - help="Parameter server container (GRPC only)", - cls=common.GradientOption, - ), - click.option( - "--parameterServerMachineType", - "parameter_server_machine_type", - metavar="", - help="Parameter server machine type (GRPC only)", - cls=common.GradientOption, - ), - click.option( - "--parameterServerCommand", - "parameter_server_command", - metavar="", - help="Parameter server command (GRPC only)", - cls=common.GradientOption, - ), - click.option( - "--parameterServerCount", - "parameter_server_count", - type=int, - help="Parameter server count (GRPC only)", - cls=common.GradientOption, - ), - click.option( - "--masterContainer", - "master_container", - metavar="", - help="Master container (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--masterMachineType", - "master_machine_type", - metavar="", - help="Master machine type (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--masterCount", - "master_count", - help="Master count (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--masterCommand", - "master_command", - metavar="", - help="Master command (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--workerContainerUser", - "worker_container_user", - help="Worker container user", - cls=common.GradientOption, - ), - click.option( - "--workerRegistryUsername", - "worker_registry_username", - help="Worker container registry username", - cls=common.GradientOption, - ), - click.option( - "--workerRegistryPassword", - "worker_registry_password", - metavar="", - help="Worker registry password", - cls=common.GradientOption, - ), - click.option( - "--workerRegistryUrl", - "worker_registry_url", - metavar="", - help="Worker registry URL", - cls=common.GradientOption, - ), - click.option( - "--parameterServerContainerUser", - "parameter_server_container_user", - help="Parameter server container user", - cls=common.GradientOption, - ), - click.option( - "--parameterServerRegistryUsername", - "parameter_server_registry_username", - help="Parameter server registry username", - cls=common.GradientOption, - ), - click.option( - "--parameterServerRegistryPassword", - "parameter_server_registry_password", - metavar="", - help="Parameter server registry password", - cls=common.GradientOption, - ), - click.option( - "--parameterServerRegistryUrl", - "parameter_server_registry_url", - metavar="", - help="Parameter server registry URL", - cls=common.GradientOption, - ), - click.option( - "--masterContainerUser", - "master_container_user", - help="Master container user (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--masterRegistryUsername", - "master_registry_username", - metavar="", - help="Master registry username (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--masterRegistryPassword", - "master_registry_password", - metavar="", - help="Master registry password (MPI only)", - cls=common.GradientOption, - ), - click.option( - "--masterRegistryUrl", - "master_registry_url", - metavar="", - help="Master registry URL (MPI only)", - cls=common.GradientOption - ), - ] - return functools.reduce(lambda x, opt: opt(x), reversed(options), f) - - -def common_experiments_create_single_node_options(f): - options = [ - click.option( - "--container", - required=True, - help="Container", - cls=common.GradientOption, - ), - click.option( - "--machineType", - "machine_type", - required=True, - metavar="", - help="Machine type", - cls=common.GradientOption, - ), - click.option( - "--command", - required=True, - metavar="", - help="Container entrypoint command", - cls=common.GradientOption, - ), - click.option( - "--containerUser", - "container_user", - help="Container user", - cls=common.GradientOption, - ), - click.option( - "--registryUsername", - "registry_username", - help="Registry username", - cls=common.GradientOption, - ), - click.option( - "--registryPassword", - "registry_password", - metavar="", - help="Registry password", - cls=common.GradientOption, - ), - click.option( - "--registryUrl", - "registry_url", - metavar="", - help="Registry URL", - cls=common.GradientOption, - ), - ] - return functools.reduce(lambda x, opt: opt(x), reversed(options), f) - - -def tensorboard_option(f): - options = [ - click.option( - "--tensorboard", - is_flag=True, - # default=experiments_commands.NoTensorboardId, - help="Creates new tensorboard for this experiment", - cls=GradientRegisterReaderOption, - ), - click.option( - "--tensorboard_set", - help="Add to existing tensorboard", - cls=GradientRegisterWriterOption, - metavar='' - ), - ] - return functools.reduce(lambda x, opt: opt(x), reversed(options), f) - - -def parse_tensorboard_options(tensorboard, tensorboard_set): - """ - :param str|bool tensorboard: - :param str|None tensorboard_set: - :rtype: str|bool - """ - if tensorboard is True: - return True - - if tensorboard_set: - return tensorboard_set - else: - return False - - -@create_experiment.command(name="multinode", help="Create multi node experiment", cls=GradientRegisterWriterCommand) -@common_experiments_create_options -@common_experiment_create_multi_node_options -@dataset_options -@tensorboard_option -@api_key_option -@common.options_file -def create_multi_node(api_key, tensorboard, tensorboard_set, options_file, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) - add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set) - - validators.validate_multi_node(kwargs) - common.del_if_value_is_none(kwargs) - experiment_type = kwargs.get('experiment_type_id') - command_class = MULTI_NODE_CREATE_EXPERIMENT_COMMANDS.get(experiment_type) - command = command_class( - api_key=api_key, - workspace_handler=get_workspace_handler(api_key), - ) - command.execute(kwargs, add_to_tensorboard=add_to_tensorboard) - - -@create_experiment.command(name="singlenode", help="Create single node experiment", cls=GradientRegisterWriterCommand) -@common_experiments_create_options -@common_experiments_create_single_node_options -@dataset_options -@tensorboard_option -@api_key_option -@common.options_file -def create_single_node(api_key, tensorboard, tensorboard_set, options_file, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) - add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set) - - common.del_if_value_is_none(kwargs) - - command = experiments_commands.CreateSingleNodeExperimentCommand( - api_key=api_key, - workspace_handler=get_workspace_handler(api_key), - ) - command.execute(kwargs, add_to_tensorboard=add_to_tensorboard) - - -@create_and_start_experiment.command(name="multinode", help="Create and start new multi node experiment", - cls=GradientRegisterWriterCommand) -@common_experiments_create_options -@common_experiment_create_multi_node_options -@click.option( - "--no-logs", - "show_logs", - is_flag=True, - flag_value=False, - default=True, - help="Don't show logs. Only create, start and exit", -) -@dataset_options -@tensorboard_option -@api_key_option -@common.options_file -@click.pass_context -def create_and_start_multi_node(ctx, api_key, show_logs, tensorboard, tensorboard_set, options_file, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) - add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set) - - validators.validate_multi_node(kwargs) - common.del_if_value_is_none(kwargs) - - experiment_type = kwargs.get('experiment_type_id') - command_class = MULTI_NODE_RUN_EXPERIMENT_COMMANDS.get(experiment_type) - - command = command_class( - api_key=api_key, - workspace_handler=get_workspace_handler(api_key), - ) - experiment_id = command.execute(kwargs, add_to_tensorboard=add_to_tensorboard) - if experiment_id and show_logs: - ctx.invoke(list_logs, experiment_id=experiment_id, line=1, limit=100, follow=True, api_key=api_key) - - -@create_and_start_experiment.command(name="singlenode", help="Create and start new single node experiment", - cls=GradientRegisterWriterCommand) -@common_experiments_create_options -@common_experiments_create_single_node_options -@click.option( - "--no-logs", - "show_logs", - is_flag=True, - flag_value=False, - default=True, - help="Don't show logs. Only create, start and exit", -) -@dataset_options -@tensorboard_option -@api_key_option -@common.options_file -@click.pass_context -def create_and_start_single_node(ctx, api_key, show_logs, tensorboard, tensorboard_set, options_file, - **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) - add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set) - - common.del_if_value_is_none(kwargs) - - command = experiments_commands.CreateAndStartSingleNodeExperimentCommand( - api_key=api_key, - workspace_handler=get_workspace_handler(api_key), - ) - experiment_id = command.execute(kwargs, add_to_tensorboard=add_to_tensorboard) - if experiment_id and show_logs: - ctx.invoke(list_logs, experiment_id=experiment_id, line=1, limit=100, follow=True, api_key=api_key) - - -@experiments_group.command("start", help="Start experiment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@click.option( - "--logs", - "show_logs", - is_flag=True, - help="Show logs", -) -@api_key_option -@common.options_file -@click.pass_context -def start_experiment(ctx, id, show_logs, api_key, options_file): - command = experiments_commands.StartExperimentCommand(api_key=api_key) - command.execute(id) - - if show_logs: - ctx.invoke(list_logs, experiment_id=id, line=1, limit=100, follow=True, api_key=api_key) - - -@experiments_group.command("stop", help="Stop experiment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@api_key_option -@common.options_file -def stop_experiment(id, api_key, options_file): - command = experiments_commands.StopExperimentCommand(api_key=api_key) - command.execute(id) - - -@experiments_group.command("list", help="List experiments") -@click.option( - "--projectId", - "-p", - "project_ids", - multiple=True, - metavar='', - help="Filter by project IDs. Multiple use", - cls=common.GradientOption, -) -@click.option( - "--tag", - "tags", - multiple=True, - cls=common.GradientOption, - help="Filter by tags. Multiple use" -) -@click.option( - "--limit", - "-l", - "exp_limit", - default=20, - help="Limit listed experiments per page", - cls=common.GradientOption, -) -@click.option( - "--offset", - "-o", - "exp_offset", - default=0, - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_experiments(project_ids, api_key, exp_limit, exp_offset, tags, options_file): - command = experiments_commands.ListExperimentsCommand(api_key=api_key) - - res = command.execute(project_id=project_ids, limit=exp_limit, offset=exp_offset, tags=tags) - for experiments_str, next_iteration in res: - click.echo(experiments_str) - if next_iteration: - click.confirm("Do you want to continue?", abort=True) - - -@experiments_group.command("details", help="Show detail of an experiment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@api_key_option -@common.options_file -def get_experiment_details(id, options_file, api_key): - command = experiments_commands.GetExperimentCommand(api_key=api_key) - command.execute(id) - - -@experiments_group.command("logs", help="List experiment logs") -@click.option( - "--id", - "experiment_id", - required=True, - cls=common.GradientOption, -) -@click.option( - "--line", - "line", - required=False, - default=0, - cls=common.GradientOption, -) -@click.option( - "--limit", - "limit", - required=False, - default=10000, - cls=common.GradientOption, -) -@click.option( - "--follow", - "follow", - required=False, - default=False, - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_logs(experiment_id, line, limit, follow, options_file, api_key=None): - command = experiments_commands.ExperimentLogsCommand(api_key=api_key) - command.execute(experiment_id, line, limit, follow) - - -@experiments_group.command("delete", help="Delete an experiment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@api_key_option -@common.options_file -def delete_experiment(id, options_file, api_key): - command = experiments_commands.DeleteExperimentCommand(api_key=api_key) - command.execute(id) - - -@experiments_tags.command("add", help="Add tags to experiment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to experiment", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to experiment", - cls=common.GradientOption -) -@api_key_option -@common.options_file -def experiment_add_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = ExperimentAddTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - -@experiments_tags.command("remove", help="Remove tags from experiment") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to remove from experiment", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want to remove from experiment", - cls=common.GradientOption -) -@api_key_option -@common.options_file -def experiment_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = ExperimentRemoveTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - -@experiments_metrics.command( - "get", - short_help="Get experiment metrics", - help="Get experiment metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "experiment_id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@click.option( - "--metric", - "metrics_list", - multiple=True, - type=str, - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help=("One or more metrics that you want to read: {}. Defaults to cpuPercentage and memoryUsage. To view available custom metrics, use command: `gradient experiments metrics list`".format(', '.join(map(str, constants.METRICS_MAP)))), - cls=common.GradientOption, -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@click.option( - "--start", - "start", - type=click.DateTime(), - help="Timestamp of first time series metric to collect", - cls=common.GradientOption, -) -@click.option( - "--end", - "end", - type=click.DateTime(), - help="Timestamp of last time series metric to collect", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def get_experiment_metrics(experiment_id, metrics_list, interval, start, end, options_file, api_key): - command = GetExperimentMetricsCommand(api_key=api_key) - command.execute(experiment_id, start, end, interval, built_in_metrics=metrics_list) - - -@experiments_metrics.command( - "list", - short_help="List experiment metrics", - help="List experiment metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "experiment_id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@click.option( - "--start", - "start", - type=click.DateTime(), - help="Timestamp of first time series metric to collect", - cls=common.GradientOption, -) -@click.option( - "--end", - "end", - type=click.DateTime(), - help="Timestamp of last time series metric to collect", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_experiment_metrics(experiment_id, interval, start, end, options_file, api_key): - command = ListExperimentMetricsCommand(api_key=api_key) - command.execute(experiment_id, start, end, interval) - - -@experiments_metrics.command( - "stream", - short_help="Watch live experiment metrics", - help="Watch live experiment metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "experiment_id", - required=True, - cls=common.GradientOption, - help="ID of the experiment", -) -@click.option( - "--metric", - "metrics_list", - multiple=True, - type=ChoiceType(constants.METRICS_MAP, case_sensitive=False), - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help="One or more metrics that you want to read. Defaults to cpuPercentage and memoryUsage", - cls=common.GradientOption, -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def get_experiment_metrics_stream(experiment_id, metrics_list, interval, options_file, api_key): - command = StreamExperimentMetricsCommand(api_key=api_key) - command.execute(experiment_id=experiment_id, interval=interval, built_in_metrics=metrics_list) diff --git a/gradient/cli/hyperparameters.py b/gradient/cli/hyperparameters.py deleted file mode 100644 index fcb103f3..00000000 --- a/gradient/cli/hyperparameters.py +++ /dev/null @@ -1,254 +0,0 @@ -import functools - -import click - -from gradient.cli import common -from gradient.cli.common import ClickGroup, validate_comma_split_option -from gradient.cli.experiments import common_experiments_create_options, get_workspace_handler, experiments_group -from gradient.commands import hyperparameters as hyperparameters_commands -from gradient.commands.hyperparameters import HyperparameterAddTagsCommand, HyperparameterRemoveTagsCommand - - -@experiments_group.group("hyperparameters", help="Manage hyperparameters", cls=ClickGroup) -def hyperparameters_group(): - pass - - -@hyperparameters_group.group("tags", help="Manage hyperparameter tags", cls=ClickGroup) -def hyperparameters_tags(): - pass - - -def common_hyperparameter_create_options(f): - options = [ - click.option( - "--tuningCommand", - "tuning_command", - required=True, - help="Tuning command", - cls=common.GradientOption, - ), - click.option( - "--workerContainer", - "worker_container", - required=True, - help="Worker container", - cls=common.GradientOption, - ), - click.option( - "--workerContainerUser", - "worker_container_user", - required=False, - help="Worker container user", - cls=common.GradientOption, - ), - click.option( - "--workerMachineType", - "worker_machine_type", - required=True, - help="Worker machine type", - cls=common.GradientOption, - ), - click.option( - "--hyperparameterServerMachineType", - "hyperparameter_server_machine_type", - required=False, - help="Hyperparameter Server machine type", - cls=common.GradientOption, - ), - click.option( - "--workerCommand", - "worker_command", - required=True, - help="Worker command", - cls=common.GradientOption, - ), - click.option( - "--workerCount", - "worker_count", - required=True, - type=int, - help="Worker count", - cls=common.GradientOption, - ), - click.option( - "--workerUseDockerfile", - "use_dockerfile", - type=bool, - is_flag=True, - default=False, - help="Flag: use dockerfile", - cls=common.GradientOption, - ), - click.option( - "--workerDockerfilePath", - "dockerfile_path", - help="Path to ", - cls=common.GradientOption, - ), - click.option( - "--workerRegistryUsername", - "worker_registry_username", - help="Worker registry username", - cls=common.GradientOption, - ), - click.option( - "--workerRegistryPassword", - "worker_registry_password", - help="Worker registry password", - cls=common.GradientOption, - ), - click.option( - "--hyperparameterServerRegistryUsername", - "hyperparameter_server_registry_username", - help="Hyperparameter server registry username", - cls=common.GradientOption, - ), - click.option( - "--hyperparameterServerRegistryPassword", - "hyperparameter_server_registry_password", - help="Hyperparameter server registry password", - cls=common.GradientOption, - ), - click.option( - "--hyperparameterServerContainer", - "hyperparameter_server_container", - help="Hyperparameter server container", - cls=common.GradientOption, - ), - click.option( - "--hyperparameterServerContainerUser", - "hyperparameter_server_container_user", - help="Hyperparameter server container user", - cls=common.GradientOption, - ), - ] - return functools.reduce(lambda x, opt: opt(x), reversed(options), f) - - -@hyperparameters_group.command("create", help="Create hyperparameter") -@common_experiments_create_options -@common_hyperparameter_create_options -@common.api_key_option -@common.options_file -def create_hyperparameter(api_key, options_file, **hyperparameter): - hyperparameter["tags"] = validate_comma_split_option(hyperparameter.pop("tags_comma"), hyperparameter.pop("tags")) - common.del_if_value_is_none(hyperparameter, del_all_falsy=True) - - command = hyperparameters_commands.CreateHyperparameterCommand( - api_key=api_key, - workspace_handler=get_workspace_handler(api_key), - ) - command.execute(hyperparameter) - - -@hyperparameters_group.command("run", help="Create and start hyperparameter tuning job") -@common_experiments_create_options -@common_hyperparameter_create_options -@common.api_key_option -@common.options_file -def create_and_start_hyperparameter(api_key, options_file, **hyperparameter): - hyperparameter["tags"] = validate_comma_split_option(hyperparameter.pop("tags_comma"), hyperparameter.pop("tags")) - common.del_if_value_is_none(hyperparameter, del_all_falsy=True) - - command = hyperparameters_commands.CreateAndStartHyperparameterCommand( - api_key=api_key, - workspace_handler=get_workspace_handler(api_key), - ) - command.execute(hyperparameter) - - -@hyperparameters_group.command("list", help="List hyperparameters") -@common.api_key_option -@common.options_file -def list_hyperparameters(api_key, options_file): - command = hyperparameters_commands.ListHyperparametersCommand(api_key=api_key) - command.execute() - - -@hyperparameters_group.command("details", help="Show details of hyperparameter") -@click.option( - "--id", - "id_", - required=True, - cls=common.GradientOption, -) -@common.api_key_option -@common.options_file -def get_hyperparameter_details(api_key, id_, options_file): - command = hyperparameters_commands.HyperparameterDetailsCommand(api_key=api_key) - command.execute(id_) - - -@hyperparameters_group.command("start", help="Start hyperparameter tuning") -@click.option( - "--id", - "id_", - required=True, - cls=common.GradientOption, -) -@common.api_key_option -@common.options_file -def start_hyperparameter_tuning(api_key, options_file, id_): - command = hyperparameters_commands.HyperparameterStartCommand(api_key=api_key) - command.execute(id_) - - -@hyperparameters_tags.command("add", help="Add tags to hyperparameter") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the hyperparameter", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to hyperparameter", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to hyperparameter", - cls=common.GradientOption -) -@common.api_key_option -@common.options_file -def hyperparameter_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = HyperparameterAddTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - -@hyperparameters_tags.command("remove", help="Remove tags from hyperparameter") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the hyperparameter", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to remove from hyperparameter", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want to remove from hyperparameter", - cls=common.GradientOption -) -@common.api_key_option -@common.options_file -def hyperparameter_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = HyperparameterRemoveTagsCommand(api_key=api_key) - command.execute(id, **kwargs) diff --git a/gradient/cli/jobs.py b/gradient/cli/jobs.py deleted file mode 100644 index eb591526..00000000 --- a/gradient/cli/jobs.py +++ /dev/null @@ -1,639 +0,0 @@ -from functools import reduce - -import click - -from gradient import clilogger -from gradient.api_sdk import constants -from gradient.cli import common -from gradient.cli.cli import cli -from gradient.cli.cli_types import json_string, ChoiceType -from gradient.cli.common import ( - api_key_option, del_if_value_is_none, ClickGroup, jsonify_dicts, - validate_comma_split_option, -) -from gradient.commands import jobs as jobs_commands -from gradient.commands.jobs import JobAddTagsCommand, JobRemoveTagsCommand, StreamJobMetricsCommand, \ - GetJobMetricsCommand, ListJobMetricsCommand -from gradient.api_sdk.workspace import WorkspaceHandler - - -def get_workspace_handler(): - logger_ = clilogger.CliLogger() - workspace_handler = WorkspaceHandler(logger_=logger_) - return workspace_handler - - -@cli.group("jobs", help="Manage gradient jobs", cls=ClickGroup) -def jobs_group(): - pass - - -@jobs_group.group("tags", help="Manage job tags", cls=ClickGroup) -def jobs_tags(): - pass - - -@jobs_group.group(name="metrics", help="Read job metrics", cls=ClickGroup) -def jobs_metrics(): - pass - - -@jobs_group.command("delete", help="Delete job") -@click.option( - "--id", - "job_id", - required=True, - help="Delete job with given ID", -) -@api_key_option -def delete_job(job_id, api_key): - command = jobs_commands.DeleteJobCommand(api_key=api_key) - command.execute(job_id) - - -@jobs_group.command("stop", help="Stop running job") -@click.option( - "--id", - "job_id", - required=True, - help="Stop job with given ID", -) -@api_key_option -def stop_job(job_id, api_key=None): - command = jobs_commands.StopJobCommand(api_key=api_key) - command.execute(job_id) - - -@jobs_group.command("list", help="List jobs with optional filtering") -@click.option( - "--project", - "project", - help="Use to filter jobs by project name", - cls=common.GradientOption, -) -@click.option( - "--projectId", - "project_id", - help="Use to filter jobs by project ID", - cls=common.GradientOption, -) -@click.option( - "--experimentId", - "experiment_id", - help="Use to filter jobs by experiment ID", - cls=common.GradientOption, -) -@click.option( - "--tag", - "tags", - multiple=True, - cls=common.GradientOption, - help="Filter by tags. Multiple use" -) -@api_key_option -@common.options_file -def list_jobs(api_key, options_file, **filters): - del_if_value_is_none(filters) - - command = jobs_commands.ListJobsCommand(api_key=api_key) - command.execute(**filters) - - -def common_jobs_create_options(f): - options = [ - click.option( - "--name", - "name", - help="Job name", - cls=common.GradientOption, - ), - click.option( - "--machineType", - "machine_type", - help="Virtual machine type", - required=True, - cls=common.GradientOption, - ), - click.option( - "--container", - "container", - default="paperspace/tensorflow-python", - help="Docker container", - required=True, - cls=common.GradientOption, - ), - click.option( - "--command", - "command", - help="Job command/entrypoint", - cls=common.GradientOption, - ), - click.option( - "--ports", - "ports", - help="Mapped ports", - cls=common.GradientOption, - ), - # TODO: make it a flag - click.option( - "--isPublic", - "is_public", - help="Flag: is job public", - cls=common.GradientOption, - ), - click.option( - "--workspace", - "workspace", - help="Path to workspace directory", - cls=common.GradientOption, - ), - click.option( - "--workingDirectory", - "working_directory", - help="Working directory for the experiment", - cls=common.GradientOption, - ), - click.option( - "--ignoreFiles", - "ignore_files", - help="Ignore certain files from uploading", - cls=common.GradientOption, - ), - click.option( - "--experimentId", - "experiment_id", - help="Experiment Id", - cls=common.GradientOption, - ), - click.option( - "--envVars", - "env_vars", - type=json_string, - help="Environmental variables", - cls=common.GradientOption, - ), - click.option( - "--useDockerfile", - "use_dockerfile", - help="Flag: using Dockerfile", - cls=common.GradientOption, - ), - # TODO: make it a flag - click.option( - "--isPreemptible", - "is_preemptible", - help="Flag: isPreemptible", - cls=common.GradientOption, - ), - click.option( - "--project", - "project", - help="Project name", - cls=common.GradientOption, - ), - click.option( - "--projectId", - "project_id", - help="Project ID", - required=True, - cls=common.GradientOption, - ), - click.option( - "--startedByUserId", - "started_by_user_id", - help="User ID", - cls=common.GradientOption, - ), - click.option( - "--relDockerfilePath", - "rel_dockerfile_path", - help="Relative path to Dockerfile", - cls=common.GradientOption, - ), - click.option( - "--registryUsername", - "registry_username", - help="Docker registry username", - cls=common.GradientOption, - ), - click.option( - "--registryPassword", - "registry_password", - help="Docker registry password", - cls=common.GradientOption, - ), - click.option( - "--cluster", - "cluster", - help="Cluster name", - cls=common.GradientOption, - ), - click.option( - "--clusterId", - "cluster_id", - help="Cluster id", - cls=common.GradientOption, - ), - click.option( - "--nodeAttrs", - "node_attrs", - type=json_string, - help="Cluster node details", - cls=common.GradientOption, - ), - click.option( - "--registryTarget", - "registry_target", - help="Docker registry target", - cls=common.GradientOption, - ), - click.option( - "--registryTargetUsername", - "registry_target_username", - help="Docker registry username", - cls=common.GradientOption, - ), - click.option( - "--registryTargetPassword", - "registry_target_password", - help="Docker registry password", - cls=common.GradientOption, - ), - click.option( - "--buildOnly", - "build_only", - type=bool, - is_flag=True, - help="Determines whether to only build and not run image (default false)", - cls=common.GradientOption, - ), - click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to experiment", - cls=common.GradientOption - ), - click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to experiment", - cls=common.GradientOption - ), - click.option( - "--dataset", - "datasets", - help="Separated by comma tags that you want add to experiment", - cls=common.GradientOption, - multiple=True, - ) - ] - return reduce(lambda x, opt: opt(x), reversed(options), f) - - -@jobs_group.command("create", help="Create job") -@common_jobs_create_options -@api_key_option -@common.options_file -@click.pass_context -def create_job(ctx, api_key, options_file, datasets=None, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) - - del_if_value_is_none(kwargs) - jsonify_dicts(kwargs) - - if datasets: - values = [] - - for value in datasets: - name, _, ref = value.partition("@") - if not (name and ref): - raise click.UsageError( - "Dataset '%s' must have an @ (ex: images@dsr8k5qzn401lb5:klfoyy9)" % value) - - dataset = {"id": ref, "name": name} - if ":" not in ref: - dataset["output"] = True - - values.append(dataset) - - kwargs["datasets"] = values - - - - command = jobs_commands.CreateJobCommand(api_key=api_key, workspace_handler=get_workspace_handler()) - job_handle = command.execute(kwargs) - if job_handle is not None: - ctx.invoke(list_logs, job_id=job_handle, line=1, limit=100, follow=True, api_key=api_key) - - -@jobs_group.command("logs", help="List job logs") -@click.option( - "--id", - "job_id", - required=True, - cls=common.GradientOption, -) -@click.option( - "--line", - "line", - required=False, - default=0, - cls=common.GradientOption, -) -@click.option( - "--limit", - "limit", - required=False, - default=10000, - cls=common.GradientOption, -) -@click.option( - "--follow", - "follow", - required=False, - default=False, - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_logs(job_id, line, limit, follow, options_file, api_key=None): - command = jobs_commands.JobLogsCommand(api_key=api_key) - command.execute(job_id, line, limit, follow) - - -@jobs_group.group("artifacts", help="Manage jobs' artifacts", cls=ClickGroup) -def artifacts(): - pass - - -@artifacts.command("destroy", help="Destroy job's artifacts") -@click.option( - "--id", - "job_id", - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--files", - "files", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def destroy_artifacts(job_id, options_file, api_key=None, files=None): - command = jobs_commands.ArtifactsDestroyCommand(api_key=api_key) - command.execute(job_id, files=files) - - -@artifacts.command("get", help="Get job's artifacts") -@click.option( - "--id", - "job_id", - cls=common.GradientOption, - help="ID of the job", -) -@api_key_option -@common.options_file -def get_artifacts(job_id, options_file, api_key=None): - command = jobs_commands.ArtifactsGetCommand(api_key=api_key) - command.execute(job_id) - - -@artifacts.command("list", help="List job's artifacts") -@click.option( - "--id", - "job_id", - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--size", - "-s", - "size", - help="Show file size", - is_flag=True, - cls=common.GradientOption, -) -@click.option( - "--links", - "-l", - "links", - help="Show file URL", - is_flag=True, - default=False, - cls=common.GradientOption, -) -@click.option( - "--files", - "files", - help="Get only given file (use at the end * as a wildcard)", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_artifacts(job_id, size, links, files, options_file, api_key=None): - command = jobs_commands.ArtifactsListCommand(api_key=api_key) - command.execute(job_id=job_id, size=size, links=links, files=files) - - -@artifacts.command("download", help="List job's artifacts") -@click.option( - "--id", - "job_id", - cls=common.GradientOption, -) -@click.option( - "--destinationDir", - "destination_directory", - required=True, - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def download_artifacts(job_id, destination_directory, options_file, api_key=None): - command = jobs_commands.DownloadArtifactsCommand(api_key=api_key) - command.execute(job_id=job_id, destination_directory=destination_directory) - - -@jobs_tags.command("add", help="Add tags to job") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to add to job", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want add to job", - cls=common.GradientOption -) -@api_key_option -@common.options_file -def job_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = JobAddTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - -@jobs_tags.command("remove", help="Remove tags from job") -@click.option( - "--id", - "id", - required=True, - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--tag", - "tags", - multiple=True, - help="One or many tags that you want to remove from job", - cls=common.GradientOption -) -@click.option( - "--tags", - "tags_comma", - help="Separated by comma tags that you want to remove from job", - cls=common.GradientOption -) -@api_key_option -@common.options_file -def job_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) - - command = JobRemoveTagsCommand(api_key=api_key) - command.execute(id, **kwargs) - - - -@jobs_metrics.command( - "get", - short_help="Get job metrics", - help="Get job metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "job_id", - required=True, - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--metric", - "metrics_list", - multiple=True, - type=str, - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help=("One or more metrics that you want to read: {}. Defaults to cpuPercentage and memoryUsage. To view available custom metrics, use command: `gradient jobs metrics list`".format(', '.join(map(str, constants.METRICS_MAP)))), - cls=common.GradientOption, -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@click.option( - "--start", - "start", - type=click.DateTime(), - help="Timestamp of first time series metric to collect", - cls=common.GradientOption, -) -@click.option( - "--end", - "end", - type=click.DateTime(), - help="Timestamp of last time series metric to collect", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def get_job_metrics(job_id, metrics_list, interval, start, end, options_file, api_key): - command = GetJobMetricsCommand(api_key=api_key) - command.execute(job_id, start, end, interval, built_in_metrics=metrics_list) - -@jobs_metrics.command( - "list", - short_help="List job metrics", - help="List job metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "job_id", - required=True, - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@click.option( - "--start", - "start", - type=click.DateTime(), - help="Timestamp of first time series metric to collect", - cls=common.GradientOption, -) -@click.option( - "--end", - "end", - type=click.DateTime(), - help="Timestamp of last time series metric to collect", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def list_job_metrics(job_id, interval, start, end, options_file, api_key): - command = ListJobMetricsCommand(api_key=api_key) - command.execute(job_id, start, end, interval) - -@jobs_metrics.command( - "stream", - short_help="Watch live job metrics", - help="Watch live job metrics. Shows CPU and RAM usage by default", -) -@click.option( - "--id", - "job_id", - required=True, - cls=common.GradientOption, - help="ID of the job", -) -@click.option( - "--metric", - "metrics_list", - multiple=True, - type=ChoiceType(constants.METRICS_MAP, case_sensitive=False), - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help="One or more metrics that you want to read. Defaults to cpuPercentage and memoryUsage", - cls=common.GradientOption, -) -@click.option( - "--interval", - "interval", - default="30s", - help="Interval", - cls=common.GradientOption, -) -@api_key_option -@common.options_file -def stream_job_metrics(job_id, metrics_list, interval, options_file, api_key): - command = StreamJobMetricsCommand(api_key=api_key) - command.execute(job_id=job_id, interval=interval, built_in_metrics=metrics_list) diff --git a/gradient/cli/tensorboards.py b/gradient/cli/tensorboards.py deleted file mode 100644 index 92162c36..00000000 --- a/gradient/cli/tensorboards.py +++ /dev/null @@ -1,154 +0,0 @@ -import click - -from gradient.cli import common -from gradient.cli.cli import cli -from gradient.commands import tensorboards as tensorboards_commands - - -@cli.group("tensorboards", help="Manage tensorboards", cls=common.ClickGroup) -def tensorboards_group(): - pass - - -@tensorboards_group.command("create", help="Create new tensorboard") -@click.option( - "--experiment", - "experiments", - multiple=True, - required=True, - metavar="[]", - help="One or more experiment IDs [--experiment id1 --experiment id2 ...]", - cls=common.GradientOption, -) -@click.option( - "--image", - "image", - help="Tensorboard Container Image, by default its set to tensorflow/tensorflow:latest", - cls=common.GradientOption, -) -@click.option( - "--username", - "username", - help="Basic Auth Username", - cls=common.GradientOption, -) -@click.option( - "--password", - "password", - help="Basic Auth Password", - cls=common.GradientOption, -) -# @click.option( -# "--instanceType", -# "instance_type", -# help="Instance type", -# cls=common.GradientOption, -# ) -# @click.option( -# "--instanceSize", -# "instance_size", -# help="Instance size", -# cls=common.GradientOption, -# ) -# @click.option( -# "--instancesCount", -# "instances_count", -# type=int, -# help="Instances count", -# cls=common.GradientOption, -# ) -@common.api_key_option -@common.options_file -def create_tensorboard(api_key, options_file, **kwargs): - command = tensorboards_commands.CreateTensorboardCommand(api_key=api_key) - command.execute(**kwargs) - - -@tensorboards_group.command("details", help="Show details of a tensorboard") -@click.option( - "--id", - "id", - metavar="", - required=True, - help="Tensorboard ID", - cls=common.GradientOption, -) -@common.api_key_option -@common.options_file -def tensorboard_details(id, api_key, options_file): - command = tensorboards_commands.GetTensorboardCommand(api_key=api_key) - command.execute(id) - - -@tensorboards_group.command("list", help="Show list of tensorboards") -@common.api_key_option -@common.options_file -def list_tensorboards(api_key, options_file): - command = tensorboards_commands.ListTensorboardsCommand(api_key=api_key) - command.execute() - - -@tensorboards_group.command("add-experiments", help="Update tensorboard experiments") -@click.option( - "--id", - "tensorboard_id", - metavar="", - required=True, - help="Tensorboard ID", - cls=common.GradientOption, -) -@click.option( - "--experiment", - "experiments", - multiple=True, - required=True, - metavar="[]", - help="One or more experiment IDs [--experiment id1 --experiment id2 ...]", - cls=common.GradientOption, -) -@common.api_key_option -@common.options_file -def add_experiments_to_tensorboard(tensorboard_id, experiments, api_key, options_file): - command = tensorboards_commands.AddExperimentToTensorboard(api_key=api_key) - command.execute(tensorboard_id, experiments) - - -@tensorboards_group.command("remove-experiments", help="Update tensorboard experiments") -@click.option( - "--id", - "id", - required=True, - metavar="", - help="Tensorboard ID", - cls=common.GradientOption, -) -@click.option( - "--experiment", - "experiments", - multiple=True, - required=True, - metavar="[]", - help="One or more experiment IDs [--experiment id1 --experiment id2 ...]", - cls=common.GradientOption, -) -@common.api_key_option -@common.options_file -def remove_experiments_to_tensorboard(api_key, options_file, **kwargs): - command = tensorboards_commands.RemoveExperimentToTensorboard(api_key=api_key) - command.execute(**kwargs) - - -@tensorboards_group.command("delete", help="Delete tensorboard") -@click.option( - "--id", - "id", - metavar="", - required=True, - help="Tensorboard ID", - cls=common.GradientOption, -) -@common.api_key_option -@common.options_file -def delete_tensorboard(api_key, options_file, **kwargs): - command = tensorboards_commands.DeleteTensorboard(api_key=api_key) - command.execute(**kwargs) diff --git a/gradient/commands/deployments.py b/gradient/commands/deployments.py deleted file mode 100644 index f8792279..00000000 --- a/gradient/commands/deployments.py +++ /dev/null @@ -1,270 +0,0 @@ -import abc -import itertools -import json -import pydoc - -import six -import terminaltables -from click import style -from halo import halo - -from gradient import exceptions, DeploymentsClient, AutoscalingMetric, AutoscalingDefinition -from gradient.api_sdk import sdk_exceptions, utils -from gradient.api_sdk.config import config -from gradient.api_sdk.utils import concatenate_urls -from gradient.cli_constants import CLI_PS_CLIENT_NAME -from gradient.cliutils import get_terminal_lines -from gradient.commands.common import DetailsCommandMixin, StreamMetricsCommand, BaseCommand, LogsCommandMixin - - -@six.add_metaclass(abc.ABCMeta) -class BaseDeploymentCommand(BaseCommand): - def _get_client(self, api_key, logger): - client = DeploymentsClient( - api_key=api_key, - logger=logger, - ps_client_name=CLI_PS_CLIENT_NAME, - ) - return client - - -class HandleAutoscalingOptions(object): - def _handle_autoscaling_options(self, kwargs): - autoscaling_metrics_and_resources = [] - metrics = kwargs.pop("metrics", None) or [] - resources = kwargs.pop("resources", None) or [] - for metric_dict in itertools.chain(resources, metrics): - metric = AutoscalingMetric( - type=metric_dict["type"], - name=metric_dict["name"], - value_type=metric_dict["value_type"], - value=metric_dict["value"], - ) - autoscaling_metrics_and_resources.append(metric) - - autoscaling_definition = AutoscalingDefinition( - min_instance_count=kwargs.pop("min_instance_count", None), - max_instance_count=kwargs.pop("max_instance_count", None), - scale_cooldown_period=kwargs.pop("scale_cooldown_period", None), - metrics=autoscaling_metrics_and_resources, - ) - kwargs["autoscaling"] = autoscaling_definition - - -class HandleWorkspaceMixin(object): - def _handle_workspace(self, instance_dict): - handler = self.workspace_handler.handle(instance_dict) - - instance_dict.pop("ignore_files", None) - instance_dict.pop("workspace", None) - if handler: - instance_dict["workspace_url"] = handler - - -class CreateDeploymentCommand(HandleAutoscalingOptions, BaseDeploymentCommand, HandleWorkspaceMixin): - def __init__(self, workspace_handler, *args, **kwargs): - super(CreateDeploymentCommand, self).__init__(*args, **kwargs) - self.workspace_handler = workspace_handler - - def execute(self, **kwargs): - self._handle_auth(kwargs) - self._handle_workspace(kwargs) - self._handle_autoscaling_options(kwargs) - with halo.Halo(text="Creating new deployment", spinner="dots"): - deployment_id = self.client.create(**kwargs) - - self.logger.log("New deployment created with id: {}".format(deployment_id)) - self.logger.log(self.get_instance_url(deployment_id)) - - def get_instance_url(self, instance_id): - url = concatenate_urls(config.WEB_URL, "{}/deployments/{}".format(self.get_namespace(), instance_id)) - return url - - def _handle_auth(self, kwargs): - if kwargs.pop("generate_auth", False): - auth_username, auth_password = utils.generate_credentials_pair(12) - kwargs["auth_username"] = auth_username - kwargs["auth_password"] = auth_password - self.logger.log("Generated credentials: \nusername:{}\npassword:{}".format(auth_password, auth_username)) - - -class ListDeploymentsCommand(BaseDeploymentCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def execute(self, **kwargs): - with halo.Halo(text=self.WAITING_FOR_RESPONSE_MESSAGE, spinner="dots"): - instances = self._get_instances(**kwargs) - - self._log_objects_list(instances) - - def _get_instances(self, **kwargs): - try: - instances = self.client.list(**kwargs) - except sdk_exceptions.GradientSdkError as e: - raise exceptions.ReceivingDataFailedError(e) - - return instances - - @staticmethod - def _get_table_data(deployments): - data = [("Name", "ID", "Endpoint", "Api Type", "Deployment Type", "Deployment State")] - for deployment in deployments: - name = deployment.name - id_ = deployment.id - endpoint = deployment.endpoint - api_type = deployment.api_type - deployment_type = deployment.deployment_type - deployment_state = deployment.state - data.append((name, id_, endpoint, api_type, deployment_type, deployment_state)) - - return data - - def _log_objects_list(self, objects): - if not objects: - self.logger.warning("No data found") - return - - table_data = self._get_table_data(objects) - table_str = self._make_table(table_data) - if len(table_str.splitlines()) > get_terminal_lines(): - pydoc.pager(table_str) - else: - self.logger.log(table_str) - - @staticmethod - def _make_table(table_data): - ascii_table = terminaltables.AsciiTable(table_data) - table_string = ascii_table.table - return table_string - - -class StartDeploymentCommand(BaseDeploymentCommand): - def execute(self, **kwargs): - self.client.start(**kwargs) - self.logger.log("Deployment started") - - -class StopDeploymentCommand(BaseDeploymentCommand): - def execute(self, **kwargs): - self.client.stop(**kwargs) - self.logger.log("Deployment stopped") - - -class DeleteDeploymentCommand(BaseDeploymentCommand): - def execute(self, **kwargs): - self.client.delete(**kwargs) - self.logger.log("Deployment deleted") - - -class UpdateDeploymentCommand(HandleAutoscalingOptions, BaseDeploymentCommand, HandleWorkspaceMixin): - def __init__(self, workspace_handler, *args, **kwargs): - super(UpdateDeploymentCommand, self).__init__(*args, **kwargs) - self.workspace_handler = workspace_handler - - def execute(self, deployment_id, **kwargs): - self._handle_workspace(kwargs) - self._handle_autoscaling_options(kwargs) - - with halo.Halo(text="Updating deployment data", spinner="dots"): - self.client.update(deployment_id, **kwargs) - - self.logger.log("Deployment data updated") - - -class GetDeploymentDetails(DetailsCommandMixin, BaseDeploymentCommand): - def _get_table_data(self, instance): - """ - :param models.Deployment instance: - """ - tags_string = ", ".join(instance.tags) - autoscaling_metrics_string = self.get_autoscaling_metrics_string(instance) - - data = ( - ("ID", instance.id), - ("Name", instance.name), - ("State", instance.state), - ("Machine type", instance.machine_type), - ("Instance count", instance.instance_count), - ("Command", instance.command), - ("Deployment type", instance.deployment_type), - ("Model ID", instance.model_id), - ("Project ID", instance.project_id), - ("Endpoint", instance.endpoint), - ("API type", instance.api_type), - ("Cluster ID", instance.cluster_id), - ("Tags", tags_string), - ("Min Instance Count", getattr(instance.autoscaling, "min_instance_count", "")), - ("Max Instance Count", getattr(instance.autoscaling, "max_instance_count", "")), - ("Scale Cooldown Period", getattr(instance.autoscaling, "scale_cooldown_period", "")), - ("Autoscaling Metrics", autoscaling_metrics_string), - ) - return data - - def get_autoscaling_metrics_string(self, instance): - if not instance.autoscaling or not instance.autoscaling.metrics: - return "" - - s = "\n".join("{}/{}:{}".format(m.name, m.value_type, m.value) - for m in instance.autoscaling.metrics) - return s - - -class DeploymentAddTagsCommand(BaseDeploymentCommand): - def execute(self, deployment_id, *args, **kwargs): - self.client.add_tags(deployment_id, **kwargs) - self.logger.log("Tags added to deployment") - - -class DeploymentRemoveTagsCommand(BaseDeploymentCommand): - def execute(self, deployment_id, *args, **kwargs): - self.client.remove_tags(deployment_id, **kwargs) - self.logger.log("Tags removed from deployment") - - -class GetDeploymentMetricsCommand(BaseDeploymentCommand): - def execute(self, deployment_id, start, end, interval, built_in_metrics, *args, **kwargs): - metrics = self.client.get_metrics( - deployment_id, - start=start, - end=end, - built_in_metrics=built_in_metrics, - interval=interval, - ) - formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True) - self.logger.log(formatted_metrics) - - -class ListDeploymentMetricsCommand(BaseDeploymentCommand): - def execute(self, deployment_id, start, end, interval, *args, **kwargs): - metrics = self.client.list_metrics( - deployment_id, - start=start, - end=end, - interval=interval, - ) - formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True) - self.logger.log(formatted_metrics) - -class StreamDeploymentMetricsCommand(StreamMetricsCommand, BaseDeploymentCommand): - pass - - -class DeploymentLogsCommand(LogsCommandMixin, BaseDeploymentCommand): - def _make_table(self, logs, experiment_id): - table_title = "Deployment %s logs" % experiment_id - table_data = [("LINE", "MESSAGE")] - table = terminaltables.AsciiTable(table_data, title=table_title) - - for log in logs: - table_data.append(self._format_row(experiment_id, log)) - - return table.table - - def _get_log_row_string(self, id, log): - log_msg = "{}\t{}".format(*self._format_row(id, log)) - return log_msg - - @staticmethod - def _format_row(id, log_row): - return (style(fg="red", text=str(log_row.line)), - log_row.message) diff --git a/gradient/commands/experiments.py b/gradient/commands/experiments.py deleted file mode 100644 index 323e9c38..00000000 --- a/gradient/commands/experiments.py +++ /dev/null @@ -1,461 +0,0 @@ -import abc -import json - -import click -import six -import terminaltables -from click import style -from halo import halo - -from gradient import api_sdk, exceptions, TensorboardClient -from gradient.api_sdk import constants, sdk_exceptions -from gradient.api_sdk.config import config -from gradient.api_sdk.utils import concatenate_urls -from gradient.cli_constants import CLI_PS_CLIENT_NAME -from gradient.clilogger import CliLogger -from gradient.cliutils import none_strings_to_none_objects -from gradient.commands import tensorboards as tensorboards_commands -from gradient.commands.common import BaseCommand, ListCommandMixin, DetailsCommandMixin, StreamMetricsCommand, \ - LogsCommandMixin - -try: - # Python 3 - from itertools import zip_longest -except ImportError: - # Python 2 - from itertools import izip_longest as zip_longest - - -@six.add_metaclass(abc.ABCMeta) -class BaseExperimentCommand(BaseCommand): - def _get_client(self, api_key, logger): - client = api_sdk.clients.ExperimentsClient( - api_key=api_key, - logger=logger, - ps_client_name=CLI_PS_CLIENT_NAME, - ) - return client - - -class TensorboardHandler(object): - def __init__(self, api_key, logger=CliLogger()): - self.api_key = api_key - self.logger = logger - - def maybe_add_to_tensorboard(self, tensorboard_id, experiment_id): - """Add experiment to existing or new tensorboard - - :param str|bool tensorboard_id: - :param str experiment_id: - """ - if isinstance(tensorboard_id, six.string_types): - self._add_experiment_to_tensorboard(tensorboard_id, experiment_id) - return - - tensorboards = self._get_tensorboards() - if len(tensorboards) == 1: - self._add_experiment_to_tensorboard( - tensorboards[0].id, experiment_id) - else: - self._create_tensorboard_with_experiment(experiment_id) - - def _add_experiment_to_tensorboard(self, tensorboard_id, experiment_id): - """Add experiment to tensorboard - - :param str tensorboard_id: - :param str experiment_id: - """ - command = tensorboards_commands.AddExperimentToTensorboard( - api_key=self.api_key) - command.execute(tensorboard_id, [experiment_id]) - - def _get_tensorboards(self): - """Get tensorboards - - :rtype: list[api_sdk.Tensorboard] - """ - tensorboard_client = TensorboardClient( - api_key=self.api_key, logger=self.logger) - tensorboards = tensorboard_client.list() - return tensorboards - - def _create_tensorboard_with_experiment(self, experiment_id): - """Create tensorboard with experiment - - :param str experiment_id: - """ - command = tensorboards_commands.CreateTensorboardCommand( - api_key=self.api_key) - command.execute(experiments=[experiment_id]) - - -@six.add_metaclass(abc.ABCMeta) -class BaseCreateExperimentCommandMixin(BaseCommand): - SPINNER_MESSAGE = "Creating new experiment" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "New experiment created with ID: {}" - - def __init__(self, workspace_handler, *args, **kwargs): - super(BaseCreateExperimentCommandMixin, self).__init__(*args, **kwargs) - self.workspace_handler = workspace_handler - - def execute(self, json_, add_to_tensorboard=False): - self._handle_workspace(json_) - self._handle_dataset_data(json_) - - with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"): - experiment_id = self._create(json_) - - self.logger.log( - self.CREATE_SUCCESS_MESSAGE_TEMPLATE.format(experiment_id)) - self.logger.log(self.get_instance_url( - experiment_id, json_["project_id"])) - - self._maybe_add_to_tensorboard( - add_to_tensorboard, experiment_id, self.api_key) - return experiment_id - - def get_instance_url(self, instance_id, project_id): - url = concatenate_urls(config.WEB_URL, "{}/projects/{}/experiments/{}".format( - self.get_namespace(), project_id, instance_id)) - return url - - def _handle_workspace(self, instance_dict): - handler = self.workspace_handler.handle(instance_dict) - - instance_dict.pop("ignore_files", None) - instance_dict.pop("workspace", None) - instance_dict.pop("workspace_archive", None) - instance_dict.pop("workspace_url", None) - if handler: - instance_dict["workspace_url"] = handler - - def _maybe_add_to_tensorboard(self, tensorboard_id, experiment_id, api_key): - """ - :param str|bool tensorboard_id: - :param str experiment_id: - :param str api_key: - """ - if tensorboard_id is not False: - tensorboard_handler = TensorboardHandler(api_key) - tensorboard_handler.maybe_add_to_tensorboard( - tensorboard_id, experiment_id) - - @staticmethod - def _handle_dataset_data(json_): - """Make list of dataset dicts""" - datasets = [ - json_.pop("dataset_uri_list", ()), - json_.pop("dataset_id_list", ()), - json_.pop("dataset_name_list", ()), - json_.pop("dataset_access_key_id_list", ()), - json_.pop("dataset_secret_access_key_list", ()), - json_.pop("dataset_endpoint_list", ()), - json_.pop("dataset_version_id_list", ()), - json_.pop("dataset_etag_list", ()), - json_.pop("dataset_volume_kind_list", ()), - json_.pop("dataset_volume_size_list", ()), - ] - - if not any(datasets): - return - else: - datasets_len = max(len(datasets[0]), len(datasets[1])) - other_dataset_param_max_len = max( - len(elem) for elem in datasets[2:]) - if datasets_len < other_dataset_param_max_len: - # there no point in defining n+1 dataset parameters of one type for n datasets - raise click.BadParameter( - "Too many dataset parameter sets ({}) for {} dataset URIs. Forgot to add one more dataset URI?" - .format(other_dataset_param_max_len, datasets_len)) - - datasets = [none_strings_to_none_objects(d) for d in datasets] - - datasets = zip_longest(*datasets, fillvalue=None) - datasets = [{"uri": dataset[0], - "id": dataset[1], - "name": dataset[2], - "aws_access_key_id": dataset[3], - "aws_secret_access_key": dataset[4], - "aws_endpoint": dataset[5], - "version_id": dataset[6], - "etag": dataset[7], - "volume_kind": dataset[8], - "volume_size": dataset[9], - } for dataset in datasets] - - json_["datasets"] = datasets - - @abc.abstractmethod - def _create(self, json_): - pass - - -class CreateSingleNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand): - def _create(self, json_): - handle = self.client.create_single_node(**json_) - return handle - - -class CreateMultiNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand): - def _create(self, json_): - handle = self.client.create_multi_node(**json_) - return handle - - -class CreateMpiMultiNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand): - def _create(self, json_): - # for MPI there is no experiment_type_id parameter in client method - json_.pop("experiment_type_id", None) - handle = self.client.create_mpi_multi_node(**json_) - return handle - - -class CreateAndStartMultiNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand): - SPINNER_MESSAGE = "Creating and starting new experiment" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "New experiment created and started with ID: {}" - - def _create(self, json_): - handle = self.client.run_multi_node(**json_) - return handle - - -class CreateAndStartMpiMultiNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand): - SPINNER_MESSAGE = "Creating and starting new experiment" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "New experiment created and started with ID: {}" - - def _create(self, json_): - # for MPI there is no experiment_type_id parameter in client method - json_.pop("experiment_type_id", None) - handle = self.client.run_mpi_multi_node(**json_) - return handle - - -class CreateAndStartSingleNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand): - SPINNER_MESSAGE = "Creating and starting new experiment" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "New experiment created and started with ID: {}" - - def _create(self, json_): - handle = self.client.run_single_node(**json_) - return handle - - -class StartExperimentCommand(BaseExperimentCommand): - def execute(self, experiment_id): - """ - :param str experiment_id: - """ - self.client.start(experiment_id) - self.logger.log("Experiment started") - - -class StopExperimentCommand(BaseExperimentCommand): - def execute(self, experiment_id): - """ - :param str experiment_id: - """ - self.client.stop(experiment_id) - self.logger.log("Experiment stopped") - - -class ListExperimentsCommand(ListCommandMixin, BaseExperimentCommand): - TOTAL_ITEMS_KEY = "totalItems" - - def _get_instances(self, **kwargs): - try: - instances, meta_data = self.client.list(get_meta=True, **kwargs) - except sdk_exceptions.GradientSdkError as e: - raise exceptions.ReceivingDataFailedError(e) - - return instances, meta_data - - def _get_table_data(self, experiments): - data = [("Name", "ID", "Status")] - for experiment in experiments: - name = experiment.name - handle = experiment.id - status = constants.ExperimentState.get_state_str(experiment.state) - data.append((name, handle, status)) - return data - - def execute(self, **kwargs): - return self._generate_data_table(**kwargs) - - -class GetExperimentCommand(DetailsCommandMixin, BaseExperimentCommand): - def _get_table_data(self, experiment): - """ - :param api_sdk.SingleNodeExperiment|api_sdk.MultiNodeExperiment|api_sdk.MpiMultiNodeExperiment experiment: - """ - if experiment.experiment_type_id == constants.ExperimentType.SINGLE_NODE: - return self._get_single_node_data(experiment) - - if experiment.experiment_type_id == constants.ExperimentType.GRPC_MULTI_NODE: - return self._get_multi_node_grpc_data(experiment) - - if experiment.experiment_type_id == constants.ExperimentType.MPI_MULTI_NODE: - return self._get_multi_node_mpi_data(experiment) - - raise ValueError("Wrong experiment type: {}".format( - experiment.experiment_type_id)) - - @staticmethod - def _get_single_node_data(experiment): - """ - :param api_sdk.SingleNodeExperiment experiment: - """ - - tags_string = ", ".join(experiment.tags) - data = ( - ("Name", experiment.name), - ("ID", experiment.id), - ("State", constants.ExperimentState.get_state_str(experiment.state)), - ("Ports", experiment.ports), - ("Project ID", experiment.project_id), - ("Worker Command", experiment.command), - ("Worker Container", experiment.container), - ("Worker Machine Type", experiment.machine_type), - ("Working Directory", experiment.working_directory), - ("Workspace URL", experiment.workspace_url), - ("Model Type", experiment.model_type), - ("Model Path", experiment.model_path), - ("Tags", tags_string), - ) - return data - - @staticmethod - def _get_multi_node_grpc_data(experiment): - """ - :param api_sdk.MultiNodeExperiment experiment: - """ - - tags_string = ", ".join(experiment.tags) - data = ( - ("Name", experiment.name), - ("ID", experiment.id), - ("State", constants.ExperimentState.get_state_str(experiment.state)), - ("Artifact directory", experiment.artifact_directory), - ("Cluster ID", experiment.cluster_id), - ("Experiment Env", experiment.experiment_env), - ("Experiment Type", constants.ExperimentType.get_type_str( - experiment.experiment_type_id)), - ("Model Type", experiment.model_type), - ("Model Path", experiment.model_path), - ("Parameter Server Command", experiment.parameter_server_command), - ("Parameter Server Container", experiment.parameter_server_container), - ("Parameter Server Count", experiment.parameter_server_count), - ("Parameter Server Machine Type", - experiment.parameter_server_machine_type), - ("Ports", experiment.ports), - ("Project ID", experiment.project_id), - ("Worker Command", experiment.worker_command), - ("Worker Container", experiment.worker_container), - ("Worker Count", experiment.worker_count), - ("Worker Machine Type", experiment.worker_machine_type), - ("Working Directory", experiment.working_directory), - ("Workspace URL", experiment.workspace_url), - ("Tags", tags_string), - ) - return data - - @staticmethod - def _get_multi_node_mpi_data(experiment): - """ - :param api_sdk.MpiMultiNodeExperiment experiment: - """ - - tags_string = ", ".join(experiment.tags) - data = ( - ("Name", experiment.name), - ("ID", experiment.id), - ("State", constants.ExperimentState.get_state_str(experiment.state)), - ("Artifact directory", experiment.artifact_directory), - ("Cluster ID", experiment.cluster_id), - ("Experiment Env", experiment.experiment_env), - ("Experiment Type", constants.ExperimentType.get_type_str( - experiment.experiment_type_id)), - ("Model Type", experiment.model_type), - ("Model Path", experiment.model_path), - ("Master Command", experiment.master_command), - ("Master Container", experiment.master_container), - ("Master Count", experiment.master_count), - ("Master Machine Type", experiment.master_machine_type), - ("Ports", experiment.ports), - ("Project ID", experiment.project_id), - ("Worker Command", experiment.worker_command), - ("Worker Container", experiment.worker_container), - ("Worker Count", experiment.worker_count), - ("Worker Machine Type", experiment.worker_machine_type), - ("Working Directory", experiment.working_directory), - ("Workspace URL", experiment.workspace_url), - ("Tags", tags_string), - ) - return data - - -class ExperimentLogsCommand(LogsCommandMixin, BaseExperimentCommand): - def _make_table(self, logs, experiment_id): - table_title = "Experiment %s logs" % experiment_id - table_data = [("JOB ID", "LINE", "MESSAGE")] - table = terminaltables.AsciiTable(table_data, title=table_title) - - for log in logs: - table_data.append(self._format_row(experiment_id, log)) - - return table.table - - def _get_log_row_string(self, id, log): - log_msg = "{}\t{}\t{}".format(*self._format_row(id, log)) - return log_msg - - @staticmethod - def _format_row(experiment_id, log_row): - return (style(fg="blue", text=experiment_id), - style(fg="red", text=str(log_row.line)), - log_row.message) - - -class DeleteExperimentCommand(BaseExperimentCommand): - def execute(self, experiment_id, *args, **kwargs): - self.client.delete(experiment_id) - self.logger.log("Experiment deleted") - - -class ExperimentAddTagsCommand(BaseExperimentCommand): - def execute(self, experiment_id, *args, **kwargs): - self.client.add_tags(experiment_id, **kwargs) - self.logger.log("Tags added to experiment") - - -class ExperimentRemoveTagsCommand(BaseExperimentCommand): - def execute(self, experiment_id, *args, **kwargs): - self.client.remove_tags(experiment_id, **kwargs) - self.logger.log("Tags removed from experiment") - - -class GetExperimentMetricsCommand(BaseExperimentCommand): - def execute(self, experiment_id, start, end, interval, built_in_metrics, *args, **kwargs): - metrics = self.client.get_metrics( - experiment_id, - start=start, - end=end, - built_in_metrics=built_in_metrics, - interval=interval, - ) - formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True) - self.logger.log(formatted_metrics) - - -class ListExperimentMetricsCommand(BaseExperimentCommand): - def execute(self, experiment_id, start, end, interval, *args, **kwargs): - metrics = self.client.list_metrics( - experiment_id, - start=start, - end=end, - interval=interval, - ) - formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True) - self.logger.log(formatted_metrics) - - -class StreamExperimentMetricsCommand(StreamMetricsCommand, BaseExperimentCommand): - pass diff --git a/gradient/commands/hyperparameters.py b/gradient/commands/hyperparameters.py deleted file mode 100644 index 7966b8dc..00000000 --- a/gradient/commands/hyperparameters.py +++ /dev/null @@ -1,95 +0,0 @@ -import abc - -import six - -from gradient import api_sdk, exceptions -from gradient.api_sdk import sdk_exceptions -from gradient.cli_constants import CLI_PS_CLIENT_NAME -from gradient.commands.common import BaseCommand, ListCommandMixin, DetailsCommandMixin -from gradient.commands.experiments import BaseCreateExperimentCommandMixin - - -@six.add_metaclass(abc.ABCMeta) -class BaseHyperparameterCommand(BaseCommand): - def _get_client(self, api_key, logger): - client = api_sdk.clients.HyperparameterJobsClient( - api_key=api_key, - logger=logger, - ps_client_name=CLI_PS_CLIENT_NAME, - ) - return client - - -class CreateHyperparameterCommand(BaseCreateExperimentCommandMixin, BaseHyperparameterCommand): - SPINNER_MESSAGE = "Creating hyperparameter tuning job" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "Hyperparameter tuning job created with ID: {}" - - def _create(self, hyperparameter): - handle = self.client.create(**hyperparameter) - return handle - - -class CreateAndStartHyperparameterCommand(BaseCreateExperimentCommandMixin, BaseHyperparameterCommand): - SPINNER_MESSAGE = "Creating and starting hyperparameter tuning job" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "Hyperparameter tuning job created and started with ID: {}" - - def _create(self, hyperparameter): - handle = self.client.run(**hyperparameter) - return handle - - -class ListHyperparametersCommand(ListCommandMixin, BaseHyperparameterCommand): - def _get_instances(self, kwargs): - try: - instances = self.client.list() - except sdk_exceptions.GradientSdkError as e: - raise exceptions.ReceivingDataFailedError(e) - - return instances - - def _get_table_data(self, objects): - data = [("Name", "ID", "Project ID")] - for obj in objects: - data.append((obj.name, obj.id, obj.project_id)) - return data - - -class HyperparameterDetailsCommand(DetailsCommandMixin, BaseHyperparameterCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def _get_table_data(self, instance): - """ - :param api_sdk.Hyperparameter instance: - """ - data = ( - ("ID", instance.id), - ("Name", instance.name), - ("Ports", instance.ports), - ("Project ID", instance.project_id), - ("Tuning command", instance.tuning_command), - ("Worker command", instance.worker_command), - ("Worker container", instance.worker_container), - ("Worker count", instance.worker_count), - ("Worker machine type", instance.worker_machine_type), - ("Worker use dockerfile", instance.use_dockerfile or False), - ("Workspace URL", instance.workspace_url), - ) - return data - - -class HyperparameterStartCommand(BaseHyperparameterCommand): - def execute(self, id_): - self.client.start(id_) - self.logger.log("Hyperparameter tuning started") - - -class HyperparameterAddTagsCommand(BaseHyperparameterCommand): - def execute(self, hyperparameter_id, *args, **kwargs): - self.client.add_tags(hyperparameter_id, **kwargs) - self.logger.log("Tags added to hyperparameter") - - -class HyperparameterRemoveTagsCommand(BaseHyperparameterCommand): - def execute(self, hyperparameter_id, *args, **kwargs): - self.client.remove_tags(hyperparameter_id, **kwargs) - self.logger.log("Tags removed from hyperparameter") diff --git a/gradient/commands/jobs.py b/gradient/commands/jobs.py deleted file mode 100644 index 03d8bd5d..00000000 --- a/gradient/commands/jobs.py +++ /dev/null @@ -1,305 +0,0 @@ -import abc -import json -import os -import pydoc - -import six -import terminaltables -from halo import halo - -from gradient import api_sdk, exceptions, JobArtifactsDownloader, cli_constants -from gradient.api_sdk import config, sdk_exceptions -from gradient.api_sdk import utils -from gradient.api_sdk.clients import http_client -from gradient.api_sdk.utils import print_dict_recursive, concatenate_urls, MultipartEncoder -from gradient.cliutils import get_terminal_lines -from gradient.commands.common import BaseCommand, StreamMetricsCommand, LogsCommandMixin - - -@six.add_metaclass(abc.ABCMeta) -class BaseJobCommand(BaseCommand): - def _get_client(self, api_key, logger_): - client = api_sdk.clients.JobsClient( - api_key=api_key, - logger=logger_, - ps_client_name=cli_constants.CLI_PS_CLIENT_NAME, - ) - return client - - -@six.add_metaclass(abc.ABCMeta) -class BaseCreateJobCommandMixin(BaseJobCommand): - SPINNER_MESSAGE = "Creating new job" - CREATE_SUCCESS_MESSAGE_TEMPLATE = "New job created with ID: {}" - - def __init__(self, workspace_handler, *args, **kwargs): - super(BaseCreateJobCommandMixin, self).__init__(*args, **kwargs) - self.workspace_handler = workspace_handler - - def execute(self, json_): - json_, data = self._handle_workspace(json_) - - with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"): - job_id = self._create(json_, data) - - self.logger.log(self.CREATE_SUCCESS_MESSAGE_TEMPLATE.format(job_id)) - self.logger.log(self.get_instance_url(job_id)) - - def get_instance_url(self, instance_id): - url = concatenate_urls(config.config.WEB_URL, "{}/jobs/{}".format(self.get_namespace(), instance_id)) - return url - - def _handle_workspace(self, instance_dict): - """ - - :param instance_dict: - :return: - """ - data = None - - self._set_project_if_not_provided(instance_dict) - workspace_url = self.workspace_handler.handle(instance_dict) - if workspace_url: - if utils.PathParser.is_local_zip_file(workspace_url): - data = self._get_multipart_data(workspace_url, instance_dict) - else: - instance_dict["workspace_file_name"] = workspace_url - - return instance_dict, data - - def _get_multipart_data(self, workspace_url, json_): - archive_basename = os.path.basename(workspace_url) - json_["workspace_file_name"] = archive_basename - job_data = self._get_files_dict(workspace_url, archive_basename) - monitor = MultipartEncoder(job_data).get_monitor() - return monitor - - def _get_files_dict(self, workspace_url, archive_basename): - job_data = {'file': (archive_basename, open(workspace_url, 'rb'), 'text/plain')} - return job_data - - @staticmethod - def _set_project_if_not_provided(json_): - if not json_.get("project_id"): - json_["project"] = "gradient-project" - - @abc.abstractmethod - def _create(self, json_, data): - pass - - -class DeleteJobCommand(BaseJobCommand): - - def execute(self, job_id): - self.client.delete(job_id) - self.logger.log("Job {} deleted".format(job_id)) - - -class StopJobCommand(BaseJobCommand): - - def execute(self, job_id): - self.client.stop(job_id) - self.logger.log("Job {} stopped".format(job_id)) - - -class ListJobsCommand(BaseJobCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def execute(self, **kwargs): - with halo.Halo(text=self.WAITING_FOR_RESPONSE_MESSAGE, spinner="dots"): - instances = self._get_instances(**kwargs) - - self._log_objects_list(instances) - - def _get_instances(self, **kwargs): - - try: - instances = self.client.list(**kwargs) - except sdk_exceptions.GradientSdkError as e: - raise exceptions.ReceivingDataFailedError(e) - - return instances - - @staticmethod - def _get_table_data(jobs): - data = [("ID", "Name", "Project", "Cluster", "Machine Type", "Created")] - for job in jobs: - id_ = job.id - name = job.name - project = job.project - cluster = job.cluster - machine_type = job.machine_type - created = job.dt_created - data.append((id_, name, project, cluster, machine_type, created)) - - return data - - def _log_objects_list(self, objects): - if not objects: - self.logger.warning("No data found") - return - - table_data = self._get_table_data(objects) - table_str = self._make_table(table_data) - if len(table_str.splitlines()) > get_terminal_lines(): - pydoc.pager(table_str) - else: - self.logger.log(table_str) - - @staticmethod - def _make_table(table_data): - ascii_table = terminaltables.AsciiTable(table_data) - table_string = ascii_table.table - return table_string - - -class JobLogsCommand(LogsCommandMixin, BaseJobCommand): - ENTITY = "Job" - - -class CreateJobCommand(BaseCreateJobCommandMixin): - - def _create(self, json_, data): - # because ignore_files is used by workspace handlers and not needed anymore (will fail if not "popped") - json_.pop("ignore_files", None) - json_.pop("workspace", None) - - return self.client.create(data=data, **json_) - - -class ArtifactsDestroyCommand(BaseJobCommand): - def execute(self, job_id, files=None): - self.client.artifacts_delete(job_id, files) - self.logger.log("Job {} artifacts deleted".format(job_id)) - - -class ArtifactsGetCommand(BaseJobCommand): - def execute(self, job_id): - artifact = self.client.artifacts_get(job_id) - - self._log_artifacts(artifact, job_id) - - def _log_artifacts(self, artifact, job_id): - if artifact: - print_dict_recursive(artifact, self.logger) - else: - self.logger.log("No artifacts found for job {}".format(job_id)) - - -class ArtifactsListCommand(BaseJobCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def execute(self, **kwargs): - with halo.Halo(text=self.WAITING_FOR_RESPONSE_MESSAGE, spinner="dots"): - try: - start_after = None - instances = [] - while True: - pagination_response = self.client.artifacts_list(start_after=start_after, **kwargs) - - if pagination_response.data: - instances.extend(pagination_response.data) - start_after = pagination_response.start_after - - if start_after is None: - break - except sdk_exceptions.GradientSdkError as e: - raise exceptions.ReceivingDataFailedError(e) - - self._log_objects_list(instances, kwargs) - - def _get_table_data(self, artifacts, kwargs): - columns = ['Files'] - - show_size = "size" in kwargs - show_url = "url" in kwargs - - if show_size: - columns.append('Size (in bytes)') - if show_url: - columns.append('URL') - - data = [tuple(columns)] - for artifact in artifacts: - row = [artifact.file] - if show_size: - row.append(artifact.size) - if show_url: - row.append(artifact.url) - data.append(tuple(row)) - return data - - def _log_objects_list(self, objects, kwargs): - if not objects: - self.logger.warning("No data found") - return - - table_data = self._get_table_data(objects, kwargs) - table_str = self._make_table(table_data) - if len(table_str.splitlines()) > get_terminal_lines(): - pydoc.pager(table_str) - else: - self.logger.log(table_str) - - @staticmethod - def _make_table(table_data): - ascii_table = terminaltables.AsciiTable(table_data) - table_string = ascii_table.table - return table_string - - -class DownloadArtifactsCommand(BaseJobCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def execute(self, job_id, destination_directory): - artifact_downloader = JobArtifactsDownloader( - self.api_key, - logger=self.logger, - ps_client_name=cli_constants.CLI_PS_CLIENT_NAME, - ) - with halo.Halo(text=self.WAITING_FOR_RESPONSE_MESSAGE, spinner="dots"): - try: - artifact_downloader.download(job_id, destination_directory) - except OSError as e: - raise sdk_exceptions.GradientSdkError(e) - - -class JobAddTagsCommand(BaseJobCommand): - def execute(self, job_id, *args, **kwargs): - self.client.add_tags(job_id, **kwargs) - self.logger.log("Tags added to job") - - -class JobRemoveTagsCommand(BaseJobCommand): - def execute(self, job_id, *args, **kwargs): - self.client.remove_tags(job_id, **kwargs) - self.logger.log("Tags removed from job") - - -class GetJobMetricsCommand(BaseJobCommand): - def execute(self, deployment_id, start, end, interval, built_in_metrics, *args, **kwargs): - metrics = self.client.get_metrics( - deployment_id, - start=start, - end=end, - built_in_metrics=built_in_metrics, - interval=interval, - ) - formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True) - self.logger.log(formatted_metrics) - - -class ListJobMetricsCommand(BaseJobCommand): - def execute(self, deployment_id, start, end, interval, *args, **kwargs): - metrics = self.client.list_metrics( - deployment_id, - start=start, - end=end, - interval=interval, - ) - formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True) - self.logger.log(formatted_metrics) - - -class StreamJobMetricsCommand(StreamMetricsCommand, BaseJobCommand): - pass diff --git a/gradient/commands/tensorboards.py b/gradient/commands/tensorboards.py deleted file mode 100644 index 5c544600..00000000 --- a/gradient/commands/tensorboards.py +++ /dev/null @@ -1,157 +0,0 @@ -import abc -import pydoc - -import halo -import six -import terminaltables - -import gradient.cli_constants -from gradient import api_sdk -from gradient.cliutils import get_terminal_lines -from . import BaseCommand, common - - -@six.add_metaclass(abc.ABCMeta) -class GetTensorboardClientCommandMixin(BaseCommand): - def _get_client(self, api_key, logger): - client = api_sdk.clients.TensorboardClient( - api_key=api_key, - logger=logger, - ps_client_name=gradient.cli_constants.CLI_PS_CLIENT_NAME, - ) - return client - - def _log_object(self, instance): - table_str = self._make_table(instance) - if len(table_str.splitlines()) > get_terminal_lines(): - pydoc.pager(table_str) - else: - self.logger.log(table_str) - - # experiment table - experiment_table_str = self._make_exp_table(instance) - if len(experiment_table_str.splitlines()) > get_terminal_lines(): - pydoc.pager(experiment_table_str) - else: - self.logger.log(experiment_table_str) - - def _make_table(self, instance): - """ - :param api_sdk.Tensorboard: - """ - data = self._get_table_data(instance) - ascii_table = terminaltables.AsciiTable(data) - table_string = ascii_table.table - return table_string - - def _make_exp_table(self, instance): - """ - :param api_sdk.Tensorboard: - """ - data = self._get_exp_table_data(instance) - ascii_table = terminaltables.AsciiTable(data) - table_string = ascii_table.table - return table_string - - @staticmethod - def _get_table_data(instance): - """ - :param api_sdk.Tensorboard instance: - """ - data = ( - ("ID", instance.id), - ("Image", instance.image), - ("URL", instance.url), - ("State", instance.state), - # ("Instance type", instance.instance.type), TODO: for now - # ("Instance size", instance.instance.size), - # ("Instance count", instance.instance.count), - ) - return data - - @staticmethod - def _get_exp_table_data(instance): - """ - :param api_sdk.Tensorboard instance: - """ - data = [["Experiments ID", "State"]] - - for e in instance.experiments: - data.append([e.get("id"), e.get('state')]) - return data - - -class CreateTensorboardCommand(GetTensorboardClientCommandMixin, common.BaseCommand): - SPINNER_MESSAGE = "Creating new tensorboard" - - def execute(self, **kwargs): - with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"): - notebook_id = self.client.create(**kwargs) - - self.logger.log("Created new tensorboard with id: {}".format(notebook_id)) - - -class GetTensorboardCommand(GetTensorboardClientCommandMixin, common.BaseCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def execute(self, id_): - with halo.Halo(text=self.WAITING_FOR_RESPONSE_MESSAGE, spinner="dots"): - instance = self._get_instance(id_) - - self._log_object(instance) - - def _get_instance(self, id_): - """ - :rtype: api_sdk.Tensorboard - """ - instance = self.client.get(id_) - return instance - - -class ListTensorboardsCommand(GetTensorboardClientCommandMixin, common.ListCommandMixin, common.BaseCommand): - WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." - - def _get_instances(self, kwargs): - instances = self.client.list() - return instances - - def _get_table_data(self, objects): - # TODO later we need to add information about state - data = [["ID", "URL", "STATE"]] - for obj in objects: - data.append([obj.id, obj.url, obj.state]) - return data - - -class AddExperimentToTensorboard(GetTensorboardClientCommandMixin, common.BaseCommand): - SPINNER_MESSAGE = "Adding experiments to tensorboard" - - def execute(self, id, experiments): - """ - :param str id: - :param list[str] experiments: - """ - with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"): - tensorboard = self.client.add_experiments(id, added_experiments=list(experiments)) - - self._log_object(tensorboard) - - -class RemoveExperimentToTensorboard(GetTensorboardClientCommandMixin, common.BaseCommand): - SPINNER_MESSAGE = "Removing experiments from tensorboard" - - def execute(self, id, **kwargs): - with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"): - tensorboard = self.client.remove_experiments(id, removed_experiments=list(kwargs.get('experiments'))) - - self._log_object(tensorboard) - - -class DeleteTensorboard(GetTensorboardClientCommandMixin, common.BaseCommand): - SPINNER_MESSAGE = "Deleting tensorboard" - - def execute(self, *args, **kwargs): - with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"): - self.client.delete(**kwargs) - - self.logger.log("Delete tensorboard ({}) ended with success".format(kwargs.get("id"))) diff --git a/gradient/examples/sdk_notebook_tutorial.ipynb b/gradient/examples/sdk_notebook_tutorial.ipynb index c7680ae5..1040075a 100644 --- a/gradient/examples/sdk_notebook_tutorial.ipynb +++ b/gradient/examples/sdk_notebook_tutorial.ipynb @@ -53,11 +53,8 @@ "metadata": {}, "outputs": [], "source": [ - "deployment_client = sdk_client.DeploymentsClient(api_key)\n", "models_client = sdk_client.ModelsClient(api_key)\n", - "jobs_client = sdk_client.JobsClient(api_key)\n", "projects_client = ProjectsClient(api_key)\n", - "experiment_client = sdk_client.ExperimentsClient(api_key)\n", "\n", "#or access them all from a single client\n", "#client = sdk_client.SdkClient(api_key)" @@ -99,116 +96,6 @@ "help(projects_client.create)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#create experiment client\n", - "experiment_client = sdk_client.ExperimentsClient(api_key)\n", - "\n", - "#list experiments in the project\n", - "print(experiment_client.list(project_id = proj_id))\n", - "help(experiment_client.list)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "single_node_path = \"/storage/models/sdk_test\"\n", - "\n", - "env_variable = {\n", - " \"EPOCHS_EVAL\":2,\n", - " \"TRAIN_EPOCHS\":3,\n", - " \"MAX_STEPS\":100,\n", - " \"EVAL_SECS\":10\n", - "}\n", - "\n", - "single_node_parameters = {\n", - " \"name\" : \"single_node_experiment-sdk\",\n", - " \"project_id\" : proj_id,\n", - " \"command\" : \"python mnist.py\",\n", - " \"machine_type\" : \"K80\",\n", - " \"experiment_env\": env_variable,\n", - " \"container\": \"tensorflow/tensorflow:1.13.1-gpu-py3\",\n", - " \"workspace_url\": \"https://github.com/Paperspace/mnist-sample.git\",\n", - " \"model_type\": \"Tensorflow\",\n", - " \"model_path\": single_node_path\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#create single node experiment\n", - "exp_id = experiment_client.create_single_node(**single_node_parameters)\n", - "print(exp_id)\n", - "\n", - "#get single node experiment object\n", - "from gradient import constants\n", - "#experiment state, created but not started\n", - "state = experiment_client.get(exp_id).state\n", - "print(\"state: \"+constants.ExperimentState.get_state_str(state))\n", - "\n", - "print()\n", - "print()\n", - "\n", - "help(experiment_client.create_single_node)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#get experiment object\n", - "print(experiment_client.get(exp_id))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#start experiment\n", - "resp = experiment_client.start(exp_id)\n", - "print(resp)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#create a log stream & print all logs for the duration of experiment\n", - "print(\"Watching state of experiment\")\n", - "state = \"created\"\n", - "while state != \"running\" or state != \"stopped\":\n", - " new_state = constants.ExperimentState.get_state_str(experiment_client.get(exp_id).state)\n", - " if new_state != state:\n", - " print(\"state: \"+state + \" new state: \"+new_state)\n", - " state = new_state\n", - " if state == \"running\": break\n", - "\n", - "log_stream = experiment_client.yield_logs(exp_id)\n", - "print(\"Streaming logs of experiment\")\n", - "try:\n", - " while True:\n", - " print(log_stream.send(None))\n", - "except:\n", - " print(\"done streaming logs\")" - ] - }, { "cell_type": "code", "execution_count": null, @@ -225,172 +112,11 @@ "outputs": [], "source": [ "#get model we just trained\n", - "model_sn = models_client.list(experiment_id = exp_id)[0]\n", + "model_sn = models_client.list()[0]\n", "\n", "#print model summary\n", "print(model_sn.summary)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#view experiment in tensorboard\n", - "%load_ext tensorboard\n", - "%tensorboard --logdir single_node_path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "##Create a dictionary of parameters for running a distributed/multinode experiment\n", - "env = {\n", - " \"EPOCHS_EVAL\":5,\n", - " \"TRAIN_EPOCHS\":10,\n", - " \"MAX_STEPS\":1000,\n", - " \"EVAL_SECS\":10\n", - "}\n", - "\n", - "multi_node_parameters = { \n", - " \"name\": \"multiEx\",\n", - " \"project_id\": proj_id,\n", - " \"experiment_type_id\": 2,\n", - " \"worker_container\": \"tensorflow/tensorflow:1.13.1-gpu-py3\",\n", - " \"worker_machine_type\": \"K80\",\n", - " \"worker_command\": \"pip install -r requirements.txt && python mnist.py\",\n", - " \"experiment_env\": env,\n", - " \"worker_count\": 2,\n", - " \"parameter_server_container\": \"tensorflow/tensorflow:1.13.1-gpu-py3\",\n", - " \"parameter_server_machine_type\": \"K80\",\n", - " \"parameter_server_command\": \"pip install -r requirements.txt && python mnist.py\",\n", - " \"parameter_server_count\": 1,\n", - " \"workspace_url\": \"https://github.com/Paperspace/mnist-sample.git\",\n", - " \"model_type\": \"Tensorflow\"\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "mn_exp_id = experiment_client.create_multi_node(**multi_node_parameters)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "experiment_client.start(mn_exp_id)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from gradient import constants\n", - "#create a log stream & print all logs for the duration of experiment\n", - "print(\"Watching state of experiment\")\n", - "state = \"created\"\n", - "while state != \"running\" or state != \"stopped\":\n", - " new_state = constants.ExperimentState.get_state_str(experiment_client.get(mn_exp_id).state)\n", - " if new_state != state:\n", - " print(\"state: \"+state)\n", - " state = new_state\n", - " if state == \"running\": break\n", - "\n", - "log_stream = experiment_client.yield_logs(mn_exp_id)\n", - "print(\"Streaming logs of experiment\")\n", - "try:\n", - " while True:\n", - " print(log_stream.send(None))\n", - "except:\n", - " print(\"done streaming logs\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run a tensorboard" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#view experiment in tensorboard\n", - "%load_ext tensorboard\n", - "%tensorboard --logdir \"/storage/models/\"+mn_exp_id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model_mn = models_client.list(experiment_id = mn_exp_id)\n", - "print(model_mn[0].summary)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model_to_deploy = None\n", - "model_to_deploy = mn_exp_id if model_mn[0]['accuracy']['result']['max'] > model_sn['accuracy']['result']['max'] else sn_exp_id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model_deploy_id = models_client.list(experiment_id = model_to_deploy)[0].model_id\n", - "\n", - "deploy_param = {\n", - " \"deployment_type\" : \"Tensorflow Serving on K8s\",\n", - " \"image_url\": \"tensorflow/serving:latest-gpu\",\n", - " \"name\": \"sdk_tutorial\",\n", - " \"machine_type\": \"K80\",\n", - " \"instance_count\": 2,\n", - " \"model_id\" : model_deploy_id\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "resp = deployment_client.create(**deploy_param)\n", - "\n", - "help(deployment_client.create)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/source/_static/gradient.css b/source/_static/gradient.css deleted file mode 100644 index e251e029..00000000 --- a/source/_static/gradient.css +++ /dev/null @@ -1,4 +0,0 @@ -div.body { - min-width: 450px; - max-width: 1200px; -} \ No newline at end of file diff --git a/source/_templates/.gitkeep b/source/_templates/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/source/cli_docs/gradient.cli.rst b/source/cli_docs/gradient.cli.rst deleted file mode 100644 index 6f00cb45..00000000 --- a/source/cli_docs/gradient.cli.rst +++ /dev/null @@ -1,12 +0,0 @@ -gradient.cli package -==================== - -Submodules ----------- - -gradient.cli.cli module ------------------------ - -.. click:: gradient.cli.cli:cli - :prog: gradient - :show-nested: diff --git a/source/conf.py b/source/conf.py deleted file mode 100644 index d043d982..00000000 --- a/source/conf.py +++ /dev/null @@ -1,194 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('..')) - - -# -- Project information ----------------------------------------------------- - -project = u'Gradient-CLI' -copyright = u'2019, Paperspace' -author = u'Paperspace' - -# The short X.Y version -version = u'0.3.0' -# The full version, including alpha/beta/rc tags -release = u'alpha' - - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'recommonmark', - 'sphinx_click.ext', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = { - '.rst': 'restructuredtext', - '.txt': 'markdown', - '.md': 'markdown', -} - -# The master toctree document. -master_doc = 'index' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = "en" - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -html_theme_options = { - 'fixed_sidebar': True, - 'page_width': '80%', -} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -html_css_files = [ - 'gradient.css' -] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'GradientPythonSDKdoc' - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'GradientPythonSDK.tex', u'Gradient Python SDK Documentation', - u'Paperspace', 'manual'), -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'gradientpythonsdk', u'Gradient Python SDK Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'GradientPythonSDK', u'Gradient Python SDK Documentation', - author, 'GradientPythonSDK', 'One line description of project.', - 'Miscellaneous'), -] - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - - -# -- Extension configuration ------------------------------------------------- -autodoc_default_options = { - 'special-members': '__init__', - 'member-order': 'bysource', -} diff --git a/source/gradient.api_sdk.clients.rst b/source/gradient.api_sdk.clients.rst index 2632c9c0..080ec4bc 100644 --- a/source/gradient.api_sdk.clients.rst +++ b/source/gradient.api_sdk.clients.rst @@ -44,22 +44,6 @@ gradient.api\_sdk.clients.dataset\_version\_client module :undoc-members: :show-inheritance: -gradient.api\_sdk.clients.deployment\_client module ---------------------------------------------------- - -.. automodule:: gradient.api_sdk.clients.deployment_client - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.clients.experiment\_client module ---------------------------------------------------- - -.. automodule:: gradient.api_sdk.clients.experiment_client - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.clients.http\_client module --------------------------------------------- @@ -68,22 +52,6 @@ gradient.api\_sdk.clients.http\_client module :undoc-members: :show-inheritance: -gradient.api\_sdk.clients.hyperparameter\_client module -------------------------------------------------------- - -.. automodule:: gradient.api_sdk.clients.hyperparameter_client - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.clients.job\_client module --------------------------------------------- - -.. automodule:: gradient.api_sdk.clients.job_client - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.clients.machine\_types\_client module ------------------------------------------------------- @@ -148,14 +116,6 @@ gradient.api\_sdk.clients.storage\_provider\_client module :undoc-members: :show-inheritance: -gradient.api\_sdk.clients.tensorboards\_client module ------------------------------------------------------ - -.. automodule:: gradient.api_sdk.clients.tensorboards_client - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.clients.workflow\_client module ------------------------------------------------- diff --git a/source/gradient.api_sdk.models.rst b/source/gradient.api_sdk.models.rst index 7c0d97ef..166d1319 100644 --- a/source/gradient.api_sdk.models.rst +++ b/source/gradient.api_sdk.models.rst @@ -44,46 +44,6 @@ gradient.api\_sdk.models.dataset\_version module :undoc-members: :show-inheritance: -gradient.api\_sdk.models.deployment module ------------------------------------------- - -.. automodule:: gradient.api_sdk.models.deployment - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.models.experiment module ------------------------------------------- - -.. automodule:: gradient.api_sdk.models.experiment - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.models.experiment\_dataset module ---------------------------------------------------- - -.. automodule:: gradient.api_sdk.models.experiment_dataset - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.models.hyperparameter module ----------------------------------------------- - -.. automodule:: gradient.api_sdk.models.hyperparameter - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.models.job module ------------------------------------ - -.. automodule:: gradient.api_sdk.models.job - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.models.log module ----------------------------------- @@ -156,14 +116,6 @@ gradient.api\_sdk.models.tag module :undoc-members: :show-inheritance: -gradient.api\_sdk.models.tensorboard module -------------------------------------------- - -.. automodule:: gradient.api_sdk.models.tensorboard - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.models.vm\_type module ---------------------------------------- diff --git a/source/gradient.api_sdk.repositories.rst b/source/gradient.api_sdk.repositories.rst index f3a54ddd..edb6f015 100644 --- a/source/gradient.api_sdk.repositories.rst +++ b/source/gradient.api_sdk.repositories.rst @@ -44,22 +44,6 @@ gradient.api\_sdk.repositories.datasets module :undoc-members: :show-inheritance: -gradient.api\_sdk.repositories.deployments module -------------------------------------------------- - -.. automodule:: gradient.api_sdk.repositories.deployments - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.repositories.experiments module -------------------------------------------------- - -.. automodule:: gradient.api_sdk.repositories.experiments - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.repositories.gradient\_deployments module ----------------------------------------------------------- @@ -68,22 +52,6 @@ gradient.api\_sdk.repositories.gradient\_deployments module :undoc-members: :show-inheritance: -gradient.api\_sdk.repositories.hyperparameter module ----------------------------------------------------- - -.. automodule:: gradient.api_sdk.repositories.hyperparameter - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.repositories.jobs module ------------------------------------------- - -.. automodule:: gradient.api_sdk.repositories.jobs - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.repositories.machine\_types module ---------------------------------------------------- @@ -148,14 +116,6 @@ gradient.api\_sdk.repositories.tags module :undoc-members: :show-inheritance: -gradient.api\_sdk.repositories.tensorboards module --------------------------------------------------- - -.. automodule:: gradient.api_sdk.repositories.tensorboards - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.repositories.workflows module ----------------------------------------------- diff --git a/source/gradient.api_sdk.rst b/source/gradient.api_sdk.rst index ec484c2b..404fdaf8 100644 --- a/source/gradient.api_sdk.rst +++ b/source/gradient.api_sdk.rst @@ -95,14 +95,6 @@ gradient.api\_sdk.validation\_messages module :undoc-members: :show-inheritance: -gradient.api\_sdk.workspace module ----------------------------------- - -.. automodule:: gradient.api_sdk.workspace - :members: - :undoc-members: - :show-inheritance: - Module contents --------------- diff --git a/source/gradient.api_sdk.serializers.rst b/source/gradient.api_sdk.serializers.rst index bfb272da..15e3fb90 100644 --- a/source/gradient.api_sdk.serializers.rst +++ b/source/gradient.api_sdk.serializers.rst @@ -52,46 +52,6 @@ gradient.api\_sdk.serializers.dataset\_version module :undoc-members: :show-inheritance: -gradient.api\_sdk.serializers.deployment module ------------------------------------------------ - -.. automodule:: gradient.api_sdk.serializers.deployment - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.serializers.experiment module ------------------------------------------------ - -.. automodule:: gradient.api_sdk.serializers.experiment - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.serializers.experiment\_dataset module --------------------------------------------------------- - -.. automodule:: gradient.api_sdk.serializers.experiment_dataset - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.serializers.hyperparameter module ---------------------------------------------------- - -.. automodule:: gradient.api_sdk.serializers.hyperparameter - :members: - :undoc-members: - :show-inheritance: - -gradient.api\_sdk.serializers.job module ----------------------------------------- - -.. automodule:: gradient.api_sdk.serializers.job - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.serializers.log module ---------------------------------------- @@ -156,14 +116,6 @@ gradient.api\_sdk.serializers.tag module :undoc-members: :show-inheritance: -gradient.api\_sdk.serializers.tensorboard module ------------------------------------------------- - -.. automodule:: gradient.api_sdk.serializers.tensorboard - :members: - :undoc-members: - :show-inheritance: - gradient.api\_sdk.serializers.utils module ------------------------------------------ diff --git a/source/gradient.cli.rst b/source/gradient.cli.rst index 6f00cb45..66e7ca1d 100644 --- a/source/gradient.cli.rst +++ b/source/gradient.cli.rst @@ -1,12 +1,149 @@ gradient.cli package ==================== +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + gradient.cli.utils + Submodules ---------- +gradient.cli.auth module +------------------------ + +.. automodule:: gradient.cli.auth + :members: + :undoc-members: + :show-inheritance: + gradient.cli.cli module ----------------------- -.. click:: gradient.cli.cli:cli - :prog: gradient - :show-nested: +.. automodule:: gradient.cli.cli + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.cli\_types module +------------------------------ + +.. automodule:: gradient.cli.cli_types + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.clusters module +---------------------------- + +.. automodule:: gradient.cli.clusters + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.common module +-------------------------- + +.. automodule:: gradient.cli.common + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.datasets module +---------------------------- + +.. automodule:: gradient.cli.datasets + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.gradient\_deployments module +----------------------------------------- + +.. automodule:: gradient.cli.gradient_deployments + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.machine\_types module +---------------------------------- + +.. automodule:: gradient.cli.machine_types + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.machines module +---------------------------- + +.. automodule:: gradient.cli.machines + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.models module +-------------------------- + +.. automodule:: gradient.cli.models + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.notebooks module +----------------------------- + +.. automodule:: gradient.cli.notebooks + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.projects module +---------------------------- + +.. automodule:: gradient.cli.projects + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.secrets module +--------------------------- + +.. automodule:: gradient.cli.secrets + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.storage\_providers module +-------------------------------------- + +.. automodule:: gradient.cli.storage_providers + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.validators module +------------------------------ + +.. automodule:: gradient.cli.validators + :members: + :undoc-members: + :show-inheritance: + +gradient.cli.workflows module +----------------------------- + +.. automodule:: gradient.cli.workflows + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: gradient.cli + :members: + :undoc-members: + :show-inheritance: diff --git a/source/gradient.commands.rst b/source/gradient.commands.rst index ba49c085..edddd7ae 100644 --- a/source/gradient.commands.rst +++ b/source/gradient.commands.rst @@ -28,22 +28,6 @@ gradient.commands.datasets module :undoc-members: :show-inheritance: -gradient.commands.deployments module ------------------------------------- - -.. automodule:: gradient.commands.deployments - :members: - :undoc-members: - :show-inheritance: - -gradient.commands.experiments module ------------------------------------- - -.. automodule:: gradient.commands.experiments - :members: - :undoc-members: - :show-inheritance: - gradient.commands.helpers module -------------------------------- @@ -52,22 +36,6 @@ gradient.commands.helpers module :undoc-members: :show-inheritance: -gradient.commands.hyperparameters module ----------------------------------------- - -.. automodule:: gradient.commands.hyperparameters - :members: - :undoc-members: - :show-inheritance: - -gradient.commands.jobs module ------------------------------ - -.. automodule:: gradient.commands.jobs - :members: - :undoc-members: - :show-inheritance: - gradient.commands.login module ------------------------------ @@ -132,14 +100,6 @@ gradient.commands.storage\_providers module :undoc-members: :show-inheritance: -gradient.commands.tensorboards module -------------------------------------- - -.. automodule:: gradient.commands.tensorboards - :members: - :undoc-members: - :show-inheritance: - gradient.commands.workflows module ---------------------------------- diff --git a/source/gradient.wizards.rst b/source/gradient.wizards.rst deleted file mode 100644 index 98c3961e..00000000 --- a/source/gradient.wizards.rst +++ /dev/null @@ -1,30 +0,0 @@ -gradient.wizards package -======================== - -Submodules ----------- - -gradient.wizards.projects module --------------------------------- - -.. automodule:: gradient.wizards.projects - :members: - :undoc-members: - :show-inheritance: - -gradient.wizards.wizard module ------------------------------- - -.. automodule:: gradient.wizards.wizard - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: gradient.wizards - :members: - :undoc-members: - :show-inheritance: diff --git a/source/index.rst b/source/index.rst deleted file mode 100644 index 31903958..00000000 --- a/source/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. Gradient Python SDK documentation master file, created by - sphinx-quickstart on Fri Jul 19 13:13:59 2019. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Gradient Python SDK's documentation! -=============================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - gradient - modules - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/tests/config_files/deployments_create.yaml b/tests/config_files/deployments_create.yaml deleted file mode 100644 index c043f26b..00000000 --- a/tests/config_files/deployments_create.yaml +++ /dev/null @@ -1,30 +0,0 @@ -authUsername: some_username -authPassword: some_password -apiKey: some_key -apiType: REST -clusterId: some_cluster_id -command: some deployment command -containerModelPath: some/container/model/path -containerUrlPath: some/container/url/path -deploymentType: TFSERVING -dockerArgs: - - some - - docker - - args -env: - key: value -imageUrl: https://www.latlmes.com/breaking/paperspace-now-has-a-100-bilion-valuation -imageUsername: some_image_username -imagePassword: some_image_password -imageServer: some.image/server -instanceCount: 666 -machineType: G1 -method: some_method -modelId: some_model_id -name: some_name -ports: 5000,6000:7000 -projectId: some_project_id -workspace: s3://some-workspace -workspaceRef: some_branch_name -workspaceUsername: username -workspacePassword: password diff --git a/tests/config_files/deployments_delete.yaml b/tests/config_files/deployments_delete.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/deployments_delete.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/deployments_details.yaml b/tests/config_files/deployments_details.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/deployments_details.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/deployments_list.yaml b/tests/config_files/deployments_list.yaml deleted file mode 100644 index 1e4c657c..00000000 --- a/tests/config_files/deployments_list.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiKey: some_key -modelId: some_model_id -projectId: some_project_id -state: BUILDING -tag: some_tag diff --git a/tests/config_files/deployments_logs.yaml b/tests/config_files/deployments_logs.yaml deleted file mode 100644 index acf259cb..00000000 --- a/tests/config_files/deployments_logs.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiKey: some_key -id: some_id -follow: 'true' -limit: 30 -line: 20 diff --git a/tests/config_files/deployments_metrics_get.yaml b/tests/config_files/deployments_metrics_get.yaml deleted file mode 100644 index 183807d7..00000000 --- a/tests/config_files/deployments_metrics_get.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiKey: some_key -end: 2020-04-02 21:37:00 -id: dev61ity7lx232 -interval: 20s -metric: -- gpuMemoryFree -- gpuMemoryUsed -start: 2020-04-01 00:00:00 diff --git a/tests/config_files/deployments_metrics_stream.yaml b/tests/config_files/deployments_metrics_stream.yaml deleted file mode 100644 index 7fba336d..00000000 --- a/tests/config_files/deployments_metrics_stream.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiKey: some_key -id: dev61ity7lx232 -interval: 20s -metric: -- gpuMemoryFree -- gpuMemoryUsed diff --git a/tests/config_files/deployments_start.yaml b/tests/config_files/deployments_start.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/deployments_start.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/deployments_stop.yaml b/tests/config_files/deployments_stop.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/deployments_stop.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/deployments_update.yaml b/tests/config_files/deployments_update.yaml deleted file mode 100644 index 24c9bbd1..00000000 --- a/tests/config_files/deployments_update.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiKey: some_key -apiType: REST -authPassword: some_password -authUsername: some_username -clusterId: some_cluster_id -command: echo Bazinga! -containerModelPath: some/container/model/path -containerUrlPath: some/container/url/path -deploymentType: TFServing -dockerArgs: -- some -- docker -- args -env: - key: value -id: some_id -imagePassword: some_image_password -imageServer: some.image/server -imageUrl: https://www.latlmes.com/breaking/paperspace-now-has-a-100-bilion-valuation -imageUsername: some_image_username -instanceCount: 666 -machineType: G1 -maxInstanceCount: 64 -method: some_method -metric: - - loss/target:2.0 - - keton/target:21.37 -minInstanceCount: "4" -modelId: some_model_id -name: some_name -ports: '5000' -projectId: some_project_id -resource: - - cpu/targetAverage:10 -scaleCooldownPeriod: 123 -workspace: s3://some-workspace -workspaceRef: some_branch_name -workspaceUsername: username -workspacePassword: password diff --git a/tests/config_files/experiments_create_multi_node.yaml b/tests/config_files/experiments_create_multi_node.yaml deleted file mode 100644 index 962afffb..00000000 --- a/tests/config_files/experiments_create_multi_node.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiKey: some_key -artifactDirectory: /artdir -clusterId: 2a -datasetUri: - - "s3://some.dataset/uri" - - "s3://some.other.dataset/uri" -datasetName: - - "some dataset name" - - null -datasetAwsAccessKeyId: - - none - - some_other_key_id -datasetAwsSecretAccessKey: - - - - some_other_secret -datasetVersionId: - - version1 - - version2 -datasetEtag: - - "some etag" - - "some other etag" -datasetVolumeKind: - - dynamic -datasetVolumeSize: - - 10Gi -experimentEnv: - key: val -experimentType: MPI -ignoreFiles: file1,file2 -isPreemptible: true -modelPath: some-model-path -modelType: some-model-type -name: multinode_mpi -masterCommand: ls -masterContainer: pscon -masterContainerUser: pscuser -masterCount: 2 -masterMachineType: psmtype -masterRegistryPassword: psrpass -masterRegistryUrl: psrurl -masterRegistryUsername: psrcus -ports: '3456' -projectId: prq70zy79 -workerCommand: wcom -workerContainer: wcon -workerContainerUser: usr -workerCount: 2 -workerMachineType: mty -workerRegistryPassword: rpass -workerRegistryUrl: rurl -workerRegistryUsername: rusr -workingDirectory: /dir -workspace: s3://some-workspace -workspaceUsername: username -workspacePassword: password -workspaceArchive: null -workspaceRef: some_branch_name diff --git a/tests/config_files/experiments_create_multi_node_ds_objs.yaml b/tests/config_files/experiments_create_multi_node_ds_objs.yaml deleted file mode 100644 index f4c85b30..00000000 --- a/tests/config_files/experiments_create_multi_node_ds_objs.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiKey: some_key -artifactDirectory: /artdir -clusterId: 2a -datasets: -- uri: "s3://some.dataset/uri" - name: "some dataset name" - awsAccessKeyId: none - awsSecretAccessKey: none - etag: "some etag" - volumeKind: "dynamic" - volumeSize: "10Gi" - versionId: version1 -- uri: "s3://some.other.dataset/uri" - name: null - awsAccessKeyId: some_other_key_id - awsSecretAccessKey: some_other_secret - etag: "some other etag" - versionId: version2 -experimentEnv: - key: val -experimentType: MPI -ignoreFiles: file1,file2 -isPreemptible: true -modelPath: some-model-path -modelType: some-model-type -name: multinode_mpi -masterCommand: ls -masterContainer: pscon -masterContainerUser: pscuser -masterCount: 2 -masterMachineType: psmtype -masterRegistryPassword: psrpass -masterRegistryUrl: psrurl -masterRegistryUsername: psrcus -ports: '3456' -projectId: prq70zy79 -workerCommand: wcom -workerContainer: wcon -workerContainerUser: usr -workerCount: 2 -workerMachineType: mty -workerRegistryPassword: rpass -workerRegistryUrl: rurl -workerRegistryUsername: rusr -workingDirectory: /dir -workspace: s3://some-workspace -workspaceUsername: username -workspacePassword: password -workspaceArchive: null -workspaceRef: some_branch_name diff --git a/tests/config_files/experiments_create_single_node.yaml b/tests/config_files/experiments_create_single_node.yaml deleted file mode 100644 index 8cc4e0f4..00000000 --- a/tests/config_files/experiments_create_single_node.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiKey: some_key -artifactDirectory: /artifact/dir/ -clusterId: 42c -command: testCommand -container: testContainer -containerUser: conUser -experimentEnv: - key: val -ignoreFiles: file1,file2 -isPreemptible: true -machineType: testType -modelPath: some-model-path -modelType: some-model-type -name: exp1 -ports: '4567' -projectId: testHandle -registryPassword: passwd -registryUrl: registryUrl -registryUsername: userName -workingDirectory: /work/dir/ -workspace: s3://some-workspace -workspaceArchive: null -workspaceRef: some_branch_name -workspaceUsername: username -workspacePassword: password diff --git a/tests/config_files/experiments_delete.yaml b/tests/config_files/experiments_delete.yaml deleted file mode 100644 index da649db4..00000000 --- a/tests/config_files/experiments_delete.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some-id diff --git a/tests/config_files/experiments_details.yaml b/tests/config_files/experiments_details.yaml deleted file mode 100644 index 242b8aef..00000000 --- a/tests/config_files/experiments_details.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: experiment-id diff --git a/tests/config_files/experiments_list.yaml b/tests/config_files/experiments_list.yaml deleted file mode 100644 index 24456f3a..00000000 --- a/tests/config_files/experiments_list.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiKey: some_key -limit: 20 -offset: 0 -projectId: -- some_id -- some_id_2 -tag: -- some_tag -- some_tag_2 diff --git a/tests/config_files/experiments_logs.yaml b/tests/config_files/experiments_logs.yaml deleted file mode 100644 index 7b9644f2..00000000 --- a/tests/config_files/experiments_logs.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiKey: some_key -id: some-id -follow: 'true' -limit: 30 -line: 20 diff --git a/tests/config_files/experiments_metrics_get.yaml b/tests/config_files/experiments_metrics_get.yaml deleted file mode 100644 index 81394c20..00000000 --- a/tests/config_files/experiments_metrics_get.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiKey: some_key -end: 2020-04-02 21:37:00 -id: esro6mbmiulvbl -interval: 20s -metric: -- gpuMemoryFree -- gpuMemoryUsed -start: 2020-04-01 00:00:00 diff --git a/tests/config_files/experiments_metrics_stream.yaml b/tests/config_files/experiments_metrics_stream.yaml deleted file mode 100644 index eebb38cf..00000000 --- a/tests/config_files/experiments_metrics_stream.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiKey: some_key -id: esro6mbmiulvbl -interval: 20s -metric: -- gpuMemoryFree -- gpuMemoryUsed diff --git a/tests/config_files/experiments_start.yaml b/tests/config_files/experiments_start.yaml deleted file mode 100644 index e50fd6a7..00000000 --- a/tests/config_files/experiments_start.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiKey: some_key -id: some-id -logs: true diff --git a/tests/config_files/experiments_stop.yaml b/tests/config_files/experiments_stop.yaml deleted file mode 100644 index da649db4..00000000 --- a/tests/config_files/experiments_stop.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some-id diff --git a/tests/config_files/hyperparameters_create.yaml b/tests/config_files/hyperparameters_create.yaml deleted file mode 100644 index e8b63e71..00000000 --- a/tests/config_files/hyperparameters_create.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiKey: some_key -artifactDirectory: some_artifact_directory -clusterId: some_cluster_id -experimentEnv: - key: val -hyperparameterServerContainer: some_hyperparameter_container -hyperparameterServerContainerUser: some_hyperparameter_container_user -hyperparameterServerMachineType: some_hyperparameter_server_machine -hyperparameterServerRegistryPassword: some_hyperparameter_registry_password -hyperparameterServerRegistryUsername: some_hyperparameter_registry_username -ignoreFiles: file2 -isPreemptible: true -modelPath: some_model_path -modelType: some_model_type -name: some_name -ports: 8080,9000:9999 -projectId: some_project_id -tuningCommand: some command -workerCommand: some worker command -workerContainer: some_worker_container -workerContainerUser: some_worker_container_user -workerCount: 666 -workerDockerfilePath: some_docker_path -workerMachineType: k80 -workerRegistryPassword: some_registry_password -workerRegistryUsername: some_registry_username -workerUseDockerfile: true -workingDirectory: some_working_directory -workspace: s3://some-path -workspaceArchive: null diff --git a/tests/config_files/hyperparameters_details.yaml b/tests/config_files/hyperparameters_details.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/hyperparameters_details.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/hyperparameters_list.yaml b/tests/config_files/hyperparameters_list.yaml deleted file mode 100644 index 8cadda1f..00000000 --- a/tests/config_files/hyperparameters_list.yaml +++ /dev/null @@ -1 +0,0 @@ -apiKey: some_key diff --git a/tests/config_files/hyperparameters_start.yaml b/tests/config_files/hyperparameters_start.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/hyperparameters_start.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/jobs_artifacts_destroy.yaml b/tests/config_files/jobs_artifacts_destroy.yaml deleted file mode 100644 index 1ae9f19b..00000000 --- a/tests/config_files/jobs_artifacts_destroy.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiKey: some_key -files: file1,file2 -id: some_id diff --git a/tests/config_files/jobs_artifacts_get.yaml b/tests/config_files/jobs_artifacts_get.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/jobs_artifacts_get.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/jobs_artifacts_list.yaml b/tests/config_files/jobs_artifacts_list.yaml deleted file mode 100644 index b04100a3..00000000 --- a/tests/config_files/jobs_artifacts_list.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiKey: some_key -files: keton*.py -id: some_id -links: true -size: true diff --git a/tests/config_files/jobs_create.yaml b/tests/config_files/jobs_create.yaml deleted file mode 100644 index 5c1ad90b..00000000 --- a/tests/config_files/jobs_create.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiKey: some_key -buildOnly: true -cluster: some_cluster_name -clusterId: some_cluster_id -command: some command -container: some_container -experimentId: some_experiment_id -ignoreFiles: file1,file2 -isPreemptible: --projectId -isPublic: --workspaceUrl -envVars: - key: val -machineType: K80 -name: some_name -nodeAttrs: - key: val -ports: 8080,9000:9900 -project: null -projectId: some_project_id -registryPassword: some_registry_password -registryUsername: some_registry_username -registryTarget: some_registry_target -registryTargetPassword: some_registry_target_password -registryTargetUsername: some_registry_target_username -relDockerfilePath: some dockerfile path -startedByUserId: some_user_id -useDockerfile: 'True' -workingDirectory: /some/path -workspace: s3://some-path diff --git a/tests/config_files/jobs_list.yaml b/tests/config_files/jobs_list.yaml deleted file mode 100644 index 5d7d9a1d..00000000 --- a/tests/config_files/jobs_list.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiKey: some_key -experimentId: null -project: null -projectId: some_project_id -tag: - - some_tag - - some_other_tag diff --git a/tests/config_files/jobs_logs.yaml b/tests/config_files/jobs_logs.yaml deleted file mode 100644 index e92d9a42..00000000 --- a/tests/config_files/jobs_logs.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiKey: some_key -follow: false -id: some_id -limit: 200 -line: 50 diff --git a/tests/config_files/jobs_metrics_get.yaml b/tests/config_files/jobs_metrics_get.yaml deleted file mode 100644 index bfff75e9..00000000 --- a/tests/config_files/jobs_metrics_get.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiKey: some_key -end: 2020-04-02 21:37:00 -id: jstkd2lapucirs -interval: 20s -metric: -- gpuMemoryFree -- gpuMemoryUsed -start: 2020-04-01 00:00:00 diff --git a/tests/config_files/jobs_metrics_stream.yaml b/tests/config_files/jobs_metrics_stream.yaml deleted file mode 100644 index a2fd197c..00000000 --- a/tests/config_files/jobs_metrics_stream.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiKey: some_key -id: jstkd2lapucirs -interval: 20s -metric: -- gpuMemoryFree -- gpuMemoryUsed diff --git a/tests/config_files/run.yaml b/tests/config_files/run.yaml deleted file mode 100644 index 93a51a1d..00000000 --- a/tests/config_files/run.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiKey: some_key -#cluster: null # commented to check if missing key does not break the app -clusterId: some_cluster_id -command: some command -container: some_container -experimentId: some_experiment_id -ignoreFiles: file1,file2 -isPreemptible: 'true' -isPublic: 'true' -envVars: - key: val -machineType: some_machine_type -module: 3 -name: some_name -nodeAttrs: - key: val2 -ports: 8080,9000:9900 -project: null -projectId: some_project_id -python-command: 3 -registryPassword: some_registry_password -registryUsername: some_registry_username -registryTarget: some_registry_target -registryTargetPassword: some_registry_target_password -registryTargetUsername: some_registry_target_username -relDockerfilePath: /some/dockerfile/path -script: - - some_script.py - - some_other_script.py -shell: 3 -startedByUserId: some_user_id -useDockerfile: 'true' -workingDirectory: /some/directory -workspace: s3://bucket/object diff --git a/tests/config_files/tensorboards_create.yaml b/tests/config_files/tensorboards_create.yaml deleted file mode 100644 index dc6e53d3..00000000 --- a/tests/config_files/tensorboards_create.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiKey: some_key -experiment: - - some_experiment_id - - some_other_experiment_id -image: some_image -password: some_password -username: some_username diff --git a/tests/config_files/tensorboards_details.yaml b/tests/config_files/tensorboards_details.yaml deleted file mode 100644 index b52f2e44..00000000 --- a/tests/config_files/tensorboards_details.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiKey: some_key -id: some_id diff --git a/tests/config_files/tensorboards_list.yaml b/tests/config_files/tensorboards_list.yaml deleted file mode 100644 index 3d7d10ab..00000000 --- a/tests/config_files/tensorboards_list.yaml +++ /dev/null @@ -1 +0,0 @@ -apiKey: null diff --git a/tests/functional/test_deployments.py b/tests/functional/test_deployments.py deleted file mode 100644 index 4727be63..00000000 --- a/tests/functional/test_deployments.py +++ /dev/null @@ -1,255 +0,0 @@ -import mock -import json -from click.testing import CliRunner - -from gradient.api_sdk.clients import http_client -from gradient.cli import cli - -EXPECTED_HEADERS = http_client.default_headers.copy() -EXPECTED_HEADERS["ps_client_name"] = "gradient-cli" - -EXPECTED_HEADERS_WITH_CHANGED_API_KEY = EXPECTED_HEADERS.copy() -EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - -URL = "https://api.paperspace.io" - - -class TestCreateDeployments(object): - LOAD_SPEC_VALUE = { - "image": "lucone83/streamlit-nginx", - "port": 8080, - "resources": { - "replicas": 1, - "instanceType": "C4" - } - } - - @mock.patch("gradient.cli.gradient_deployments.load_spec", return_value=LOAD_SPEC_VALUE) - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.create_deployment") - def test_create_deployments(self, method, load_spec): - STDOUT = "Created deployment: 5c229375-6f77-41b1-afbc-acd00cac9b77" - method.return_value = {"id": "5c229375-6f77-41b1-afbc-acd00cac9b77"} - - result = CliRunner().invoke( - cli.cli, - ["deployments", "create"] + [ - "--name=test-deployment", - "--projectId=prsmlkp15", - "--spec=./deployment.yaml", - "--apiKey=some_key" - ] - ) - - assert STDOUT in result.output - - -class TestUpdateDeployments(object): - LOAD_SPEC_VALUE = { - "image": "lucone83/streamlit-nginx", - "port": 8080, - "resources": { - "replicas": 1, - "instanceType": "C4" - } - } - - @mock.patch("gradient.cli.gradient_deployments.load_spec", return_value=LOAD_SPEC_VALUE) - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.update_deployment") - def test_update_deployments(self, method, load_spec): - STDOUT = "Updated deployment: 5c229375-6f77-41b1-afbc-acd00cac9b77" - method.return_value = {"id": "5c229375-6f77-41b1-afbc-acd00cac9b77"} - - result = CliRunner().invoke( - cli.cli, - ["deployments", "update"] + [ - "--id=5c229375-6f77-41b1-afbc-acd00cac9b77" - "--name=test-deployment", - "--projectId=prsmlkp15", - "--spec=./deployment.yaml", - "--clusterId=cluster_id_1", - "--apiKey=some_key"] - ) - - assert STDOUT in result.output - - -class TestListDeployments(object): - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.list_deployments") - def test_list_deployments(self, method): - STDOUT = """+-----------------+--------------------------------------+ -| Name | ID | -+-----------------+--------------------------------------+ -| test-fV3opaPJHl | 58dac4ff-fd7c-4da0-97a7-f114c304b3eb | -+-----------------+--------------------------------------+ -""" - - method.return_value = [ - { - "id": "58dac4ff-fd7c-4da0-97a7-f114c304b3eb", - "name": "test-fV3opaPJHl", - "deploymentSpecs": { - "nodes": [ - { - "id": "efe770ee-6f9d-42cb-b802-66e6dd5d646c", - "data": { - "image": "bash:5", - "port": 8000, - "resources": { - "instanceType": "C10", - "replicas": 1 - } - }, - "endpointUrl": "hash.cluster.paperspacegradient.com", - "actor": { - "avatarUrl": None, - "fullName": None - }, - "deploymentRuns": { - "nodes": [] - } - } - ] - } - } - ] - - result = CliRunner().invoke(cli.cli, ["deployments", "list", "--apiKey=some_key"]) - - assert STDOUT in result.output - - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.list_deployments") - def test_list_deployments_no_nodes(self, method): - STDOUT = "No deployments found" - - method.return_value = [] - - result = CliRunner().invoke(cli.cli, ["deployments", "list", "--apiKey=some_key"]) - - assert STDOUT in result.output - - -class TestGetDeployment(object): - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.get_deployment") - def test_list_deployments(self, method): - STDOUT = """{ - "id": "efe770ee-6f9d-42cb-b802-66e6dd5d646c", - "name": "test-deployment-2", - "deploymentSpecs": [ - { - "id": "03fe1b5c-e82c-42fe-9f48-4020939aaa58", - "data": { - "image": "lucone83/streamlit-nginx", - "port": 8080, - "resources": { - "instanceType": "Air", - "replicas": 1 - }, - "command": null, - "env": [ - { - "name": "ENV", - "value": "VAR" - } - ], - "models": null - }, - "endpointUrl": "d5c2293756f7741b1afbcacd00cac9b77.null", - "actor": { - "avatarUrl": null, - "fullName": null - }, - "cluster": { - "id": "cl9tz9no4" - }, - "deploymentRuns": [] - } - ] -} -""" - - method.return_value = { - "deployment": { - "id": "efe770ee-6f9d-42cb-b802-66e6dd5d646c", - "name": "test-deployment-2", - "deploymentSpecs": { - "nodes": [ - { - "id": "03fe1b5c-e82c-42fe-9f48-4020939aaa58", - "data": { - "image": "lucone83/streamlit-nginx", - "port": 8080, - "resources": { - "instanceType": "Air", - "replicas": 1 - }, - "command": None, - "env": [ - { - "name": "ENV", - "value": "VAR" - } - ], - "models": None - }, - "endpointUrl": "d5c2293756f7741b1afbcacd00cac9b77.null", - "actor": { - "avatarUrl": None, - "fullName": None - }, - "cluster": { - "id": "cl9tz9no4" - }, - "deploymentRuns": { - "nodes": [] - } - } - ] - } - } - } - - result = CliRunner().invoke( - cli.cli, ["deployments", "get", "--id=efe770ee-6f9d-42cb-b802-66e6dd5d646c", "--apiKey=some_key"]) - - assert json.loads(STDOUT) == json.loads(result.output) - - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.get_deployment") - def test_get_deployment_no_deployment(self, method): - STDOUT = "Deployment not found" - - method.return_value = { - "deployment": None - } - - result = CliRunner().invoke( - cli.cli, ["deployments", "get", "--id=efe770ee-6f9d-42cb-b802-66e6dd5d646c", "--apiKey=some_key"]) - - assert STDOUT in result.output - - -class TestDeleteDeployment(object): - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.delete_deployment") - def test_delete_deployments(self, method): - STDOUT = "Deleted deployment: 5c229375-6f77-41b1-afbc-acd00cac9b77" - - method.return_value = { - "deployment": { - "id": "5c229375-6f77-41b1-afbc-acd00cac9b77" - } - } - - result = CliRunner().invoke( - cli.cli, ["deployments", "delete", "--id=5c229375-6f77-41b1-afbc-acd00cac9b77"]) - - assert STDOUT in result.output - - @mock.patch("gradient.api_sdk.repositories.gradient_deployments.delete_deployment") - def test_delete_deployment_no_deployment(self, method): - STDOUT = "Deployment not found" - - method.return_value = None - - result = CliRunner().invoke( - cli.cli, ["deployments", "delete", "--id=5c229375-6f77-41b1-afbc-acd00cac9b78", "--apiKey=some_key"]) - - assert STDOUT in result.output diff --git a/tests/functional/test_experiments.py b/tests/functional/test_experiments.py deleted file mode 100644 index 5ca886f9..00000000 --- a/tests/functional/test_experiments.py +++ /dev/null @@ -1,2167 +0,0 @@ -import copy -import json -import os -import shutil -import tempfile -import zipfile - -import mock -import pytest -from click.testing import CliRunner - -from gradient.api_sdk import constants, sdk_exceptions -from gradient.api_sdk.clients import http_client -from gradient.api_sdk.clients.http_client import default_headers -from gradient.api_sdk.validation_messages import EXPERIMENT_MODEL_PATH_VALIDATION_ERROR -from gradient.cli import cli -from tests import example_responses, MockResponse -from tests.unit.test_archiver_class import create_test_dir_tree - -EXPECTED_HEADERS = default_headers.copy() -EXPECTED_HEADERS["ps_client_name"] = "gradient-cli" - -EXPECTED_HEADERS_WITH_CHANGED_API_KEY = EXPECTED_HEADERS.copy() -EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - - -@pytest.fixture -def temporary_directory_for_extracted_files(): - temp_dir_path = os.path.join(tempfile.gettempdir(), "extracted_files") - shutil.rmtree(temp_dir_path, ignore_errors=True) - - yield temp_dir_path - - shutil.rmtree(temp_dir_path, ignore_errors=True) - - -@pytest.fixture -def temporary_zip_file_path(): - zip_file_path = os.path.join(tempfile.gettempdir(), "workspace.zip") - - try: - os.remove(zip_file_path) - except OSError: - pass - - yield zip_file_path - - try: - os.remove(zip_file_path) - except OSError: - pass - - -@pytest.fixture -def basic_options_metrics_stream_websocket_connection_iterator(): - def generator(self): - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "memoryUsage", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640736, "value": "0"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640736, "value": "0"}}}""" - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "memoryUsage", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640738, "value": "0"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640738, "value": "0"}}}""" - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "cpuPercentage", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640958, "value": "0.004048304444444915"}, - "mljob-esba290c1osdth-0-worker": {"time_stamp": 1587640958, - "value": "33.81072210402445"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640958, - "value": "62.25938679226199"}}}""" - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "memoryUsage", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640960, "value": "236097536"}, - "mljob-esba290c1osdth-0-worker": {"time_stamp": 1587640960, "value": "165785600"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640960, "value": "130957312"}}}""" - - raise sdk_exceptions.EndWebsocketStream("keton") - - return generator - - -@pytest.fixture -def all_options_metrics_stream_websocket_connection_iterator(): - def generator(self): - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "gpuMemoryUsed", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640736, "value": "0"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640736, "value": "0"}}}""" - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "gpuMemoryUsed", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640738, "value": "0"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640738, "value": "0"}}}""" - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "gpuMemoryFree", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640958, "value": "1234"}, - "mljob-esba290c1osdth-0-worker": {"time_stamp": 1587640958, - "value": "234"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640958, - "value": "345"}}}""" - yield """{"handle": "esba290c1osdth", - "object_type": "experiment", - "chart_name": "gpuMemoryUsed", - "pod_metrics": {"mljob-esba290c1osdth-0-ps": {"time_stamp": 1587640960, "value": "236097536"}, - "mljob-esba290c1osdth-0-worker": {"time_stamp": 1587640960, "value": "165785600"}, - "mljob-esba290c1osdth-1-worker": {"time_stamp": 1587640960, "value": "130957312"}}}""" - - raise sdk_exceptions.EndWebsocketStream("keton") - - return generator - - -class TestExperimentsCreateSingleNode(object): - URL = "https://services.paperspace.io/experiments/v1/experiments/" - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/" - BASIC_OPTIONS_COMMAND = [ - "experiments", "create", "singlenode", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", "s3://some-workspace", - ] - BASIC_OPTIONS_COMMAND_WITH_LOCAL_WORKSPACE = [ - "experiments", "create", "singlenode", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", # local path added in test - ] - FULL_OPTIONS_COMMAND = [ - "experiments", "create", "singlenode", - "--name", "exp1", - "--ports", "4567", - "--workspace", "s3://some-workspace", - "--workspaceRef", "some_branch_name", - "--workspaceUsername", "username", - "--workspacePassword", "password", - "--workingDirectory", "/work/dir/", - "--artifactDirectory", "/artifact/dir/", - "--clusterId", "42c", - "--experimentEnv", '{"key":"val"}', - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--containerUser", "conUser", - "--registryUsername", "userName", - "--registryPassword", "passwd", - "--registryUrl", "registryUrl", - "--apiKey", "some_key", - "--modelPath", "some-model-path", - "--modelType", "some-model-type", - "--ignoreFiles", "file1,file2", - "--isPreemptible", - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "experiments", "create", "singlenode", - "--optionsFile", # path added in test, - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE_AND_SOME_VALUES_OVERWRITTEN_IN_LINE = [ - "experiments", "create", "singlenode", - "--name", "some_other_name", - "--optionsFile", # path added in test, - ] - BASIC_OPTIONS_REQUEST = { - "projectHandle": u"testHandle", - "container": u"testContainer", - "machineType": u"testType", - "command": u"dGVzdENvbW1hbmQ=", - "experimentTypeId": constants.ExperimentType.SINGLE_NODE, - "workspaceUrl": u"s3://some-workspace", - } - FULL_OPTIONS_REQUEST = { - "name": u"exp1", - "ports": "4567", - "workspaceUrl": u"s3://some-workspace", - "workspaceRef": "some_branch_name", - "workspaceUsername": u"username", - "workspacePassword": u"password", - "workingDirectory": u"/work/dir/", - "artifactDirectory": u"/artifact/dir/", - "clusterId": "42c", - "experimentEnv": {u"key": u"val"}, - "projectHandle": u"testHandle", - "container": u"testContainer", - "machineType": u"testType", - "command": u"dGVzdENvbW1hbmQ=", - "containerUser": u"conUser", - "registryUsername": u"userName", - "registryPassword": u"passwd", - "registryUrl": u"registryUrl", - "experimentTypeId": constants.ExperimentType.SINGLE_NODE, - "modelPath": "some-model-path", - "modelType": "some-model-type", - "isPreemptible": True, - } - RESPONSE_JSON_200 = {"handle": "sadkfhlskdjh", "message": "success"} - EXPECTED_STDOUT = "New experiment created with ID: sadkfhlskdjh\n" - - RESPONSE_JSON_404_PROJECT_NOT_FOUND = {"details": {"handle": "wrong_handle"}, "error": "Project not found"} - RESPONSE_CONTENT_404_PROJECT_NOT_FOUND = b'{"details":{"handle":"wrong_handle"},"error":"Project not found"}\n' - EXPECTED_STDOUT_PROJECT_NOT_FOUND = "handle: wrong_handle\nProject not found\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_basic_options(self, - post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=self.BASIC_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_config_file( - self, post_patched, create_single_node_experiment_config_path): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - command = self.FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [create_single_node_experiment_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_config_file_and_overwrite_options_with_values_provided_in_terminal( - self, post_patched, create_single_node_experiment_config_path): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - request_json = copy.deepcopy(self.FULL_OPTIONS_REQUEST) - request_json["name"] = "some_other_name" - request_json["projectHandle"] = "some_other_project_id" - - command = self.FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE_AND_SOME_VALUES_OVERWRITTEN_IN_LINE[:] - command = command[:] + [create_single_node_experiment_config_path, "--projectId", "some_other_project_id"] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=request_json, - params=None, - files=None, - data=None) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_full_options(self, - post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.FULL_OPTIONS_COMMAND) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_wrong_project_id_was_given(self, post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_404_PROJECT_NOT_FOUND, 404, - self.RESPONSE_CONTENT_404_PROJECT_NOT_FOUND) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=self.BASIC_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert self.EXPECTED_STDOUT_PROJECT_NOT_FOUND in result.output, result.exc_info[1] - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.commands.experiments.TensorboardHandler") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_use_tensorboard_handler_with_true_value_when_tensorboard_option_was_used_without_value( - self, post_patched, tensorboard_handler_class, get_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - get_patched.return_value = MockResponse(example_responses.GET_V1_CLUSTER_DETAILS_RESPONSE) - command = self.FULL_OPTIONS_COMMAND[:] + ["--tensorboard=some_tensorboard_id"] - tensorboard_handler = mock.MagicMock() - tensorboard_handler_class.return_value = tensorboard_handler - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - tensorboard_handler_class.assert_called_once_with("some_key") - tensorboard_handler.maybe_add_to_tensorboard.assert_called_once_with("some_tensorboard_id", "sadkfhlskdjh") - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.commands.experiments.TensorboardHandler") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_use_tensorboard_handler_with_tb_id_when_tensorboard_option_was_used_with_tb_id( - self, post_patched, tensorboard_handler_class, get_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - get_patched.return_value = MockResponse(example_responses.GET_V1_CLUSTER_DETAILS_RESPONSE) - command = self.FULL_OPTIONS_COMMAND[:] + ["--tensorboard"] - tensorboard_handler = mock.MagicMock() - tensorboard_handler_class.return_value = tensorboard_handler - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - tensorboard_handler_class.assert_called_once_with("some_key") - tensorboard_handler.maybe_add_to_tensorboard.assert_called_once_with(True, "sadkfhlskdjh") - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.commands.experiments.TensorboardHandler") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_full_options( - self, post_patched, tensorboard_handler_class, get_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - get_patched.return_value = MockResponse(example_responses.GET_V1_CLUSTER_DETAILS_RESPONSE) - command = self.FULL_OPTIONS_COMMAND[:] + ["--tensorboard"] - tensorboard_handler = mock.MagicMock() - tensorboard_handler_class.return_value = tensorboard_handler - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - tensorboard_handler_class.assert_called_once_with("some_key") - tensorboard_handler.maybe_add_to_tensorboard.assert_called_once_with(True, "sadkfhlskdjh") - - @mock.patch("gradient.api_sdk.workspace.s3_uploader.MultipartEncoderWithProgressbar") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_zip_and_upload_local_workspace_when_local_path_was_passed_to_workspace_option( - self, get_patched, - post_patched, - multipart_encoder_cls_patched, - temporary_directory_for_extracted_files, - temporary_zip_file_path, - ): - multipart_encoder_patched = mock.MagicMock() - multipart_encoder_content_type = mock.MagicMock() - # multipart_encoder_patched.content_type = multipart_encoder_content_type - multipart_monitor = mock.MagicMock() - multipart_monitor.content_type = multipart_encoder_content_type - multipart_encoder_patched.get_monitor.return_value = multipart_monitor - multipart_encoder_cls_patched.return_value = multipart_encoder_patched - - headers_for_uploading_to_s3 = { - "Content-Type": multipart_encoder_content_type - } - - get_patched.return_value = MockResponse(example_responses.GET_PRESIGNED_URL_FOR_S3_BUCKET_RESPONSE_JSON) - post_patched.side_effect = [ - MockResponse(status_code=204), - MockResponse(status_code=201), - ] - - workspace_path = create_test_dir_tree() - zip_file_name = os.path.basename(temporary_zip_file_path) - command = self.BASIC_OPTIONS_COMMAND_WITH_LOCAL_WORKSPACE[:] + [workspace_path] - create_experiment_request_json = self.BASIC_OPTIONS_REQUEST.copy() - create_experiment_request_json["workspaceUrl"] = \ - "s3://ps-projects/" + example_responses.GET_PRESIGNED_URL_FOR_S3_BUCKET_RESPONSE_JSON["data"]["fields"][ - "key"] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - # assert self.EXPECTED_STDOUT in result.output, result.exc_info - - with zipfile.ZipFile(temporary_zip_file_path) as zip_handler: - zip_handler.extractall(temporary_directory_for_extracted_files) - - file1_path = os.path.join(temporary_directory_for_extracted_files, "file1.txt") - assert os.path.exists(file1_path) - assert os.path.isfile(file1_path) - with open(file1_path) as h: - assert h.read() == "keton" - - file2_path = os.path.join(temporary_directory_for_extracted_files, "subdir1", "file2.jpg") - assert os.path.exists(file2_path) - assert os.path.isfile(file2_path) - with open(file2_path) as h: - assert h.read() == "keton" - - get_patched.assert_called_once_with( - "https://services.paperspace.io/experiments/v1/workspace/get_presigned_url", - headers=EXPECTED_HEADERS, - json=None, - params={"projectHandle": "testHandle", "workspaceName": zip_file_name}, - ) - - post_patched.assert_has_calls( - [ - mock.call( - "https://ps-projects.s3.amazonaws.com/", - json=None, - params=None, - headers=headers_for_uploading_to_s3, - files=None, - data=multipart_monitor, - ), - mock.call( - self.URL, - json=create_experiment_request_json, - params=None, - headers=EXPECTED_HEADERS, - files=None, - data=None, - ), - ] - ) - - assert result.exit_code == 0 - - -class TestExperimentsCreateMultiNode(object): - URL = "https://services.paperspace.io/experiments/v1/experiments/" - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/" - BASIC_OPTIONS_COMMAND = [ - "experiments", "create", "multinode", - "--projectId", "prq70zy79", - "--experimentType", "GRPC", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--parameterServerContainer", "pscon", - "--parameterServerMachineType", "psmtype", - "--parameterServerCommand", "ls", - "--parameterServerCount", 2, - "--workerContainerUser", "usr", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - ] - FULL_OPTIONS_COMMAND = [ - "experiments", "create", "multinode", - "--name", "multinode_mpi", - "--ports", 3456, - "--workspace", "s3://some-workspace", - "--workspaceRef", "some_branch_name", - "--workspaceUsername", "username", - "--workspacePassword", "password", - "--workingDirectory", "/dir", - "--artifactDirectory", "/artdir", - "--clusterId", '2a', - "--experimentEnv", '{"key":"val"}', - "--projectId", "prq70zy79", - "--experimentType", "MPI", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--masterContainer", "pscon", - "--masterMachineType", "psmtype", - "--masterCommand", "ls", - "--masterCount", 2, - "--workerContainerUser", "usr", - "--workerRegistryUsername", "rusr", - "--workerRegistryPassword", "rpass", - "--workerRegistryUrl", "rurl", - "--masterContainerUser", "pscuser", - "--masterRegistryUsername", "psrcus", - "--masterRegistryPassword", "psrpass", - "--masterRegistryUrl", "psrurl", - "--apiKey", "some_key", - "--modelPath", "some-model-path", - "--modelType", "some-model-type", - "--ignoreFiles", "file1,file2", - "--isPreemptible", - "--datasetUri", "s3://some.dataset/uri", - "--datasetName", "some dataset name", - "--datasetAwsAccessKeyId", "none", - "--datasetAwsSecretAccessKey", "none", - "--datasetVersionId", "version1", - "--datasetEtag", "some etag", - "--datasetUri", "s3://some.other.dataset/uri", - "--datasetName", "none", - "--datasetAwsAccessKeyId", "some_other_key_id", - "--datasetAwsSecretAccessKey", "some_other_secret", - "--datasetVersionId", "version2", - "--datasetEtag", "some other etag", - "--datasetVolumeKind", "dynamic", - "--datasetVolumeSize", "10Gi", - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "experiments", "create", "multinode", - "--optionsFile", # path added in test, - ] - BASIC_OPTIONS_REQUEST = { - u"projectHandle": u"prq70zy79", - u"experimentTypeId": 2, - u"workerContainer": u"wcon", - u"workerMachineType": u"mty", - u"workerCommand": u"d2NvbQ==", - u"workerCount": 2, - u"parameterServerContainer": u"pscon", - u"parameterServerMachineType": u"psmtype", - u"parameterServerCommand": u"bHM=", - u"parameterServerCount": 2, - u"workerContainerUser": u"usr", - u"workspaceUrl": u"https://github.com/Paperspace/gradient-cli.git", - } - FULL_OPTIONS_REQUEST = { - "name": u"multinode_mpi", - "ports": "3456", - "workspaceUrl": u"s3://some-workspace", - "workspaceRef": "some_branch_name", - "workspaceUsername": u"username", - "workspacePassword": u"password", - "workingDirectory": u"/dir", - "artifactDirectory": u"/artdir", - "clusterId": '2a', - "experimentEnv": {"key": "val"}, - "projectHandle": "prq70zy79", - "experimentTypeId": 3, - "workerContainer": u"wcon", - "workerMachineType": u"mty", - "workerCommand": u"d2NvbQ==", - "workerCount": 2, - "masterContainer": u"pscon", - "masterMachineType": u"psmtype", - "masterCommand": u"bHM=", - "masterCount": 2, - "workerContainerUser": u"usr", - "workerRegistryUsername": u"rusr", - "workerRegistryPassword": u"rpass", - "workerRegistryUrl": u"rurl", - "masterContainerUser": u"pscuser", - "masterRegistryUsername": u"psrcus", - "masterRegistryPassword": u"psrpass", - "masterRegistryUrl": u"psrurl", - "isPreemptible": True, - "modelPath": "some-model-path", - "modelType": "some-model-type", - "datasets": [ - { - "uri": "s3://some.dataset/uri", - "name": "some dataset name", - "etag": "some etag", - "versionId": "version1", - "volumeOptions": { - "kind": "dynamic", - "size": "10Gi", - }, - }, - { - "uri": "s3://some.other.dataset/uri", - "awsAccessKeyId": "some_other_key_id", - "awsSecretAccessKey": "some_other_secret", - "etag": "some other etag", - "versionId": "version2", - }, - ] - } - RESPONSE_JSON_200 = {"handle": "sadkfhlskdjh", "message": "success"} - RESPONSE_CONTENT_200 = b'{"handle":"sadkfhlskdjh","message":"success"}\n' - EXPECTED_STDOUT = "New experiment created with ID: sadkfhlskdjh\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_basic_options(self, - post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=self.BASIC_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_config_file( - self, post_patched, create_multi_node_experiment_config_path): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - command = self.FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [create_multi_node_experiment_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_full_options(self, - post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.FULL_OPTIONS_COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert self.EXPECTED_STDOUT in result.output - assert result.exit_code == 0 - - @mock.patch("gradient.commands.experiments.TensorboardHandler") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_use_tensorboard_handler_with_true_value_when_tensorboard_option_was_used_without_value( - self, post_patched, tensorboard_handler_class): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - command = self.FULL_OPTIONS_COMMAND[:] + ["--tensorboard"] - tensorboard_handler = mock.MagicMock() - tensorboard_handler_class.return_value = tensorboard_handler - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - tensorboard_handler_class.assert_called_once_with("some_key") - tensorboard_handler.maybe_add_to_tensorboard.assert_called_once_with(True, "sadkfhlskdjh") - - @mock.patch("gradient.commands.experiments.TensorboardHandler") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_use_tensorboard_handler_with_tb_id_when_tensorboard_option_was_used_with_tb_id( - self, post_patched, tensorboard_handler_class): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - command = self.FULL_OPTIONS_COMMAND[:] + ["--tensorboard=some_tensorboard_id"] - tensorboard_handler = mock.MagicMock() - tensorboard_handler_class.return_value = tensorboard_handler - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - tensorboard_handler_class.assert_called_once_with("some_key") - tensorboard_handler.maybe_add_to_tensorboard.assert_called_once_with("some_tensorboard_id", "sadkfhlskdjh") - - @mock.patch("gradient.commands.experiments.TensorboardHandler") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_full_options( - self, post_patched, tensorboard_handler_class): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - command = self.FULL_OPTIONS_COMMAND[:] + ["--tensorboard"] - tensorboard_handler = mock.MagicMock() - tensorboard_handler_class.return_value = tensorboard_handler - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - assert EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key" - - tensorboard_handler_class.assert_called_once_with("some_key") - tensorboard_handler.maybe_add_to_tensorboard.assert_called_once_with(True, "sadkfhlskdjh") - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_config_file_with_dataset_defined_as_list_of_objects( - self, post_patched, create_multi_node_experiment_ds_objects_config_path): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200) - command = self.FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [create_multi_node_experiment_ds_objects_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.FULL_OPTIONS_REQUEST, - params=None, - files=None, - data=None) - assert result.exit_code == 0 - - -class TestExperimentsCreateAndStartSingleNode(TestExperimentsCreateSingleNode): - URL = "https://services.paperspace.io/experiments/v1/experiments/run/" - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/run/" - BASIC_OPTIONS_COMMAND = [ - "experiments", "run", "singlenode", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", "s3://some-workspace", - "--no-logs", - ] - BASIC_OPTIONS_COMMAND_WITH_LOCAL_WORKSPACE = [ - "experiments", "run", "singlenode", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", # local path added in test - ] - FULL_OPTIONS_COMMAND = [ - "experiments", "run", "singlenode", - "--name", "exp1", - "--ports", 4567, - "--workspace", "s3://some-workspace", - "--workspaceRef", "some_branch_name", - "--workspaceUsername", "username", - "--workspacePassword", "password", - "--workingDirectory", "/work/dir/", - "--artifactDirectory", "/artifact/dir/", - "--clusterId", "42c", - "--experimentEnv", '{"key":"val"}', - "--projectId", 987654, - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--containerUser", "conUser", - "--registryUsername", "userName", - "--registryPassword", "passwd", - "--registryUrl", "registryUrl", - "--apiKey", "some_key", - "--no-logs", - "--modelPath", "some-model-path", - "--modelType", "some-model-type", - "--ignoreFiles", "file1,file2", - "--isPreemptible", - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "experiments", "run", "singlenode", - "--optionsFile", # path added in test, - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE_AND_SOME_VALUES_OVERWRITTEN_IN_LINE = [ - "experiments", "run", "singlenode", - "--name", "some_other_name", - "--optionsFile", # path added in test, - ] - EXPECTED_STDOUT = "New experiment created and started with ID: sadkfhlskdjh\n" - - -class TestExperimentsCreateAndStartMultiNode(TestExperimentsCreateMultiNode): - URL = "https://services.paperspace.io/experiments/v1/experiments/run/" - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/run/" - BASIC_OPTIONS_COMMAND = [ - "experiments", "run", "multinode", - "--projectId", "prq70zy79", - "--experimentType", "GRPC", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--parameterServerContainer", "pscon", - "--parameterServerMachineType", "psmtype", - "--parameterServerCommand", "ls", - "--parameterServerCount", 2, - "--workerContainerUser", "usr", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--no-logs", - ] - FULL_OPTIONS_COMMAND = [ - "experiments", "run", "multinode", - "--name", "multinode_mpi", - "--ports", 3456, - "--workspace", "s3://some-workspace", - "--workspaceRef", "some_branch_name", - "--workspaceUsername", "username", - "--workspacePassword", "password", - "--workingDirectory", "/dir", - "--artifactDirectory", "/artdir", - "--clusterId", '2a', - "--experimentEnv", '{"key":"val"}', - "--projectId", 34, - "--projectId", "prq70zy79", - "--experimentType", "MPI", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--masterContainer", "pscon", - "--masterMachineType", "psmtype", - "--masterCommand", "ls", - "--masterCount", 2, - "--workerContainerUser", "usr", - "--workerRegistryUsername", "rusr", - "--workerRegistryPassword", "rpass", - "--workerRegistryUrl", "rurl", - "--masterContainerUser", "pscuser", - "--masterRegistryUsername", "psrcus", - "--masterRegistryPassword", "psrpass", - "--masterRegistryUrl", "psrurl", - "--apiKey", "some_key", - "--no-logs", - "--modelPath", "some-model-path", - "--modelType", "some-model-type", - "--ignoreFiles", "file1,file2", - "--isPreemptible", - "--datasetUri", "s3://some.dataset/uri", - "--datasetName", "some dataset name", - "--datasetAwsAccessKeyId", "none", - "--datasetAwsSecretAccessKey", "none", - "--datasetVersionId", "version1", - "--datasetEtag", "some etag", - "--datasetUri", "s3://some.other.dataset/uri", - "--datasetName", "none", - "--datasetAwsAccessKeyId", "some_other_key_id", - "--datasetAwsSecretAccessKey", "some_other_secret", - "--datasetVersionId", "version2", - "--datasetEtag", "some other etag", - "--datasetVolumeKind", "dynamic", - "--datasetVolumeSize", "10Gi", - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "experiments", "run", "multinode", - "--no-logs", - "--optionsFile", # path added in test - ] - BASIC_OPTIONS_COMMAND_WHEN_CLUSTER_ID_WAS_SET = [ - "experiments", "run", "multinode", - "--projectId", "prq70zy79", - "--experimentType", "GRPC", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--parameterServerContainer", "pscon", - "--parameterServerMachineType", "psmtype", - "--parameterServerCommand", "ls", - "--parameterServerCount", 2, - "--workerContainerUser", "usr", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--no-logs", - "--clusterId", "some_cluster_id", - ] - EXPECTED_STDOUT = "New experiment created and started with ID: sadkfhlskdjh\n" - - -class TestExperimentDetail(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/experiment-id/" - - COMMAND = ["experiments", "details", "--id", "experiment-id"] - COMMAND_WITH_API_KEY = ["experiments", "details", "--id", "experiment-id", "--apiKey", "some_key"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "details", "--optionsFile", ] # path added in test - - MULTI_NODE_DETAILS_STDOUT = """+---------------------+--------------------------+ -| Name | some_name | -+---------------------+--------------------------+ -| ID | emarbao6t6tsn | -| State | created | -| Artifact directory | /some/artifact/directory | -| Cluster ID | clqr4b0ox | -| Experiment Env | {'key': 'value'} | -| Experiment Type | MPI multi node | -| Model Type | some_type | -| Model Path | /some/model/path | -| Master Command | None | -| Master Container | None | -| Master Count | None | -| Master Machine Type | None | -| Ports | 5000 | -| Project ID | pr85u3sfa | -| Worker Command | None | -| Worker Container | None | -| Worker Count | None | -| Worker Machine Type | None | -| Working Directory | /some/working/directory | -| Workspace URL | some.url | -| Tags | tag1, tag2 | -+---------------------+--------------------------+ -""" - SINGLE_NODE_DETAILS_STDOUT = """+---------------------+----------------+ -| Name | dsfads | -+---------------------+----------------+ -| ID | esro6mbmiulvbl | -| State | pending | -| Ports | 5000 | -| Project ID | prq70zy79 | -| Worker Command | sadas | -| Worker Container | asd | -| Worker Machine Type | C2 | -| Working Directory | None | -| Workspace URL | None | -| Model Type | None | -| Model Path | None | -| Tags | tag1, tag2 | -+---------------------+----------------+ -""" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_single_node_experiment_details_in_a_table(self, get_patched): - get_patched.return_value = MockResponse(example_responses.DETAILS_OF_SINGLE_NODE_EXPERIMENT_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.SINGLE_NODE_DETAILS_STDOUT, result.exc_info - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert result.exit_code == 0 - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_with_api_key_passed_in_terminal(self, get_patched): - get_patched.return_value = MockResponse(example_responses.DETAILS_OF_SINGLE_NODE_EXPERIMENT_RESPONSE_JSON) - expected_headers = EXPECTED_HEADERS.copy() - expected_headers["X-API-Key"] = "some_key" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY) - - assert result.output == self.SINGLE_NODE_DETAILS_STDOUT, result.exc_info[1] - get_patched.assert_called_once_with(self.URL_V2, - headers=expected_headers, - json=None, - params=None) - - assert result.exit_code == 0 - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_read_options_from_config_file(self, get_patched, experiment_details_config_path): - get_patched.return_value = MockResponse(example_responses.DETAILS_OF_SINGLE_NODE_EXPERIMENT_RESPONSE_JSON) - expected_headers = EXPECTED_HEADERS.copy() - expected_headers["X-API-Key"] = "some_key" - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [experiment_details_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.SINGLE_NODE_DETAILS_STDOUT, result.exc_info[1] - get_patched.assert_called_once_with(self.URL_V2, - headers=expected_headers, - json=None, - params=None) - - assert result.exit_code == 0 - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_multi_node_experiment_details_in_a_table(self, get_patched): - get_patched.return_value = MockResponse(example_responses.DETAILS_OF_MULTI_NODE_EXPERIMENT_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert result.output.strip() == self.MULTI_NODE_DETAILS_STDOUT.strip(), result.exc_info[1] - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_request_content_when_response_data_was_malformed(self, get_patched): - get_patched.return_value = MockResponse({}, content="fake content") - expected_output = "Error parsing response data: fake content\n" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert result.output == expected_output, result.exc_info[1] - assert result.exit_code == 0 - - -class TestExperimentList(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/" - COMMAND = ["experiments", "list"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "list", "--optionsFile", ] # path added in test - LIST_JSON = example_responses.LIST_OF_EXPERIMENTS_RESPONSE_JSON - DETAILS_STDOUT = """+---------------+---------------+---------+ -| Name | ID | Status | -+---------------+---------------+---------+ -| dsfads | ea2lfbbpdyzsq | pending | -| dsfads | em6btk2vtb7it | pending | -| multinode_mpi | ew69ls0vy3eto | pending | -+---------------+---------------+---------+ - -Do you want to continue? [y/N]: -Aborted! -""" - RESPONSE_JSON_WHEN_WRONG_API_KEY_WAS_USED = {"details": "Incorrect API Key provided", "error": "Forbidden"} - EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED = "Failed to fetch data: Incorrect API Key provided\nForbidden\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_list_of_experiments(self, get_patched): - get_patched.return_value = MockResponse(self.LIST_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.DETAILS_STDOUT - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params={"limit": 20, "offset": 0}) - - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.commands.common.pydoc") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_paginate_list_when_output_table_len_is_gt_lines_in_terminal(self, get_patched, - pydoc_patched): - list_json = {"data": self.LIST_JSON["data"] * 40} - get_patched.return_value = MockResponse(list_json) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params={"limit": 20, "offset": 0}) - - assert result.exit_code == 1 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_list_of_experiments_filtered_with_two_projects(self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_OF_EXPERIMENTS_FILTERED_WITH_TWO_PROJECTS) - - runner = CliRunner() - result = runner.invoke(cli.cli, ["experiments", "list", "--projectId", "handle1", "-p", "handle2"]) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params={"limit": 20, - "offset": 0, - "projectHandle[0]": u"handle1", - "projectHandle[1]": u"handle2"}) - - assert result.output == example_responses.LIST_OF_EXPERIMENTS_FILTERED_WITH_TWO_PROJECTS_STDOUT - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_list_of_experiments_filtered_with_two_projects_but_none_found( - self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_OF_EXPERIMENTS_FILTERED_BUT_NONE_FOUND) - - runner = CliRunner() - result = runner.invoke(cli.cli, ["experiments", "list", "--projectId", "handle1", "-p", "handle2", - "--tag", "some_tag", "--tag", "some_tag_2"]) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params={"limit": 20, - "offset": 0, - "projectHandle[0]": u"handle1", - "projectHandle[1]": u"handle2", - "tag": ("some_tag", "some_tag_2"), - }) - - assert result.output == "No data found\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_proper_message_when_wrong_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.RESPONSE_JSON_WHEN_WRONG_API_KEY_WAS_USED, - status_code=403) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params={"limit": 20, "offset": 0}) - - assert result.output == self.EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_defined_in_a_config_file(self, get_patched, experiments_list_config_path): - get_patched.return_value = MockResponse(json_data=self.RESPONSE_JSON_WHEN_WRONG_API_KEY_WAS_USED, - status_code=403) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [experiments_list_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"limit": 20, - "offset": 0, - "projectHandle[0]": "some_id", - "projectHandle[1]": "some_id_2", - "tag": ("some_tag", "some_tag_2"), - }, - ) - - assert result.output == self.EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - -class TestStartExperiment(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/some-id/start/" - COMMAND = ["experiments", "start", "--id", "some-id"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "start", "--optionsFile", ] # path added in test - COMMAND_WITH_API_KEY = ["experiments", "start", "--id", "some-id", "--apiKey", "some_key"] - RESPONSE_JSON = {"message": "success"} - START_STDOUT = "Experiment started\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_put_request_and_print_confirmation(self, put_patched): - put_patched.return_value = MockResponse(self.RESPONSE_JSON) - expected_headers = http_client.default_headers.copy() - expected_headers["X-API-Key"] = "some_key" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.START_STDOUT, result.exc_info - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None, - data=None, - ) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_put_request_with_changed_api_key_when_api_key_option_was_provided(self, put_patched): - put_patched.return_value = MockResponse(self.RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY) - - assert result.output == self.START_STDOUT - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None, - ) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_read_options_from_config_file(self, put_patched, experiments_start_config_path): - put_patched.return_value = MockResponse(self.RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [experiments_start_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.START_STDOUT - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None, - ) - - -class TestStopExperiment(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/experiments/some-id/stop/" - COMMAND = ["experiments", "stop", "--id", "some-id"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "stop", "--optionsFile", ] # path added in test - COMMAND_WITH_API_KEY = ["experiments", "stop", "--id", "some-id", "--apiKey", "some_key"] - RESPONSE_JSON = {"message": "success"} - EXPECTED_STDOUT = "Experiment stopped\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_put_request_and_print_confirmation(self, put_patched): - put_patched.return_value = MockResponse(self.RESPONSE_JSON) - expected_headers = http_client.default_headers.copy() - expected_headers["X-API-Key"] = "some_key" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None, - data=None, - ) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_put_request_with_changed_api_key_when_api_key_option_was_provided(self, put_patched): - put_patched.return_value = MockResponse(self.RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY) - - assert result.output == self.EXPECTED_STDOUT - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None, - ) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_read_options_from_config_file(self, put_patched, experiments_stop_config_path): - put_patched.return_value = MockResponse(self.RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [experiments_stop_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None, - ) - - -class TestDeleteExperiment(object): - URL = "https://services.paperspace.io/experiments/v2/experiments/some-id/" - COMMAND = ["experiments", "delete", "--id", "some-id"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "delete", "--optionsFile", ] # path added in test - COMMAND_WITH_API_KEY = ["experiments", "delete", "--id", "some-id", "--apiKey", "some_key"] - RESPONSE_JSON = {"message": "success"} - EXPECTED_STDOUT = "Experiment deleted\n" - - NOT_FOUND_JSON_RESPONSE = {"details": "Experiment not found", "error": "Object not found"} - NOT_FOUND_EXPECTED_STDOUT = "Failed to delete resource: Experiment not found\nObject not found\n" - INVALID_API_KEY_RESPONSE_JSON = {"details": "Incorrect API Key provided", "error": "Forbidden"} - INVALID_API_KEY_STDOUT = "Failed to delete resource: Incorrect API Key provided\nForbidden\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_send_delete_request_and_print_confirmation(self, delete_patched): - delete_patched.return_value = MockResponse(self.RESPONSE_JSON, 204) - expected_headers = http_client.default_headers.copy() - expected_headers["X-API-Key"] = "some_key" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - delete_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_send_delete_request_with_changed_api_key_when_api_key_option_was_provided(self, delete_patched): - delete_patched.return_value = MockResponse(self.RESPONSE_JSON, 204) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY) - - assert result.output == self.EXPECTED_STDOUT - delete_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_read_options_from_config_file(self, delete_patched, experiments_delete_config_path): - delete_patched.return_value = MockResponse(self.RESPONSE_JSON, 204) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [experiments_delete_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT - delete_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_print_proper_message_when_experiment_was_not_found(self, delete_patched): - delete_patched.return_value = MockResponse(self.NOT_FOUND_JSON_RESPONSE, 404) - expected_headers = http_client.default_headers.copy() - expected_headers["X-API-Key"] = "some_key" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.NOT_FOUND_EXPECTED_STDOUT, result.exc_info - delete_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_send_print_proper_message_when_wrong_api_key_was_used(self, delete_patched): - delete_patched.return_value = MockResponse(self.INVALID_API_KEY_RESPONSE_JSON, 403) - expected_headers = http_client.default_headers.copy() - expected_headers["X-API-Key"] = "some_key" - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.INVALID_API_KEY_STDOUT, result.exc_info - delete_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - -class TestExperimentLogs(object): - URL = "https://logs.paperspace.io/jobs/logs" - COMMAND = ["experiments", "logs", "--id", "some_id"] - COMMAND_WITH_FOLLOW = ["experiments", "logs", "--id", "some_id", "--follow", "True"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "logs", "--optionsFile", ] # path added in test - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_all_received_logs_when_logs_command_was_used(self, get_patched): - get_patched.return_value = MockResponse(json_data=example_responses.LIST_OF_LOGS_FOR_EXPERIMENT) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert "I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA" in result.output - # This one checks if trailing \n was removed from log line. - # There were empty lines printed if log line had a new line character at the end so we rstrip lines now - assert "|\n| " not in result.output - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_should_read_options_from_config_file(self, get_patched, experiments_logs_config_path): - get_patched.return_value = MockResponse(json_data=example_responses.LIST_OF_LOGS_FOR_EXPERIMENT) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [experiments_logs_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"line": 20, "limit": 30, "experimentId": "some-id"}) - assert "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels" \ - "-idx1-ubyte.gz to /tmp/tmpbrss4txl.gz" in result.output - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_all_received_logs_when_logs_command_was_used_with_follow_flag( - self, get_patched): - get_patched.return_value = MockResponse(json_data=example_responses.LIST_OF_LOGS_FOR_EXPERIMENT) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_FOLLOW) - - assert "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels" \ - "-idx1-ubyte.gz to /tmp/tmpbrss4txl.gz" in result.output - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_error_message_when_wrong_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse(content="Authentication failed", - status_code=401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_FOLLOW) - - assert "Awaiting logs...\nFailed to fetch data: Authentication failed\n" in result.output - - -class TestExperimentValidation(object): - - @pytest.mark.parametrize( - "command, expected_message", [ - ( - [ - "experiments", "create", "singlenode", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--modelPath", "some/model/path" - ], - EXPERIMENT_MODEL_PATH_VALIDATION_ERROR - ), ( - [ - "experiments", "create", "multinode", - "--name", "multinode", - "--projectId", "prq70zy79", - "--experimentType", "GRPC", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--parameterServerContainer", "pscon", - "--parameterServerMachineType", "psmtype", - "--parameterServerCommand", "ls", - "--parameterServerCount", 2, - "--workerContainerUser", "usr", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--modelPath", "some/model/path" - ], - EXPERIMENT_MODEL_PATH_VALIDATION_ERROR - ), ( - [ - "experiments", "run", "singlenode", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--no-logs", - "--modelPath", "some/model/path" - ], - EXPERIMENT_MODEL_PATH_VALIDATION_ERROR - ), ( - [ - "experiments", "run", "multinode", - "--projectId", "prq70zy79", - "--experimentType", "GRPC", - "--workerContainer", "wcon", - "--workerMachineType", "mty", - "--workerCommand", "wcom", - "--workerCount", 2, - "--parameterServerContainer", "pscon", - "--parameterServerMachineType", "psmtype", - "--parameterServerCommand", "ls", - "--parameterServerCount", 2, - "--workerContainerUser", "usr", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--no-logs", - "--modelPath", "some/model/path" - ], - EXPERIMENT_MODEL_PATH_VALIDATION_ERROR - ), - ] - ) - def test_experiment_create_argument_validation_error(self, command, expected_message): - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert expected_message in result.output - - -class TestExperimentsMetricsGetCommand(object): - GET_EXPERIMENT_URL = "https://services.paperspace.io/experiments/v2/experiments/esro6mbmiulvbl/" - LIST_JOBS_URL = "https://api.paperspace.io/jobs/getJobList/" - GET_METRICS_URL = "https://aws-testing.paperspace.io/metrics/api/v1/range" - BASIC_OPTIONS_COMMAND = [ - "experiments", "metrics", "get", - "--id", "esro6mbmiulvbl", - ] - ALL_OPTIONS_COMMAND = [ - "experiments", "metrics", "get", - "--id", "esro6mbmiulvbl", - "--metric", "gpuMemoryFree", - "--metric", "gpuMemoryUsed", - "--interval", "20s", - "--start", "2020-04-01", - "--end", "2020-04-02 21:37:00", - "--apiKey", "some_key", - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "experiments", "metrics", "get", - "--optionsFile", # path added in test, - ] - - GET_JOB_LIST_REQUEST_PARAMS = {'filter': '{"filter": {"where": {"experimentId": "esro6mbmiulvbl"}}}'} - BASIC_COMMAND_GET_METRICS_REQUEST_PARAMS = { - "start": "2020-04-02T21:37:00Z", - "handle": "esro6mbmiulvbl", - "interval": "30s", - "charts": "cpuPercentage,memoryUsage", - "objecttype": "experiment", - } - ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS = { - "start": "2020-04-01T00:00:00Z", - "handle": "esro6mbmiulvbl", - "interval": "20s", - "charts": "gpuMemoryFree,gpuMemoryUsed", - "objecttype": "experiment", - "end": "2020-04-02T21:37:00Z", - } - - GET_EXPERIMENT_RESPONSE_JSON = example_responses.DETAILS_OF_SINGLE_NODE_EXPERIMENT_RESPONSE_JSON - GET_LIST_OF_JOBS_RESPONSE_JSON = example_responses.LIST_JOBS_RESPONSE_JSON - GET_METRICS_RESPONSE_JSON = example_responses.EXPERIMENTS_METRICS_GET_RESPONSE - - EXPECTED_STDOUT = """{ - "cpuPercentage": { - "mljob-esro6mbmiulvbl-0-worker": [ - { - "time_stamp": 1587375065, - "value": "0" - }, - { - "time_stamp": 1587375095, - "value": "0" - }, - { - "time_stamp": 1587375125, - "value": "0" - }, - { - "time_stamp": 1587375155, - "value": "0" - } - ], - "mljob-esro6mbmiulvbl-1-worker": [ - { - "time_stamp": 1587375065, - "value": "0" - }, - { - "time_stamp": 1587375095, - "value": "0" - }, - { - "time_stamp": 1587375125, - "value": "0" - }, - { - "time_stamp": 1587375155, - "value": "0" - } - ] - }, - "memoryUsage": { - "mljob-esro6mbmiulvbl-0-worker": [ - { - "time_stamp": 1587375005, - "value": "0" - }, - { - "time_stamp": 1587375035, - "value": "0" - }, - { - "time_stamp": 1587375065, - "value": "761856" - }, - { - "time_stamp": 1587375095, - "value": "761856" - }, - { - "time_stamp": 1587375125, - "value": "761856" - } - ], - "mljob-esro6mbmiulvbl-1-worker": [ - { - "time_stamp": 1587375005, - "value": "0" - }, - { - "time_stamp": 1587375035, - "value": "0" - }, - { - "time_stamp": 1587375065, - "value": "761856" - }, - { - "time_stamp": 1587375095, - "value": "761856" - }, - { - "time_stamp": 1587375125, - "value": "761856" - } - ] - } -} -""" - - EXPECTED_STDOUT_WHEN_INVALID_API_KEY_WAS_USED = "Failed to fetch data: Incorrect API Key provided\nForbidden\n" - EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND = "Failed to fetch data: Experiment not found\nObject not found\n" - EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_STARTED = "Experiment has not started yet\n" - EXPECTED_STDOUT_WHEN_NO_METRICS_WERE_FOUND = "{}\n" - EXPECTED_STDOUT_WHEN_ERROR_CODE_WAS_RETURNED_WITHOUT_ERROR_MESSAGE = "Failed to fetch data\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_all_available_metrics_when_metrics_get_command_was_used_with_basic_options(self, get_patched): - get_patched.side_effect = [ - MockResponse(self.GET_EXPERIMENT_RESPONSE_JSON), - MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON), - MockResponse(self.GET_METRICS_RESPONSE_JSON), - ] - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT.strip()), result.exc_info - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS, - ), - mock.call( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS, - ), - mock.call( - self.GET_METRICS_URL, - json=None, - params=self.BASIC_COMMAND_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_metrics_when_metrics_get_command_was_used_with_all_options(self, get_patched): - get_patched.side_effect = [ - MockResponse(self.GET_EXPERIMENT_RESPONSE_JSON), - MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON), - MockResponse(self.GET_METRICS_RESPONSE_JSON), - ] - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - # comparing objects instead of strings because Py2 and Py3 produce slightly different outputs - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT.strip()), result.exc_info - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_metrics_when_metrics_get_was_executed_and_options_file_was_used( - self, get_patched, experiments_metrics_get_config_path): - get_patched.side_effect = [ - MockResponse(self.GET_EXPERIMENT_RESPONSE_JSON), - MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON), - MockResponse(self.GET_METRICS_RESPONSE_JSON), - ] - command = self.FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [experiments_metrics_get_config_path] - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - # comparing objects instead of strings because Py2 and Py3 produce slightly different outputs - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT.strip()), result.exc_info - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_invalid_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse({"details": "Incorrect API Key provided", "error": "Forbidden"}, - status_code=403) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_INVALID_API_KEY_WAS_USED, result.exc_info - - get_patched.assert_called_once_with( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_experiment_was_not_found(self, get_patched): - get_patched.side_effect = [ - MockResponse({"details": "Experiment not found", "error": "Object not found"}, 404), - ] - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND, result.exc_info - - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_experiment_was_not_started_and_no_jobs_were_found(self, get_patched): - get_patched.side_effect = [ - MockResponse(self.GET_EXPERIMENT_RESPONSE_JSON), - MockResponse({"jobList": []}), - ] - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_STARTED, result.exc_info - - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_message_when_experiment_was_no_metrics_were_returned(self, get_patched): - get_patched.side_effect = [ - MockResponse(self.GET_EXPERIMENT_RESPONSE_JSON), - MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON), - MockResponse(example_responses.EXPERIMENTS_METRICS_GET_RESPONSE_WHEN_NO_DATA_WAS_FOUND), - ] - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_NO_METRICS_WERE_FOUND, result.exc_info - - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_error_code_was_returned_without_error_message(self, get_patched): - get_patched.side_effect = [ - MockResponse(self.GET_EXPERIMENT_RESPONSE_JSON), - MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON), - MockResponse(status_code=500), - ] - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_ERROR_CODE_WAS_RETURNED_WITHOUT_ERROR_MESSAGE, result.exc_info - - get_patched.assert_has_calls( - [ - mock.call( - self.GET_EXPERIMENT_URL, - json=None, - params=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - mock.call( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ), - ] - ) - - assert result.exit_code == 0, result.exc_info - - -class TestExperimentsMetricsStreamCommand(object): - LIST_JOBS_URL = "https://api.paperspace.io/jobs/getJobList/" - GET_METRICS_URL = "https://aws-testing.paperspace.io/metrics/api/v1/stream" - BASIC_OPTIONS_COMMAND = [ - "experiments", "metrics", "stream", - "--id", "esro6mbmiulvbl", - ] - ALL_OPTIONS_COMMAND = [ - "experiments", "metrics", "stream", - "--id", "esro6mbmiulvbl", - "--metric", "gpuMemoryFree", - "--metric", "gpuMemoryUsed", - "--interval", "20s", - "--apiKey", "some_key", - ] - ALL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "experiments", "metrics", "stream", - "--optionsFile", # path added in test, - ] - - GET_JOB_LIST_REQUEST_PARAMS = {'filter': '{"filter": {"where": {"experimentId": "esro6mbmiulvbl"}}}'} - BASIC_COMMAND_CHART_DESCRIPTOR = '{"chart_names": ["cpuPercentage", "memoryUsage"], "handles": ["esro6mbmiulvbl"]' \ - ', "object_type": "experiment", "poll_interval": "30s"}' - - ALL_COMMANDS_CHART_DESCRIPTOR = '{"chart_names": ["gpuMemoryFree", "gpuMemoryUsed"], "handles": ["esro6mbmiulvbl"' \ - '], "object_type": "experiment", "poll_interval": "20s"}' - - GET_LIST_OF_JOBS_RESPONSE_JSON = example_responses.LIST_JOBS_RESPONSE_JSON - - EXPECTED_TABLE_1 = """+-------------------------------+---------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+-------------------------------+---------------+-------------+ -| mljob-esba290c1osdth-0-ps | | 0 | -| mljob-esba290c1osdth-1-worker | | 0 | -+-------------------------------+---------------+-------------+""" - EXPECTED_TABLE_2 = """+-------------------------------+---------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+-------------------------------+---------------+-------------+ -| mljob-esba290c1osdth-0-ps | | 0 | -| mljob-esba290c1osdth-1-worker | | 0 | -+-------------------------------+---------------+-------------+""" - EXPECTED_TABLE_3 = """+-------------------------------+----------------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+-------------------------------+----------------------+-------------+ -| mljob-esba290c1osdth-0-ps | 0.004048304444444915 | 0 | -| mljob-esba290c1osdth-0-worker | 33.81072210402445 | | -| mljob-esba290c1osdth-1-worker | 62.25938679226199 | 0 | -+-------------------------------+----------------------+-------------+""" - EXPECTED_TABLE_4 = """+-------------------------------+----------------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+-------------------------------+----------------------+-------------+ -| mljob-esba290c1osdth-0-ps | 0.004048304444444915 | 236097536 | -| mljob-esba290c1osdth-0-worker | 33.81072210402445 | 165785600 | -| mljob-esba290c1osdth-1-worker | 62.25938679226199 | 130957312 | -+-------------------------------+----------------------+-------------+""" - - ALL_OPTIONS_EXPECTED_TABLE_1 = """+-------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+-------------------------------+---------------+---------------+ -| mljob-esba290c1osdth-0-ps | | 0 | -| mljob-esba290c1osdth-1-worker | | 0 | -+-------------------------------+---------------+---------------+""" - ALL_OPTIONS_EXPECTED_TABLE_2 = """+-------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+-------------------------------+---------------+---------------+ -| mljob-esba290c1osdth-0-ps | | 0 | -| mljob-esba290c1osdth-1-worker | | 0 | -+-------------------------------+---------------+---------------+""" - ALL_OPTIONS_EXPECTED_TABLE_3 = """+-------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+-------------------------------+---------------+---------------+ -| mljob-esba290c1osdth-0-ps | 1234 | 0 | -| mljob-esba290c1osdth-0-worker | 234 | | -| mljob-esba290c1osdth-1-worker | 345 | 0 | -+-------------------------------+---------------+---------------+""" - ALL_OPTIONS_EXPECTED_TABLE_4 = """+-------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+-------------------------------+---------------+---------------+ -| mljob-esba290c1osdth-0-ps | 1234 | 236097536 | -| mljob-esba290c1osdth-0-worker | 234 | 165785600 | -| mljob-esba290c1osdth-1-worker | 345 | 130957312 | -+-------------------------------+---------------+---------------+""" - - EXPECTED_STDOUT_WHEN_INVALID_API_KEY_WAS_USED = "Failed to fetch data: Incorrect API Key provided\nForbidden\n" - EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND = "Experiment has not started yet\n" - EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_STARTED = "Experiment has not started yet\n" - EXPECTED_STDOUT_WHEN_NO_METRICS_WERE_FOUND = "{}\n" - EXPECTED_STDOUT_WHEN_ERROR_CODE_WAS_RETURNED_WITHOUT_ERROR_MESSAGE = "Failed to fetch data\n" - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_all_available_metrics_when_metrics_get_command_was_used_with_basic_options( - self, get_patched, create_ws_connection_patched, terminal_printer_cls_patched, - basic_options_metrics_stream_websocket_connection_iterator): - get_patched.return_value = MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON) - - ws_connection_instance_mock = mock.MagicMock() - ws_connection_instance_mock.__iter__ = basic_options_metrics_stream_websocket_connection_iterator - create_ws_connection_patched.return_value = ws_connection_instance_mock - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - terminal_printer_cls_patched().init.assert_called_once() - terminal_printer_cls_patched().rewrite_screen.assert_has_calls([ - mock.call(self.EXPECTED_TABLE_1), - mock.call(self.EXPECTED_TABLE_2), - mock.call(self.EXPECTED_TABLE_3), - mock.call(self.EXPECTED_TABLE_4), - ]) - terminal_printer_cls_patched().cleanup.assert_called_once() - - get_patched.assert_called_once_with( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS, - ) - ws_connection_instance_mock.send.assert_called_once_with(self.BASIC_COMMAND_CHART_DESCRIPTOR) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_metrics_when_metrics_get_command_was_used_with_all_options( - self, get_patched, create_ws_connection_patched, terminal_printer_cls_patched, - all_options_metrics_stream_websocket_connection_iterator): - get_patched.return_value = MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON) - - ws_connection_instance_mock = mock.MagicMock() - ws_connection_instance_mock.__iter__ = all_options_metrics_stream_websocket_connection_iterator - create_ws_connection_patched.return_value = ws_connection_instance_mock - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - terminal_printer_cls_patched().init.assert_called_once() - terminal_printer_cls_patched().rewrite_screen.assert_has_calls([ - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_1), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_2), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_3), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_4), - ]) - terminal_printer_cls_patched().cleanup.assert_called_once() - - get_patched.assert_called_once_with( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - ws_connection_instance_mock.send.assert_called_once_with(self.ALL_COMMANDS_CHART_DESCRIPTOR) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_metrics_when_metrics_get_was_executed_and_options_file_was_used( - self, get_patched, create_ws_connection_patched, terminal_printer_cls_patched, - all_options_metrics_stream_websocket_connection_iterator, - experiments_metrics_stream_config_path): - get_patched.return_value = MockResponse(self.GET_LIST_OF_JOBS_RESPONSE_JSON) - ws_connection_instance_mock = mock.MagicMock() - ws_connection_instance_mock.__iter__ = all_options_metrics_stream_websocket_connection_iterator - create_ws_connection_patched.return_value = ws_connection_instance_mock - - command = self.ALL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [experiments_metrics_stream_config_path] - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - terminal_printer_cls_patched().init.assert_called_once() - terminal_printer_cls_patched().rewrite_screen.assert_has_calls([ - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_1), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_2), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_3), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_4), - ]) - terminal_printer_cls_patched().cleanup.assert_called_once() - - get_patched.assert_called_once_with( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - ws_connection_instance_mock.send.assert_called_once_with(self.ALL_COMMANDS_CHART_DESCRIPTOR) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_invalid_api_key_was_used( - self, get_patched, create_ws_connection_patched, terminal_printer_cls_patched): - get_patched.return_value = MockResponse({"status": 400, "message": "Invalid API token"}, 400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert "Failed to fetch data: Invalid API token\n" == result.output, result.exc_info - - get_patched.assert_called_once_with( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - create_ws_connection_patched.assert_not_called() - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_experiment_was_not_found( - self, get_patched, create_ws_connection_patched, terminal_printer_cls_patched): - get_patched.return_value = MockResponse({"jobList": []}) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND, result.exc_info - - get_patched.assert_called_once_with( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - create_ws_connection_patched.assert_not_called() - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_valid_error_message_when_experiment_was_not_started_and_no_jobs_were_found( - self, get_patched, create_ws_connection_patched, terminal_printer_cls_patched): - get_patched.return_value = MockResponse({"jobList": []}) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND, result.exc_info - - get_patched.assert_called_once_with( - self.LIST_JOBS_URL, - json=None, - params=self.GET_JOB_LIST_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - create_ws_connection_patched.assert_not_called() - assert result.exit_code == 0, result.exc_info diff --git a/tests/functional/test_hyperparameters.py b/tests/functional/test_hyperparameters.py deleted file mode 100644 index 003e3183..00000000 --- a/tests/functional/test_hyperparameters.py +++ /dev/null @@ -1,901 +0,0 @@ -import mock -from click.testing import CliRunner - -from gradient.api_sdk import constants -from gradient.api_sdk.clients.http_client import default_headers -from gradient.cli import cli -from tests import MockResponse, example_responses - -EXPECTED_HEADERS = default_headers.copy() -EXPECTED_HEADERS["ps_client_name"] = "gradient-cli" - -EXPECTED_HEADERS_WITH_CHANGED_API_KEY = EXPECTED_HEADERS.copy() -EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - - -class TestCreateHyperparameters(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/hyperopt/" - TAGS_URL = "https://api.paperspace.io/entityTags/updateTags" - COMMAND = [ - "experiments", "hyperparameters", "create", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "1", - "--projectId", "some_project_id", - ] - COMMAND_WITH_TAGS = [ - "experiments", "hyperparameters", "create", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "1", - "--projectId", "some_project_id", - "--tag", "test0", - "--tag", "test1", - "--tags", "test2,test3", - ] - EXPECTED_REQUEST_JSON = { - "workerContainer": "some_container", - "workerMachineType": "k80", - "name": "some_name", - "tuningCommand": "c29tZSBjb21tYW5k", - "workerCount": 1, - "workerCommand": "c29tZSB3b3JrZXIgY29tbWFuZA==", - "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING, - "projectHandle": "some_project_id", - } - TAGS_JSON = { - "entity": "experiment", - "entityId": "eshgvasywz9k1w", - "tags": ["test0", "test1", "test2", "test3"] - } - UPDATE_TAGS_RESPONSE_JSON_200 = example_responses.UPDATE_TAGS_RESPONSE - - COMMAND_WHEN_ALL_PARAMETERS_WERE_USED = [ - "experiments", "hyperparameters", "create", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_worker_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "666", - "--projectId", "some_project_id", - "--workerRegistryUsername", "some_registry_username", - "--workerRegistryPassword", "some_registry_password", - "--workerContainerUser", "some_worker_container_user", - "--hyperparameterServerRegistryUsername", "some_hyperparameter_registry_username", - "--hyperparameterServerRegistryPassword", "some_hyperparameter_registry_password", - "--hyperparameterServerContainer", "some_hyperparameter_container", - "--hyperparameterServerContainerUser", "some_hyperparameter_container_user", - "--hyperparameterServerMachineType", "some_hyperparameter_server_machine", - "--modelPath", "some_model_path", - "--modelType", "some_model_type", - "--ignoreFiles", "file1,file2", - "--isPreemptible", - "--artifactDirectory", "some_artifact_directory", - "--clusterId", "some_cluster_id", - "--experimentEnv", "{\"key\":\"val\"}", - "--ignoreFiles", "file2", - "--ports", "8080,9000:9999", - "--workerDockerfilePath", "some_docker_path", - "--workerUseDockerfile", - "--workingDirectory", "some_working_directory", - "--workspace", "s3://some-path", - ] - EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED = { - "workerContainer": "some_worker_container", - "workerMachineType": "k80", - "name": "some_name", - "tuningCommand": "c29tZSBjb21tYW5k", - "workerCount": 666, - "workerCommand": "c29tZSB3b3JrZXIgY29tbWFuZA==", - "workerRegistryUsername": "some_registry_username", - "workerRegistryPassword": "some_registry_password", - "workerContainerUser": "some_worker_container_user", - "projectHandle": "some_project_id", - "hyperparameterServerRegistryUsername": "some_hyperparameter_registry_username", - "hyperparameterServerRegistryPassword": "some_hyperparameter_registry_password", - "hyperparameterServerContainer": "some_hyperparameter_container", - "hyperparameterServerContainerUser": "some_hyperparameter_container_user", - "hyperparameterServerMachineType": "some_hyperparameter_server_machine", - "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING, - "modelPath": "some_model_path", - "modelType": "some_model_type", - "isPreemptible": True, - "dockerfilePath": "some_docker_path", - "artifactDirectory": "some_artifact_directory", - "clusterId": "some_cluster_id", - "experimentEnv": {"key": "val"}, - "ports": "8080,9000:9999", - "useDockerfile": True, - "workingDirectory": "some_working_directory", - "workspaceUrl": "s3://some-path", - } - COMMAND_WITH_OPTIONS_FILE = ["experiments", "hyperparameters", "create", "--optionsFile", ] # path added in test - - EXPECTED_RESPONSE = {"handle": "eshgvasywz9k1w", "message": "success"} - EXPECTED_STDOUT = "Hyperparameter tuning job created with ID: eshgvasywz9k1w\n" \ - "https://console.paperspace.com/projects/some_project_id/experiments/eshgvasywz9k1w\n" - - EXPECTED_RESPONSE_JSON_WITH_ERROR = { - "details": { - "projectHandle": ["Missing data for required field."], - }, - "error": "Experiment data error", - } - EXPECTED_STDOUT_WHEN_ERROR_RECEIVED = "Failed to create resource: projectHandle: Missing data for required field." \ - "\nExperiment data error\n" - - COMMAND_WITH_API_KEY_PARAMETER_USED = [ - "experiments", "hyperparameters", "create", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "1", - "--projectId", "some_project_id", - "--apiKey", "some_key", - ] - EXPECTED_REQUEST_JSON_WHEN_API_KEY_PARAMETERS_WAS_USED = { - "workerContainer": "some_container", - "workerMachineType": "k80", - "name": "some_name", - "tuningCommand": "some command", - "workerCount": 1, - "workerCommand": "some worker command", - "projectHandle": "pr4yxj956", - "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING, - } - - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"details": "Incorrect API Key provided", "error": "Forbidden"} - EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED = "Failed to create resource: Incorrect API Key provided\nForbidden\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_get_request_and_print_proper_message_when_create_command_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_get_request_and_print_proper_message_when_create_command_was_used_with_all_options(self, - post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WHEN_ALL_PARAMETERS_WERE_USED) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) - - assert result.output == self.EXPECTED_STDOUT - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_proper_message_when_error_message_received(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WITH_ERROR, 400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT_WHEN_ERROR_RECEIVED - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_proper_message_when_wrong_api_key_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 403) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_yaml_file(self, post_patched, hyperparameters_create_config_path): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [hyperparameters_create_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED, - params=None, - files=None, - data=None) - - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_request_and_print_proper_message_when_error_code_returned_without_json_data(self, - post_patched): - post_patched.return_value = MockResponse(status_code=500) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == "Failed to create resource\n" - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_tag_hyperopt_experiment(self, post_patched, get_patched, put_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE) - get_patched.return_value = MockResponse({}, ) - put_patched.return_value = MockResponse(self.UPDATE_TAGS_RESPONSE_JSON_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_TAGS) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - put_patched.assert_called_once_with( - self.TAGS_URL, - headers=EXPECTED_HEADERS, - json=self.TAGS_JSON, - params=None, - data=None, - ) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - assert result.exit_code == 0 - - -class TestCreateAndStartHyperparameters(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/hyperopt/create_and_start/" - COMMAND = [ - "experiments", "hyperparameters", "run", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "1", - "--projectId", "some_project_id", - ] - EXPECTED_REQUEST_JSON = { - "workerContainer": "some_container", - "workerMachineType": "k80", - "name": "some_name", - "tuningCommand": "c29tZSBjb21tYW5k", - "workerCount": 1, - "workerCommand": "c29tZSB3b3JrZXIgY29tbWFuZA==", - "projectHandle": "some_project_id", - "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING, - } - - COMMAND_WHEN_ALL_PARAMETERS_WERE_USED = [ - "experiments", "hyperparameters", "run", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_worker_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "666", - "--projectId", "some_project_id", - "--workerRegistryUsername", "some_registry_username", - "--workerRegistryPassword", "some_registry_password", - "--workerContainerUser", "some_worker_container_user", - "--hyperparameterServerRegistryUsername", "some_hyperparameter_registry_username", - "--hyperparameterServerRegistryPassword", "some_hyperparameter_registry_password", - "--hyperparameterServerContainer", "some_hyperparameter_container", - "--hyperparameterServerContainerUser", "some_hyperparameter_container_user", - "--hyperparameterServerMachineType", "some_hyperparameter_server_machine", - "--modelPath", "some_model_path", - "--modelType", "some_model_type", - "--ignoreFiles", "file1,file2", - "--isPreemptible", - "--artifactDirectory", "some_artifact_directory", - "--clusterId", "some_cluster_id", - "--experimentEnv", "{\"key\":\"val\"}", - "--ignoreFiles", "file2", - "--ports", "8080,9000:9999", - "--workerDockerfilePath", "some_docker_path", - "--workerUseDockerfile", - "--workingDirectory", "some_working_directory", - "--workspace", "s3://some-path", - ] - EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED = { - "workerContainer": "some_worker_container", - "workerMachineType": "k80", - "name": "some_name", - "tuningCommand": "c29tZSBjb21tYW5k", - "workerCount": 666, - "workerCommand": "c29tZSB3b3JrZXIgY29tbWFuZA==", - "workerRegistryUsername": "some_registry_username", - "workerRegistryPassword": "some_registry_password", - "workerContainerUser": "some_worker_container_user", - "projectHandle": "some_project_id", - "hyperparameterServerRegistryUsername": "some_hyperparameter_registry_username", - "hyperparameterServerRegistryPassword": "some_hyperparameter_registry_password", - "hyperparameterServerContainer": "some_hyperparameter_container", - "hyperparameterServerContainerUser": "some_hyperparameter_container_user", - "hyperparameterServerMachineType": "some_hyperparameter_server_machine", - "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING, - "modelPath": "some_model_path", - "modelType": "some_model_type", - "isPreemptible": True, - "dockerfilePath": "some_docker_path", - "artifactDirectory": "some_artifact_directory", - "clusterId": "some_cluster_id", - "experimentEnv": {"key": "val"}, - "ports": "8080,9000:9999", - "useDockerfile": True, - "workingDirectory": "some_working_directory", - "workspaceUrl": "s3://some-path", - } - COMMAND_WITH_OPTIONS_FILE = ["experiments", "hyperparameters", "run", "--optionsFile", ] # path added in test - - EXPECTED_RESPONSE = {"handle": "eshgvasywz9k1w", "message": "success"} - EXPECTED_STDOUT = "Hyperparameter tuning job created and started with ID: eshgvasywz9k1w\n" \ - "https://console.paperspace.com/projects/some_project_id/experiments/eshgvasywz9k1w\n" - - EXPECTED_RESPONSE_JSON_WITH_ERROR = { - "details": { - "projectHandle": ["Missing data for required field."], - }, - "error": "Experiment data error", - } - EXPECTED_STDOUT_WHEN_ERROR_RECEIVED = "Failed to create resource: " \ - "projectHandle: Missing data for required field.\nExperiment data error\n" - - COMMAND_WITH_API_KEY_PARAMETER_USED = [ - "experiments", "hyperparameters", "run", - "--name", "some_name", - "--tuningCommand", "some command", - "--workerContainer", "some_container", - "--workerMachineType", "k80", - "--workerCommand", "some worker command", - "--workerCount", "1", - "--projectId", "some_project_id", - "--apiKey", "some_key", - ] - EXPECTED_REQUEST_JSON_WHEN_API_KEY_PARAMETERS_WAS_USED = { - "workerContainer": "some_container", - "workerMachineType": "k80", - "name": "some_name", - "tuningCommand": "some command", - "workerCount": 1, - "workerCommand": "some worker command", - "projectHandle": "pr4yxj956", - "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING, - } - - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"details": "Incorrect API Key provided", "error": "Forbidden"} - EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED = "Failed to create resource: Incorrect API Key provided\nForbidden\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_get_request_and_print_proper_message_when_create_command_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_get_request_and_print_proper_message_when_create_command_was_used_with_all_options(self, - post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WHEN_ALL_PARAMETERS_WERE_USED) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_proper_message_when_error_message_received(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WITH_ERROR, 400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT_WHEN_ERROR_RECEIVED - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_proper_message_when_wrong_api_key_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 403) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == self.EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_request_and_print_proper_message_when_error_code_returned_without_json_data(self, - post_patched): - post_patched.return_value = MockResponse(status_code=500) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - files=None, - data=None) - - assert result.output == "Failed to create resource\n" - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_yaml_file(self, post_patched, hyperparameters_create_config_path): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [hyperparameters_create_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED, - params=None, - files=None, - data=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - -class TestStartHyperparameters(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/hyperopt/some_id/start/" - COMMAND = [ - "experiments", "hyperparameters", "start", - "--id", "some_id", - ] - - COMMAND_WITH_OPTIONS_FILE = ["experiments", "hyperparameters", "start", "--optionsFile", ] # path added in test - - EXPECTED_RESPONSE = {"message": "success"} - EXPECTED_STDOUT = "Hyperparameter tuning started\n" - - EXPECTED_RESPONSE_JSON_WITH_ERROR = {"error": "Could not find cluster meeting requirements"} - EXPECTED_STDOUT_WHEN_ERROR_RECEIVED = "Failed to start hyperparameter tuning job: Could not find cluster meeting requirements\n" - - COMMAND_WITH_API_KEY_PARAMETER_USED = [ - "experiments", "hyperparameters", "start", - "--id", "some_id", - "--apiKey", "some_key", - ] - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"details": "Incorrect API Key provided", "error": "Forbidden"} - EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED = "Failed to start hyperparameter tuning job: Incorrect API Key provided\nForbidden\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_get_request_and_print_proper_message_when_start_command_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None, - data=None, - ) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) - - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None, - ) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_read_options_from_yaml_file(self, post_patched, hyperparameters_start_config_path): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE, 201) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [hyperparameters_start_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None, - ) - - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_print_proper_message_when_error_message_received(self, put_patched): - put_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WITH_ERROR, 400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None, - data=None, - ) - - assert result.output == self.EXPECTED_STDOUT_WHEN_ERROR_RECEIVED - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_print_proper_message_when_wrong_api_key_was_used(self, put_patched): - put_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 403) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None, - data=None, - ) - - assert result.output == self.EXPECTED_STDOUT_WHEN_WRONG_API_KEY_WAS_USED - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_request_and_print_proper_message_when_error_code_returned_without_json_data(self, put_patched): - put_patched.return_value = MockResponse(status_code=500) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - put_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None, - data=None, - ) - - assert result.output == "Failed to start hyperparameter tuning job\n" - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - -class TestHyperparametersList(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/hyperopt/" - COMMAND = ["experiments", "hyperparameters", "list"] - COMMAND_WITH_OPTIONS_FILE = ["experiments", "hyperparameters", "list", "--optionsFile", ] # path added in test - EXPECTED_REQUEST_PARAMS = {"limit": -1} - - COMMAND_WITH_API_KEY_PARAMETER_USED = ["experiments", "hyperparameters", "list", "--apiKey", "some_key"] - - EXPECTED_RESPONSE_JSON_WHEN_NO_OBJECTS_WERE_FOUND = { - "data": [], - "message": "success", - "meta": { - "filter": [], - "limit": -1, - "offset": 0, - "totalItems": 0, - }, - } - - EXPECTED_STDOUT = """+-----------+----------------+------------+ -| Name | ID | Project ID | -+-----------+----------------+------------+ -| some_name | es3dn6fu16r4kk | pr4yxj956 | -| some_name | eshlqek7wzvrxa | pr4yxj956 | -| some_name | esdwnui5qsk8qm | pr4yxj956 | -| some_name | eshz1z9k37w4nm | pr4yxj956 | -+-----------+----------------+------------+ -""" - - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"status": 401, "message": "No such API token"} - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_list_of_hyperparameters(self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_HYPERPARAMETERS_RESPONSE_JSON, 200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_REQUEST_PARAMS) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_HYPERPARAMETERS_RESPONSE_JSON, 200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.EXPECTED_REQUEST_PARAMS) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml(self, get_patched, hyperparameters_list_config_path): - get_patched.return_value = MockResponse(example_responses.LIST_HYPERPARAMETERS_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [hyperparameters_list_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.EXPECTED_REQUEST_PARAMS) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_proper_message_when_no_objects_were_found( - self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WHEN_NO_OBJECTS_WERE_FOUND, 200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_REQUEST_PARAMS) - - assert result.output == "No data found\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_proper_message_when_wrong_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_REQUEST_PARAMS) - - assert result.output == "Failed to fetch data: No such API token\n" - - -class TestHyperparametersDetails(object): - URL_V2 = "https://services.paperspace.io/experiments/v2/hyperopt/some_id/" - COMMAND = ["experiments", "hyperparameters", "details", "--id", "some_id"] - - COMMAND_WITH_OPTIONS_FILE = ["experiments", "hyperparameters", "details", "--optionsFile", ] # path added in test - COMMAND_WITH_API_KEY_PARAMETER_USED = [ - "experiments", "hyperparameters", "details", - "--id", "some_id", - "--apiKey", "some_key", - ] - - EXPECTED_RESPONSE_JSON_WHEN_NO_OBJECT_WAS_NOT_FOUND = {"error": "Hyperopt not found"} - - EXPECTED_STDOUT = """+-----------------------+---------------------+ -| ID | ess6t3fjs2hb1g | -+-----------------------+---------------------+ -| Name | some_name | -| Ports | 5000 | -| Project ID | pr4yxj956 | -| Tuning command | some command | -| Worker command | some worker command | -| Worker container | some_container | -| Worker count | 1 | -| Worker machine type | k80 | -| Worker use dockerfile | False | -| Workspace URL | none | -+-----------------------+---------------------+ -""" - - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"status": 401, "message": "No such API token"} - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_details_of_hyperparameters_job(self, get_patched): - get_patched.return_value = MockResponse(example_responses.HYPERPARAMETERS_DETAILS_RESPONSE_JSON, 200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, get_patched): - get_patched.return_value = MockResponse(example_responses.HYPERPARAMETERS_DETAILS_RESPONSE_JSON, 200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - - assert result.output == self.EXPECTED_STDOUT - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file( - self, get_patched, hyperparameters_details_config_path): - get_patched.return_value = MockResponse(example_responses.HYPERPARAMETERS_DETAILS_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [hyperparameters_details_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_proper_message_when_no_objects_were_found(self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WHEN_NO_OBJECT_WAS_NOT_FOUND, 404) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert result.output == "Failed to fetch data: Hyperopt not found\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_proper_message_when_wrong_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert result.output == "Failed to fetch data: No such API token\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_proper_message_when_got_error_response_without_data(self, get_patched): - get_patched.return_value = MockResponse(status_code=500) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_called_once_with(self.URL_V2, - headers=EXPECTED_HEADERS, - json=None, - params=None) - - assert result.output == "Failed to fetch data\n" diff --git a/tests/functional/test_jobs.py b/tests/functional/test_jobs.py deleted file mode 100644 index 02844b12..00000000 --- a/tests/functional/test_jobs.py +++ /dev/null @@ -1,1210 +0,0 @@ -import json -import os -import shutil -import tempfile - -import mock -import pytest -from click.testing import CliRunner - -from gradient.api_sdk import sdk_exceptions -from gradient.api_sdk.clients.http_client import default_headers -from gradient.cli import cli -from tests import example_responses, MockResponse -from tests.example_responses import LIST_JOB_FILES_RESPONSE_JSON - -EXPECTED_HEADERS = default_headers.copy() -EXPECTED_HEADERS["ps_client_name"] = "gradient-cli" - -EXPECTED_HEADERS_WITH_CHANGED_API_KEY = EXPECTED_HEADERS.copy() -EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - - -@pytest.fixture -def basic_options_metrics_stream_websocket_connection_iterator(): - def generator(self): - yield """{"handle": "jstkd2lapucirs", "object_type": "mljob", "chart_name": "memoryUsage", - "pod_metrics": {"mljob-ecgrgm7ok8chv-0-worker": {"time_stamp": 1588155670, "value": "5881856"}}}""" - - yield """{"handle": "jstkd2lapucirs", "object_type": "mljob", "chart_name": "cpuPercentage", - "pod_metrics": {"mljob-ecgrgm7ok8chv-0-worker": {"time_stamp": 1588155670, "value": "0"}}}""" - - yield """{"handle": "jstkd2lapucirs", "object_type": "mljob", "chart_name": "memoryUsage", - "pod_metrics": {"mljob-ecgrgm7ok8chv-0-worker": {"time_stamp": 1588155700, "value": "5881857"}}}""" - - raise sdk_exceptions.EndWebsocketStream("keton") - - return generator - - -@pytest.fixture -def all_options_metrics_stream_websocket_connection_iterator(): - def generator(self): - yield """{"handle": "jstkd2lapucirs", "object_type": "mljob", "chart_name": "gpuMemoryUsed", - "pod_metrics": {"mljob-ecgrgm7ok8chv-0-worker": {"time_stamp": 1588155670, "value": "5881856"}}}""" - - yield """{"handle": "jstkd2lapucirs", "object_type": "mljob", "chart_name": "gpuMemoryFree", - "pod_metrics": {"mljob-ecgrgm7ok8chv-0-worker": {"time_stamp": 1588155670, "value": "0"}}}""" - - yield """{"handle": "jstkd2lapucirs", "object_type": "mljob", "chart_name": "gpuMemoryUsed", - "pod_metrics": {"mljob-ecgrgm7ok8chv-0-worker": {"time_stamp": 1588155700, "value": "5881857"}}}""" - - raise sdk_exceptions.EndWebsocketStream("keton") - - return generator - - -class TestJobs(object): - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to fetch data: Invalid API token\n" - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"status": 400, "message": "Invalid API token"} - - EXPECTED_HEADERS = default_headers.copy() - EXPECTED_HEADERS_WITH_CHANGED_API_KEY = default_headers.copy() - EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - - -class TestListJobs(TestJobs): - URL = "https://api.paperspace.io/jobs/getJobList/" - BASIC_COMMAND = ["jobs", "list"] - COMMAND_WITH_OPTIONS_FILE = ["jobs", "list", "--optionsFile", ] # path added in test - - EXPECTED_RESPONSE_JSON = example_responses.LIST_JOBS_RESPONSE_JSON - EXPECTED_STDOUT = """+----------------+---------------------------+-------------------+----------------+--------------+--------------------------+ -| ID | Name | Project | Cluster | Machine Type | Created | -+----------------+---------------------------+-------------------+----------------+--------------+--------------------------+ -| jsxeeba5qq99yn | job 1 | keton | PS Jobs on GCP | K80 | 2019-03-25T14:51:16.118Z | -| jfl063dsv634h | job 2 | keton | PS Jobs on GCP | P100 | 2019-03-25T14:54:30.866Z | -| jsvau8w47k78zm | Clone - jfl063dsv634h | keton | PS Jobs on GCP | P100 | 2019-03-25T15:04:43.844Z | -| j2eq99xhvgtum | keton1-worker-1 | keton | PS Jobs on GCP | P100 | 2019-03-25T15:07:30.383Z | -| jzzinybinuxf9 | keton2-worker-1 | keton | PS Jobs on GCP | P100 | 2019-03-25T15:18:51.461Z | -| jsb37duc1zlbz0 | keton4-worker-1 | keton | PS Jobs on GCP | P100 | 2019-03-25T15:29:04.601Z | -| jq41vipwy18f7 | keton4-parameter_server-1 | keton | PS Jobs on GCP | P100 | 2019-03-25T15:29:06.765Z | -| jsigkjnyb6m3qm | Test1-worker-1 | keton | PS Jobs on GCP | K80 | 2019-04-02T15:17:05.618Z | -| j4g76vuppxqao | job 1 | paperspace-python | PS Jobs on GCP | K80 | 2019-04-04T15:12:34.414Z | -| jsbnvdhwb46vr9 | job 2 | paperspace-python | PS Jobs on GCP | G1 | 2019-04-24T09:09:53.645Z | -| jt8alwzv28kha | job 3 | paperspace-python | PS Jobs on GCP | G1 | 2019-04-24T10:18:30.620Z | -+----------------+---------------------------+-------------------+----------------+--------------+--------------------------+ -""" - - BASIC_COMMAND_WITH_API_KEY = ["jobs", "list", "--apiKey", "some_key"] - - RESPONSE_JSON_WHEN_NO_JOBS_WERE_FOUND = [] - EXPECTED_STDOUT_WHEN_NO_JOBS_WERE_FOUND = "No data found\n" - - BASIC_COMMAND_WITH_FILTERING = [ - "jobs", "list", - "--projectId", "some_project_id", - "--tag", "some_tag", - "--tag", "some_other_tag", - ] - EXPECTED_PARAMS_WITHOUT_FILTERING = { - "filter": "{\"filter\": {\"where\": {}}}", - } - EXPECTED_REQUEST_PARAMS_WITH_FILTERING = { - "filter": "{\"filter\": {\"where\": {\"projectId\": \"some_project_id\"}}}", - "modelName": "team", - "tagFilter[0]": "some_tag", - "tagFilter[1]": "some_other_tag", - } - - BASIC_COMMAND_WITH_MUTUALLY_EXCLUSIVE_FILTERS = [ - "jobs", "list", - "--project", "some_project_name", - "--projectId", "some_project_id", - ] - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_post_request_and_print_table_when_jobs_list_was_used(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.BASIC_COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_PARAMS_WITHOUT_FILTERING) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_post_request_when_jobs_list_was_used_with_api_key_option(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.BASIC_COMMAND_WITH_API_KEY) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.EXPECTED_PARAMS_WITHOUT_FILTERING) - assert result.output == self.EXPECTED_STDOUT - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file(self, get_patched, jobs_list_config_path): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [jobs_list_config_path] - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, command) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.EXPECTED_REQUEST_PARAMS_WITH_FILTERING) - assert result.output == self.EXPECTED_STDOUT - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_post_request_when_jobs_list_was_used_with_wrong_api_key(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.RESPONSE_JSON_WITH_WRONG_API_TOKEN, status_code=400) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.BASIC_COMMAND_WITH_API_KEY) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.EXPECTED_PARAMS_WITHOUT_FILTERING) - assert result.output == self.EXPECTED_STDOUT_WITH_WRONG_API_TOKEN - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_error_message_when_no_job_was_not_found(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.RESPONSE_JSON_WHEN_NO_JOBS_WERE_FOUND) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.BASIC_COMMAND) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_PARAMS_WITHOUT_FILTERING) - assert result.output == self.EXPECTED_STDOUT_WHEN_NO_JOBS_WERE_FOUND - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_error_message_when_error_status_code_received_but_no_content_was_provided(self, get_patched): - get_patched.return_value = MockResponse(status_code=400) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.BASIC_COMMAND) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_PARAMS_WITHOUT_FILTERING) - assert result.output == "Failed to fetch data\n" - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_post_request_when_jobs_list_was_used_with_filter_options(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON, status_code=200) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.BASIC_COMMAND_WITH_FILTERING) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=self.EXPECTED_REQUEST_PARAMS_WITH_FILTERING) - assert result.output == self.EXPECTED_STDOUT - assert result.exit_code == 0 - - -class TestJobLogs(TestJobs): - URL = "https://logs.paperspace.io/jobs/logs" - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"status": 400, "message": "Invalid API token"} - EXPECTED_RESPONSE_JSON = example_responses.LIST_OF_LOGS_FOR_JOB - COMMAND_WITHOUT_REQUIRED_PARAMETERS = ["jobs", "logs"] - COMMAND_WITH_ALL_OPTIONS = [ - "jobs", "logs", - "--id", "some_id", - "--apiKey", "some_key", - "--line", "50", - "--limit", "200", - # "--follow", "True", - ] - - COMMAND_WITH_OPTIONS_FILE = ["jobs", "logs", "--optionsFile", ] # path added in test - REQUEST_WITH_OPTIONS_FILE = {"jobId": "some_id", "line": 50, "limit": 200} - - EXPECTED_STDOUT = """+Job some_id logs------------------------------------------------------------------------+ -| LINE | MESSAGE | -+------+---------------------------------------------------------------------------------+ -| 1 | Traceback (most recent call last): | -| 2 | File "generate_figures.py", line 15, in | -| 3 | import dnnlib.tflib as tflib | -| 4 | File "/paperspace/dnnlib/tflib/__init__.py", line 8, in | -| 5 | from . import autosummary | -| 6 | File "/paperspace/dnnlib/tflib/autosummary.py", line 31, in | -| 7 | from . import tfutil | -| 8 | File "/paperspace/dnnlib/tflib/tfutil.py", line 34, in | -| 9 | def shape_to_list(shape: Iterable[tf.Dimension]) -> List[Union[int, None]]: | -| 10 | AttributeError: module \'tensorflow\' has no attribute 'Dimension' | -| 11 | PSEOF | -+------+---------------------------------------------------------------------------------+ -""" - - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to fetch data: Invalid API token\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_get_request_and_print_available_logs(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.COMMAND_WITH_ALL_OPTIONS) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.REQUEST_WITH_OPTIONS_FILE) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_get_request_when_log_list_was_used_with_wrong_api_key(self, get_patched): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.COMMAND_WITH_ALL_OPTIONS) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.REQUEST_WITH_OPTIONS_FILE) - assert result.output == self.EXPECTED_STDOUT - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file(self, get_patched, jobs_logs_config_path): - get_patched.return_value = MockResponse(json_data=self.EXPECTED_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [jobs_logs_config_path] - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, command) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.REQUEST_WITH_OPTIONS_FILE) - assert result.output == self.EXPECTED_STDOUT - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_print_error_message_when_error_status_code_received_but_no_content_was_provided(self, get_patched): - get_patched.return_value = MockResponse(status_code=400) - - cli_runner = CliRunner() - result = cli_runner.invoke(cli.cli, self.COMMAND_WITH_ALL_OPTIONS) - - get_patched.assert_called_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.REQUEST_WITH_OPTIONS_FILE) - assert result.output == "Failed to fetch data\n" - assert result.exit_code == 0 - - -class TestDestroyJobArtifactsCommands(TestJobs): - runner = CliRunner() - URL = "https://api.paperspace.io" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_valid_post_request_when_destroying_artifacts_with_files_specified(self, post_patched): - post_patched.return_value = MockResponse(status_code=200) - job_id = "some_job_id" - file_names = "some_file_names" - result = self.runner.invoke(cli.cli, ["jobs", "artifacts", "destroy", "--id", job_id, "--files", file_names, - "--apiKey", "some_key"]) - - assert result.exit_code == 0, result.exc_info - post_patched.assert_called_with("{}/jobs/{}/artifactsDestroy".format(self.URL, job_id), - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"files": file_names}, - data=None) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_valid_post_request_when_destroying_artifacts_without_files_specified(self, post_patched): - post_patched.return_value = MockResponse(status_code=200) - job_id = "some_job_id" - result = self.runner.invoke(cli.cli, ["jobs", "artifacts", "destroy", "--id", job_id, "--apiKey", "some_key"]) - - post_patched.assert_called_with("{}/jobs/{}/artifactsDestroy".format(self.URL, job_id), - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_yaml_file(self, post_patched, jobs_artifacts_destroy_config_path): - post_patched.return_value = MockResponse() - command = ["jobs", "artifacts", "destroy", "--optionsFile", jobs_artifacts_destroy_config_path] - - result = self.runner.invoke(cli.cli, command) - - assert result.exit_code == 0, result.exc_info - post_patched.assert_called_with("{}/jobs/some_id/artifactsDestroy".format(self.URL), - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"files": "file1,file2"}, - data=None) - - -class TestGetJobArtifacts(TestJobs): - runner = CliRunner() - URL = "https://api.paperspace.io" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_send_valid_get_request_and_receive_json_response(self, get_patched): - get_patched.return_value = MockResponse() - job_id = "some_job_id" - result = self.runner.invoke(cli.cli, ["jobs", "artifacts", "get", "--id", job_id, "--apiKey", "some_key"]) - - get_patched.assert_called_with("{}/jobs/artifactsGet".format(self.URL), - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"jobId": job_id}) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file(self, get_patched, jobs_artifacts_get_config_path): - get_patched.return_value = MockResponse() - command = ["jobs", "artifacts", "get", "--optionsFile", jobs_artifacts_get_config_path] - - result = self.runner.invoke(cli.cli, command) - - get_patched.assert_called_with("{}/jobs/artifactsGet".format(self.URL), - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"jobId": "some_id"}) - assert result.exit_code == 0 - - -class TestListJobArtifacts(TestJobs): - runner = CliRunner() - URL = "https://api.paperspace.io" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_get_request_with_all_parameters_for_a_list_of_artifacts(self, get_patched): - get_patched.return_value = MockResponse(LIST_JOB_FILES_RESPONSE_JSON) - job_id = "some_job_id" - result = self.runner.invoke(cli.cli, - ["jobs", "artifacts", "list", "--id", job_id, "--apiKey", "some_key", "--size", - "--links", - "--files", "foo"]) - - get_patched.assert_called_with("{}/jobs/artifactsListV2".format(self.URL), - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"jobId": job_id, - "size": True, - "links": True, - "files": "foo"}) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file(self, get_patched, jobs_artifacts_list_config_path): - get_patched.return_value = MockResponse(LIST_JOB_FILES_RESPONSE_JSON) - command = ["jobs", "artifacts", "list", "--optionsFile", jobs_artifacts_list_config_path] - result = self.runner.invoke(cli.cli, command) - - get_patched.assert_called_with("{}/jobs/artifactsListV2".format(self.URL), - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"files": "keton*.py", - "size": True, - "links": True, - "jobId": "some_id"}) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @pytest.mark.parametrize('option,param', [("--size", "size"), - ("-s", "size"), - ("--links", "links"), - ("-l", "links")]) - def test_should_send_valid_get_request_with_valid_param_for_a_list_of_artifacts_for_both_formats_of_param(self, - get_patched, - option, - param): - get_patched.return_value = MockResponse(LIST_JOB_FILES_RESPONSE_JSON, status_code=200) - job_id = "some_job_id" - result = self.runner.invoke(cli.cli, - ["jobs", "artifacts", "list", "--id", job_id, "--apiKey", "some_key"] + [option]) - - get_patched.assert_called_with("{}/jobs/artifactsListV2".format(self.URL), - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params={"jobId": job_id, - param: True}) - assert result.exit_code == 0 - - -class TestJobsCreate(object): - URL = "https://api.paperspace.io" - TAGS_URL = "https://api.paperspace.io/entityTags/updateTags" - EXPECTED_HEADERS = default_headers.copy() - EXPECTED_HEADERS_WITH_CHANGED_API_KEY = default_headers.copy() - EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - BASIC_OPTIONS_COMMAND = [ - "jobs", "create", - "--name", "exp1", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - ] - BASIC_OPTIONS_COMMAND_WITH_TAGS = [ - "jobs", "create", - "--name", "exp1", - "--projectId", "testHandle", - "--container", "testContainer", - "--machineType", "testType", - "--command", "testCommand", - "--workspace", "https://github.com/Paperspace/gradient-cli.git", - "--tag", "test0", - "--tag", "test1", - "--tags", "test2,test3", - ] - FULL_OPTIONS_COMMAND = [ - "jobs", "create", - "--apiKey", "some_key", - "--cluster", "some_cluster_name", - "--clusterId", "some_cluster_id", - "--command", "some command", - "--container", "some_container", - "--experimentId", "some_experiment_id", - "--ignoreFiles", "file1,file2", - "--isPreemptible", "True", - "--projectId", "some_project_id", - "--isPublic", "True", - "--envVars", '{"key":"val"}', - "--machineType", "K80", - "--name", "some_name", - "--nodeAttrs", '{"key":"val"}', - "--ports", "8080,9000:9900", - "--projectId", "some_project_id", - "--registryPassword", "some_registry_password", - "--registryUsername", "some_registry_username", - "--registryTarget", "some_registry_target", - "--registryTargetPassword", "some_registry_target_password", - "--registryTargetUsername", "some_registry_target_username", - "--relDockerfilePath", "some dockerfile path", - "--startedByUserId", "some_user_id", - "--useDockerfile", "True", - "--workingDirectory", "/some/path", - "--buildOnly", - "--workspace", "s3://some-path", - ] - BASIC_OPTIONS_REQUEST = { - "name": u"exp1", - "projectId": u"testHandle", - "container": u"testContainer", - "machineType": u"testType", - "command": u"testCommand", - "workspaceFileName": u"https://github.com/Paperspace/gradient-cli.git", - } - FULL_OPTIONS_REQUEST = { - "clusterId": "some_cluster_id", - "experimentId": "some_experiment_id", - "cluster": "some_cluster_name", - "startedByUserId": "some_user_id", - "isPreemptible": True, - "container": "some_container", - "workingDirectory": "/some/path", - "projectId": "some_project_id", - "registryTargetUsername": "some_registry_target_username", - "machineType": "K80", - "registryTargetPassword": "some_registry_target_password", - "registryTarget": "some_registry_target", - "isPublic": True, - "workspaceFileName": "s3://some-path", - "envVars": '{"key": "val"}', - "useDockerfile": True, - "name": "some_name", - "relDockerfilePath": "some dockerfile path", - "targetNodeAttrs": {"key": "val"}, - "command": "some command", - "ports": "8080,9000:9900", - "buildOnly": True, - "registryUsername": "some_registry_username", - "registryPassword": "some_registry_password", - } - TAGS_JSON = { - "entity": "job", - "entityId": "sadkfhlskdjh", - "tags": ["test0", "test1", "test2", "test3"] - } - RESPONSE_JSON_200 = {"id": "sadkfhlskdjh", "message": "success"} - UPDATE_TAGS_RESPONSE_JSON_200 = example_responses.UPDATE_TAGS_RESPONSE - RESPONSE_CONTENT_200 = b'{"handle":"sadkfhlskdjh","message":"success"}\n' - EXPECTED_STDOUT = u'New job created with ID: sadkfhlskdjh\n' - EXPECTED_STDOUT_TAGS = u'New job created with ID: sadkfhlskdjh\n' \ - u'https://console.paperspace.com/jobs/sadkfhlskdjh\n' - - RESPONSE_JSON_404_PROJECT_NOT_FOUND = {"details": {"handle": "wrong_handle"}, "error": "Project not found"} - RESPONSE_CONTENT_404_PROJECT_NOT_FOUND = b'{"details":{"handle":"wrong_handle"},"error":"Project not found"}\n' - EXPECTED_STDOUT_PROJECT_NOT_FOUND = "Project not found\nhandle: wrong_handle\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_job_was_run_with_basic_options(self, post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL + '/jobs/createJob/', - headers=EXPECTED_HEADERS, - json=None, - params=self.BASIC_OPTIONS_REQUEST, - files=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_print_message_when_create_job_was_run_with_all_options(self, post_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.FULL_OPTIONS_COMMAND) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL + '/jobs/createJob/', - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.FULL_OPTIONS_REQUEST, - files=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_yaml_file(self, post_patched, jobs_create_config_path): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) - command = ["jobs", "create", "--optionsFile", jobs_create_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert self.EXPECTED_STDOUT in result.output, result.exc_info - post_patched.assert_called_once_with(self.URL + '/jobs/createJob/', - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=self.FULL_OPTIONS_REQUEST, - files=None, - data=None) - assert result.exit_code == 0 - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_proper_data_and_tag_job(self, post_patched, get_patched, put_patched): - post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) - get_patched.return_value = MockResponse({}, 200, "fake content") - put_patched.return_value = MockResponse(self.UPDATE_TAGS_RESPONSE_JSON_200, 200, "fake content") - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND_WITH_TAGS) - - post_patched.assert_called_once_with(self.URL + '/jobs/createJob/', - headers=EXPECTED_HEADERS, - json=None, - params=self.BASIC_OPTIONS_REQUEST, - files=None, - data=None) - - put_patched.assert_called_once_with( - self.TAGS_URL, - headers=EXPECTED_HEADERS, - json=self.TAGS_JSON, - params=None, - data=None, - ) - - assert result.output == self.EXPECTED_STDOUT_TAGS, result.exc_info - assert result.exit_code == 0 - - -class TestDownloadJobArtifacts(TestJobs): - runner = CliRunner() - LIST_FILES_URL = "https://api.paperspace.io/jobs/artifactsListV2" - DESTINATION_DIR_NAME = "dest" - DESTINATION_DIR_PATH = os.path.join(tempfile.gettempdir(), "dest") - - COMMAND = ["jobs", "artifacts", "download", "--id", "some_job_id", "--destinationDir", DESTINATION_DIR_PATH] - - @classmethod - def teardown_method(cls): - shutil.rmtree(cls.DESTINATION_DIR_PATH) - - @mock.patch("gradient.api_sdk.s3_downloader.requests.get") - def test_should_get_a_list_of_files_and_download_them_to_defined_directory_when_download_command_was_executed( - self, get_patched, - ): - file_response_mock = mock.MagicMock() - file_response_mock.content = "\"Hello Paperspace!\n\"" - file_response_mock_2 = mock.MagicMock() - file_response_mock_2.content = "\"Hello Paperspace 2\n\"" - file_response_mock_3 = mock.MagicMock() - file_response_mock_3.content = "\"Elo\n\"" - get_patched.side_effect = [ - MockResponse(LIST_JOB_FILES_RESPONSE_JSON), - file_response_mock, - file_response_mock_2, - file_response_mock_3, - ] - - result = self.runner.invoke(cli.cli, self.COMMAND) - - get_patched.assert_has_calls([ - mock.call(self.LIST_FILES_URL, - headers=EXPECTED_HEADERS, - json=None, - params={"links": True, "jobId": "some_job_id"}), - mock.call("https://ps-projects.s3.amazonaws.com/some/path/artifacts/hello.txt?AWSAccessKeyId=" - "some_aws_access_key_id&Expires=713274132&Signature=7CT5k6buEmZe5k5E7g6BXMs2xV4%3D&" - "response-content-disposition=attachment%3Bfilename%3D%22hello.txt%22&x-amz-security-token=" - "some_amz_security_token"), - mock.call("https://ps-projects.s3.amazonaws.com/some/path/artifacts/hello2.txt?AWSAccessKeyId=" - "some_aws_access_key_id&Expires=713274132&Signature=L1lI47cNyiROzdYkf%2FF3Cm3165E%3D&" - "response-content-disposition=attachment%3Bfilename%3D%22hello2.txt%22&x-amz-security-token=" - "some_amz_security_token"), - mock.call("https://ps-projects.s3.amazonaws.com/some/path/artifacts/keton/elo.txt?AWSAccessKeyId=" - "some_aws_access_key_id&Expires=713274132&Signature=tHriojGx03S%2FKkVGQGVI5CQRFTo%3D&" - "response-content-disposition=attachment%3Bfilename%3D%22elo.txt%22&x-amz-security-token=" - "some_amz_security_token"), - ]) - assert os.path.exists(self.DESTINATION_DIR_PATH) - assert os.path.isdir(self.DESTINATION_DIR_PATH) - assert os.path.exists(os.path.join(self.DESTINATION_DIR_PATH, "keton")) - assert os.path.isdir(os.path.join(self.DESTINATION_DIR_PATH, "keton")) - - hello_txt_path = os.path.join(self.DESTINATION_DIR_PATH, "hello.txt") - assert os.path.exists(hello_txt_path) - assert not os.path.isdir(hello_txt_path) - with open(hello_txt_path) as h: - assert h.read() == "\"Hello Paperspace!\n\"" - - hello2_txt_path = os.path.join(self.DESTINATION_DIR_PATH, "hello2.txt") - assert os.path.exists(hello2_txt_path) - assert not os.path.isdir(hello2_txt_path) - with open(hello2_txt_path) as h: - assert h.read() == "\"Hello Paperspace 2\n\"" - - elo_txt_path = os.path.join(self.DESTINATION_DIR_PATH, "keton", "elo.txt") - assert os.path.exists(elo_txt_path) - assert not os.path.isdir(elo_txt_path) - with open(elo_txt_path) as h: - assert h.read() == "\"Elo\n\"" - - assert result.exit_code == 0 - - -class TestJobsMetricsGetCommand(object): - GET_JOB_URL = "https://api.paperspace.io/jobs/getPublicJob" - GET_METRICS_URL = "https://aws-testing.paperspace.io/metrics/api/v1/range" - BASIC_OPTIONS_COMMAND = [ - "jobs", "metrics", "get", - "--id", "jstkd2lapucirs", - ] - ALL_OPTIONS_COMMAND = [ - "jobs", "metrics", "get", - "--id", "jstkd2lapucirs", - "--metric", "gpuMemoryFree", - "--metric", "gpuMemoryUsed", - "--interval", "20s", - "--start", "2020-04-01", - "--end", "2020-04-02 21:37:00", - "--apiKey", "some_key", - ] - FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "jobs", "metrics", "get", - "--optionsFile", # path added in test, - ] - - GET_JOB_REQUEST_JSON = {"jobId": "jstkd2lapucirs"} - BASIC_COMMAND_GET_METRICS_REQUEST_PARAMS = { - "start": "2020-04-29T10:11:07Z", - "handle": "jstkd2lapucirs", - "interval": "30s", - "charts": "cpuPercentage,memoryUsage", - "objecttype": "mljob", - } - ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS = { - "start": "2020-04-01T00:00:00Z", - "handle": "jstkd2lapucirs", - "interval": "20s", - "charts": "gpuMemoryFree,gpuMemoryUsed", - "objecttype": "mljob", - "end": "2020-04-02T21:37:00Z", - } - - GET_JOB_RESPONSE_JSON = example_responses.GET_JOB_RESPONSE - GET_METRICS_RESPONSE_JSON = example_responses.GET_JOB_METRICS_RESPONSE - GET_JOB_RESPONSE_JSON_WHEN_NO_JOBS_WERE_FOUND = { - "error": { - "name": "ApplicationError", - "status": 404, - "message": "No such job", - }, - } - - EXPECTED_STDOUT = """{ - "cpuPercentage": { - "mljob-ecgrgm7ok8chv-0-worker": [ - { - "time_stamp": 1588155157, - "value": "0" - }, - { - "time_stamp": 1588155187, - "value": "0" - }, - { - "time_stamp": 1588155217, - "value": "0" - } - ] - }, - "memoryUsage": { - "mljob-ecgrgm7ok8chv-0-worker": [ - { - "time_stamp": 1588155097, - "value": "0" - }, - { - "time_stamp": 1588155127, - "value": "5881856" - }, - { - "time_stamp": 1588155157, - "value": "5881856" - }, - { - "time_stamp": 1588155187, - "value": "5881856" - } - ] - } -} -""" - - EXPECTED_STDOUT_WHEN_INVALID_API_KEY_WAS_USED = "Failed to fetch data: Invalid API token\n" - EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND = "Failed to fetch data: No such job\n" - EXPECTED_STDOUT_WHEN_NO_METRICS_FOUND = """{ - "cpuPercentage": null, - "memoryUsage": null -} -""" - EXPECTED_STDOUT_WHEN_ERROR_CODE_WAS_RETURNED_WITHOUT_ERROR_MESSAGE = "Failed to fetch data\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_all_available_metrics_when_metrics_get_command_was_used_with_basic_options( - self, post_patched, get_patched): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - get_patched.return_value = MockResponse(self.GET_METRICS_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT.strip()), \ - str(result.output) + str(result.exc_info) - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS, - ) - get_patched.assert_called_once_with( - self.GET_METRICS_URL, - json=None, - params=self.BASIC_COMMAND_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS, - ) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_metrics_when_metrics_get_command_was_used_with_all_options( - self, post_patched, get_patched): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - get_patched.return_value = MockResponse(self.GET_METRICS_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - # comparing objects instead of strings because Py2 and Py3 produce slightly different outputs - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT.strip()), result.exc_info - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - get_patched.assert_called_once_with( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_metrics_when_metrics_get_was_executed_and_options_file_was_used( - self, post_patched, get_patched, jobs_metrics_get_config_path): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - get_patched.return_value = MockResponse(self.GET_METRICS_RESPONSE_JSON) - - command = self.FULL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [jobs_metrics_get_config_path] - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - # comparing objects instead of strings because Py2 and Py3 produce slightly different outputs - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT.strip()), result.exc_info - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - get_patched.assert_called_once_with( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_valid_error_message_when_invalid_api_key_was_used(self, post_patched): - post_patched.return_value = MockResponse({"status": 400, "message": "Invalid API token"}, status_code=400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_INVALID_API_KEY_WAS_USED, result.exc_info - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_valid_error_message_when_job_was_not_found(self, post_patched): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON_WHEN_NO_JOBS_WERE_FOUND, status_code=400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_EXPERIMENT_WAS_NOT_FOUND, result.exc_info - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_valid_message_when_was_no_metrics_were_returned( - self, post_patched, get_patched): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - get_patched.return_value = MockResponse(example_responses.JOBS_METRICS_GET_RESPONSE_WHEN_NO_DATA_WAS_FOUND) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert json.loads(result.output.strip()) == json.loads(self.EXPECTED_STDOUT_WHEN_NO_METRICS_FOUND.strip()), \ - result.exc_info - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - get_patched.assert_called_once_with( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_valid_error_message_when_error_code_was_returned_without_error_message( - self, post_patched, get_patched): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - get_patched.return_value = MockResponse(status_code=500) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_ERROR_CODE_WAS_RETURNED_WITHOUT_ERROR_MESSAGE, result.exc_info - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - get_patched.assert_called_once_with( - self.GET_METRICS_URL, - json=None, - params=self.ALL_COMMANDS_GET_METRICS_REQUEST_PARAMS, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - assert result.exit_code == 0, result.exc_info - - -class TestJobsMetricsStreamCommand(object): - GET_JOB_URL = "https://api.paperspace.io/jobs/getPublicJob" - GET_METRICS_URL = "https://aws-testing.paperspace.io/metrics/api/v1/stream" - BASIC_OPTIONS_COMMAND = [ - "jobs", "metrics", "stream", - "--id", "jstkd2lapucirs", - ] - ALL_OPTIONS_COMMAND = [ - "jobs", "metrics", "stream", - "--id", "jstkd2lapucirs", - "--metric", "gpuMemoryFree", - "--metric", "gpuMemoryUsed", - "--interval", "20s", - "--apiKey", "some_key", - ] - ALL_OPTIONS_COMMAND_WITH_OPTIONS_FILE = [ - "jobs", "metrics", "stream", - "--optionsFile", # path added in test, - ] - - GET_JOB_REQUEST_JSON = {"jobId": "jstkd2lapucirs"} - BASIC_COMMAND_CHART_DESCRIPTOR = '{"chart_names": ["cpuPercentage", "memoryUsage"], "handles": ["jstkd2lapucirs"]' \ - ', "object_type": "mljob", "poll_interval": "30s"}' - ALL_COMMANDS_CHART_DESCRIPTOR = '{"chart_names": ["gpuMemoryFree", "gpuMemoryUsed"], "handles": ["jstkd2lapucirs"' \ - '], "object_type": "mljob", "poll_interval": "20s"}' - - GET_JOB_RESPONSE_JSON = example_responses.GET_JOB_RESPONSE - - EXPECTED_TABLE_1 = """+------------------------------+---------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+------------------------------+---------------+-------------+ -| mljob-ecgrgm7ok8chv-0-worker | | 5881856 | -+------------------------------+---------------+-------------+""" - EXPECTED_TABLE_2 = """+------------------------------+---------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+------------------------------+---------------+-------------+ -| mljob-ecgrgm7ok8chv-0-worker | 0 | 5881856 | -+------------------------------+---------------+-------------+""" - EXPECTED_TABLE_3 = """+------------------------------+---------------+-------------+ -| Pod | cpuPercentage | memoryUsage | -+------------------------------+---------------+-------------+ -| mljob-ecgrgm7ok8chv-0-worker | 0 | 5881857 | -+------------------------------+---------------+-------------+""" - - ALL_OPTIONS_EXPECTED_TABLE_1 = """+------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+------------------------------+---------------+---------------+ -| mljob-ecgrgm7ok8chv-0-worker | | 5881856 | -+------------------------------+---------------+---------------+""" - ALL_OPTIONS_EXPECTED_TABLE_2 = """+------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+------------------------------+---------------+---------------+ -| mljob-ecgrgm7ok8chv-0-worker | 0 | 5881856 | -+------------------------------+---------------+---------------+""" - ALL_OPTIONS_EXPECTED_TABLE_3 = """+------------------------------+---------------+---------------+ -| Pod | gpuMemoryFree | gpuMemoryUsed | -+------------------------------+---------------+---------------+ -| mljob-ecgrgm7ok8chv-0-worker | 0 | 5881857 | -+------------------------------+---------------+---------------+""" - - EXPECTED_STDOUT_WHEN_INVALID_API_KEY_WAS_USED = "Failed to fetch data: Incorrect API Key provided\nForbidden\n" - EXPECTED_STDOUT_WHEN_JOB_WAS_NOT_FOUND = "Failed to fetch data: No such job\n" - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_all_available_metrics_when_metrics_get_command_was_used_with_basic_options( - self, post_patched, create_ws_connection_patched, terminal_printer_cls_patched, - basic_options_metrics_stream_websocket_connection_iterator): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - - ws_connection_instance_mock = mock.MagicMock() - ws_connection_instance_mock.__iter__ = basic_options_metrics_stream_websocket_connection_iterator - create_ws_connection_patched.return_value = ws_connection_instance_mock - - runner = CliRunner() - result = runner.invoke(cli.cli, self.BASIC_OPTIONS_COMMAND) - - terminal_printer_cls_patched().init.assert_called_once() - terminal_printer_cls_patched().rewrite_screen.assert_has_calls([ - mock.call(self.EXPECTED_TABLE_1), - mock.call(self.EXPECTED_TABLE_2), - mock.call(self.EXPECTED_TABLE_3), - ]) - terminal_printer_cls_patched().cleanup.assert_called_once() - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS, - ) - ws_connection_instance_mock.send.assert_called_once_with(self.BASIC_COMMAND_CHART_DESCRIPTOR) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_metrics_when_metrics_get_command_was_used_with_all_options( - self, post_patched, create_ws_connection_patched, terminal_printer_cls_patched, - all_options_metrics_stream_websocket_connection_iterator): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - - ws_connection_instance_mock = mock.MagicMock() - ws_connection_instance_mock.__iter__ = all_options_metrics_stream_websocket_connection_iterator - create_ws_connection_patched.return_value = ws_connection_instance_mock - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - terminal_printer_cls_patched().init.assert_called_once() - terminal_printer_cls_patched().rewrite_screen.assert_has_calls([ - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_1), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_2), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_3), - ]) - terminal_printer_cls_patched().cleanup.assert_called_once() - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - ws_connection_instance_mock.send.assert_called_once_with(self.ALL_COMMANDS_CHART_DESCRIPTOR) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_metrics_when_metrics_get_was_executed_and_options_file_was_used( - self, post_patched, create_ws_connection_patched, terminal_printer_cls_patched, - all_options_metrics_stream_websocket_connection_iterator, - jobs_metrics_stream_config_path): - post_patched.return_value = MockResponse(self.GET_JOB_RESPONSE_JSON) - ws_connection_instance_mock = mock.MagicMock() - ws_connection_instance_mock.__iter__ = all_options_metrics_stream_websocket_connection_iterator - create_ws_connection_patched.return_value = ws_connection_instance_mock - - command = self.ALL_OPTIONS_COMMAND_WITH_OPTIONS_FILE[:] + [jobs_metrics_stream_config_path] - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - terminal_printer_cls_patched().init.assert_called_once() - terminal_printer_cls_patched().rewrite_screen.assert_has_calls([ - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_1), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_2), - mock.call(self.ALL_OPTIONS_EXPECTED_TABLE_3), - ]) - terminal_printer_cls_patched().cleanup.assert_called_once() - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - ws_connection_instance_mock.send.assert_called_once_with(self.ALL_COMMANDS_CHART_DESCRIPTOR) - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_valid_error_message_when_invalid_api_key_was_used( - self, post_patched, create_ws_connection_patched, terminal_printer_cls_patched): - post_patched.return_value = MockResponse({"status": 400, "message": "Invalid API token"}, 400) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert "Failed to fetch data: Invalid API token\n" == result.output, result.exc_info - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - create_ws_connection_patched.assert_not_called() - assert result.exit_code == 0, result.exc_info - - @mock.patch("gradient.commands.common.TerminalPrinter") - @mock.patch("gradient.api_sdk.repositories.common.websocket.create_connection") - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_print_valid_error_message_when_job_was_not_found( - self, post_patched, create_ws_connection_patched, terminal_printer_cls_patched): - post_patched.return_value = MockResponse( - {"error": {"name": "ApplicationError", "status": 404, "message": "No such job"}}, - 404, - ) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.ALL_OPTIONS_COMMAND) - - assert result.output == self.EXPECTED_STDOUT_WHEN_JOB_WAS_NOT_FOUND, result.exc_info - - post_patched.assert_called_once_with( - self.GET_JOB_URL, - json=self.GET_JOB_REQUEST_JSON, - params=None, - data=None, - files=None, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - ) - - create_ws_connection_patched.assert_not_called() - assert result.exit_code == 0, result.exc_info diff --git a/tests/functional/test_tags.py b/tests/functional/test_tags.py index f3524e73..e52f596c 100644 --- a/tests/functional/test_tags.py +++ b/tests/functional/test_tags.py @@ -39,10 +39,6 @@ class TestEntityAddTags(object): @pytest.mark.parametrize( "entity_command, entity, result_entity", [ - # ("deployments", "deployment", "deployment"), - ("experiments", "experiment", "experiment"), - ("experiments hyperparameters", "experiment", "hyperparameter"), - ("jobs", "job", "job"), ("machines", "machine", "machine"), ("models", "mlModel", "ml model"), ("notebooks", "notebook", "notebook"), @@ -80,10 +76,6 @@ def test_should_send_proper_data_and_success(self, get_patched, put_patched, ent @pytest.mark.parametrize( "entity_command, entity, result_entity", [ - # ("deployments", "deployment", "deployment"), - ("experiments", "experiment", "experiment"), - ("experiments hyperparameters", "experiment", "hyperparameter"), - ("jobs", "job", "job"), ("machines", "machine", "machine"), ("models", "mlModel", "ml model"), ("notebooks", "notebook", "notebook"), @@ -147,10 +139,6 @@ class TestEntityRemoveTags(object): @pytest.mark.parametrize( "entity_command, entity, result_entity", [ - # ("deployments", "deployment", "deployment"), - ("experiments", "experiment", "experiment"), - ("experiments hyperparameters", "experiment", "hyperparameter"), - ("jobs", "job", "job"), ("machines", "machine", "machine"), ("models", "mlModel", "ml model"), ("notebooks", "notebook", "notebook"), @@ -189,10 +177,6 @@ def test_should_send_proper_data_and_success(self, get_patched, put_patched, ent @pytest.mark.parametrize( "entity_command, entity, result_entity", [ - # ("deployments", "deployment", "deployment"), - ("experiments", "experiment", "experiment"), - ("experiments hyperparameters", "experiment", "hyperparameter"), - ("jobs", "job", "job"), ("machines", "machine", "machine"), ("models", "mlModel", "ml model"), ("notebooks", "notebook", "notebook"), diff --git a/tests/functional/test_tensorboards.py b/tests/functional/test_tensorboards.py deleted file mode 100644 index fe0656cd..00000000 --- a/tests/functional/test_tensorboards.py +++ /dev/null @@ -1,462 +0,0 @@ -import mock -from click.testing import CliRunner - -from gradient.api_sdk.clients.http_client import default_headers -from gradient.cli import cli -from tests import MockResponse, example_responses - -EXPECTED_HEADERS = default_headers.copy() -EXPECTED_HEADERS["ps_client_name"] = "gradient-cli" - -EXPECTED_HEADERS_WITH_CHANGED_API_KEY = EXPECTED_HEADERS.copy() -EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] = "some_key" - - -class TestTensorboardsCreate(object): - URL = "https://services.paperspace.io/tensorboards/v1/" - COMMAND = [ - "tensorboards", - "create", - "--experiment", "some_experiment_id", - "--experiment", "some_other_experiment_id", - ] - EXPECTED_REQUEST_JSON = {"experiments": ["some_experiment_id", "some_other_experiment_id"]} - EXPECTED_RESPONSE_JSON = example_responses.TENSORBOARD_CREATE_RESPONSE_JSON - EXPECTED_STDOUT = """Created new tensorboard with id: some_id\n""" - - COMMAND_WITH_API_KEY_CHANGED = [ - "tensorboards", - "create", - "--experiment", "some_experiment_id", - "--experiment", "some_other_experiment_id", - "--apiKey", "some_key", - ] - COMMAND_WITH_ALL_OPTIONS = [ - "tensorboards", - "create", - "--experiment", "some_experiment_id", - "--experiment", "some_other_experiment_id", - "--image", "some_image", - "--username", "some_username", - "--password", "some_password", - "--apiKey", "some_key", - ] - - EXPECTED_REQUEST_JSON_WITH_ALL_OPTIONS = { - "experiments": [ - "some_experiment_id", - "some_other_experiment_id", - ], - "image": "some_image", - "username": "some_username", - "password": "some_password" - } - COMMAND_WITH_OPTIONS_FILE_USED = ["tensorboards", "create", "--optionsFile", ] # path added in test - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"title": "Invalid credentials provided"} - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to create resource: Invalid credentials provided\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_valid_request_when_command_was_executed_with_required_options(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - data=None, - files=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_request_with_changed_api_key_when_api_key_option_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_CHANGED) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON, - data=None, - files=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_send_valid_request_when_command_was_executed_with_all_options(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_ALL_OPTIONS) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON_WITH_ALL_OPTIONS, - data=None, - files=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.post") - def test_should_read_options_from_yaml_file(self, post_patched, tensorboards_create_config_path): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE_USED[:] + [tensorboards_create_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON_WITH_ALL_OPTIONS, - data=None, - files=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - -class TestTensorboardsDetail(object): - URL = "https://services.paperspace.io/tensorboards/v1/some_id/" - COMMAND = [ - "tensorboards", - "details", - "--id", "some_id", - ] - EXPECTED_RESPONSE_JSON = example_responses.TENSORBOARD_DETAIL_RESPONSE_JSON - EXPECTED_STDOUT = """+-------+--------------------------------------------------------+ -| ID | some_id | -+-------+--------------------------------------------------------+ -| Image | tensorflow/tensorflow:latest-py3 | -| URL | https://aws-testing.paperspace.io/tensorboard/some_id/ | -| State | Some State | -+-------+--------------------------------------------------------+ -+--------------------------+------------+ -| Experiments ID | State | -+--------------------------+------------+ -| some_experiment_id | Some State | -| some_other_experiment_id | Some State | -+--------------------------+------------+ -""" - # TODO later change response to contain information about state: | State | 1 | - - COMMAND_WITH_API_KEY_CHANGED = [ - "tensorboards", - "details", - "--id", "some_id", - "--apiKey", "some_key", - ] - COMMAND_WITH_OPTIONS_FILE_USED = ["tensorboards", "details", "--optionsFile", ] # path added in test - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"title": "Invalid credentials provided"} - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to create resource: Invalid credentials provided\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_request_when_command_was_executed_with_required_options(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_request_with_changed_api_key_when_api_key_option_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_CHANGED) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file(self, post_patched, tensorboards_details_config_path): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE_USED[:] + [tensorboards_details_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - -class TestTensorboardsList(object): - URL = "https://services.paperspace.io/tensorboards/v1/" - COMMAND = ["tensorboards", "list"] - EXPECTED_RESPONSE_JSON = example_responses.TENSORBOARD_LIST_RESPONSE_JSON - EXPECTED_STDOUT = """+-----------------+----------------------------------------------------------------+------------+ -| ID | URL | STATE | -+-----------------+----------------------------------------------------------------+------------+ -| tbrs2kcjman4ly | https://aws-testing.paperspace.io/tensorboard/tbrs2kcjman4ly/ | Some State | -| tbskzep6d9po04d | https://aws-testing.paperspace.io/tensorboard/tbskzep6d9po04d/ | Some State | -| tbsaq6hggzxcnet | https://aws-testing.paperspace.io/tensorboard/tbsaq6hggzxcnet/ | Some State | -| tbwuzalec7ik58 | https://aws-testing.paperspace.io/tensorboard/tbwuzalec7ik58/ | Some State | -+-----------------+----------------------------------------------------------------+------------+ -""" - - COMMAND_WITH_API_KEY_CHANGED = ["tensorboards", "list", "--apiKey", "some_key"] - COMMAND_WITH_OPTIONS_FILE_USED = ["tensorboards", "list", "--optionsFile", ] # path added in test - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"title": "Invalid credentials provided"} - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to create resource: Invalid credentials provided\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_valid_request_when_command_was_executed_with_required_options(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_request_with_changed_api_key_when_api_key_option_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_CHANGED) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_read_options_from_yaml_file(self, post_patched, tensorboards_details_config_path): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE_USED[:] + [tensorboards_details_config_path] - - runner = CliRunner() - result = runner.invoke(cli.cli, command) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - post_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - -class TestTensorboardsAddExperiment(object): - URL = "https://services.paperspace.io/tensorboards/v1/some_id" - COMMAND = ["tensorboards", "add-experiments", "--id", "some_id", "--experiment", "some_third_experiment_id"] - - EXPECTED_RESPONSE_JSON = example_responses.TENSORBOARD_UPDATE_RESPONSE_JSON - EXPECTED_REQUEST_JSON = {"added_experiments": ["some_third_experiment_id"], "removed_experiments": []} - EXPECTED_STDOUT = """+-------+--------------------------------------------------------+ -| ID | some_id | -+-------+--------------------------------------------------------+ -| Image | tensorflow/tensorflow:latest-py3 | -| URL | https://aws-testing.paperspace.io/tensorboard/some_id/ | -| State | Some State | -+-------+--------------------------------------------------------+ -+--------------------------+------------+ -| Experiments ID | State | -+--------------------------+------------+ -| some_experiment_id | Some State | -| some_other_experiment_id | Some State | -| some_third_experiment_id | Some State | -+--------------------------+------------+ -""" - - COMMAND_WITH_API_KEY_CHANGED = [ - "tensorboards", "add-experiments", "--id", "some_id", - "--experiment", "some_third_experiment_id", "--apiKey", "some_key" - ] - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"title": "Invalid credentials provided"} - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to update resource: Invalid credentials provided\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_valid_request_when_command_was_executed_with_required_options(self, put_patched): - put_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - put_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - data=None, - ) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_request_with_changed_api_key_when_api_key_option_was_used(self, put_patched): - put_patched.return_value = MockResponse(self.RESPONSE_JSON_WITH_WRONG_API_TOKEN, status_code=401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_CHANGED) - - assert result.output == self.EXPECTED_STDOUT_WITH_WRONG_API_TOKEN, result.exc_info - put_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON, - params=None, - data=None, - ) - - -class TestTensorboardsRemoveExperiment(object): - URL = "https://services.paperspace.io/tensorboards/v1/some_id" - COMMAND = ["tensorboards", "remove-experiments", "--id", "some_id", "--experiment", "some_other_experiment_id"] - - EXPECTED_RESPONSE_JSON = example_responses.TENSORBOARD_UPDATE_REMOVE_RESPONSE_JSON - EXPECTED_REQUEST_JSON = {"added_experiments": [], "removed_experiments": ["some_other_experiment_id"]} - EXPECTED_STDOUT = """+-------+--------------------------------------------------------+ -| ID | some_id | -+-------+--------------------------------------------------------+ -| Image | tensorflow/tensorflow:latest-py3 | -| URL | https://aws-testing.paperspace.io/tensorboard/some_id/ | -| State | Some State | -+-------+--------------------------------------------------------+ -+--------------------+------------+ -| Experiments ID | State | -+--------------------+------------+ -| some_experiment_id | Some State | -+--------------------+------------+ -""" - - COMMAND_WITH_API_KEY_CHANGED = [ - "tensorboards", "remove-experiments", "--id", "some_id", - "--experiment", "some_other_experiment_id", "--apiKey", "some_key" - ] - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"title": "Invalid credentials provided"} - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to update resource: Invalid credentials provided\n" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_valid_request_when_command_was_executed_with_required_options(self, put_patched): - put_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - put_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON, - params=None, - data=None, - ) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.put") - def test_should_send_request_with_changed_api_key_when_api_key_option_was_used(self, put_patched): - put_patched.return_value = MockResponse(self.RESPONSE_JSON_WITH_WRONG_API_TOKEN, status_code=401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_CHANGED) - - assert result.output == self.EXPECTED_STDOUT_WITH_WRONG_API_TOKEN, result.exc_info - put_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=self.EXPECTED_REQUEST_JSON, - params=None, - data=None, - ) - - -class TestTensorboardsDelete(object): - URL = "https://services.paperspace.io/tensorboards/v1/some_id" - COMMAND = ["tensorboards", "delete", "--id", "some_id"] - - EXPECTED_RESPONSE_JSON = example_responses.TENSORBOARD_DELETE_RESPONSE_JSON - EXPECTED_REQUEST_JSON = {"added_experiments": [], "removed_experiments": ["some_other_experiment_id"]} - EXPECTED_STDOUT = "Delete tensorboard (some_id) ended with success\n" - - COMMAND_WITH_API_KEY_CHANGED = [ - "tensorboards", "delete", "--id", "some_id", "--apiKey", "some_key" - ] - - RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"title": "Invalid credentials provided"} - EXPECTED_STDOUT_WITH_WRONG_API_TOKEN = "Failed to delete resource: Invalid credentials provided\n" - - RESPONSE_JSON_WITH_WRONG_ACCESS = { - "error": "You don't have access to tensorboard some_id", - "title": "401 Unauthorized" - } - EXPECTED_WRONG_ACCESS_RESPONSE = """Failed to delete resource: You don't have access to tensorboard some_id -401 Unauthorized -""" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_send_valid_request_when_command_was_executed_with_required_options(self, delete_patched): - delete_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON, status_code=200) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_STDOUT, result.exc_info - delete_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None - ) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_send_request_with_changed_api_key_when_api_key_option_was_used(self, delete_patched): - delete_patched.return_value = MockResponse(self.RESPONSE_JSON_WITH_WRONG_API_TOKEN, status_code=401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_CHANGED) - - assert result.output == self.EXPECTED_STDOUT_WITH_WRONG_API_TOKEN, result.exc_info - delete_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, - json=None, - params=None - ) - - @mock.patch("gradient.api_sdk.clients.http_client.requests.delete") - def test_should_send_request_to_remove_tensorboard_without_proper_access(self, delete_patched): - delete_patched.return_value = MockResponse(self.RESPONSE_JSON_WITH_WRONG_ACCESS, status_code=401) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND) - - assert result.output == self.EXPECTED_WRONG_ACCESS_RESPONSE, result.exc_info - delete_patched.assert_called_once_with( - self.URL, - headers=EXPECTED_HEADERS, - json=None, - params=None - ) - assert EXPECTED_HEADERS["X-API-Key"] != "some_key" diff --git a/tests/functional/test_uploaders.py b/tests/functional/test_uploaders.py deleted file mode 100644 index b9a935e1..00000000 --- a/tests/functional/test_uploaders.py +++ /dev/null @@ -1,155 +0,0 @@ -import os -import sys -import tempfile - -import mock -from pytest import fixture - -from gradient.api_sdk.s3_uploader import S3FileUploader, ExperimentFileUploader, S3ModelFileUploader, \ - ExperimentWorkspaceDirectoryUploader, S3ModelUploader - -open_path = "builtins.open" -if sys.version_info[0] < 3: - open_path = "__builtin__.open" - - -@fixture -def client(): - client = mock.MagicMock() - client.get = mock.MagicMock() - client.put = mock.MagicMock() - client.post = mock.MagicMock() - return client - - -@fixture -def file_uploader(client): - mock_get_client = mock.MagicMock() - mock_get_client.return_value = client - - mock_encoder_class = mock.Mock(spec=S3FileUploader.DEFAULT_MULTIPART_ENCODER_CLS) - uploader = S3FileUploader(multipart_encoder_cls=mock_encoder_class) - uploader._get_client = mock_get_client - return uploader - - -def get_ps_exp_valid_response(mock_url): - mock_response = mock.MagicMock() - mock_response.status_code = 200 - mock_response.ok = True - mock_response.json.return_value = { - "message": "success", - "data": { - "url": mock_url, - "bucket_name": "bucketname", - "fields": { - "key": "foo_key" - } - } - } - return mock_response - - -class TestS3FileUploader(object): - def test_should_post_file_data_and_return_valid_url(self, client, file_uploader): - upload_url = "s3://url" - _mock_open = mock.mock_open(read_data="data") - with mock.patch(open_path, _mock_open) as mock_file: - file_uploader.upload("filename", upload_url, {"key": "foo"}) - - file_uploader._get_client.assert_called_with(upload_url) - client.post.assert_called() - - -class TestExperimentFileUploader(object): - def test_should_post_file_data_and_return_valid_url(self, client, file_uploader): - upload_url = "s3://url" - mock_response = get_ps_exp_valid_response(upload_url) - - mock_api_client = mock.MagicMock() - mock_api_client.get.return_value = mock_response - - uploader = ExperimentFileUploader("api_key", uploader=file_uploader) - uploader.experiments_api = mock_api_client - - _mock_open = mock.mock_open(read_data="data") - with mock.patch(open_path, _mock_open) as mock_file: - uploader.upload("foo", "pjHandle", "clHandle") - - file_uploader._get_client.assert_called_with(upload_url) - client.post.assert_called() - - -class TestS3ModelFileUploader(object): - def test_should_post_file_data_and_return_valid_url(self, client, file_uploader): - upload_url = "s3://url" - - mock_response = mock.MagicMock() - mock_response.status_code = 200 - mock_response.ok = True - mock_response.json.return_value = upload_url - - mock_api_client = mock.MagicMock() - mock_api_client.get.return_value = mock_response - - uploader = S3ModelFileUploader("api_key", s3uploader=file_uploader) - uploader.ps_api_client = mock_api_client - _mock_open = mock.mock_open(read_data="data") - - with mock.patch(open_path, _mock_open) as mock_file: - uploader.upload("foo", "mdHandle") - - file_uploader._get_client.assert_called_with(upload_url) - client.post.assert_called() - - -class TestS3ModelUploader(object): - - @mock.patch("os.path.isdir") - @mock.patch("gradient.api_sdk.archivers.ZipArchiver.archive") - def test_should_post_file_data_and_return_valid_url(self, archive_patched, is_dir_patched, client, file_uploader): - upload_url = "s3://url" - - mock_response = mock.MagicMock() - mock_response.status_code = 200 - mock_response.ok = True - mock_response.json.return_value = upload_url - - mock_api_client = mock.MagicMock() - mock_api_client.get.return_value = mock_response - - uploader = S3ModelUploader("api_key", s3uploader=file_uploader) - uploader.ps_api_client = mock_api_client - _mock_open = mock.mock_open(read_data="data") - - is_dir_patched.return_value = True - - with mock.patch(open_path, _mock_open) as mock_file: - uploader.upload("foo", "mdHandle") - - archive_patched.assert_called_once_with("foo", os.path.join(tempfile.gettempdir(), "model.zip")) - file_uploader._get_client.assert_called_with(upload_url) - client.post.assert_called() - - -class TestExperimentWorkspaceDirectoryUploader(object): - - def test_should_post_file_data_and_return_valid_url(self, client, file_uploader): - upload_url = "s3://url" - - mock_response = get_ps_exp_valid_response(upload_url) - - mock_api_client = mock.MagicMock() - mock_api_client.get.return_value = mock_response - project_uploader = ExperimentFileUploader("api_key", uploader=file_uploader) - project_uploader.experiments_api = mock_api_client - - uploader = ExperimentWorkspaceDirectoryUploader(api_key="api_key", archiver=mock.MagicMock(), - project_uploader=project_uploader) - - _mock_open = mock.mock_open(read_data="data") - with mock.patch(open_path, _mock_open) as mock_file: - uploader.upload("foo", "mdHandle") - - file_uploader._get_client.assert_called_with(upload_url) - client.post.assert_called() diff --git a/tests/unit/test_archiver_class.py b/tests/unit/test_archiver_class.py index 19452e4b..ac3a6ecf 100644 --- a/tests/unit/test_archiver_class.py +++ b/tests/unit/test_archiver_class.py @@ -63,7 +63,8 @@ def test_should_get_valid_excluded_paths_when_empty_list_was_passed_to_get_exclu def test_should_get_valid_excluded_paths_when_list_of_files_was_passed_to_get_excluded_paths(self): archiver = gradient.api_sdk.archivers.ZipArchiver() - excluded = archiver.get_excluded_paths(["some_file", "some_dir/some_other_file"]) + excluded = archiver.get_excluded_paths( + ["some_file", "some_dir/some_other_file"]) assert excluded == { os.path.join(".git", "*"), @@ -127,7 +128,8 @@ def test_should_add_files_to_archive_when_zip_archiver_was_used(self): zip_file.extractall(temp_dir_for_extracted_files) archiver2 = gradient.api_sdk.archivers.ZipArchiver() archiver2.default_excluded_paths = [] - paths_in_extracted_dir = archiver2.get_file_paths(temp_dir_for_extracted_files) + paths_in_extracted_dir = archiver2.get_file_paths( + temp_dir_for_extracted_files) finally: shutil.rmtree(temp_dir) @@ -144,54 +146,3 @@ def test_should_upload_file_to_s3_and_get_bucket_url_when_upload_was_executed(se uploader.upload(file_path, "s3://some.url", {"key": "some_key"}) post_patched.assert_called_once() - - -class TestExperimentWorkspaceDirectoryUploader(object): - WORKSPACE_DIR_PATH = "/some/workspace/dir/path/" - TEMP_DIR_PATH = "/some/temp/dir/path/" - - @mock.patch("gradient.api_sdk.s3_uploader.ExperimentFileUploader") - @mock.patch("gradient.api_sdk.s3_uploader.ZipArchiver") - def test_class_with_default_params(self, zip_archiver_cls, s3_project_file_uploader_cls): - zip_archiver = mock.MagicMock() - zip_archiver_cls.return_value = zip_archiver - s3_project_file_uploader = mock.MagicMock() - s3_project_file_uploader.upload.return_value = "s3://url/to/bucket" - s3_project_file_uploader_cls.return_value = s3_project_file_uploader - archive_path = os.path.join(tempfile.gettempdir(), "temp.zip") - - uploader = gradient.ExperimentWorkspaceDirectoryUploader("some_api_key", ps_client_name="some_client_name") - bucket_url = uploader.upload(self.WORKSPACE_DIR_PATH, "some_project_id") - - s3_project_file_uploader_cls.assert_called_once_with("some_api_key", ps_client_name="some_client_name") - zip_archiver.archive.assert_called_once_with(self.WORKSPACE_DIR_PATH, archive_path, exclude=None) - s3_project_file_uploader.upload.assert_called_once_with(archive_path, "some_project_id") - assert bucket_url == "s3://url/to/bucket" - - def test_should_run_upload_(self): - zip_archiver = mock.MagicMock() - s3_project_file_uploader = mock.MagicMock() - s3_project_file_uploader.upload.return_value = "s3://url/to/bucket" - temp_file_name = "some_temp_file_name.zip" - archive_path = os.path.join(self.TEMP_DIR_PATH, temp_file_name) - - uploader = gradient.ExperimentWorkspaceDirectoryUploader( - "some_api_key", - temp_dir=self.TEMP_DIR_PATH, - archiver=zip_archiver, - project_uploader=s3_project_file_uploader, - ) - bucket_url = uploader.upload( - self.WORKSPACE_DIR_PATH, - "some_project_id", - exclude=["file1", "dir/file2"], - temp_file_name=temp_file_name, - ) - - zip_archiver.archive.assert_called_once_with( - self.WORKSPACE_DIR_PATH, - os.path.join(self.TEMP_DIR_PATH, temp_file_name), - exclude=["file1", "dir/file2"], - ) - s3_project_file_uploader.upload.assert_called_once_with(archive_path, "some_project_id") - assert bucket_url == "s3://url/to/bucket" diff --git a/tests/unit/test_deployment_autoscaling_option_validator.py b/tests/unit/test_deployment_autoscaling_option_validator.py deleted file mode 100644 index c054e709..00000000 --- a/tests/unit/test_deployment_autoscaling_option_validator.py +++ /dev/null @@ -1,37 +0,0 @@ -import click -import pytest - -# TODO: This import was messing up how click was accessing the new version of deployments cli -# Since we're removing it soon, I'm just going to comment these tests out -# from gradient.cli.deployments import validate_autoscaling_metric_or_resource - - -# def test_should_return_none_if_none_was_provided(): -# rv = validate_autoscaling_metric_or_resource(1, 2, None, "Metric") -# assert rv is None - - -# def test_should_raise_bad_parameter_if_provided_value_was_not_valid(): -# with pytest.raises(click.BadParameter): -# validate_autoscaling_metric_or_resource(1, 2, ["cpu"], "Metric") - - -# def test_should_return_valid_structure_if_value_format_was_correct(): -# expected_value = ( -# { -# "type": "Metric", -# "name": "cpu", -# "value_type": "target", -# "value": 60, -# }, -# { -# "type": "Metric", -# "name": "gpu", -# "value_type": "target", -# "value": 40, -# }, -# ) - -# rv = validate_autoscaling_metric_or_resource(1, 2, ["cpu/target:60", "gpu/target:40"], "Metric") - -# assert rv == expected_value diff --git a/tests/unit/test_experiments_commands.py b/tests/unit/test_experiments_commands.py deleted file mode 100644 index 87bcd36d..00000000 --- a/tests/unit/test_experiments_commands.py +++ /dev/null @@ -1,79 +0,0 @@ -import pytest -from click import BadParameter - -from gradient.commands.experiments import BaseCreateExperimentCommandMixin - - -class TestHandleDatasetDataMethod(object): - def test_should_transform_lists_of_datasets_parameters_to_list_of_dicts_and_leave_other_data_intact(self): - input_data = { - "some_key": "some_value", - "dataset_uri_list": ["uri1", "uri2"], - "dataset_id_list": [None, None, "test"], - "dataset_name_list": ["name1", None, "name2"], - "dataset_access_key_id_list": [None, "key_id"], - "dataset_secret_access_key_list": ["none", "secret"], - # "dataset_version_id_list": [] # this key does not exist if parameter was not used - "dataset_etag_list": ["etag"], - "dataset_volume_kind_list": [], - } - expected_data = { - "some_key": "some_value", - "datasets": [ - { - "uri": "uri1", - "id": None, - "name": "name1", - "aws_access_key_id": None, - "aws_secret_access_key": None, - "aws_endpoint": None, - "version_id": None, - "etag": "etag", - "volume_kind": None, - "volume_size": None, - }, - { - "uri": "uri2", - "id": None, - "name": None, - "aws_access_key_id": "key_id", - "aws_secret_access_key": "secret", - "aws_endpoint": None, - "version_id": None, - "etag": None, - "volume_kind": None, - "volume_size": None, - }, - { - "uri": None, - "id": "test", - "name": "name2", - "aws_access_key_id": None, - "aws_secret_access_key": None, - "aws_endpoint": None, - "version_id": None, - "etag": None, - "volume_kind": None, - "volume_size": None, - }, - ] - - } - - BaseCreateExperimentCommandMixin._handle_dataset_data(input_data) - - assert expected_data == input_data - - def test_should_raise_exception_if_number_of_any_dataset_parameter_is_greater_than_number_of_dataset_uris(self): - input_data = { - "some_key": "some_value", - "dataset_uri_list": ["uri1", "uri2"], - "dataset_name_list": ["name1", None], - "dataset_access_key_id_list": [None, "key_id"], - "dataset_secret_access_key_list": ["none", "secret"], - # "dataset_version_id_list": [] # this key does not exist if parameter was not used - "dataset_etag_list": ["etag", "etag2", "etag3"], # more etags than uris - } - - with pytest.raises(BadParameter): - BaseCreateExperimentCommandMixin._handle_dataset_data(input_data) diff --git a/tests/unit/test_tensorboard_handler.py b/tests/unit/test_tensorboard_handler.py deleted file mode 100644 index cceffc85..00000000 --- a/tests/unit/test_tensorboard_handler.py +++ /dev/null @@ -1,101 +0,0 @@ -import mock -import pytest - -from gradient.commands.experiments import TensorboardHandler - - -@pytest.fixture() -def tensorboard_handler(): - tensorboard_handler = TensorboardHandler("some_key") - return tensorboard_handler - - -class TestTensorboardHandler(object): - def test_should_add_experiment_to_tensorboard_when_tensorboard_is_a_string(self, tensorboard_handler): - tensorboard_handler._add_experiment_to_tensorboard = mock.MagicMock() - tensorboard_handler._create_tensorboard_with_experiment = mock.MagicMock() - tensorboard_handler._get_tensorboards = mock.MagicMock() - - tensorboard_handler.maybe_add_to_tensorboard("some_tensorboard_id", "some_experiment_id") - - tensorboard_handler._add_experiment_to_tensorboard.assert_called_once_with("some_tensorboard_id", - "some_experiment_id") - tensorboard_handler._create_tensorboard_with_experiment.assert_not_called() - tensorboard_handler._get_tensorboards.assert_not_called() - - def test_should_add_experiment_to_tensorboard_when_tensorboard_is_not_a_string_and_one_tb_exists( - self, tensorboard_handler): - tensorboard_handler._add_experiment_to_tensorboard = mock.MagicMock() - tensorboard_handler._create_tensorboard_with_experiment = mock.MagicMock() - fake_tensorboard = mock.Mock() - fake_tensorboard.id = "some_tensorboard_id" - tensorboards = [fake_tensorboard] - tensorboard_handler._get_tensorboards = mock.MagicMock(return_value=tensorboards) - - tensorboard_handler.maybe_add_to_tensorboard(True, "some_experiment_id") - - tensorboard_handler._get_tensorboards.assert_called_once() - tensorboard_handler._add_experiment_to_tensorboard.assert_called_once_with("some_tensorboard_id", - "some_experiment_id") - tensorboard_handler._create_tensorboard_with_experiment.assert_not_called() - - def test_should_add_experiment_to_tensorboard_when_tensorboard_is_not_a_string_and_more_than_one_tb_exists( - self, tensorboard_handler): - tensorboard_handler._add_experiment_to_tensorboard = mock.MagicMock() - tensorboard_handler._create_tensorboard_with_experiment = mock.MagicMock() - fake_tensorboard = mock.Mock() - fake_tensorboard.id = "some_tensorboard_id" - tensorboards = [fake_tensorboard, fake_tensorboard] - tensorboard_handler._get_tensorboards = mock.MagicMock(return_value=tensorboards) - - tensorboard_handler.maybe_add_to_tensorboard(True, "some_experiment_id") - - tensorboard_handler._get_tensorboards.assert_called_once() - tensorboard_handler._add_experiment_to_tensorboard.assert_not_called() - tensorboard_handler._create_tensorboard_with_experiment.assert_called_once_with("some_experiment_id") - - def test_should_add_experiment_to_tensorboard_when_tensorboard_is_not_a_string_and_no_tb_exists( - self, tensorboard_handler): - tensorboard_handler._add_experiment_to_tensorboard = mock.MagicMock() - tensorboard_handler._create_tensorboard_with_experiment = mock.MagicMock() - tensorboards = [] - tensorboard_handler._get_tensorboards = mock.MagicMock(return_value=tensorboards) - - tensorboard_handler.maybe_add_to_tensorboard(True, "some_experiment_id") - - tensorboard_handler._get_tensorboards.assert_called_once() - tensorboard_handler._add_experiment_to_tensorboard.assert_not_called() - tensorboard_handler._create_tensorboard_with_experiment.assert_called_once_with("some_experiment_id") - - @mock.patch("gradient.commands.tensorboards.AddExperimentToTensorboard") - def test_should_execute_proper_command_when__add_experiment_to_tensorboard_was_executed( - self, command_class_patched, tensorboard_handler): - command = mock.MagicMock() - command_class_patched.return_value = command - - tensorboard_handler._add_experiment_to_tensorboard("tensorboard_id", "experiment_id") - - command_class_patched.assert_called_once_with(api_key="some_key") - command.execute.assert_called_once_with("tensorboard_id", ["experiment_id"]) - - @mock.patch("gradient.commands.experiments.TensorboardClient") - def test_should_execute_proper_command_when__get_tensorboards_was_executed( - self, client_class_patched, tensorboard_handler): - client = mock.MagicMock() - client_class_patched.return_value = client - - tensorboard_handler._get_tensorboards() - - client_class_patched.assert_called_once_with(api_key="some_key", logger=tensorboard_handler.logger) - client.list.assert_called_once_with() - - @mock.patch("gradient.commands.tensorboards.CreateTensorboardCommand") - def test_should_execute_proper_command_when__create_tensorboard_with_experiment( - self, client_class_patched, tensorboard_handler): - command = mock.MagicMock() - client_class_patched.return_value = command - - tensorboard_handler._create_tensorboard_with_experiment("experiment_id") - - client_class_patched.assert_called_once_with(api_key=tensorboard_handler.api_key) - command.execute.assert_called_once_with(experiments=["experiment_id"]) diff --git a/tests/unit/test_workspace.py b/tests/unit/test_workspace.py deleted file mode 100644 index 52b72b30..00000000 --- a/tests/unit/test_workspace.py +++ /dev/null @@ -1,90 +0,0 @@ -import os - -import mock -import pytest - -import gradient.api_sdk.utils -import gradient.cliutils -from gradient.api_sdk.workspace import S3WorkspaceHandler -from gradient.cli_constants import CLI_PS_CLIENT_NAME - -MOCK_BUCKET_NAME = 'bucket_name' -MOCK_OBJECT_KEY = 'object_key' -mock_upload_data = { - "bucket_name": MOCK_BUCKET_NAME, - "fields": { - "key": MOCK_OBJECT_KEY - } -} - -mock_upload_response = { - "message": "success", - "data": mock_upload_data -} - - -@pytest.fixture -def workspace_handler(): - s3_workspace_handler = S3WorkspaceHandler("some_key", client_name=CLI_PS_CLIENT_NAME) - s3_workspace_handler._upload = mock.MagicMock(return_value="s3://{}/{}".format(MOCK_BUCKET_NAME, MOCK_OBJECT_KEY)) - return s3_workspace_handler - - -class TestWorkspace(object): - @mock.patch("gradient.utils.PathParser.parse_path", return_value=gradient.api_sdk.utils.PathParser.S3_URL) - @mock.patch("gradient.api_sdk.workspace.S3WorkspaceHandler._upload") - def test_dont_upload_if_s3_url_provided(self, _, __, workspace_handler): - workspace_handler._upload = mock.MagicMock() - - workspace_handler.handle({'workspace': 's3://some-path'}) - - workspace_handler._upload.assert_not_called() - - def test_zip_files_and_receive_s3_response_when_no_dir_provided(self, workspace_handler): - archive_name = 'foo.zip' - - workspace_handler._zip_workspace = mock.MagicMock() - workspace_handler._zip_workspace.return_value = archive_name - - response_url = workspace_handler.handle({"projectHandle": "some_project_id"}) - - workspace_handler._zip_workspace.assert_not_called() - workspace_handler._upload.assert_not_called() - assert response_url is None - - @mock.patch("gradient.utils.PathParser.is_local_dir", return_value=True) - def test_zip_files_and_receive_s3_response_when_workspace_dir_provided(self, _, workspace_handler): - archive_name = 'foo.zip' - - workspace_handler._zip_workspace = mock.MagicMock() - workspace_handler._zip_workspace.return_value = archive_name - - response_url = workspace_handler.handle({"projectHandle": "some_project_id", "workspace": "foo/bar"}) - - workspace_handler._zip_workspace.assert_called_once() - workspace_handler._upload.assert_called_once() - workspace_handler._upload.assert_called_with(archive_name, "some_project_id", cluster_id=None) - assert response_url == 's3://{}/{}'.format(MOCK_BUCKET_NAME, MOCK_OBJECT_KEY) - - @mock.patch("gradient.utils.PathParser.is_local_zip_file", return_value=True) - def test_dont_zip_files_and_receive_s3_response_when_workspace_archive_provided(self, _, workspace_handler): - workspace_handler._zip_workspace = mock.MagicMock() - - response_url = workspace_handler.handle({"projectHandle": "some_project_id", "workspace": "foo.zip"}) - - workspace_handler._zip_workspace.assert_not_called() - workspace_handler._upload.assert_called_once() - workspace_handler._upload.assert_called_with(os.path.abspath('foo.zip'), "some_project_id", cluster_id=None) - assert response_url == 's3://{}/{}'.format(MOCK_BUCKET_NAME, MOCK_OBJECT_KEY) - - @mock.patch("gradient.utils.PathParser.is_local_zip_file", return_value=True) - def test_dont_zip_files_and_receive_s3_response_when_workspace_archive_provided_with_workspace(self, _, - workspace_handler): - workspace_handler._zip_workspace = mock.MagicMock() - - response_url = workspace_handler.handle({'projectHandle': 'some_project_id', 'workspace': 'foo.zip'}) - - workspace_handler._zip_workspace.assert_not_called() - workspace_handler._upload.assert_called_once() - workspace_handler._upload.assert_called_with(os.path.abspath('foo.zip'), "some_project_id", cluster_id=None) - assert response_url == 's3://{}/{}'.format(MOCK_BUCKET_NAME, MOCK_OBJECT_KEY) From ac65d8f74d67c79e7e5ffe5ebf6da082bd5a01f7 Mon Sep 17 00:00:00 2001 From: Ben Batha Date: Tue, 12 Apr 2022 12:02:36 -0400 Subject: [PATCH 2/3] fixup! feat: retire gradient v1 and v2 --- gradient/api_sdk/clients/model_client.py | 8 +- gradient/api_sdk/models/model.py | 2 - gradient/api_sdk/repositories/models.py | 19 +- gradient/api_sdk/serializers/model.py | 16 +- gradient/api_sdk/serializers/utils.py | 31 +- gradient/api_sdk/utils.py | 25 +- gradient/cli/clusters.py | 2 +- gradient/cli/machines.py | 25 +- gradient/cli/models.py | 30 +- gradient/cli/notebooks.py | 43 +- gradient/cli/projects.py | 15 +- gradient/cli/validators.py | 17 - gradient/commands/models.py | 6 +- tests/conftest.py | 252 -- tests/example_responses.py | 4492 +--------------------- tests/functional/test_models.py | 194 +- 16 files changed, 307 insertions(+), 4870 deletions(-) diff --git a/gradient/api_sdk/clients/model_client.py b/gradient/api_sdk/clients/model_client.py index d913dd7f..e15cdca8 100644 --- a/gradient/api_sdk/clients/model_client.py +++ b/gradient/api_sdk/clients/model_client.py @@ -7,10 +7,9 @@ class ModelsClient(TagsSupportMixin, BaseClient): entity = "mlModel" - def list(self, experiment_id=None, project_id=None, tags=None): + def list(self, project_id=None, tags=None): """Get list of models - :param str experiment_id: Experiment ID to filter models :param str project_id: Project ID to filter models :param list[str]|tuple[str] tags: tags to filter models @@ -18,7 +17,7 @@ def list(self, experiment_id=None, project_id=None, tags=None): :rtype: list[models.Model] """ repository = self.build_repository(repositories.ListModels) - models_list = repository.list(experiment_id=experiment_id, project_id=project_id, tags=tags) + models_list = repository.list(project_id=project_id, tags=tags) return models_list def delete(self, model_id): @@ -114,5 +113,6 @@ def get_model_files(self, model_id, links=False, size=False): :rtype: list[models.ModelFile] """ repository = self.build_repository(repositories.ListModelFiles) - models_list = repository.list(model_id=model_id, links=links, size=size) + models_list = repository.list( + model_id=model_id, links=links, size=size) return models_list diff --git a/gradient/api_sdk/models/model.py b/gradient/api_sdk/models/model.py index 5ce431d3..955d6d31 100644 --- a/gradient/api_sdk/models/model.py +++ b/gradient/api_sdk/models/model.py @@ -9,7 +9,6 @@ class Model(object): :param str id: :param str name: :param str project_id: - :param str experiment_id: :param str cluster_id: :param list tags: :param str model_type: @@ -23,7 +22,6 @@ class Model(object): id = attr.ib(type=str, default=None) name = attr.ib(type=str, default=None) project_id = attr.ib(type=str, default=None) - experiment_id = attr.ib(type=str, default=None) cluster_id = attr.ib(type=str, default=None) tags = attr.ib(type=list, factory=list) model_type = attr.ib(type=str, default=None) diff --git a/gradient/api_sdk/repositories/models.py b/gradient/api_sdk/repositories/models.py index d46cc72c..ac35e1ad 100644 --- a/gradient/api_sdk/repositories/models.py +++ b/gradient/api_sdk/repositories/models.py @@ -26,20 +26,18 @@ def get_request_url(self, **kwargs): return "/mlModels/getModelList/" def _parse_objects(self, data, **kwargs): - experiments = [] + models = [] for model_dict in data["modelList"]: - experiment = self._parse_object(model_dict) - experiments.append(experiment) + model = self._parse_object(model_dict) + models.append(model) - return experiments + return models def _get_request_params(self, kwargs): return {"limit": -1} def _get_request_json(self, kwargs): filters = {} - if kwargs.get("experiment_id"): - filters["experimentId"] = kwargs.get("experiment_id") if kwargs.get("project_id"): filters["projectId"] = kwargs.get("project_id") @@ -89,7 +87,8 @@ def _get_request_json(self, instance_dict): return None def create(self, instance, data=None, path=None, cluster_id=None): - model_id = super(UploadModel, self).create(instance, data=data, path=path) + model_id = super(UploadModel, self).create( + instance, data=data, path=path) try: self._upload_model(path, model_id, cluster_id=cluster_id) except BaseException: @@ -105,7 +104,8 @@ def _upload_model(self, file_path, model_id, cluster_id=None): model_uploader.upload(file_path, model_id, cluster_id=cluster_id) def _delete_model(self, model_id): - repository = DeleteModel(self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) + repository = DeleteModel( + self.api_key, logger=self.logger, ps_client_name=self.ps_client_name) repository.delete(model_id) @@ -123,7 +123,8 @@ def _get_request_json(self, instance_dict): return None def create(self, instance, data=None, path=None): - model_id = super(CreateModel, self).create(instance, data=data, path=path) + model_id = super(CreateModel, self).create( + instance, data=data, path=path) return model_id diff --git a/gradient/api_sdk/serializers/model.py b/gradient/api_sdk/serializers/model.py index a46d6aac..9a2122ca 100644 --- a/gradient/api_sdk/serializers/model.py +++ b/gradient/api_sdk/serializers/model.py @@ -9,17 +9,21 @@ class Model(BaseSchema): id = marshmallow.fields.Str() name = marshmallow.fields.Str() - project_id = marshmallow.fields.Str(dump_to="projectId", load_from="projectId") - experiment_id = marshmallow.fields.Str(dump_to="experimentId", load_from="experimentId") + project_id = marshmallow.fields.Str( + dump_to="projectId", load_from="projectId") tags = marshmallow.fields.List(marshmallow.fields.Str(), load_only=True) - model_type = marshmallow.fields.Str(dump_to="modelType", load_from="modelType") + model_type = marshmallow.fields.Str( + dump_to="modelType", load_from="modelType") url = marshmallow.fields.Str() - model_path = marshmallow.fields.Str(dump_to="modelPath", load_from="modelPath") - deployment_state = marshmallow.fields.Str(dump_to="deploymentState", load_from="deploymentState") + model_path = marshmallow.fields.Str( + dump_to="modelPath", load_from="modelPath") + deployment_state = marshmallow.fields.Str( + dump_to="deploymentState", load_from="deploymentState") summary = marshmallow.fields.Dict() detail = marshmallow.fields.Dict() notes = marshmallow.fields.Str() - dataset_ref = marshmallow.fields.Str(dump_to="datasetRef", load_from="datasetRef") + dataset_ref = marshmallow.fields.Str( + dump_to="datasetRef", load_from="datasetRef") class ModelFileSchema(BaseSchema): diff --git a/gradient/api_sdk/serializers/utils.py b/gradient/api_sdk/serializers/utils.py index 57b8de50..3856dcd1 100644 --- a/gradient/api_sdk/serializers/utils.py +++ b/gradient/api_sdk/serializers/utils.py @@ -3,38 +3,13 @@ from .base import BaseSchema from .. import models -from ..sdk_exceptions import GradientSdkError -from ..serializers import SingleNodeExperimentSchema, MultiNodeExperimentSchema, HyperparameterSchema, \ - MpiMultiNodeExperimentSchema - - -EXPERIMENT_ID_TO_EXPERIMENT_SERIALIZER_MAPPING = { - 1: SingleNodeExperimentSchema, - 2: MultiNodeExperimentSchema, - 3: MpiMultiNodeExperimentSchema, - 4: HyperparameterSchema, -} - - -def get_serializer_for_experiment(experiment_dict): - """ - :param dict experiment_dict: - :rtype: serializers.BaseExperimentSchema - """ - experiment_type_id = experiment_dict["experimentTypeId"] - - try: - serializer = EXPERIMENT_ID_TO_EXPERIMENT_SERIALIZER_MAPPING[experiment_type_id] - except KeyError as e: - raise GradientSdkError("No experiment type with ID: {}".format(str(e))) - - return serializer def paginate_schema(schema): class PaginationSchema(BaseSchema): MODEL = models.Pagination data = marshmallow.fields.Nested(schema, many=True, dump_only=True) - start_after = marshmallow.fields.String(dump_to='startAfter', load_from='startAfter') + start_after = marshmallow.fields.String( + dump_to='startAfter', load_from='startAfter') - return PaginationSchema() \ No newline at end of file + return PaginationSchema() diff --git a/gradient/api_sdk/utils.py b/gradient/api_sdk/utils.py index 9e7883ad..e120f307 100644 --- a/gradient/api_sdk/utils.py +++ b/gradient/api_sdk/utils.py @@ -61,19 +61,6 @@ def print_dict_recursive(input_dict, logger, indent=0, tabulator=" "): logger.log("%s%s" % (tabulator * (indent + 1), val)) -class ExperimentsClientHelpersMixin(object): - def _get_experiment_type_id(self, value): - if isinstance(value, int): - return value - - try: - experiment_type_id = constants.MULTI_NODE_EXPERIMENT_TYPES_MAP[value] - except KeyError as e: - raise sdk_exceptions.GradientSdkError("Invalid experiment type: {}".format(e)) - - return experiment_type_id - - def validate_auth_options(auth_username, auth_password, generate_auth): if generate_auth and any((auth_username, auth_password)): raise sdk_exceptions.InvalidParametersError( @@ -82,11 +69,13 @@ def validate_auth_options(auth_username, auth_password, generate_auth): # checking if both or none auth parameters were used if len([val for val in (auth_username, auth_password) if not bool(val)]) == 1: - raise sdk_exceptions.InvalidParametersError("auth_username and auth_password have to be used together") + raise sdk_exceptions.InvalidParametersError( + "auth_username and auth_password have to be used together") def generate_credential(n): - cred = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(n)) + cred = ''.join(random.choice(string.ascii_uppercase + + string.ascii_lowercase + string.digits) for _ in range(n)) return cred @@ -126,7 +115,8 @@ def concatenate_urls(fst_part, snd_part): class MultipartEncoder(object): def __init__(self, fields): mp_encoder = encoder.MultipartEncoder(fields=fields) - self.monitor = encoder.MultipartEncoderMonitor(mp_encoder, callback=self._create_callback(mp_encoder)) + self.monitor = encoder.MultipartEncoderMonitor( + mp_encoder, callback=self._create_callback(mp_encoder)) def get_monitor(self): return self.monitor @@ -177,7 +167,8 @@ def parse_path(cls, path): if cls.is_http_url(path): return cls.HTTP_URL - raise sdk_exceptions.WrongPathError("Given path is neither local path, nor valid URL") + raise sdk_exceptions.WrongPathError( + "Given path is neither local path, nor valid URL") @staticmethod def is_local_dir(path): diff --git a/gradient/cli/clusters.py b/gradient/cli/clusters.py index 07097f0e..62279fff 100644 --- a/gradient/cli/clusters.py +++ b/gradient/cli/clusters.py @@ -17,7 +17,7 @@ def clusters(): "-l", "cluster_limit", default=20, - help="Limit listed experiments per page", + help="Limit listed clusters per page", cls=common.GradientOption, ) @click.option( diff --git a/gradient/cli/machines.py b/gradient/cli/machines.py index b5b1c063..754c5866 100644 --- a/gradient/cli/machines.py +++ b/gradient/cli/machines.py @@ -189,19 +189,20 @@ def check_machine_availability(region, machine_type, api_key, options_file): "--tag", "tags", multiple=True, - help="One or many tags that you want to add to experiment", + help="One or many tags that you want to add to machine", cls=common.GradientOption ) @click.option( "--tags", "tags_comma", - help="Separated by comma tags that you want add to experiment", + help="Separated by comma tags that you want add to machine", cls=common.GradientOption ) @api_key_option @common.options_file def create_machine(api_key, options_file, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags")) + kwargs["tags"] = validate_comma_split_option( + kwargs.pop("tags_comma"), kwargs.pop("tags")) del_if_value_is_none(kwargs) assign_public_ip = kwargs.get("assign_public_ip") @@ -217,7 +218,8 @@ def create_machine(api_key, options_file, **kwargs): validate_mutually_exclusive([user_id], [email, password, first_name, last_name], "--userId is mutually exclusive with --email, --password, --firstName and --lastName") - command = machines_commands.CreateMachineCommand(api_key=api_key, logger=clilogger.CliLogger()) + command = machines_commands.CreateMachineCommand( + api_key=api_key, logger=clilogger.CliLogger()) command.execute(kwargs) @@ -247,7 +249,8 @@ def create_machine(api_key, options_file, **kwargs): @api_key_option @common.options_file def destroy_machine(machine_id, release_public_ip, api_key, options_file): - command = machines_commands.DestroyMachineCommand(api_key=api_key, logger=clilogger.CliLogger()) + command = machines_commands.DestroyMachineCommand( + api_key=api_key, logger=clilogger.CliLogger()) command.execute(machine_id, release_public_ip) @@ -424,7 +427,8 @@ def list_machines(api_key, params, options_file, **kwargs): validate_mutually_exclusive(params.values(), kwargs.values(), "You can use either --params dictionary or single filter arguments") - command = machines_commands.ListMachinesCommand(api_key=api_key, logger=clilogger.CliLogger()) + command = machines_commands.ListMachinesCommand( + api_key=api_key, logger=clilogger.CliLogger()) filters = params or kwargs command.execute(**filters) @@ -445,7 +449,8 @@ def list_machines(api_key, params, options_file, **kwargs): @api_key_option @common.options_file def restart_machine(machine_id, api_key, options_file): - command = machines_commands.RestartMachineCommand(api_key=api_key, logger=clilogger.CliLogger()) + command = machines_commands.RestartMachineCommand( + api_key=api_key, logger=clilogger.CliLogger()) command.execute(machine_id) @@ -652,7 +657,8 @@ def wait_for_machine_state(machine_id, state, api_key, options_file): @api_key_option @common.options_file def machine_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = machines_commands.MachineAddTagsCommand(api_key=api_key) command.execute(id, **kwargs) @@ -682,7 +688,8 @@ def machine_add_tag(id, options_file, api_key, **kwargs): @api_key_option @common.options_file def machine_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = machines_commands.MachineRemoveTagsCommand(api_key=api_key) command.execute(id, **kwargs) diff --git a/gradient/cli/models.py b/gradient/cli/models.py index 6450a36f..aa4f1bc7 100644 --- a/gradient/cli/models.py +++ b/gradient/cli/models.py @@ -19,12 +19,6 @@ def model_tags(): @models_group.command("list", help="List models with optional filtering") -@click.option( - "--experimentId", - "experiment_id", - help="Use to filter by experiment ID", - cls=common.GradientOption, -) @click.option( "--projectId", "project_id", @@ -40,9 +34,9 @@ def model_tags(): ) @common.api_key_option @common.options_file -def list_models(api_key, experiment_id, project_id, tags, options_file): +def list_models(api_key, project_id, tags, options_file): command = models_commands.ListModelsCommand(api_key=api_key) - command.execute(experiment_id=experiment_id, project_id=project_id, tags=tags) + command.execute(project_id=project_id, tags=tags) @models_group.command("delete", help="Delete model") @@ -106,19 +100,20 @@ def delete_model(api_key, model_id, options_file): "--tag", "tags", multiple=True, - help="One or many tags that you want to add to experiment", + help="One or many tags that you want to add to model", cls=common.GradientOption ) @click.option( "--tags", "tags_comma", - help="Separated by comma tags that you want add to experiment", + help="Separated by comma tags that you want add to model", cls=common.GradientOption ) @common.api_key_option @common.options_file def create_model(api_key, options_file, **model): - model["tags"] = validate_comma_split_option(model.pop("tags_comma"), model.pop("tags")) + model["tags"] = validate_comma_split_option( + model.pop("tags_comma"), model.pop("tags")) command = models_commands.CreateModel(api_key=api_key) command.execute(**model) @@ -174,19 +169,20 @@ def create_model(api_key, options_file, **model): "--tag", "tags", multiple=True, - help="One or many tags that you want to add to experiment", + help="One or many tags that you want to add to model", cls=common.GradientOption ) @click.option( "--tags", "tags_comma", - help="Separated by comma tags that you want add to experiment", + help="Separated by comma tags that you want add to model", cls=common.GradientOption ) @common.api_key_option @common.options_file def upload_model(api_key, options_file, **model): - model["tags"] = validate_comma_split_option(model.pop("tags_comma"), model.pop("tags")) + model["tags"] = validate_comma_split_option( + model.pop("tags_comma"), model.pop("tags")) command = models_commands.UploadModel(api_key=api_key) command.execute(**model) @@ -252,7 +248,8 @@ def download_model_files(model_id, destination_directory, api_key, options_file) @common.api_key_option @common.options_file def ml_model_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = models_commands.MLModelAddTagsCommand(api_key=api_key) command.execute(id, **kwargs) @@ -282,7 +279,8 @@ def ml_model_add_tag(id, options_file, api_key, **kwargs): @common.api_key_option @common.options_file def ml_model_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = models_commands.MLModelRemoveTagsCommand(api_key=api_key) command.execute(id, **kwargs) diff --git a/gradient/cli/notebooks.py b/gradient/cli/notebooks.py index 89eda118..a3cf54ad 100644 --- a/gradient/cli/notebooks.py +++ b/gradient/cli/notebooks.py @@ -150,22 +150,24 @@ def notebook_metrics(): "--tag", "tags", multiple=True, - help="One or many tags that you want to add to experiment", + help="One or many tags that you want to add to notebook", cls=common.GradientOption ) @click.option( "--tags", "tags_comma", - help="Separated by comma tags that you want add to experiment", + help="Separated by comma tags that you want add to notebook", cls=common.GradientOption ) @common.api_key_option @common.options_file def create_notebook(api_key, options_file, **notebook): - notebook["tags"] = validate_comma_split_option(notebook.pop("tags_comma"), notebook.pop("tags")) + notebook["tags"] = validate_comma_split_option( + notebook.pop("tags_comma"), notebook.pop("tags")) command = notebooks.CreateNotebookCommand(api_key=api_key) command.execute(**notebook) + @notebooks_group.command("start", help="Start notebook") @click.option( "--id", @@ -209,23 +211,24 @@ def create_notebook(api_key, options_file, **notebook): "--tag", "tags", multiple=True, - help="One or many tags that you want to add to experiment", + help="One or many tags that you want to add to notebook", cls=common.GradientOption ) @click.option( "--tags", "tags_comma", - help="Separated by comma tags that you want add to experiment", + help="Separated by comma tags that you want add to notebook", cls=common.GradientOption ) - @common.api_key_option @common.options_file def start_notebook(api_key, options_file, **notebook): - notebook["tags"] = validate_comma_split_option(notebook.pop("tags_comma"), notebook.pop("tags")) + notebook["tags"] = validate_comma_split_option( + notebook.pop("tags_comma"), notebook.pop("tags")) command = notebooks.StartNotebookCommand(api_key=api_key) command.execute(**notebook) + @notebooks_group.command("fork", help="Fork existing notebook") @click.option( "--id", @@ -322,7 +325,8 @@ def show_notebook(id, api_key, options_file): @common.api_key_option @common.options_file def notebook_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = notebooks.NotebookAddTagsCommand(api_key=api_key) command.execute(id, **kwargs) @@ -352,7 +356,8 @@ def notebook_add_tag(id, options_file, api_key, **kwargs): @common.api_key_option @common.options_file def notebook_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = notebooks.NotebookRemoveTagsCommand(api_key=api_key) command.execute(id, **kwargs) @@ -375,8 +380,10 @@ def notebook_remove_tags(id, options_file, api_key, **kwargs): "metrics_list", multiple=True, type=str, - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), - help=("One or more metrics that you want to read: {}. Defaults to cpuPercentage and memoryUsage. To view available custom metrics, use command: `gradient notebooks metrics list`".format(', '.join(map(str, constants.METRICS_MAP)))), + default=(constants.BuiltinMetrics.cpu_percentage, + constants.BuiltinMetrics.memory_usage), + help=("One or more metrics that you want to read: {}. Defaults to cpuPercentage and memoryUsage. To view available custom metrics, use command: `gradient notebooks metrics list`".format( + ', '.join(map(str, constants.METRICS_MAP)))), cls=common.GradientOption, ) @click.option( @@ -404,7 +411,8 @@ def notebook_remove_tags(id, options_file, api_key, **kwargs): @common.options_file def get_deployment_metrics(notebook_id, metrics_list, interval, start, end, options_file, api_key): command = GetNotebookMetricsCommand(api_key=api_key) - command.execute(notebook_id, start, end, interval, built_in_metrics=metrics_list) + command.execute(notebook_id, start, end, interval, + built_in_metrics=metrics_list) @notebook_metrics.command( @@ -464,7 +472,8 @@ def list_deployment_metrics(notebook_id, interval, start, end, options_file, api "metrics_list", multiple=True, type=ChoiceType(constants.METRICS_MAP, case_sensitive=False), - default=(constants.BuiltinMetrics.cpu_percentage, constants.BuiltinMetrics.memory_usage), + default=(constants.BuiltinMetrics.cpu_percentage, + constants.BuiltinMetrics.memory_usage), help="One or more metrics that you want to read. Defaults to cpuPercentage and memoryUsage", cls=common.GradientOption, ) @@ -479,7 +488,9 @@ def list_deployment_metrics(notebook_id, interval, start, end, options_file, api @common.options_file def stream_model_deployment_metrics(notebook_id, metrics_list, interval, options_file, api_key): command = StreamNotebookMetricsCommand(api_key=api_key) - command.execute(notebook_id=notebook_id, interval=interval, built_in_metrics=metrics_list) + command.execute(notebook_id=notebook_id, interval=interval, + built_in_metrics=metrics_list) + @notebooks_group.command("stop", help="Stop running notebook") @click.option( @@ -534,7 +545,9 @@ def artifacts(): @common.options_file def list_artifacts(notebook_id, size, links, files, options_file, api_key=None): command = notebooks.ArtifactsListCommand(api_key=api_key) - command.execute(notebook_id=notebook_id, size=size, links=links, files=files) + command.execute(notebook_id=notebook_id, size=size, + links=links, files=files) + @notebooks_group.command("logs", help="List notebook logs") @click.option( diff --git a/gradient/cli/projects.py b/gradient/cli/projects.py index 4d71b2d4..37ab0472 100644 --- a/gradient/cli/projects.py +++ b/gradient/cli/projects.py @@ -55,19 +55,20 @@ def list_projects(api_key, tags, options_file): "--tag", "tags", multiple=True, - help="One or many tags that you want to add to experiment", + help="One or many tags that you want to add to project", cls=common.GradientOption ) @click.option( "--tags", "tags_comma", - help="Separated by comma tags that you want add to experiment", + help="Separated by comma tags that you want add to project", cls=common.GradientOption ) @common.api_key_option @common.options_file def create_project(api_key, options_file, **project): - project["tags"] = validate_comma_split_option(project.pop("tags_comma"), project.pop("tags")) + project["tags"] = validate_comma_split_option( + project.pop("tags_comma"), project.pop("tags")) command = projects_commands.CreateProjectCommand(api_key) command.execute(project) @@ -86,7 +87,7 @@ def create_project(project_id, api_key, options_file): command.execute(project_id) -@projects_group.command("delete", help="Delete project and all its experiments") +@projects_group.command("delete", help="Delete project and all its nested resources") @click.option( "--id", "project_id", @@ -125,7 +126,8 @@ def delete_project(project_id, options_file, api_key): @common.api_key_option @common.options_file def project_add_tag(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = projects_commands.ProjectAddTagsCommand(api_key=api_key) command.execute(id, **kwargs) @@ -155,7 +157,8 @@ def project_add_tag(id, options_file, api_key, **kwargs): @common.api_key_option @common.options_file def project_remove_tags(id, options_file, api_key, **kwargs): - kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) + kwargs["tags"] = validate_comma_split_option(kwargs.pop( + "tags_comma"), kwargs.pop("tags"), raise_if_no_values=True) command = projects_commands.ProjectRemoveTagsCommand(api_key=api_key) command.execute(id, **kwargs) diff --git a/gradient/cli/validators.py b/gradient/cli/validators.py index 6027d221..13e0086d 100644 --- a/gradient/cli/validators.py +++ b/gradient/cli/validators.py @@ -4,14 +4,6 @@ from gradient.api_sdk import constants -REQUIRED_PARAMS_PER_EXPERIMENT_TYPE = { - constants.ExperimentType.GRPC_MULTI_NODE: ["parameter_server_container", "parameter_server_machine_type", - "parameter_server_command", "parameter_server_count"], - constants.ExperimentType.MPI_MULTI_NODE: ["master_container", "master_machine_type", "master_command", - "master_count"], -} - - def validate_mutually_exclusive(options_1, options_2, error_message): used_option_in_options_1 = any(option is not None for option in options_1) used_option_in_options_2 = any(option is not None for option in options_2) @@ -25,12 +17,3 @@ def validate_email(ctx, param, value): raise click.BadParameter("Bad email address format") return value - - -def validate_multi_node(params): - experiment_type = params.get('experiment_type_id') - required_params = REQUIRED_PARAMS_PER_EXPERIMENT_TYPE.get(experiment_type) - - for param_name in required_params: - if not params.get(param_name): - raise click.UsageError("Param %s is required for this experiment type" % param_name) diff --git a/gradient/commands/models.py b/gradient/commands/models.py index 8d718fc4..f17a7f9d 100644 --- a/gradient/commands/models.py +++ b/gradient/commands/models.py @@ -31,14 +31,13 @@ def _get_instances(self, kwargs): return instances def _get_table_data(self, models): - data = [("Name", "ID", "Model Type", "Project ID", "Experiment ID")] + data = [("Name", "ID", "Model Type", "Project ID")] for model in models: name = model.name id_ = model.id project_id = model.project_id - experiment_id = model.experiment_id model_type = model.model_type - data.append((name, id_, model_type, project_id, experiment_id)) + data.append((name, id_, model_type, project_id)) return data @@ -80,7 +79,6 @@ def _get_table_data(self, instance): ("ID", instance.id), ("Name", instance.name), ("Project ID", instance.project_id), - ("Experiment ID", instance.experiment_id), ("Model Type", instance.model_type), ("URL", instance.url), ("Deployment State", instance.deployment_state), diff --git a/tests/conftest.py b/tests/conftest.py index 5f72a592..187940b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,230 +6,6 @@ from pathlib2 import Path -@pytest.fixture -def create_single_node_experiment_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_create_single_node.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def create_multi_node_experiment_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_create_multi_node.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def create_multi_node_experiment_ds_objects_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_create_multi_node_ds_objs.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiment_details_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_details.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_list_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_list.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_start_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_start.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_stop_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_stop.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_logs_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_logs.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_metrics_get_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_metrics_get.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_metrics_stream_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_metrics_stream.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_create_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_create.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_update_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_update.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_list_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_list.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_start_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_start.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_stop_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_delete.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_delete_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_stop.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_details_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_details.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_metrics_get_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_metrics_get.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_metrics_stream_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_metrics_stream.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def deployments_logs_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "deployments_logs.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def hyperparameters_create_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "hyperparameters_create.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def hyperparameters_start_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "hyperparameters_start.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def hyperparameters_list_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "hyperparameters_list.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def hyperparameters_details_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "hyperparameters_details.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_list_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_list.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_logs_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_logs.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_artifacts_destroy_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_artifacts_destroy.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_artifacts_get_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_artifacts_get.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_artifacts_list_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_artifacts_list.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_create_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_create.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_metrics_get_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_metrics_get.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def jobs_metrics_stream_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "jobs_metrics_stream.yaml" - return str(fixture_dir.resolve()) - - @pytest.fixture def machines_availability_config_path(): p = Path(__file__) @@ -356,13 +132,6 @@ def projects_delete_config_path(): return str(fixture_dir.resolve()) -@pytest.fixture -def run_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "run.yaml" - return str(fixture_dir.resolve()) - - @pytest.fixture def notebooks_create_config_path(): p = Path(__file__) @@ -405,27 +174,6 @@ def notebooks_metrics_stream_config_path(): return str(fixture_dir.resolve()) -@pytest.fixture -def tensorboards_create_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "tensorboards_create.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def tensorboards_details_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "tensorboards_details.yaml" - return str(fixture_dir.resolve()) - - -@pytest.fixture -def experiments_delete_config_path(): - p = Path(__file__) - fixture_dir = p.parent / "config_files" / "experiments_delete.yaml" - return str(fixture_dir.resolve()) - - @pytest.fixture def models_upload_config_path(): p = Path(__file__) diff --git a/tests/example_responses.py b/tests/example_responses.py index f7291090..3438895d 100644 --- a/tests/example_responses.py +++ b/tests/example_responses.py @@ -1,979 +1,3 @@ -LIST_OF_EXPERIMENTS_RESPONSE_JSON = { - "data": [ - { - "dtCreated": "2019-03-21T07:47:05.616096+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-21T07:47:05.616096+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 6315, - "experimentTemplateId": 60, - "experimentTypeId": 1, - "handle": "ea2lfbbpdyzsq", - "id": 6292, - "projectHandle": "prq70zy79", - "projectId": 612, - "started_by_user_id": 1, - "state": 1, - "templateHistory": { - "dtCreated": "2019-03-21T07:47:04.925852+00:00", - "dtDeleted": None, - "experimentTemplateId": 60, - "id": 6315, - "params": { - "experimentTypeId": 1, - "name": "dsfads", - "ports": 5000, - "project_handle": "prq70zy79", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "sadas" - }, - "triggerEvent": None, - "triggerEventId": None - }, - "datasets": [ - { - "uri": "s3://some/dataset", - "volume_options": { - "kind": "dynamic", - "size": "10Gi", - }, - }, - ], - }, - { - "dtCreated": "2019-03-21T07:46:57.706055+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-21T07:46:57.706055+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 6314, - "experimentTemplateId": 60, - "experimentTypeId": 1, - "handle": "em6btk2vtb7it", - "id": 6291, - "projectHandle": "prq70zy79", - "projectId": 612, - "started_by_user_id": 1, - "state": 1, - "templateHistory": { - "dtCreated": "2019-03-21T07:46:56.949590+00:00", - "dtDeleted": None, - "experimentTemplateId": 60, - "id": 6314, - "params": { - "experimentTypeId": 1, - "name": "dsfads", - "ports": 5000, - "project_handle": "prq70zy79", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "sadas" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-20T19:56:50.154853+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-20T19:56:50.154853+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 6297, - "experimentTemplateId": 60, - "experimentTypeId": 3, - "handle": "ew69ls0vy3eto", - "id": 6286, - "projectHandle": "prq70zy79", - "projectId": 612, - "started_by_user_id": 1, - "state": 1, - "templateHistory": { - "dtCreated": "2019-03-20T19:56:49.427354+00:00", - "dtDeleted": None, - "experimentTemplateId": 60, - "id": 6297, - "params": { - "artifactDirectory": "/artdir", - "clusterId": 2, - "experimentEnv": { - "key": "val" - }, - "experimentTypeId": 3, - "name": "multinode_mpi", - "parameter_server_command": "ls", - "parameter_server_container": "pscon", - "parameter_server_container_user": "pscuser", - "parameter_server_count": 2, - "parameter_server_machine_type": "psmtype", - "parameter_server_registry_password": "psrpass", - "parameter_server_registry_username": "psrcus", - "ports": 3456, - "project_handle": "prq70zy79", - "project_id": 34, - "trigger_event_id": 12, - "worker_command": "wcom", - "worker_container": "wcon", - "worker_container_user": "usr", - "worker_count": 2, - "worker_machine_type": "mty", - "worker_registry_password": "rpass", - "worker_registry_username": "rusr", - "workingDirectory": "/dir", - "workspaceUrl": "wurl" - }, - "triggerEvent": { - "dtCreated": "2019-03-11T14:47:57+00:00", - "eventData": { - "author": { - "email": "bluckey@paperspace.com", - "login": "ultrabluewolf", - "name": "Britney Luckey" - }, - "branch": "feature/test-1", - "message": "Update readme #2", - "repo_node_id": "MDEwOlJlcG9zaXRvcnkxNzQ3MjI3NDc=", - "sender": { - "id": 4633049, - "login": "ultrabluewolf" - }, - "sha": "daa117a00cd1e0e9b1b55695031e698a560cca29", - "timestamp": "2019-03-11T10:47:57-04:00" - }, - "id": 12, - "type": "github" - }, - "triggerEventId": 12 - } - } - ], - "message": "success", - "meta": { - "filter": [], - "limit": 11, - "offset": 0, - "totalItems": 27 - } -} - -LIST_OF_EXPERIMENTS_FILTERED_WITH_TWO_PROJECTS = { - "data": [ - { - "data": [ - { - "dtCreated": "2019-03-21T08:48:49.449397+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-21T08:48:49.449397+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": "Expecting value: line 1 column 1 (char 0)", - "experimentTemplateHistoryId": 297, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "esj8mcpaayh5kx", - "id": 295, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 1, - "state": 7, - "templateHistory": { - "dtCreated": "2019-03-21T08:48:48.436994+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 297, - "params": { - "artifactDirectory": "sagdf", - "clusterId": 342, - "experimentEnv": { - "k": "v" - }, - "experimentTypeId": 1, - "name": "dsfads", - "ports": 3245, - "project_handle": "pr4yxj956", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "C2", - "workingDirectory": "sadfas", - "workspaceUrl": "sadfds" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-21T08:48:29.553574+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-21T08:48:29.553574+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": "{'name': 'Error', 'status': 400, 'message': 'Invalid machine type'}", - "experimentTemplateHistoryId": 296, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "estun7jhqta8sm", - "id": 294, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 1, - "state": 7, - "templateHistory": { - "dtCreated": "2019-03-21T08:48:28.467167+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 296, - "params": { - "artifactDirectory": "sagdf", - "clusterId": 342, - "experimentEnv": { - "k": "v" - }, - "experimentTypeId": 1, - "name": "dsfads", - "ports": 3245, - "project_handle": "pr4yxj956", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "c2", - "workingDirectory": "sadfas", - "workspaceUrl": "sadfds" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-21T08:44:42.581919+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-21T08:44:42.581919+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": "{'name': 'Error', 'status': 400, 'message': 'Invalid machine type'}", - "experimentTemplateHistoryId": 295, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "es08mwh7ye6zpm", - "id": 293, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 1, - "state": 7, - "templateHistory": { - "dtCreated": "2019-03-21T08:44:41.366528+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 295, - "params": { - "artifactDirectory": "sagdf", - "clusterId": 342, - "experimentEnv": { - "k": "v" - }, - "experimentTypeId": 1, - "name": "dsfads", - "ports": 3245, - "project_handle": "pr4yxj956", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "sadas", - "workingDirectory": "sadfas", - "workspaceUrl": "sadfds" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-19T16:35:28.425184+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-19T16:38:40.664560+00:00", - "dtModified": "2019-03-19T16:35:28.425184+00:00", - "dtProvisioningFinished": "2019-03-19T16:37:37.516138+00:00", - "dtProvisioningStarted": "2019-03-19T16:37:37.516138+00:00", - "dtStarted": "2019-03-19T16:37:37.516138+00:00", - "dtTeardownFinished": "2019-03-19T16:38:40.664560+00:00", - "dtTeardownStarted": "2019-03-19T16:38:40.664560+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 38, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "eshdufkt9ti8mx", - "id": 38, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-19T16:35:28.425184+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 38, - "params": { - "command": "python2 hello.py", - "container": "paperspace/tensorflow-python", - "project": "paperspace-python", - "workspaceFileName": "hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-19T14:52:42.476592+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-19T14:53:00.850701+00:00", - "dtModified": "2019-03-19T14:52:42.476592+00:00", - "dtProvisioningFinished": "2019-03-19T14:52:46.844619+00:00", - "dtProvisioningStarted": "2019-03-19T14:52:46.844619+00:00", - "dtStarted": "2019-03-19T14:52:46.844619+00:00", - "dtTeardownFinished": "2019-03-19T14:53:00.850701+00:00", - "dtTeardownStarted": "2019-03-19T14:53:00.850701+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 34, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "e07h0ym8fewm", - "id": 34, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-19T14:52:42.476592+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 34, - "params": { - "command": "python2 hello.py", - "container": "paperspace/tensorflow-python", - "project": "paperspace-python", - "workspaceFileName": "hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-19T14:46:21.605202+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-19T14:46:40.360111+00:00", - "dtModified": "2019-03-19T14:46:21.605202+00:00", - "dtProvisioningFinished": "2019-03-19T14:46:26.094384+00:00", - "dtProvisioningStarted": "2019-03-19T14:46:26.094384+00:00", - "dtStarted": "2019-03-19T14:46:26.094384+00:00", - "dtTeardownFinished": "2019-03-19T14:46:40.360111+00:00", - "dtTeardownStarted": "2019-03-19T14:46:40.360111+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 33, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "e0fq31vyla039", - "id": 33, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-19T14:46:21.605202+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 33, - "params": { - "command": "python3 hello.py", - "container": "paperspace/tensorflow-python", - "project": "paperspace-python", - "workspaceFileName": "hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-19T14:38:38.227536+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-19T14:42:41.723400+00:00", - "dtModified": "2019-03-19T14:38:38.227536+00:00", - "dtProvisioningFinished": "2019-03-19T14:41:34.630690+00:00", - "dtProvisioningStarted": "2019-03-19T14:41:34.630690+00:00", - "dtStarted": "2019-03-19T14:41:34.630690+00:00", - "dtTeardownFinished": "2019-03-19T14:42:41.723400+00:00", - "dtTeardownStarted": "2019-03-19T14:42:41.723400+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "esxz3ihfl9qxs2", - "id": 32, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-19T14:38:38.227536+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 32, - "params": { - "command": "python3 hello.py", - "container": "paperspace/tensorflow-python", - "project": "paperspace-python", - "workspaceFileName": "hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-14T14:06:27.383232+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-14T14:10:05.315380+00:00", - "dtModified": "2019-03-14T14:06:27.383232+00:00", - "dtProvisioningFinished": "2019-03-14T14:10:00.097887+00:00", - "dtProvisioningStarted": "2019-03-14T14:09:02.736309+00:00", - "dtStarted": "2019-03-14T14:10:00.097887+00:00", - "dtTeardownFinished": "2019-03-14T14:10:05.809122+00:00", - "dtTeardownStarted": "2019-03-14T14:10:05.462304+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32862, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "ecqq0rz0gok9i", - "id": 32860, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 32862, - "params": { - "clusterId": "clmtkpnm2", - "command": "python3 hello.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "K80", - "name": "job 5", - "projectId": "pr4yxj956", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr4yxj956/jw276004uoaiv/hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-14T11:22:14.345466+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-14T11:23:17.763554+00:00", - "dtModified": "2019-03-14T11:22:14.345466+00:00", - "dtProvisioningFinished": "2019-03-14T11:23:12.549387+00:00", - "dtProvisioningStarted": "2019-03-14T11:22:18.820247+00:00", - "dtStarted": "2019-03-14T11:23:12.549387+00:00", - "dtTeardownFinished": "2019-03-14T11:23:18.063367+00:00", - "dtTeardownStarted": "2019-03-14T11:23:17.900575+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32859, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "es310784c1o1x6", - "id": 32857, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 32859, - "params": { - "clusterId": "clmtkpnm2", - "command": "python3 hello.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "K80", - "name": "job 4", - "projectId": "pr4yxj956", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr4yxj956/jpngp435ekbp9/hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-09T11:05:34.409959+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-09T11:09:38.903533+00:00", - "dtModified": "2019-03-09T11:05:34.409959+00:00", - "dtProvisioningFinished": "2019-03-09T11:09:33.686660+00:00", - "dtProvisioningStarted": "2019-03-09T11:08:34.681334+00:00", - "dtStarted": "2019-03-09T11:09:33.686660+00:00", - "dtTeardownFinished": "2019-03-09T11:09:39.194718+00:00", - "dtTeardownStarted": "2019-03-09T11:09:39.036278+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32351, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "esabwtehoekn6w", - "id": 32349, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 32351, - "params": { - "clusterId": "clmtkpnm2", - "command": "python3 hello.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "K80", - "name": "job 3", - "projectId": "pr4yxj956", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr4yxj956/jafb4f2w30afo/hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-06T13:56:55.165302+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-06T13:57:13.165854+00:00", - "dtModified": "2019-03-06T13:56:55.165302+00:00", - "dtProvisioningFinished": "2019-03-06T13:57:07.955608+00:00", - "dtProvisioningStarted": "2019-03-06T13:57:00.905506+00:00", - "dtStarted": "2019-03-06T13:57:07.955608+00:00", - "dtTeardownFinished": "2019-03-06T13:57:13.454509+00:00", - "dtTeardownStarted": "2019-03-06T13:57:13.299681+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 31991, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "ea1d6t082wq7i", - "id": 31989, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 31991, - "params": { - "clusterId": "clmtkpnm2", - "command": "python3 hello.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "K80", - "name": "job 2", - "projectId": "pr4yxj956", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr4yxj956/jst4l3hoi0hcyn/hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-06T13:52:48.106674+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-06T13:56:29.890444+00:00", - "dtModified": "2019-03-06T13:52:48.106674+00:00", - "dtProvisioningFinished": "2019-03-06T13:56:24.678388+00:00", - "dtProvisioningStarted": "2019-03-06T13:55:32.325678+00:00", - "dtStarted": "2019-03-06T13:56:24.678388+00:00", - "dtTeardownFinished": "2019-03-06T13:56:30.178571+00:00", - "dtTeardownStarted": "2019-03-06T13:56:30.024471+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 31989, - "experimentTemplateId": 8, - "experimentTypeId": 1, - "handle": "esbipxwngbmji4", - "id": 31987, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 31989, - "params": { - "clusterId": "clmtkpnm2", - "command": "python3 hello.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "K80", - "name": "job 1", - "projectId": "pr4yxj956", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr4yxj956/j5cidd1koia24/hello.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "pr4yxj956" - }, - "totalItems": 12 - } - }, - { - "data": [ - { - "dtCreated": "2019-03-25T14:35:05.738112+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-25T14:35:21.314193+00:00", - "dtModified": "2019-03-25T14:35:05.738112+00:00", - "dtProvisioningFinished": "2019-03-25T14:35:12.777000+00:00", - "dtProvisioningStarted": "2019-03-25T14:35:12.777000+00:00", - "dtStarted": "2019-03-25T14:35:12.777000+00:00", - "dtTeardownFinished": "2019-03-25T14:35:21.314193+00:00", - "dtTeardownStarted": "2019-03-25T14:35:21.314193+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 33353, - "experimentTemplateId": 2257, - "experimentTypeId": 1, - "handle": "e8lufvr2bh8ld", - "id": 33351, - "projectHandle": "pr1opq9uz", - "projectId": 15548, - "started_by_user_id": 199654, - "state": 7, - "templateHistory": { - "dtCreated": "2019-03-25T14:35:05.738112+00:00", - "dtDeleted": None, - "experimentTemplateId": 2257, - "id": 33353, - "params": { - "command": "nvidia-smi", - "container": "Test-Container", - "isPreemptible": False, - "machineType": "C2", - "projectId": "pr1opq9uz", - "workspaceFileName": "none" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-25T14:20:07.758313+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-25T14:20:07.758313+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": "Expecting value: line 1 column 1 (char 0)", - "experimentTemplateHistoryId": 33352, - "experimentTemplateId": 2257, - "experimentTypeId": 1, - "handle": "esasysj42i2j2s", - "id": 33350, - "projectHandle": "pr1opq9uz", - "projectId": 15548, - "started_by_user_id": 199654, - "state": 7, - "templateHistory": { - "dtCreated": "2019-03-25T14:20:06.517435+00:00", - "dtDeleted": None, - "experimentTemplateId": 2257, - "id": 33352, - "params": { - "experimentTypeId": 1, - "name": "dsfads", - "ports": 5000, - "project_handle": "pr1opq9uz", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "C2" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-15T11:46:32.978544+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-15T11:46:50.541797+00:00", - "dtModified": "2019-03-15T11:46:32.978544+00:00", - "dtProvisioningFinished": "2019-03-15T11:46:45.501290+00:00", - "dtProvisioningStarted": "2019-03-15T11:46:41.431717+00:00", - "dtStarted": "2019-03-15T11:46:45.501290+00:00", - "dtTeardownFinished": "2019-03-15T11:46:50.598748+00:00", - "dtTeardownStarted": "2019-03-15T11:46:50.561267+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32879, - "experimentTemplateId": 2257, - "experimentTypeId": 1, - "handle": "estgm33hfbqf5k", - "id": 32877, - "projectHandle": "pr1opq9uz", - "projectId": 15548, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 2257, - "id": 32879, - "params": { - "clusterId": "cls28l0qm", - "command": "python3 myscript.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "P5000", - "name": "job 4", - "projectId": "pr1opq9uz", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr1opq9uz/j8bxv3ni2zc17/myscript.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-15T11:46:32.513024+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-15T11:46:50.363855+00:00", - "dtModified": "2019-03-15T11:46:32.513024+00:00", - "dtProvisioningFinished": "2019-03-15T11:46:45.324783+00:00", - "dtProvisioningStarted": "2019-03-15T11:46:40.947880+00:00", - "dtStarted": "2019-03-15T11:46:45.324783+00:00", - "dtTeardownFinished": "2019-03-15T11:46:50.419813+00:00", - "dtTeardownStarted": "2019-03-15T11:46:50.383192+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32878, - "experimentTemplateId": 2257, - "experimentTypeId": 1, - "handle": "esbo3pisqq3t68", - "id": 32876, - "projectHandle": "pr1opq9uz", - "projectId": 15548, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 2257, - "id": 32878, - "params": { - "clusterId": "cls28l0qm", - "command": "python3 myscript.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "P5000", - "name": "job 3", - "projectId": "pr1opq9uz", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr1opq9uz/jpk82qnq71w9c/myscript.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-15T11:46:16.422317+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-15T11:46:30.820071+00:00", - "dtModified": "2019-03-15T11:46:16.422317+00:00", - "dtProvisioningFinished": "2019-03-15T11:46:25.776927+00:00", - "dtProvisioningStarted": "2019-03-15T11:46:21.616491+00:00", - "dtStarted": "2019-03-15T11:46:25.776927+00:00", - "dtTeardownFinished": "2019-03-15T11:46:30.873777+00:00", - "dtTeardownStarted": "2019-03-15T11:46:30.839238+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32877, - "experimentTemplateId": 2257, - "experimentTypeId": 1, - "handle": "eso8s7e014681", - "id": 32875, - "projectHandle": "pr1opq9uz", - "projectId": 15548, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 2257, - "id": 32877, - "params": { - "clusterId": "cls28l0qm", - "command": "python3 test_remote.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "P5000", - "name": "job 2", - "projectId": "pr1opq9uz", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr1opq9uz/jsuqgzne50tlg1/test_remote.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-03-15T11:26:14.607276+00:00", - "dtDeleted": None, - "dtFinished": "2019-03-15T11:26:32.872835+00:00", - "dtModified": "2019-03-15T11:26:14.607276+00:00", - "dtProvisioningFinished": "2019-03-15T11:26:27.823314+00:00", - "dtProvisioningStarted": "2019-03-15T11:26:23.367983+00:00", - "dtStarted": "2019-03-15T11:26:27.823314+00:00", - "dtTeardownFinished": "2019-03-15T11:26:32.930538+00:00", - "dtTeardownStarted": "2019-03-15T11:26:32.897522+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 32876, - "experimentTemplateId": 2257, - "experimentTypeId": 1, - "handle": "esml2ioetny5o6", - "id": 32874, - "projectHandle": "pr1opq9uz", - "projectId": 15548, - "started_by_user_id": 199654, - "state": 5, - "templateHistory": { - "dtCreated": "2019-03-22T23:31:15.372159+00:00", - "dtDeleted": None, - "experimentTemplateId": 2257, - "id": 32876, - "params": { - "clusterId": "cls28l0qm", - "command": "python3 myscript.py", - "containerUrl": "paperspace/tensorflow-python", - "isPreemptible": False, - "machineType": "P5000", - "name": "job 1", - "projectId": "pr1opq9uz", - "startedByUserId": "u3z4be26", - "workspaceUrl": "s3://ps-projects/pr1opq9uz/jsnze3m7s2xlpr/myscript.py.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "pr1opq9uz" - }, - "totalItems": 6 - } - } - ], - "message": "success", - "meta": { - "filter": [], - "limit": 1000000, - "offset": 0, - "totalItems": 18 - } -} - -LIST_OF_EXPERIMENTS_FILTERED_WITH_TWO_PROJECTS_STDOUT = """+--------+----------------+---------+ -| Name | ID | Status | -+--------+----------------+---------+ -| dsfads | esj8mcpaayh5kx | failed | -| dsfads | estun7jhqta8sm | failed | -| dsfads | es08mwh7ye6zpm | failed | -| None | eshdufkt9ti8mx | stopped | -| None | e07h0ym8fewm | stopped | -| None | e0fq31vyla039 | stopped | -| None | esxz3ihfl9qxs2 | stopped | -| job 5 | ecqq0rz0gok9i | stopped | -| job 4 | es310784c1o1x6 | stopped | -| job 3 | esabwtehoekn6w | stopped | -| job 2 | ea1d6t082wq7i | stopped | -| job 1 | esbipxwngbmji4 | stopped | -| None | e8lufvr2bh8ld | failed | -| dsfads | esasysj42i2j2s | failed | -| job 4 | estgm33hfbqf5k | stopped | -| job 3 | esbo3pisqq3t68 | stopped | -| job 2 | eso8s7e014681 | stopped | -| job 1 | esml2ioetny5o6 | stopped | -+--------+----------------+---------+ - -""" - -LIST_OF_EXPERIMENTS_FILTERED_BUT_NONE_FOUND = { - "data": [ - { - "data": [], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "pr4yxj95" - }, - "totalItems": 0 - } - }, - { - "data": [], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "pr1opq9u" - }, - "totalItems": 0 - } - } - ], - "message": "success", - "meta": { - "filter": [], - "limit": 1000000, - "offset": 0, - "totalItems": 0 - } -} - UPDATE_TAGS_RESPONSE = [ { "tagId": 1, @@ -1102,300 +126,6 @@ ] } -CREATE_DEPLOYMENT_WITH_BASIC_OPTIONS_RESPONSE = { - "deployment": { - "id": "sadkfhlskdjh", - "ownerId": "tejxgn5bt", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", - "state": "Stopped", - "startedByUserId": "ukgvw4i8", - "startedByUserEmail": "batman.bin.supraman@paperspace.com", - "deploymentType": "TFServing", - "modelId": "mosu30xm7q8vb0p", - "modelUrl": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "name": "some_name", - "tag": None, - "params": None, - "code": None, - "cluster": "PS Jobs on GCP", - "machineType": "K80", - "imageUrl": "https://www.latlmes.com/breaking/paperspace-now-has-a-100-bilion-valuation", - "imageUsername": None, - "imagePassword": None, - "workspaceUrl": None, - "workspaceUsername": None, - "workspacePassword": None, - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": "-----", - "annotations": {}, - "oauthKey": "-----", - "oauthSecret": "some_oauth_secret", - "serviceType": "model", - "apiType": "REST", - "endpoint": "https://development-services.paperspace.io/model-serving/desanw1jptk7woh:predict", - "ports": None, - "dtCreated": "2019-04-04T10:27:22.659Z", - "dtModified": "2019-04-04T10:27:22.659Z", - "dtBuildStarted": None, - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": None, - "dtDeleted": None, - "isDeleted": False - } -} - -LIST_DEPLOYMENTS = { - "deploymentList": [ - { - "id": "dev61ity7lx232", - "ownerId": "tejxgn5bt", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", - "state": "Stopped", - "startedByUserId": "ukgvw4i8", - "startedByUserEmail": "bartosz@paperspace.com", - "deploymentType": "TFServing", - "modelId": "mosu30xm7q8vb0p", - "modelUrl": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "name": "some_name", - "tag": "some_tag", - "params": None, - "code": "some_code", - "cluster": "PS Jobs on GCP", - "machineType": "K80", - "imageUrl": "https://github.com/Paperspace/mnist-sample", - "imageUsername": "some_username", - "imagePassword": "some_password", - "workspaceUrl": "https://www.example.com", - "workspaceUsername": "some_username", - "workspacePassword": "some_password", - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": "some_auth_token", - "annotations": {}, - "oauthKey": "some_oauth_key", - "oauthSecret": "some_oauth_secret", - "serviceType": "some_service_type", - "apiType": "some_api_type", - "endpoint": "https://development-services.paperspace.io/model-serving/dev61ity7lx232:predict", - "ports": "5000", - "dtCreated": "2019-04-04T10:53:56.665Z", - "dtModified": "2019-04-04T10:53:56.729Z", - "dtBuildStarted": None, - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": "2019-04-04T10:53:56.711Z", - "dtDeleted": None, - "isDeleted": False, - "metricsURL": "aws-testing.paperspace.io", - }, - { - "id": "desanw1jptk7woh", - "ownerId": "tejxgn5bt", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", - "state": "Stopped", - "startedByUserId": "ukgvw4i8", - "startedByUserEmail": "bartosz@paperspace.com", - "deploymentType": "TFServing", - "modelId": "mosu30xm7q8vb0p", - "modelUrl": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "name": "some_name", - "tag": None, - "params": None, - "code": None, - "cluster": "PS Jobs on GCP", - "machineType": "K80", - "imageUrl": "https://github.com/Paperspace/mnist-sample", - "imageUsername": None, - "imagePassword": None, - "workspaceUrl": None, - "workspaceUsername": None, - "workspacePassword": None, - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": "-----", - "annotations": {}, - "oauthKey": "-----", - "oauthSecret": "-----", - "serviceType": "model", - "apiType": "REST", - "endpoint": "https://development-services.paperspace.io/model-serving/desanw1jptk7woh:predict", - "ports": None, - "dtCreated": "2019-04-04T10:27:22.659Z", - "dtModified": "2019-04-04T10:27:22.726Z", - "dtBuildStarted": None, - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": "2019-04-04T10:27:22.710Z", - "dtDeleted": None, - "isDeleted": False - }, - { - "id": "desfnnrqt1v633v", - "ownerId": "tejxgn5bt", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", - "state": "Stopped", - "startedByUserId": "ukgvw4i8", - "startedByUserEmail": "bartosz@paperspace.com", - "deploymentType": "TFServing", - "modelId": "mosu30xm7q8vb0p", - "modelUrl": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "name": "some_name", - "tag": None, - "params": None, - "code": None, - "cluster": "PS Jobs on GCP", - "machineType": "K80", - "imageUrl": "https://github.com/Paperspace/mnist-sample", - "imageUsername": None, - "imagePassword": None, - "workspaceUrl": None, - "workspaceUsername": None, - "workspacePassword": None, - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": "-----", - "annotations": {}, - "oauthKey": "-----", - "oauthSecret": "-----", - "serviceType": "model", - "apiType": "REST", - "endpoint": "https://development-services.paperspace.io/model-serving/desfnnrqt1v633v:predict", - "ports": None, - "dtCreated": "2019-04-04T10:27:10.180Z", - "dtModified": "2019-04-04T10:27:10.248Z", - "dtBuildStarted": None, - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": "2019-04-04T10:27:10.230Z", - "dtDeleted": None, - "isDeleted": False - }, - { - "id": "desdyn55d2e02su", - "ownerId": "tejxgn5bt", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", - "state": "Stopped", - "startedByUserId": "ukgvw4i8", - "startedByUserEmail": "bartosz@paperspace.com", - "deploymentType": "TFServing", - "modelId": "mosu30xm7q8vb0p", - "modelUrl": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "name": "some_name", - "tag": None, - "params": None, - "code": None, - "cluster": "PS Jobs on GCP", - "machineType": "K80", - "imageUrl": "https://github.com/Paperspace/mnist-sample", - "imageUsername": None, - "imagePassword": None, - "workspaceUrl": None, - "workspaceUsername": None, - "workspacePassword": None, - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": "-----", - "annotations": {}, - "oauthKey": "-----", - "oauthSecret": "-----", - "serviceType": "model", - "apiType": "REST", - "endpoint": "https://development-services.paperspace.io/model-serving/desdyn55d2e02su:predict", - "ports": None, - "dtCreated": "2019-04-04T10:11:10.204Z", - "dtModified": "2019-04-04T10:11:10.281Z", - "dtBuildStarted": None, - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": "2019-04-04T10:11:10.260Z", - "dtDeleted": None, - "isDeleted": False - }, - { - "id": "des3tmqa3s627o9", - "ownerId": "tejxgn5bt", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", - "state": "Stopped", - "startedByUserId": "ukgvw4i8", - "startedByUserEmail": "bartosz@paperspace.com", - "deploymentType": "TFServing", - "modelId": "mosu30xm7q8vb0p", - "modelUrl": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "name": "some_name", - "tag": None, - "params": None, - "code": None, - "cluster": "PS Jobs on GCP", - "machineType": "K80", - "imageUrl": "https://github.com/Paperspace/mnist-sample", - "imageUsername": None, - "imagePassword": None, - "workspaceUrl": None, - "workspaceUsername": None, - "workspacePassword": None, - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": "-----", - "annotations": {}, - "oauthKey": "-----", - "oauthSecret": "-----", - "serviceType": "model", - "apiType": "REST", - "endpoint": "https://development-services.paperspace.io/model-serving/des3tmqa3s627o9:predict", - "ports": None, - "dtCreated": "2019-04-04T09:29:22.277Z", - "dtModified": "2019-04-04T09:29:22.345Z", - "dtBuildStarted": None, - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": "2019-04-04T09:29:22.327Z", - "dtDeleted": None, - "isDeleted": False - } - ], - "total": 5, - "displayTotal": 5, - "runningTotal": 0 -} - CREATE_MACHINE_RESPONSE = { "id": "psclbvqpc", "name": "some_machine", @@ -1590,57 +320,6 @@ "repoNodeId": None, "repoName": None, "repoUrl": None, - "experiments": { - "data": [ - { - "dtCreated": "2019-04-05T15:10:55.692629+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-04-05T15:10:55.692629+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 22159, - "experimentTemplateId": 60, - "experimentTypeId": 1, - "handle": "estgcoux8igx32", - "id": 22123, - "projectHandle": "prq70zy79", - "projectId": 612, - "started_by_user_id": 1655, - "state": 1, - "templateHistory": { - "dtCreated": "2019-04-05T15:10:54.923725+00:00", - "dtDeleted": None, - "experimentTemplateId": 60, - "id": 22159, - "params": { - "is_preemptible": False, - "name": "dsfads", - "ports": 5000, - "project_handle": "prq70zy79", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "C2", - "worker_use_dockerfile": False, - "workspaceUrl": "example.com" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "prq70zy79" - }, - "totalItems": 1 - } - } }, { "name": "keton", @@ -1651,58 +330,6 @@ "repoNodeId": None, "repoName": None, "repoUrl": None, - "experiments": { - "data": [ - { - "dtCreated": "2019-04-02T15:17:03.393886+00:00", - "dtDeleted": None, - "dtFinished": "2019-04-02T17:02:54.654569+00:00", - "dtModified": "2019-04-02T15:17:03.393886+00:00", - "dtProvisioningFinished": "2019-04-02T15:17:10.978198+00:00", - "dtProvisioningStarted": "2019-04-02T15:17:10.978198+00:00", - "dtStarted": "2019-04-02T15:17:10.978198+00:00", - "dtTeardownFinished": "2019-04-02T17:02:54.654569+00:00", - "dtTeardownStarted": "2019-04-02T17:02:54.654569+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 22130, - "experimentTemplateId": 174, - "experimentTypeId": 1, - "handle": "ehla1kvbwzaco", - "id": 22094, - "projectHandle": "prmr22ve0", - "projectId": 626, - "started_by_user_id": 1655, - "state": 5, - "templateHistory": { - "dtCreated": "2019-04-02T15:17:02.663449+00:00", - "dtDeleted": None, - "experimentTemplateId": 174, - "id": 22130, - "params": { - "is_preemptible": False, - "model_path": "/artifacts", - "model_type": "Tensorflow", - "name": "Test1", - "ports": 5000, - "project_handle": "prmr22ve0", - "worker_command": "python mnist.py --data_format=channels_last", - "worker_container": "tensorflow/tensorflow:1.13.1-py3", - "worker_machine_type": "K80", - "workspaceUrl": "https://github.com/Paperspace/mnist-sample" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "prmr22ve0" - }, - "totalItems": 1 - } - } }, { "name": "paperspace-python", @@ -1713,53 +340,6 @@ "repoNodeId": None, "repoName": None, "repoUrl": None, - "experiments": { - "data": [ - { - "dtCreated": "2019-04-24T10:18:30.523193+00:00", - "dtDeleted": None, - "dtFinished": "2019-04-24T10:18:43.613748+00:00", - "dtModified": "2019-04-24T10:18:30.523193+00:00", - "dtProvisioningFinished": "2019-04-24T10:18:35.010792+00:00", - "dtProvisioningStarted": "2019-04-24T10:18:35.010792+00:00", - "dtStarted": "2019-04-24T10:18:35.010792+00:00", - "dtTeardownFinished": "2019-04-24T10:18:43.613748+00:00", - "dtTeardownStarted": "2019-04-24T10:18:43.613748+00:00", - "experimentError": None, - "experimentTemplateHistoryId": 22311, - "experimentTemplateId": 186, - "experimentTypeId": 1, - "handle": "es47og38wzhnuo", - "id": 22270, - "projectHandle": "przhbct98", - "projectId": 649, - "started_by_user_id": 1655, - "state": 7, - "templateHistory": { - "dtCreated": "2019-04-24T10:18:30.523193+00:00", - "dtDeleted": None, - "experimentTemplateId": 186, - "id": 22311, - "params": { - "command": ". test.sh\npython2 hello.py", - "container": "paperspace/tensorflow-python", - "machineType": "G1", - "project": "paperspace-python", - "workspaceFileName": "temp.zip" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "przhbct98" - }, - "totalItems": 1 - } - } } ], "meta": { @@ -1767,802 +347,11 @@ } } -LIST_JOBS_RESPONSE_JSON = { - "jobList": [ +LIST_MODELS_RESPONSE_JSON = { + "modelList": [ { - "id": "jsxeeba5qq99yn", - "name": "job 1", - "state": "Error", - "workspaceUrl": "none", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "nvidia-smi", + "id": "mosu30xm7q8vb0p", "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "K80", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "K80 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": "Error quering for experiment for job: sql: Scan error on column index 4, name \"model_path\": unsupported Scan, storing driver.Value type into type *string", - "dtCreated": "2019-03-25T14:51:16.118Z", - "dtModified": "2019-03-25T14:51:16.118Z", - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtFinished": "2019-03-27T13:53:34.188Z", - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtDeleted": None, - "exitCode": None, - "queuePosition": None, - "seqNum": 1, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553525480", - "fqdn": "jsxeeba5qq99yn.dgradient.paperspace.com", - "ports": None, - "isPublic": None, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": None, - "cpuCount": None, - "cpuModel": None, - "cpuFlags": None, - "cpuMem": None, - "gpuName": None, - "gpuSerial": None, - "gpuDevice": None, - "gpuDriver": None, - "gpuCount": None, - "gpuMem": None, - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "aws-testing.paperspace.io", - "customMetrics": None, - "experimentId": "esk8lny3pxyqd6", - "tags": [ - "tag1", - "tag2", - ], - }, - { - "id": "jfl063dsv634h", - "name": "job 2", - "state": "Stopped", - "workspaceUrl": "none", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "nvidia-smi", - "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "P100", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "P100 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-03-25T14:54:30.866Z", - "dtModified": "2019-03-25T14:54:30.866Z", - "dtProvisioningStarted": "2019-03-25T14:59:15.818Z", - "dtProvisioningFinished": "2019-03-25T14:59:20.542Z", - "dtStarted": "2019-03-25T14:59:20.542Z", - "dtFinished": "2019-03-25T14:59:25.631Z", - "dtTeardownStarted": "2019-03-25T14:59:25.669Z", - "dtTeardownFinished": "2019-03-25T14:59:25.758Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 2, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553525723", - "fqdn": "jfl063dsv634h.dgradient.paperspace.com", - "ports": None, - "isPublic": None, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553525723", - "cpuCount": 4, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "24683148 kB", - "gpuName": "Tesla P100-PCIE-16GB", - "gpuSerial": "0324317067114", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "16280 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "ejd2v80p7cw6m" - }, - { - "id": "jsvau8w47k78zm", - "name": "Clone - jfl063dsv634h", - "state": "Stopped", - "workspaceUrl": "none", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "nvidia-smi", - "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "P100", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "P100 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": "jfl063dsv634h", - "jobError": None, - "dtCreated": "2019-03-25T15:04:43.844Z", - "dtModified": "2019-03-25T15:04:43.844Z", - "dtProvisioningStarted": "2019-03-25T15:07:43.854Z", - "dtProvisioningFinished": "2019-03-25T15:07:48.435Z", - "dtStarted": "2019-03-25T15:07:48.435Z", - "dtFinished": "2019-03-25T15:07:53.523Z", - "dtTeardownStarted": "2019-03-25T15:07:53.561Z", - "dtTeardownFinished": "2019-03-25T15:07:53.649Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 3, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553526309", - "fqdn": "jsvau8w47k78zm.dgradient.paperspace.com", - "ports": None, - "isPublic": None, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553526309", - "cpuCount": 4, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "24683148 kB", - "gpuName": "Tesla P100-PCIE-16GB", - "gpuSerial": "0324317067114", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "16280 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": None - }, - { - "id": "j2eq99xhvgtum", - "name": "keton1-worker-1", - "state": "Pending", - "workspaceUrl": None, - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "echo keton", - "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "P100", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "P100 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-03-25T15:07:30.383Z", - "dtModified": "2019-03-25T15:07:30.383Z", - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtFinished": "2019-03-25T15:07:30.383Z", - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtDeleted": None, - "exitCode": None, - "queuePosition": None, - "seqNum": 4, - "storageRegion": "GCP West", - "clusterMachine": None, - "fqdn": "j2eq99xhvgtum.dgradient.paperspace.com", - "ports": "5000", - "isPublic": False, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": None, - "cpuCount": None, - "cpuModel": None, - "cpuFlags": None, - "cpuMem": None, - "gpuName": None, - "gpuSerial": None, - "gpuDevice": None, - "gpuDriver": None, - "gpuCount": None, - "gpuMem": None, - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "esibxync23szaq" - }, - { - "id": "jzzinybinuxf9", - "name": "keton2-worker-1", - "state": "Stopped", - "workspaceUrl": "git+https://github.com/Paperspace/multinode-mnist", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "echo keton", - "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "P100", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "P100 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-03-25T15:18:51.461Z", - "dtModified": "2019-03-25T15:18:51.461Z", - "dtProvisioningStarted": "2019-03-25T15:18:58.089Z", - "dtProvisioningFinished": "2019-03-25T15:19:03.246Z", - "dtStarted": "2019-03-25T15:19:03.246Z", - "dtFinished": "2019-03-25T15:19:08.337Z", - "dtTeardownStarted": "2019-03-25T15:19:08.374Z", - "dtTeardownFinished": "2019-03-25T15:19:08.461Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 5, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553526309", - "fqdn": "jzzinybinuxf9.dgradient.paperspace.com", - "ports": "5000", - "isPublic": False, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553526309", - "cpuCount": 4, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "24683148 kB", - "gpuName": "Tesla P100-PCIE-16GB", - "gpuSerial": "0324317067114", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "16280 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "ep6hmawh97q0v" - }, - { - "id": "jsb37duc1zlbz0", - "name": "keton4-worker-1", - "state": "Stopped", - "workspaceUrl": "git+https://github.com/Paperspace/multinode-mnist", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "echo siema", - "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "P100", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "P100 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-03-25T15:29:04.601Z", - "dtModified": "2019-03-25T15:29:04.601Z", - "dtProvisioningStarted": "2019-03-25T15:30:42.529Z", - "dtProvisioningFinished": "2019-03-25T15:30:48.252Z", - "dtStarted": "2019-03-25T15:30:48.252Z", - "dtFinished": "2019-03-25T15:30:53.349Z", - "dtTeardownStarted": "2019-03-25T15:30:53.387Z", - "dtTeardownFinished": "2019-03-25T15:30:53.470Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 6, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553526309", - "fqdn": "jsb37duc1zlbz0.dgradient.paperspace.com", - "ports": "3456", - "isPublic": False, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553526309", - "cpuCount": 4, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "24683148 kB", - "gpuName": "Tesla P100-PCIE-16GB", - "gpuSerial": "0324317067114", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "16280 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "esgeuvkdokyom2" - }, - { - "id": "jq41vipwy18f7", - "name": "keton4-parameter_server-1", - "state": "Stopped", - "workspaceUrl": "git+https://github.com/Paperspace/multinode-mnist", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "ls", - "projectId": "prmr22ve0", - "project": "keton", - "container": "Test-Container", - "containerUrl": "paperspace/tensorflow:1.5.0-gpu", - "baseContainer": "Test-Container", - "baseContainerUrl": "paperspace/tensorflow:1.5.0-gpu", - "machineType": "P100", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "P100 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-03-25T15:29:06.765Z", - "dtModified": "2019-03-25T15:29:06.765Z", - "dtProvisioningStarted": "2019-03-25T15:30:41.416Z", - "dtProvisioningFinished": "2019-03-25T15:30:48.004Z", - "dtStarted": "2019-03-25T15:30:48.004Z", - "dtFinished": "2019-03-25T15:30:53.097Z", - "dtTeardownStarted": "2019-03-25T15:30:53.135Z", - "dtTeardownFinished": "2019-03-25T15:30:53.223Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 7, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553526384", - "fqdn": "jq41vipwy18f7.dgradient.paperspace.com", - "ports": "3456", - "isPublic": False, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553526384", - "cpuCount": 4, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "24683148 kB", - "gpuName": "Tesla P100-PCIE-16GB", - "gpuSerial": "0324317004340", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "16280 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "esgeuvkdokyom2" - }, - { - "id": "jsigkjnyb6m3qm", - "name": "Test1-worker-1", - "state": "Stopped", - "workspaceUrl": "git+https://github.com/Paperspace/mnist-sample", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "python mnist.py --data_format=channels_last", - "projectId": "prmr22ve0", - "project": "keton", - "container": "tensorflow/tensorflow:1.13.1-py3", - "containerUrl": "tensorflow/tensorflow:1.13.1-py3", - "baseContainer": "tensorflow/tensorflow:1.13.1-py3", - "baseContainerUrl": "tensorflow/tensorflow:1.13.1-py3", - "machineType": "K80", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "K80 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-04-02T15:17:05.618Z", - "dtModified": "2019-04-02T15:17:05.618Z", - "dtProvisioningStarted": "2019-04-02T15:17:11.018Z", - "dtProvisioningFinished": "2019-04-02T15:17:56.754Z", - "dtStarted": "2019-04-02T15:17:56.754Z", - "dtFinished": "2019-04-02T17:02:26.950Z", - "dtTeardownStarted": "2019-04-02T17:02:26.987Z", - "dtTeardownFinished": "2019-04-02T17:02:54.596Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 8, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553793082-82415ed3", - "fqdn": "jsigkjnyb6m3qm.dgradient.paperspace.com", - "ports": "5000:5000", - "isPublic": False, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553793082-82415ed3", - "cpuCount": 2, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "12297216 kB", - "gpuName": "Tesla K80", - "gpuSerial": "0320617087317", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "11441 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "ehla1kvbwzaco" - }, - { - "id": "j4g76vuppxqao", - "name": "job 1", - "state": "Stopped", - "workspaceUrl": "s3://ps-projects-development/przhbct98/j4g76vuppxqao/__init__.py.zip", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "echo keton", - "projectId": "przhbct98", - "project": "paperspace-python", - "container": "paperspace/tensorflow-python", - "containerUrl": "paperspace/tensorflow-python", - "baseContainer": "paperspace/tensorflow-python", - "baseContainerUrl": "paperspace/tensorflow-python", - "machineType": "K80", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "K80 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-04-04T15:12:34.414Z", - "dtModified": "2019-04-04T15:12:34.414Z", - "dtProvisioningStarted": "2019-04-04T15:12:41.338Z", - "dtProvisioningFinished": "2019-04-04T15:12:47.492Z", - "dtStarted": "2019-04-04T15:12:47.492Z", - "dtFinished": "2019-04-04T15:12:52.582Z", - "dtTeardownStarted": "2019-04-04T15:12:52.621Z", - "dtTeardownFinished": "2019-04-04T15:12:52.752Z", - "dtDeleted": None, - "exitCode": 0, - "queuePosition": None, - "seqNum": 1, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1553793082-82415ed3", - "fqdn": "j4g76vuppxqao.dgradient.paperspace.com", - "ports": None, - "isPublic": None, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1553793082-82415ed3", - "cpuCount": 2, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "12297216 kB", - "gpuName": "Tesla K80", - "gpuSerial": "0320617087317", - "gpuDevice": "/dev/nvidia0", - "gpuDriver": "410.48", - "gpuCount": 1, - "gpuMem": "11441 MiB", - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "esfmkbql393ut0" - }, - { - "id": "jsbnvdhwb46vr9", - "name": "job 2", - "state": "Failed", - "workspaceUrl": "s3://ps-projects-development/przhbct98/jsbnvdhwb46vr9/temp.zip", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": ". test.sh\npython2 hello.py", - "projectId": "przhbct98", - "project": "paperspace-python", - "container": "paperspace/tensorflow-python", - "containerUrl": "paperspace/tensorflow-python", - "baseContainer": "paperspace/tensorflow-python", - "baseContainerUrl": "paperspace/tensorflow-python", - "machineType": "G1", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "G1 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-04-24T09:09:53.645Z", - "dtModified": "2019-04-24T09:09:53.645Z", - "dtProvisioningStarted": "2019-04-24T09:10:50.771Z", - "dtProvisioningFinished": "2019-04-24T09:11:50.968Z", - "dtStarted": "2019-04-24T09:11:50.968Z", - "dtFinished": "2019-04-24T09:11:56.092Z", - "dtTeardownStarted": "2019-04-24T09:11:56.150Z", - "dtTeardownFinished": "2019-04-24T09:11:56.346Z", - "dtDeleted": None, - "exitCode": 2, - "queuePosition": None, - "seqNum": 2, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1556074006", - "fqdn": "jsbnvdhwb46vr9.dgradient.paperspace.com", - "ports": None, - "isPublic": None, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1556074006", - "cpuCount": 1, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "1783384 kB", - "gpuName": None, - "gpuSerial": None, - "gpuDevice": None, - "gpuDriver": None, - "gpuCount": None, - "gpuMem": None, - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "esmnlol3tavvvf" - }, - { - "id": "jt8alwzv28kha", - "name": "job 3", - "state": "Failed", - "workspaceUrl": "s3://ps-projects-development/przhbct98/jt8alwzv28kha/temp.zip", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": ". test.sh\npython2 hello.py", - "projectId": "przhbct98", - "project": "paperspace-python", - "container": "paperspace/tensorflow-python", - "containerUrl": "paperspace/tensorflow-python", - "baseContainer": "paperspace/tensorflow-python", - "baseContainerUrl": "paperspace/tensorflow-python", - "machineType": "G1", - "cluster": "PS Jobs on GCP", - "clusterId": "clkyczmyz", - "usageRate": "G1 hourly", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2019-04-24T10:18:30.620Z", - "dtModified": "2019-04-24T10:18:30.620Z", - "dtProvisioningStarted": "2019-04-24T10:18:35.057Z", - "dtProvisioningFinished": "2019-04-24T10:18:38.241Z", - "dtStarted": "2019-04-24T10:18:38.241Z", - "dtFinished": "2019-04-24T10:18:43.348Z", - "dtTeardownStarted": "2019-04-24T10:18:43.394Z", - "dtTeardownFinished": "2019-04-24T10:18:43.544Z", - "dtDeleted": None, - "exitCode": 2, - "queuePosition": None, - "seqNum": 3, - "storageRegion": "GCP West", - "clusterMachine": "gradient-host-1556074006", - "fqdn": "jt8alwzv28kha.dgradient.paperspace.com", - "ports": None, - "isPublic": None, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": "gradient-host-1556074006", - "cpuCount": 1, - "cpuModel": "Intel(R) Xeon(R) CPU @ 2.30GHz", - "cpuFlags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat arch_capabilities", - "cpuMem": "1783384 kB", - "gpuName": None, - "gpuSerial": None, - "gpuDevice": None, - "gpuDriver": None, - "gpuCount": None, - "gpuMem": None, - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "metrics-gcp-dev.paperspace.io", - "customMetrics": None, - "experimentId": "es47og38wzhnuo" - } - ] -} - -LIST_MODELS_RESPONSE_JSON = { - "modelList": [ - { - "id": "mosu30xm7q8vb0p", - "projectId": "prmr22ve0", - "experimentId": "ehla1kvbwzaco", "modelType": "Tensorflow", "name": None, "tag": None, @@ -3114,1557 +903,101 @@ "checkpoint": 5785 }, { - "value": 0, - "checkpoint": 6956 - }, - { - "value": 0, - "checkpoint": 8116 - }, - { - "value": 0, - "checkpoint": 9262 - }, - { - "value": 0, - "checkpoint": 10397 - }, - { - "value": 0, - "checkpoint": 11566 - }, - { - "value": 0, - "checkpoint": 12000 - } - ] - }, - { - "operation": "var", - "checkpoints": [ - { - "value": 0, - "checkpoint": 1175 - }, - { - "value": 0, - "checkpoint": 2323 - }, - { - "value": 0, - "checkpoint": 3471 - }, - { - "value": 0, - "checkpoint": 4606 - }, - { - "value": 0, - "checkpoint": 5785 - }, - { - "value": 0, - "checkpoint": 6956 - }, - { - "value": 0, - "checkpoint": 8116 - }, - { - "value": 0, - "checkpoint": 9262 - }, - { - "value": 0, - "checkpoint": 10397 - }, - { - "value": 0, - "checkpoint": 11566 - }, - { - "value": 0, - "checkpoint": 12000 - } - ] - } - ], - "scalar": "accuracy" - } - ], - "session": "eval" - } - ], - "params": None, - "url": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", - "notes": None, - "isDeleted": False, - "isPublic": False, - "dtCreated": "2019-04-02T17:02:47.157Z", - "dtModified": "2019-04-02T17:02:54.273Z", - "dtUploaded": "2019-04-02T17:02:54.273Z", - "dtDeleted": None, - "modelPath": "/artifacts" - } - ], - "total": 1, - "displayTotal": 1 -} - -LIST_OF_LOGS_FOR_JOB = [ - { - "line": 1, - "timestamp": "2019-04-03T15:56:35.457Z", - "message": "Traceback (most recent call last):" - }, { - "line": 2, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " File \"generate_figures.py\", line 15, in " - }, { - "line": 3, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " import dnnlib.tflib as tflib" - }, { - "line": 4, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " File \"/paperspace/dnnlib/tflib/__init__.py\", line 8, in " - }, { - "line": 5, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " from . import autosummary" - }, { - "line": 6, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " File \"/paperspace/dnnlib/tflib/autosummary.py\", line 31, in " - }, { - "line": 7, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " from . import tfutil" - }, { - "line": 8, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " File \"/paperspace/dnnlib/tflib/tfutil.py\", line 34, in " - }, { - "line": 9, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": " def shape_to_list(shape: Iterable[tf.Dimension]) -> List[Union[int, None]]:" - }, { - "line": 10, - "timestamp": "2019-04-03T15:56:35.458Z", - "message": "AttributeError: module \'tensorflow\' has no attribute \'Dimension\'" - }, { - "line": 11, - "timestamp": "2019-04-03T15:56:46.168Z", - "message": "PSEOF" - } -] - -LIST_HYPERPARAMETERS_RESPONSE_JSON = { - "data": [ - { - "dtCreated": "2019-05-13T11:29:43.155736+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-05-13T11:29:43.155736+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 45987, - "experimentTemplateId": 8, - "experimentTypeId": 4, - "handle": "es3dn6fu16r4kk", - "id": 45980, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 1, - "templateHistory": { - "dtCreated": "2019-05-13T11:29:41.933178+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 45987, - "params": { - "is_preemptible": False, - "name": "some_name", - "ports": 5000, - "project_handle": "pr4yxj956", - "tuning_command": "some command", - "worker_command": "some worker command", - "worker_container": "some_container", - "worker_count": 1, - "worker_machine_type": "k80", - "worker_use_dockerfile": False, - "workspaceUrl": "none" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-05-13T11:29:40.196982+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-05-13T11:29:40.196982+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 45986, - "experimentTemplateId": 8, - "experimentTypeId": 4, - "handle": "eshlqek7wzvrxa", - "id": 45979, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 1, - "templateHistory": { - "dtCreated": "2019-05-13T11:29:38.882999+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 45986, - "params": { - "is_preemptible": False, - "name": "some_name", - "ports": 5000, - "project_handle": "pr4yxj956", - "tuning_command": "some command", - "worker_command": "some worker command", - "worker_container": "some_container", - "worker_count": 1, - "worker_machine_type": "k80", - "worker_use_dockerfile": False, - "workspaceUrl": "none" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-05-13T11:25:03.760490+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-05-13T11:25:03.760490+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 45985, - "experimentTemplateId": 8, - "experimentTypeId": 4, - "handle": "esdwnui5qsk8qm", - "id": 45978, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 1, - "templateHistory": { - "dtCreated": "2019-05-13T11:25:02.489045+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 45985, - "params": { - "is_preemptible": False, - "name": "some_name", - "ports": 5000, - "project_handle": "pr4yxj956", - "tuning_command": "some command", - "worker_command": "some worker command", - "worker_container": "some_container", - "worker_count": 1, - "worker_machine_type": "k80", - "worker_use_dockerfile": False, - "workspaceUrl": "none" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - { - "dtCreated": "2019-05-13T11:23:53.803162+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-05-13T11:23:53.803162+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 45984, - "experimentTemplateId": 8, - "experimentTypeId": 4, - "handle": "eshz1z9k37w4nm", - "id": 45977, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 1, - "templateHistory": { - "dtCreated": "2019-05-13T11:23:52.778310+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 45984, - "params": { - "is_preemptible": False, - "name": "some_name", - "ports": 5000, - "project_handle": "pr4yxj956", - "tuning_command": "some command", - "worker_command": "some worker command", - "worker_container": "some_container", - "worker_count": 1, - "worker_machine_type": "k80", - "worker_use_dockerfile": False, - "workspaceUrl": "none" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - ], - "message": "success", - "meta": { - "filter": [], - "limit": 11, - "offset": 0, - "totalItems": 15 - } -} - -HYPERPARAMETERS_DETAILS_RESPONSE_JSON = { - "data": { - "dtCreated": "2019-05-13T10:57:32.828135+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-05-13T10:57:32.828135+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 45973, - "experimentTemplateId": 8, - "experimentTypeId": 4, - "handle": "ess6t3fjs2hb1g", - "id": 45966, - "projectHandle": "pr4yxj956", - "projectId": 14898, - "started_by_user_id": 199654, - "state": 1, - "templateHistory": { - "dtCreated": "2019-05-13T10:57:31.592657+00:00", - "dtDeleted": None, - "experimentTemplateId": 8, - "id": 45973, - "params": { - "is_preemptible": False, - "name": "some_name", - "ports": 5000, - "project_handle": "pr4yxj956", - "tuning_command": "some command", - "worker_command": "some worker command", - "worker_container": "some_container", - "worker_count": 1, - "worker_machine_type": "k80", - "worker_use_dockerfile": False, - "workspaceUrl": "none" - }, - "triggerEvent": None, - "triggerEventId": None - } - }, - "message": "success" -} - -LIST_OF_LOGS_FOR_EXPERIMENT = [ - { - "jobId": "jsy2ibsz1l026y", - "line": 1, - "timestamp": "2019-07-08T12:40:59.139Z", - "message": "2019-07-08 12:40:59.139494: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 2, - "timestamp": "2019-07-08T12:40:59.271Z", - "message": "2019-07-08 12:40:59.270783: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 3, - "timestamp": "2019-07-08T12:40:59.271Z", - "message": "2019-07-08 12:40:59.271379: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x5642a00 executing computations on platform CUDA. Devices:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 4, - "timestamp": "2019-07-08T12:40:59.271Z", - "message": "2019-07-08 12:40:59.271427: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): Tesla K80, Compute Capability 3.7" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 5, - "timestamp": "2019-07-08T12:40:59.274Z", - "message": "2019-07-08 12:40:59.274589: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 6, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "2019-07-08 12:40:59.274952: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x56aba20 executing computations on platform Host. Devices:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 7, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "2019-07-08 12:40:59.274983: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): \u003cundefined\u003e, \u003cundefined\u003e" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 8, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "2019-07-08 12:40:59.275358: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 9, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 10, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "pciBusID: 0000:00:04.0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 11, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "totalMemory: 11.17GiB freeMemory: 11.09GiB" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 12, - "timestamp": "2019-07-08T12:40:59.275Z", - "message": "2019-07-08 12:40:59.275392: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 13, - "timestamp": "2019-07-08T12:40:59.276Z", - "message": "2019-07-08 12:40:59.276439: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 14, - "timestamp": "2019-07-08T12:40:59.276Z", - "message": "2019-07-08 12:40:59.276473: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0 " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 15, - "timestamp": "2019-07-08T12:40:59.276Z", - "message": "2019-07-08 12:40:59.276483: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 16, - "timestamp": "2019-07-08T12:40:59.276Z", - "message": "2019-07-08 12:40:59.276778: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:0 with 10787 MB memory) -\u003e physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 17, - "timestamp": "2019-07-08T12:40:59.28Z", - "message": "WARNING: Logging before flag parsing goes to stderr." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 18, - "timestamp": "2019-07-08T12:40:59.28Z", - "message": "I0708 12:40:59.280403 140451862599424 mnist.py:265] ==================== Environment Variables ====================" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 19, - "timestamp": "2019-07-08T12:40:59.28Z", - "message": "I0708 12:40:59.280700 140451862599424 mnist.py:267] NVIDIA_VISIBLE_DEVICES: all" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 20, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.280864 140451862599424 mnist.py:267] PS_DOCKER_AGENT: 6136343261613365" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 21, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.280993 140451862599424 mnist.py:267] EVAL_SECS: 10" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 22, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281114 140451862599424 mnist.py:267] PATH: /usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 23, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281230 140451862599424 mnist.py:267] PS_HOST_PRIVATE_IP_ADDRESS: 10.138.0.15" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 24, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281354 140451862599424 mnist.py:267] PS_HOSTNAME: gradient-host-1562589161" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 25, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281510 140451862599424 mnist.py:267] HOSTNAME: 9b41cad9ccda" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 26, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281623 140451862599424 mnist.py:267] EPOCHS_EVAL: 5" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 27, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281736 140451862599424 mnist.py:267] PS_JOBSPACE: /storage" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 28, - "timestamp": "2019-07-08T12:40:59.281Z", - "message": "I0708 12:40:59.281847 140451862599424 mnist.py:267] PS_JOB_RUNNER: 1" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 29, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.281965 140451862599424 mnist.py:267] NVIDIA_REQUIRE_CUDA: cuda\u003e=10.0 brand=tesla,driver\u003e=384,driver\u003c385 brand=tesla,driver\u003e=410,driver\u003c411" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 30, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282079 140451862599424 mnist.py:267] CUDA_PKG_VERSION: 10-0=10.0.130-1" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 31, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282189 140451862599424 mnist.py:267] LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 32, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282299 140451862599424 mnist.py:267] TYPE: master" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 33, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282416 140451862599424 mnist.py:267] PS_DATASETS: /datasets" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 34, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282531 140451862599424 mnist.py:267] TRAIN_EPOCHS: 10" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 35, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282641 140451862599424 mnist.py:267] MAX_STEPS: 1000" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 36, - "timestamp": "2019-07-08T12:40:59.282Z", - "message": "I0708 12:40:59.282753 140451862599424 mnist.py:267] PS_HOME: /paperspace" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 37, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.282861 140451862599424 mnist.py:267] INDEX: 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 38, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.282970 140451862599424 mnist.py:267] CUDA_VERSION: 10.0.130" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 39, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283078 140451862599424 mnist.py:267] PORTS: 5000" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 40, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283186 140451862599424 mnist.py:267] PS_ARTIFACTS: /artifacts" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 41, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283293 140451862599424 mnist.py:267] NVIDIA_DRIVER_CAPABILITIES: compute,utility" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 42, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283408 140451862599424 mnist.py:267] PS_HOST_PUBLIC_IP_ADDRESS: 35.233.198.134" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 43, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283516 140451862599424 mnist.py:267] LANG: C.UTF-8" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 44, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283623 140451862599424 mnist.py:267] PS_JOB_ID: jsy2ibsz1l026y" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 45, - "timestamp": "2019-07-08T12:40:59.283Z", - "message": "I0708 12:40:59.283730 140451862599424 mnist.py:267] HOME: /root" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 46, - "timestamp": "2019-07-08T12:40:59.862Z", - "message": "I0708 12:40:59.862003 140451862599424 run_config.py:532] Initializing RunConfig with distribution strategies." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 47, - "timestamp": "2019-07-08T12:40:59.862Z", - "message": "I0708 12:40:59.862317 140451862599424 estimator_training.py:166] Not using Distribute Coordinator." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 48, - "timestamp": "2019-07-08T12:40:59.863Z", - "message": "I0708 12:40:59.862842 140451862599424 estimator.py:201] Using config: {'_master': '', '_num_worker_replicas': 1, '_num_ps_replicas': 0, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_train_distribute': \u003ctensorflow.contrib.distribute.python.one_device_strategy.OneDeviceStrategy object at 0x7fbcb5adecc0\u003e, '_protocol': None, '_session_config': allow_soft_placement: True" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 49, - "timestamp": "2019-07-08T12:40:59.863Z", - "message": ", '_tf_random_seed': None, '_global_id_in_cluster': 0, '_is_chief': True, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': '/paperspace/models/mnist', '_save_summary_steps': 100, '_service': None, '_experimental_distribute': None, '_cluster_spec': \u003ctensorflow.python.training.server_lib.ClusterSpec object at 0x7fbcb34f7828\u003e, '_save_checkpoints_steps': 600, '_evaluation_master': '', '_eval_distribute': None, '_task_id': 0, '_distribute_coordinator_mode': None, '_task_type': 'worker', '_device_fn': None}" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 50, - "timestamp": "2019-07-08T12:40:59.863Z", - "message": "I0708 12:40:59.863682 140451862599424 estimator_training.py:185] Not using Distribute Coordinator." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 51, - "timestamp": "2019-07-08T12:40:59.864Z", - "message": "I0708 12:40:59.864074 140451862599424 training.py:610] Running training and evaluation locally (non-distributed)." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 52, - "timestamp": "2019-07-08T12:40:59.864Z", - "message": "I0708 12:40:59.864597 140451862599424 training.py:698] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 600 or save_checkpoints_secs None." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 53, - "timestamp": "2019-07-08T12:41:01.276Z", - "message": "W0708 12:41:01.276278 140451862599424 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/data/ops/dataset_ops.py:1419: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 54, - "timestamp": "2019-07-08T12:41:01.276Z", - "message": "Instructions for updating:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 55, - "timestamp": "2019-07-08T12:41:01.276Z", - "message": "Colocations handled automatically by placer." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 56, - "timestamp": "2019-07-08T12:41:01.292Z", - "message": "I0708 12:41:01.291953 140451862599424 estimator.py:1111] Calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 57, - "timestamp": "2019-07-08T12:41:01.389Z", - "message": "W0708 12:41:01.389566 140451862599424 deprecation.py:506] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 58, - "timestamp": "2019-07-08T12:41:01.389Z", - "message": "Instructions for updating:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 59, - "timestamp": "2019-07-08T12:41:01.389Z", - "message": "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 60, - "timestamp": "2019-07-08T12:41:01.537Z", - "message": "W0708 12:41:01.536658 140451862599424 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 61, - "timestamp": "2019-07-08T12:41:01.537Z", - "message": "Instructions for updating:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 62, - "timestamp": "2019-07-08T12:41:01.537Z", - "message": "Use tf.cast instead." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 63, - "timestamp": "2019-07-08T12:41:01.86Z", - "message": "I0708 12:41:01.860109 140451862599424 estimator.py:1113] Done calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 64, - "timestamp": "2019-07-08T12:41:01.909Z", - "message": "I0708 12:41:01.909625 140451862599424 basic_session_run_hooks.py:527] Create CheckpointSaverHook." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 65, - "timestamp": "2019-07-08T12:41:02.104Z", - "message": "I0708 12:41:02.104314 140451862599424 monitored_session.py:222] Graph was finalized." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 66, - "timestamp": "2019-07-08T12:41:02.105Z", - "message": "2019-07-08 12:41:02.105148: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 67, - "timestamp": "2019-07-08T12:41:02.105Z", - "message": "2019-07-08 12:41:02.105239: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 68, - "timestamp": "2019-07-08T12:41:02.105Z", - "message": "2019-07-08 12:41:02.105274: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0 " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 69, - "timestamp": "2019-07-08T12:41:02.105Z", - "message": "2019-07-08 12:41:02.105290: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 70, - "timestamp": "2019-07-08T12:41:02.105Z", - "message": "2019-07-08 12:41:02.105612: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10787 MB memory) -\u003e physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 71, - "timestamp": "2019-07-08T12:41:03.052Z", - "message": "I0708 12:41:03.051607 140451862599424 session_manager.py:491] Running local_init_op." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 72, - "timestamp": "2019-07-08T12:41:03.064Z", - "message": "I0708 12:41:03.064292 140451862599424 session_manager.py:493] Done running local_init_op." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 73, - "timestamp": "2019-07-08T12:41:03.405Z", - "message": "I0708 12:41:03.404671 140451862599424 basic_session_run_hooks.py:594] Saving checkpoints for 0 into /paperspace/models/mnist/model.ckpt." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 74, - "timestamp": "2019-07-08T12:41:03.616Z", - "message": "I0708 12:41:03.616098 140451862599424 util.py:164] Initialize strategy" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 75, - "timestamp": "2019-07-08T12:41:03.715Z", - "message": "2019-07-08 12:41:03.715695: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 76, - "timestamp": "2019-07-08T12:41:09.915Z", - "message": "I0708 12:41:09.915456 140451862599424 basic_session_run_hooks.py:249] cross_entropy = 2.3025289, learning_rate = 1e-04, train_accuracy = 0.08" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 77, - "timestamp": "2019-07-08T12:41:09.916Z", - "message": "I0708 12:41:09.915941 140451862599424 basic_session_run_hooks.py:249] loss = 2.3025289, step = 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 78, - "timestamp": "2019-07-08T12:41:11.313Z", - "message": "I0708 12:41:11.313229 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 71.5095" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 79, - "timestamp": "2019-07-08T12:41:11.314Z", - "message": "I0708 12:41:11.314374 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.3830725, learning_rate = 1e-04, train_accuracy = 0.49 (1.399 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 80, - "timestamp": "2019-07-08T12:41:11.314Z", - "message": "I0708 12:41:11.314673 140451862599424 basic_session_run_hooks.py:247] loss = 0.3830725, step = 100 (1.399 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 81, - "timestamp": "2019-07-08T12:41:12.369Z", - "message": "I0708 12:41:12.368999 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 94.7225" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 82, - "timestamp": "2019-07-08T12:41:12.37Z", - "message": "I0708 12:41:12.370402 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.19424663, learning_rate = 1e-04, train_accuracy = 0.6433333 (1.056 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 83, - "timestamp": "2019-07-08T12:41:12.371Z", - "message": "I0708 12:41:12.370768 140451862599424 basic_session_run_hooks.py:247] loss = 0.19424663, step = 200 (1.056 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 84, - "timestamp": "2019-07-08T12:41:13.424Z", - "message": "I0708 12:41:13.423866 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 94.7932" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 85, - "timestamp": "2019-07-08T12:41:13.425Z", - "message": "I0708 12:41:13.425122 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.20186995, learning_rate = 1e-04, train_accuracy = 0.7125 (1.055 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 86, - "timestamp": "2019-07-08T12:41:13.425Z", - "message": "I0708 12:41:13.425472 140451862599424 basic_session_run_hooks.py:247] loss = 0.20186995, step = 300 (1.055 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 87, - "timestamp": "2019-07-08T12:41:14.457Z", - "message": "I0708 12:41:14.457170 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 96.7791" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 88, - "timestamp": "2019-07-08T12:41:14.458Z", - "message": "I0708 12:41:14.458280 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.14686151, learning_rate = 1e-04, train_accuracy = 0.762 (1.033 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 89, - "timestamp": "2019-07-08T12:41:14.458Z", - "message": "I0708 12:41:14.458536 140451862599424 basic_session_run_hooks.py:247] loss = 0.14686151, step = 400 (1.033 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 90, - "timestamp": "2019-07-08T12:41:15.531Z", - "message": "I0708 12:41:15.530984 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 93.1259" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 91, - "timestamp": "2019-07-08T12:41:15.532Z", - "message": "I0708 12:41:15.532083 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.115687564, learning_rate = 1e-04, train_accuracy = 0.795 (1.074 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 92, - "timestamp": "2019-07-08T12:41:15.532Z", - "message": "I0708 12:41:15.532342 140451862599424 basic_session_run_hooks.py:247] loss = 0.115687564, step = 500 (1.074 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 93, - "timestamp": "2019-07-08T12:41:16.563Z", - "message": "I0708 12:41:16.563513 140451862599424 basic_session_run_hooks.py:594] Saving checkpoints for 600 into /paperspace/models/mnist/model.ckpt." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 94, - "timestamp": "2019-07-08T12:41:17.176Z", - "message": "I0708 12:41:17.176506 140451862599424 estimator.py:1111] Calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 95, - "timestamp": "2019-07-08T12:41:17.385Z", - "message": "I0708 12:41:17.385438 140451862599424 estimator.py:1113] Done calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 96, - "timestamp": "2019-07-08T12:41:17.41Z", - "message": "I0708 12:41:17.410446 140451862599424 evaluation.py:257] Starting evaluation at 2019-07-08T12:41:17Z" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 97, - "timestamp": "2019-07-08T12:41:17.517Z", - "message": "I0708 12:41:17.517657 140451862599424 monitored_session.py:222] Graph was finalized." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 98, - "timestamp": "2019-07-08T12:41:17.518Z", - "message": "2019-07-08 12:41:17.518354: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 99, - "timestamp": "2019-07-08T12:41:17.518Z", - "message": "2019-07-08 12:41:17.518419: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 100, - "timestamp": "2019-07-08T12:41:17.518Z", - "message": "2019-07-08 12:41:17.518438: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0 " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 101, - "timestamp": "2019-07-08T12:41:17.518Z", - "message": "2019-07-08 12:41:17.518451: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 102, - "timestamp": "2019-07-08T12:41:17.518Z", - "message": "2019-07-08 12:41:17.518626: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10787 MB memory) -\u003e physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 103, - "timestamp": "2019-07-08T12:41:17.519Z", - "message": "W0708 12:41:17.519085 140451862599424 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 104, - "timestamp": "2019-07-08T12:41:17.519Z", - "message": "Instructions for updating:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 105, - "timestamp": "2019-07-08T12:41:17.519Z", - "message": "Use standard file APIs to check for files with this prefix." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 106, - "timestamp": "2019-07-08T12:41:17.52Z", - "message": "I0708 12:41:17.520449 140451862599424 saver.py:1270] Restoring parameters from /paperspace/models/mnist/model.ckpt-600" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 107, - "timestamp": "2019-07-08T12:41:17.572Z", - "message": "I0708 12:41:17.572486 140451862599424 session_manager.py:491] Running local_init_op." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 108, - "timestamp": "2019-07-08T12:41:17.584Z", - "message": "I0708 12:41:17.584268 140451862599424 session_manager.py:493] Done running local_init_op." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 109, - "timestamp": "2019-07-08T12:41:19.062Z", - "message": "I0708 12:41:19.062499 140451862599424 evaluation.py:277] Finished evaluation at 2019-07-08-12:41:19" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 110, - "timestamp": "2019-07-08T12:41:19.063Z", - "message": "I0708 12:41:19.062822 140451862599424 estimator.py:1979] Saving dict for global step 600: accuracy = 0.9691, global_step = 600, loss = 0.10409811" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 111, - "timestamp": "2019-07-08T12:41:19.113Z", - "message": "I0708 12:41:19.113427 140451862599424 estimator.py:2039] Saving 'checkpoint_path' summary for global step 600: /paperspace/models/mnist/model.ckpt-600" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 112, - "timestamp": "2019-07-08T12:41:19.126Z", - "message": "I0708 12:41:19.125805 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 27.817" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 113, - "timestamp": "2019-07-08T12:41:19.126Z", - "message": "I0708 12:41:19.126667 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.108451165, learning_rate = 1e-04, train_accuracy = 0.82285714 (3.595 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 114, - "timestamp": "2019-07-08T12:41:19.127Z", - "message": "I0708 12:41:19.126947 140451862599424 basic_session_run_hooks.py:247] loss = 0.108451165, step = 600 (3.595 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 115, - "timestamp": "2019-07-08T12:41:20.185Z", - "message": "I0708 12:41:20.185327 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 94.39" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 116, - "timestamp": "2019-07-08T12:41:20.186Z", - "message": "I0708 12:41:20.186514 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.122035645, learning_rate = 1e-04, train_accuracy = 0.8425 (1.060 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 117, - "timestamp": "2019-07-08T12:41:20.186Z", - "message": "I0708 12:41:20.186774 140451862599424 basic_session_run_hooks.py:247] loss = 0.122035645, step = 700 (1.060 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 118, - "timestamp": "2019-07-08T12:41:21.256Z", - "message": "I0708 12:41:21.255784 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 93.4188" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 119, - "timestamp": "2019-07-08T12:41:21.257Z", - "message": "I0708 12:41:21.256838 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.07952806, learning_rate = 1e-04, train_accuracy = 0.8577778 (1.070 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 120, - "timestamp": "2019-07-08T12:41:21.257Z", - "message": "I0708 12:41:21.257101 140451862599424 basic_session_run_hooks.py:247] loss = 0.07952806, step = 800 (1.070 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 121, - "timestamp": "2019-07-08T12:41:22.288Z", - "message": "I0708 12:41:22.287836 140451862599424 basic_session_run_hooks.py:680] global_step/sec: 96.8942" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 122, - "timestamp": "2019-07-08T12:41:22.289Z", - "message": "I0708 12:41:22.288971 140451862599424 basic_session_run_hooks.py:247] cross_entropy = 0.12382953, learning_rate = 1e-04, train_accuracy = 0.868 (1.032 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 123, - "timestamp": "2019-07-08T12:41:22.289Z", - "message": "I0708 12:41:22.289424 140451862599424 basic_session_run_hooks.py:247] loss = 0.12382953, step = 900 (1.032 sec)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 124, - "timestamp": "2019-07-08T12:41:23.297Z", - "message": "I0708 12:41:23.297554 140451862599424 basic_session_run_hooks.py:594] Saving checkpoints for 1000 into /paperspace/models/mnist/model.ckpt." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 125, - "timestamp": "2019-07-08T12:41:23.399Z", - "message": "I0708 12:41:23.398564 140451862599424 training.py:525] Skip the current checkpoint eval due to throttle secs (10 secs)." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 126, - "timestamp": "2019-07-08T12:41:23.451Z", - "message": "I0708 12:41:23.450761 140451862599424 estimator.py:1111] Calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 127, - "timestamp": "2019-07-08T12:41:23.655Z", - "message": "I0708 12:41:23.655447 140451862599424 estimator.py:1113] Done calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 128, - "timestamp": "2019-07-08T12:41:23.681Z", - "message": "I0708 12:41:23.681046 140451862599424 evaluation.py:257] Starting evaluation at 2019-07-08T12:41:23Z" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 129, - "timestamp": "2019-07-08T12:41:23.872Z", - "message": "I0708 12:41:23.871646 140451862599424 monitored_session.py:222] Graph was finalized." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 130, - "timestamp": "2019-07-08T12:41:23.872Z", - "message": "2019-07-08 12:41:23.872269: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 131, - "timestamp": "2019-07-08T12:41:23.872Z", - "message": "2019-07-08 12:41:23.872324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 132, - "timestamp": "2019-07-08T12:41:23.872Z", - "message": "2019-07-08 12:41:23.872338: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0 " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 133, - "timestamp": "2019-07-08T12:41:23.872Z", - "message": "2019-07-08 12:41:23.872353: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 134, - "timestamp": "2019-07-08T12:41:23.872Z", - "message": "2019-07-08 12:41:23.872548: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10787 MB memory) -\u003e physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 135, - "timestamp": "2019-07-08T12:41:23.874Z", - "message": "I0708 12:41:23.874099 140451862599424 saver.py:1270] Restoring parameters from /paperspace/models/mnist/model.ckpt-1000" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 136, - "timestamp": "2019-07-08T12:41:23.932Z", - "message": "I0708 12:41:23.932351 140451862599424 session_manager.py:491] Running local_init_op." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 137, - "timestamp": "2019-07-08T12:41:23.946Z", - "message": "I0708 12:41:23.946145 140451862599424 session_manager.py:493] Done running local_init_op." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 138, - "timestamp": "2019-07-08T12:41:25.49Z", - "message": "I0708 12:41:25.490408 140451862599424 evaluation.py:277] Finished evaluation at 2019-07-08-12:41:25" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 139, - "timestamp": "2019-07-08T12:41:25.491Z", - "message": "I0708 12:41:25.490850 140451862599424 estimator.py:1979] Saving dict for global step 1000: accuracy = 0.9787, global_step = 1000, loss = 0.0693267" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 140, - "timestamp": "2019-07-08T12:41:25.491Z", - "message": "I0708 12:41:25.491545 140451862599424 estimator.py:2039] Saving 'checkpoint_path' summary for global step 1000: /paperspace/models/mnist/model.ckpt-1000" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 141, - "timestamp": "2019-07-08T12:41:25.492Z", - "message": "I0708 12:41:25.492296 140451862599424 training.py:928] Calling exporter with the `is_the_final_export=True`." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 142, - "timestamp": "2019-07-08T12:41:25.492Z", - "message": "I0708 12:41:25.492531 140451862599424 util.py:168] Finalize strategy." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 143, - "timestamp": "2019-07-08T12:41:25.572Z", - "message": "I0708 12:41:25.571871 140451862599424 estimator.py:359] Loss for final step: 0.03955281." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 144, - "timestamp": "2019-07-08T12:41:25.572Z", - "message": "I0708 12:41:25.572580 140451862599424 mnist.py:240] Starting to Export model to /paperspace/models" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 145, - "timestamp": "2019-07-08T12:41:25.583Z", - "message": "I0708 12:41:25.583298 140451862599424 estimator.py:1111] Calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 146, - "timestamp": "2019-07-08T12:41:25.744Z", - "message": "I0708 12:41:25.744162 140451862599424 estimator.py:1113] Done calling model_fn." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 147, - "timestamp": "2019-07-08T12:41:25.744Z", - "message": "W0708 12:41:25.744491 140451862599424 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 148, - "timestamp": "2019-07-08T12:41:25.744Z", - "message": "Instructions for updating:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 149, - "timestamp": "2019-07-08T12:41:25.744Z", - "message": "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 150, - "timestamp": "2019-07-08T12:41:25.745Z", - "message": "I0708 12:41:25.745018 140451862599424 export.py:587] Signatures INCLUDED in export for Regress: None" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 151, - "timestamp": "2019-07-08T12:41:25.745Z", - "message": "I0708 12:41:25.745139 140451862599424 export.py:587] Signatures INCLUDED in export for Train: None" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 152, - "timestamp": "2019-07-08T12:41:25.745Z", - "message": "I0708 12:41:25.745274 140451862599424 export.py:587] Signatures INCLUDED in export for Predict: ['serving_default', 'classify']" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 153, - "timestamp": "2019-07-08T12:41:25.745Z", - "message": "I0708 12:41:25.745424 140451862599424 export.py:587] Signatures INCLUDED in export for Classify: None" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 154, - "timestamp": "2019-07-08T12:41:25.745Z", - "message": "I0708 12:41:25.745549 140451862599424 export.py:587] Signatures INCLUDED in export for Eval: None" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 155, - "timestamp": "2019-07-08T12:41:25.746Z", - "message": "2019-07-08 12:41:25.746041: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 156, - "timestamp": "2019-07-08T12:41:25.746Z", - "message": "2019-07-08 12:41:25.746099: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 157, - "timestamp": "2019-07-08T12:41:25.746Z", - "message": "2019-07-08 12:41:25.746110: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0 " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 158, - "timestamp": "2019-07-08T12:41:25.746Z", - "message": "2019-07-08 12:41:25.746126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N " - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 159, - "timestamp": "2019-07-08T12:41:25.746Z", - "message": "2019-07-08 12:41:25.746278: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10787 MB memory) -\u003e physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 160, - "timestamp": "2019-07-08T12:41:25.779Z", - "message": "I0708 12:41:25.778900 140451862599424 saver.py:1270] Restoring parameters from /paperspace/models/mnist/model.ckpt-1000" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 161, - "timestamp": "2019-07-08T12:41:25.814Z", - "message": "I0708 12:41:25.814109 140451862599424 builder_impl.py:654] Assets added to graph." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 162, - "timestamp": "2019-07-08T12:41:25.814Z", - "message": "I0708 12:41:25.814344 140451862599424 builder_impl.py:449] No assets to write." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 163, - "timestamp": "2019-07-08T12:41:25.878Z", - "message": "I0708 12:41:25.877661 140451862599424 builder_impl.py:414] SavedModel written to: /paperspace/models/temp-b'1562589685'/saved_model.pb" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 164, - "timestamp": "2019-07-08T12:41:25.878Z", - "message": "I0708 12:41:25.878680 140451862599424 mnist.py:247] Model Exported" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 165, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "Gradient SDK not installed. Distributed training is not possible" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 166, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 167, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 168, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "For more information, please see:" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 169, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 170, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": " * https://github.com/tensorflow/addons" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 171, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "If you depend on functionality not listed there, please file an issue." - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 172, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 173, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz to /tmp/tmp5a21acuc.gz" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 174, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz to /tmp/tmpl6cdsahk.gz" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 175, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz to /tmp/tmplh4ob02m.gz" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 176, - "timestamp": "2019-07-08T12:41:25.879Z", - "message": "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz to /tmp/tmpbrss4txl.gz" - }, - { - "jobId": "jsy2ibsz1l026y", - "line": 177, - "timestamp": "2019-07-08T12:41:36.525Z", - "message": "PSEOF" - } -] - -DETAILS_OF_SINGLE_NODE_EXPERIMENT_RESPONSE_JSON = { - "data": { - "dtCreated": "2020-04-02T21:37:00.000000+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-03-22T13:22:27.200591+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 21814, - "experimentTemplateId": 60, - "experimentTypeId": 1, - "handle": "esro6mbmiulvbl", - "id": 21784, - "projectHandle": "prq70zy79", - "projectId": 612, - "started_by_user_id": 1655, - "state": 1, - "tags": [ - { - "name": "tag1", - }, - { - "name": "tag2", - }, - ], - "templateHistory": { - "dtCreated": "2019-03-22T13:22:26.375543+00:00", - "dtDeleted": None, - "experimentTemplateId": 60, - "id": 21814, - "params": { - "experimentTypeId": 1, - "name": "dsfads", - "ports": 5000, - "project_handle": "prq70zy79", - "worker_command": "sadas", - "worker_container": "asd", - "worker_machine_type": "C2" - }, - "triggerEvent": None, - "triggerEventId": None - }, - "datasets": [ - { - "uri": "s3://some/dataset", - "volume_options": { - "kind": "dynamic", - "size": "10Gi", - }, - }, - ], - }, - "message": "success" -} - -DETAILS_OF_MULTI_NODE_EXPERIMENT_RESPONSE_JSON = { - "data": { - "dtCreated": "2019-12-16T16:18:47.881765+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2019-12-16T16:18:47.881765+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 34335, - "experimentTemplateId": 439, - "experimentTypeId": 3, - "handle": "emarbao6t6tsn", - "id": 34272, - "projectHandle": "pr85u3sfa", - "projectId": 1824, - "started_by_user_id": 1655, - "state": 10, - "tags": [ - { - "name": "tag1", - }, - { - "name": "tag2", - }, - ], - "templateHistory": { - "cliCommand": "gradient experiments create multinode --name some_name --projectId pr85u3sfa --experimentType MPI --workerContainer python --workerMachineType c5.xlarge --workerCommand \"echo worker\" --workerCount 2 --masterContainer python --masterMachineType c5.xlarge --masterCommand \"echo master\" --masterCount 1 --workspaceUrl some.url --workingDirectory /some/working/directory --clusterId clqr4b0ox --experimentEnv '{\"key\": \"value\"}' --ports 5000 --modelType some_type --modelPath /some/model/path", - "dtCreated": "2019-12-16T16:18:46.577973+00:00", + "value": 0, + "checkpoint": 6956 + }, + { + "value": 0, + "checkpoint": 8116 + }, + { + "value": 0, + "checkpoint": 9262 + }, + { + "value": 0, + "checkpoint": 10397 + }, + { + "value": 0, + "checkpoint": 11566 + }, + { + "value": 0, + "checkpoint": 12000 + } + ] + }, + { + "operation": "var", + "checkpoints": [ + { + "value": 0, + "checkpoint": 1175 + }, + { + "value": 0, + "checkpoint": 2323 + }, + { + "value": 0, + "checkpoint": 3471 + }, + { + "value": 0, + "checkpoint": 4606 + }, + { + "value": 0, + "checkpoint": 5785 + }, + { + "value": 0, + "checkpoint": 6956 + }, + { + "value": 0, + "checkpoint": 8116 + }, + { + "value": 0, + "checkpoint": 9262 + }, + { + "value": 0, + "checkpoint": 10397 + }, + { + "value": 0, + "checkpoint": 11566 + }, + { + "value": 0, + "checkpoint": 12000 + } + ] + } + ], + "scalar": "accuracy" + } + ], + "session": "eval" + } + ], + "params": None, + "url": "s3://ps-projects-development/prmr22ve0/ehla1kvbwzaco/model/", + "notes": None, + "isDeleted": False, + "isPublic": False, + "dtCreated": "2019-04-02T17:02:47.157Z", + "dtModified": "2019-04-02T17:02:54.273Z", + "dtUploaded": "2019-04-02T17:02:54.273Z", "dtDeleted": None, - "experimentTemplateId": 439, - "id": 34335, - "params": { - "artifact_directory": "/some/artifact/directory", - "cluster_id": "clqr4b0ox", - "experiment_env": { - "key": "value" - }, - "is_preemptible": False, - "model_path": "/some/model/path", - "model_type": "some_type", - "name": "some_name", - "ports": "5000", - "project_handle": "pr85u3sfa", - "version": "v2", - "working_directory": "/some/working/directory", - "workspace_url": "some.url" - }, - "triggerEvent": None, - "triggerEventId": None + "modelPath": "/artifacts" } - }, - "message": "success" + ], + "total": 1, + "displayTotal": 1 } NOTEBOOK_GET_RESPONSE = { @@ -5527,216 +1860,6 @@ "displayTotal": 9 } -TENSORBOARD_CREATE_RESPONSE_JSON = { - "message": "success", - "data": { - "cluster_id": "some_cluster_id", - "url": None, - "id": "some_id", - "instance": None, - "dt_created": "2019-09-18T12:10:09.817102+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "some_project_id", - "id": "some_experiment_id" - }, { - "project_id": "some_project_id", - "id": "some_other_experiment_id" - } - ], - "dt_deleted": None - } -} - -TENSORBOARD_DETAIL_RESPONSE_JSON = { - "message": "success", - "data": { - "cluster_id": "some_cluster_id", - "url": "https://aws-testing.paperspace.io/tensorboard/some_id/", - "id": "some_id", - "instance": { - "type": "cpu", - "size": "large", - "count": 2 - }, - "dt_created": "2019-09-18T12:10:09.817102+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "some_project_id", - "id": "some_experiment_id", - "state": "Some State" - }, { - "project_id": "some_project_id", - "id": "some_other_experiment_id", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - } -} - -TENSORBOARD_UPDATE_RESPONSE_JSON = { - "message": "success", - "data": { - "cluster_id": "some_cluster_id", - "url": "https://aws-testing.paperspace.io/tensorboard/some_id/", - "id": "some_id", - "instance": { - "type": "cpu", - "size": "large", - "count": 2 - }, - "dt_created": "2019-09-18T12:10:09.817102+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "some_project_id", - "id": "some_experiment_id", - "state": "Some State" - }, - { - "project_id": "some_project_id", - "id": "some_other_experiment_id", - "state": "Some State" - }, - { - "project_id": "some_project_id", - "id": "some_third_experiment_id", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - } -} - -TENSORBOARD_UPDATE_REMOVE_RESPONSE_JSON = { - "message": "success", - "data": { - "cluster_id": "some_cluster_id", - "url": "https://aws-testing.paperspace.io/tensorboard/some_id/", - "id": "some_id", - "instance": { - "type": "cpu", - "size": "large", - "count": 2 - }, - "dt_created": "2019-09-18T12:10:09.817102+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "some_project_id", - "id": "some_experiment_id", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - } -} - -TENSORBOARD_LIST_RESPONSE_JSON = { - "message": "success", - "data": [ - { - "cluster_id": "clmtkpnm2", - "url": "https://aws-testing.paperspace.io/tensorboard/tbrs2kcjman4ly/", - "id": "tbrs2kcjman4ly", - "instance": None, - "dt_created": "2019-09-16T09:28:10.415478+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "prrdpg67x", - "id": "ers2kcjman4ly", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - }, - { - "cluster_id": "clmtkpnm2", - "url": "https://aws-testing.paperspace.io/tensorboard/tbskzep6d9po04d/", - "id": "tbskzep6d9po04d", - "instance": None, - "dt_created": "2019-09-16T09:28:56.908450+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "prrdpg67x", - "id": "ers2kcjman4ly", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - }, - { - "cluster_id": "clmtkpnm2", - "url": "https://aws-testing.paperspace.io/tensorboard/tbsaq6hggzxcnet/", - "id": "tbsaq6hggzxcnet", - "instance": None, - "dt_created": "2019-09-16T12:59:20.788663+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "prrdpg67x", - "id": "ers2kcjman4ly", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - }, - { - "cluster_id": "clmtkpnm2", - "url": "https://aws-testing.paperspace.io/tensorboard/tbwuzalec7ik58/", - "id": "tbwuzalec7ik58", - "instance": { - "size": "large", - "type": "cpu", - "count": 2 - }, - "dt_created": "2019-09-17T07:43:39.679876+00:00", - "image": "tensorflow/tensorflow:latest-py3", - "experiments": [ - { - "project_id": "prrdpg67x", - "id": "ers2kcjman4ly", - "state": "Some State" - } - ], - "dt_deleted": None, - "state": "Some State" - } - ] -} - -TENSORBOARD_DELETE_RESPONSE_JSON = { - "message": "success" -} - -LIST_JOB_FILES_RESPONSE_JSON = { - 'data': [ - { - "file": "hello.txt", - "url": "https://ps-projects.s3.amazonaws.com/some/path/artifacts/hello.txt?AWSAccessKeyId=some_aws_access_key_id&Expires=713274132&Signature=7CT5k6buEmZe5k5E7g6BXMs2xV4%3D&response-content-disposition=attachment%3Bfilename%3D%22hello.txt%22&x-amz-security-token=some_amz_security_token" - }, - { - "file": "hello2.txt", - "url": "https://ps-projects.s3.amazonaws.com/some/path/artifacts/hello2.txt?AWSAccessKeyId=some_aws_access_key_id&Expires=713274132&Signature=L1lI47cNyiROzdYkf%2FF3Cm3165E%3D&response-content-disposition=attachment%3Bfilename%3D%22hello2.txt%22&x-amz-security-token=some_amz_security_token" - }, - { - "file": "keton/elo.txt", - "url": "https://ps-projects.s3.amazonaws.com/some/path/artifacts/keton/elo.txt?AWSAccessKeyId=some_aws_access_key_id&Expires=713274132&Signature=tHriojGx03S%2FKkVGQGVI5CQRFTo%3D&response-content-disposition=attachment%3Bfilename%3D%22elo.txt%22&x-amz-security-token=some_amz_security_token" - } - ], -} - GET_PRESIGNED_URL_FOR_S3_BUCKET_RESPONSE_JSON = { "data": { "bucket_name": "ps-projects", @@ -5762,7 +1885,6 @@ MODEL_CREATE_RESPONSE_JSON_V2 = { "id": "some_model_id", "projectId": None, - "experimentId": None, "updatedByUserId": "ukgvw4i8", "updatedByUserEmail": "asd@paperspace.com", "modelType": "Custom", @@ -5791,7 +1913,6 @@ MODEL_UPLOAD_RESPONSE_JSON = { "id": "some_model_id", "projectId": None, - "experimentId": None, "updatedByUserId": "ukgvw4i8", "updatedByUserEmail": "some_email@paperspace.com", "modelType": "Tensorflow", @@ -5816,7 +1937,6 @@ { "id": "some_id", "projectId": "some_project_id", - "experimentId": "some_experiment_id", "updatedByUserId": "some_user_id", "updatedByUserEmail": "paperspace@paperspace.com", "modelType": "Tensorflow", @@ -5846,7 +1966,6 @@ { "id": "some_id", "projectId": "some_project_id", - "experimentId": "some_experiment_id", "updatedByUserId": "some_user_id", "updatedByUserEmail": "paperspace@paperspace.com", "modelType": "Tensorflow", @@ -5890,100 +2009,6 @@ } ] -GET_DEPLOYMENT_DETAILS_JSON_RESPONSE = { - "deploymentList": [ - { - "id": "some_id", - "ownerId": "some_owner_id", - "projectId": "some_project_id", - "experimentId": "some_experiment_id", - "state": "Stopped", - "startedByUserId": "some_user_id", - "startedByUserEmail": "some@address.com", - "deploymentType": "TFServing", - "deploymentTypeDescription": "Tensorflow Serving on K8s", - "modelId": "some_model_id", - "modelUrl": "s3://ps-projects/asdf/some/model/url", - "name": "some_name", - "tag": None, - "params": None, - "code": None, - "cluster": "KPS Jobs", - "clusterId": "some_cluster_id", - "machineType": "p3.2xlarge", - "imageUrl": "tensorflow/serving", - "imageUsername": None, - "imagePassword": None, - "imageServer": None, - "workspaceUrl": None, - "workspaceUsername": None, - "workspacePassword": None, - "instanceCount": 1, - "runningCount": 0, - "error": None, - "authToken": None, - "annotations": {}, - "oauthKey": None, - "oauthSecret": None, - "serviceType": "model", - "apiType": "REST", - "endpoint": "https://paperspace.io/model-serving/some_id:predict", - "ports": None, - "dtCreated": "2020-01-13T22:21:28.210Z", - "dtModified": "2020-01-14T00:32:27.577Z", - "dtBuildStarted": "2020-01-13T22:21:28.482Z", - "dtBuildFinished": None, - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtStopped": "2020-01-13T22:48:40.107Z", - "dtDeleted": None, - "isDeleted": False, - "modelType": "Tensorflow", - "modelPath": None, - "command": "some deployment command", - "args": None, - "env": None, - "containerModelPath": None, - "containerUrlPath": None, - "endpointUrlPath": None, - "method": None, - "tags": ["tag1", "tag2"], - "metricsURL": "aws-testing.paperspace.io", - "autoscaling": { - "minInstanceCount": 4, - "maxInstanceCount": 64, - "scaleCooldownPeriod": 123, - "metrics": [ - { - "type": "Resource", - "name": "cpu", - "valueType": "targetAverage", - "value": 10, - }, - { - "type": "Metric", - "name": "loss", - "valueType": "target", - "value": 2, - }, - { - "type": "Metric", - "name": "keton", - "valueType": "target", - "value": 21.37, - }, - ], - }, - }, - ], - "total": 129, - "displayTotal": 129, - "runningTotal": 0 -} - GET_CLUSTER_DETAILS_RESPONSE = { "id": "some_cluster_id", "name": "EKS testing", @@ -6058,66 +2083,6 @@ "buildPullRequests": True, "buildForks": False, "buildBranches": "default", - "experiments": { - "data": [ - { - "dtCreated": "2020-02-13T16:02:22.978027+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2020-02-13T16:02:22.978027+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 70590, - "experimentTemplateId": 7156, - "experimentTypeId": 2, - "handle": "some_handle", - "id": 70557, - "projectHandle": "some_id", - "projectId": 49107, - "started_by_user_id": 192354, - "state": 1, - "templateHistory": { - "cliCommand": "gradient experiments create multinode --name some_name --projectId some_id --experimentType GRPC --workerContainer python --workerMachineType c5.xlarge --workerCommand \"python script.py\" --workerCount 2 --parameterServerContainer python --parameterServerMachineType c5.xlarge --parameterServerCommand \"python script.py\" --parameterServerCount 1 --workspaceUrl s3://some_workspace.url --clusterId some_cluster_id --ports 5000:5000", - "dtCreated": "2020-02-13T16:02:21.263923+00:00", - "dtDeleted": None, - "experimentTemplateId": 7156, - "id": 70590, - "params": { - "cluster_id": "some_cluster_id", - "is_preemptible": False, - "name": "some_name", - "parameter_server_command": "python script.py", - "parameter_server_container": "python", - "parameter_server_count": 1, - "parameter_server_machine_type": "c5.xlarge", - "parameter_server_use_dockerfile": False, - "ports": "5000:5000", - "project_handle": "some_id", - "version": "v2", - "worker_command": "python script.py", - "worker_container": "python", - "worker_count": 2, - "worker_machine_type": "c5.xlarge", - "worker_use_dockerfile": False, - "workspace_url": "s3://some_workspace.url" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "some_id" - }, - "totalItems": 40 - } - } } ], "meta": { @@ -6147,66 +2112,6 @@ "buildPullRequests": True, "buildForks": False, "buildBranches": "default", - "experiments": { - "data": [ - { - "dtCreated": "2020-02-13T16:02:22.978027+00:00", - "dtDeleted": None, - "dtFinished": None, - "dtModified": "2020-02-13T16:02:22.978027+00:00", - "dtProvisioningFinished": None, - "dtProvisioningStarted": None, - "dtStarted": None, - "dtTeardownFinished": None, - "dtTeardownStarted": None, - "experimentError": None, - "experimentTemplateHistoryId": 70590, - "experimentTemplateId": 7156, - "experimentTypeId": 2, - "handle": "some_handle", - "id": 70557, - "projectHandle": "some_id", - "projectId": 49107, - "started_by_user_id": 192354, - "state": 1, - "templateHistory": { - "cliCommand": "gradient experiments create multinode --name some_name --projectId some_id --experimentType GRPC --workerContainer python --workerMachineType c5.xlarge --workerCommand \"python script.py\" --workerCount 2 --parameterServerContainer python --parameterServerMachineType c5.xlarge --parameterServerCommand \"python script.py\" --parameterServerCount 1 --workspaceUrl s3://some_workspace.url --clusterId some_cluster_id --ports 5000:5000", - "dtCreated": "2020-02-13T16:02:21.263923+00:00", - "dtDeleted": None, - "experimentTemplateId": 7156, - "id": 70590, - "params": { - "cluster_id": "some_cluster_id", - "is_preemptible": False, - "name": "some_name", - "parameter_server_command": "python script.py", - "parameter_server_container": "python", - "parameter_server_count": 1, - "parameter_server_machine_type": "c5.xlarge", - "parameter_server_use_dockerfile": False, - "ports": "5000:5000", - "project_handle": "some_id", - "version": "v2", - "worker_command": "python script.py", - "worker_container": "python", - "worker_count": 2, - "worker_machine_type": "c5.xlarge", - "worker_use_dockerfile": False, - "workspace_url": "s3://some_workspace.url" - }, - "triggerEvent": None, - "triggerEventId": None - } - } - ], - "meta": { - "itemGroup": { - "key": "projectHandle", - "value": "some_id" - }, - "totalItems": 40 - } - } } ], "meta": { @@ -7792,211 +3697,6 @@ ] } -EXPERIMENTS_METRICS_GET_RESPONSE = { - "handle": "ess57ozwb5ijpt", - "object_type": "experiment", - "charts": { - "cpuPercentage": { - "mljob-esro6mbmiulvbl-0-worker": [ - {"time_stamp": 1587375065, "value": "0"}, - {"time_stamp": 1587375095, "value": "0"}, - {"time_stamp": 1587375125, "value": "0"}, - {"time_stamp": 1587375155, "value": "0"}, - ], - "mljob-esro6mbmiulvbl-1-worker": [ - {"time_stamp": 1587375065, "value": "0"}, - {"time_stamp": 1587375095, "value": "0"}, - {"time_stamp": 1587375125, "value": "0"}, - {"time_stamp": 1587375155, "value": "0"}, - ], - }, - "memoryUsage": { - "mljob-esro6mbmiulvbl-0-worker": [ - {"time_stamp": 1587375005, "value": "0"}, - {"time_stamp": 1587375035, "value": "0"}, - {"time_stamp": 1587375065, "value": "761856"}, - {"time_stamp": 1587375095, "value": "761856"}, - {"time_stamp": 1587375125, "value": "761856"}, - ], - "mljob-esro6mbmiulvbl-1-worker": [ - {"time_stamp": 1587375005, "value": "0"}, - {"time_stamp": 1587375035, "value": "0"}, - {"time_stamp": 1587375065, "value": "761856"}, - {"time_stamp": 1587375095, "value": "761856"}, - {"time_stamp": 1587375125, "value": "761856"}, - ], - }, - }, -} - -EXPERIMENTS_METRICS_GET_RESPONSE_WHEN_NO_DATA_WAS_FOUND = { - "handle": "ess57ozwb5ijpt", - "object_type": "experiment", - "charts": {}, -} - -DEPLOYMENTS_METRICS_GET_RESPONSE = { - "handle": "desgffa3mtgepvm", - "object_type": "modelDeployment", - "charts": { - "cpuPercentage": { - "desgffa3mtgepvm-0": [ - {"time_stamp": 1587340800, "value": "0.0388702066666724"}, - {"time_stamp": 1587370800, "value": "0.04452898888887249"}, - {"time_stamp": 1587400800, "value": "0.044658617777757724"}, - {"time_stamp": 1587430800, "value": "0.04922275555555997"}, - {"time_stamp": 1587460800, "value": "0.0589409911111084"}, - {"time_stamp": 1587490800, "value": "0.02873176888891117"}, - {"time_stamp": 1587520800, "value": "0.042048226666666876"}, - {"time_stamp": 1587550800, "value": "0.04952780222222625"}, - ], - "desgffa3mtgepvm-1": [ - {"time_stamp": 1587340800, "value": "0.05044751111111307"}, - {"time_stamp": 1587370800, "value": "0.04381767555555724"}, - {"time_stamp": 1587400800, "value": "0.03436263111110646"}, - {"time_stamp": 1587430800, "value": "0.048889264444432624"}, - {"time_stamp": 1587460800, "value": "0.041525960000020255"}, - {"time_stamp": 1587490800, "value": "0.04574227333332853"}, - {"time_stamp": 1587520800, "value": "0.03383691777780011"}, - {"time_stamp": 1587550800, "value": "0.045942304444426756"}, - ], - }, - "memoryUsage": { - "desgffa3mtgepvm-0": [ - {"time_stamp": 1587340800, "value": "34910208"}, - {"time_stamp": 1587370800, "value": "34910208"}, - {"time_stamp": 1587400800, "value": "34914304"}, - {"time_stamp": 1587430800, "value": "34914304"}, - {"time_stamp": 1587460800, "value": "34914304"}, - {"time_stamp": 1587490800, "value": "34914304"}, - {"time_stamp": 1587520800, "value": "34914304"}, - {"time_stamp": 1587550800, "value": "34914304"}, - ], - "desgffa3mtgepvm-1": [ - {"time_stamp": 1587340800, "value": "35942400"}, - {"time_stamp": 1587370800, "value": "35942400"}, - {"time_stamp": 1587400800, "value": "35942400"}, - {"time_stamp": 1587430800, "value": "35942400"}, - {"time_stamp": 1587460800, "value": "35942400"}, - {"time_stamp": 1587490800, "value": "35942400"}, - {"time_stamp": 1587520800, "value": "35942400"}, - {"time_stamp": 1587550800, "value": "35942400"}, - ], - }, - }, -} - -GET_JOB_RESPONSE = { - "job": { - "id": "jstkd2lapucirs", - "name": "ufoym/deepo:all-py36-worker-1", - "state": "Running", - "workspaceUrl": "git+https://github.com/dte/stylegan.git", - "workingDirectory": "/paperspace", - "artifactsDirectory": "/artifacts", - "entrypoint": "sleep 3600 ; python generate_figures.py", - "projectId": "pr85u3sfa", - "project": "keton", - "container": "ufoym/deepo:all-py36", - "containerUrl": "ufoym/deepo:all-py36", - "baseContainer": "ufoym/deepo:all-py36", - "baseContainerUrl": "ufoym/deepo:all-py36", - "machineType": "c5.xlarge", - "cluster": "KPS Jobs", - "clusterId": "clqr4b0ox", - "usageRate": "Employee", - "startedByUserId": "ukgvw4i8", - "parentJobId": None, - "jobError": None, - "dtCreated": "2020-04-29T10:11:07.924Z", - "dtModified": "2020-04-29T10:11:07.924Z", - "dtProvisioningStarted": None, - "dtProvisioningFinished": None, - "dtStarted": None, - "dtFinished": None, - "dtTeardownStarted": None, - "dtTeardownFinished": None, - "dtDeleted": None, - "exitCode": None, - "queuePosition": None, - "seqNum": 155, - "storageRegion": "Private", - "clusterMachine": None, - "fqdn": "undefined.dgradient.paperspace.com", - "ports": "5000:5000", "isPublic": False, - "containerUser": None, - "hasCode": None, - "codeUploaded": None, - "codeCommit": None, - "runTillCancelled": None, - "pushOnCompletion": None, - "newImageName": None, - "cpuHostname": None, - "cpuCount": None, - "cpuModel": None, - "cpuFlags": None, - "cpuMem": None, - "gpuName": None, - "gpuSerial": None, - "gpuDevice": None, - "gpuDriver": None, - "gpuCount": None, - "gpuMem": None, - "tpuType": None, - "tpuName": None, - "tpuGrpcUrl": None, - "tpuTFVersion": None, - "tpuDatasetDir": None, - "tpuModelDir": None, - "targetNodeAttrs": None, - "jobEnv": None, - "sharedMemMBytes": None, - "shutdownTimeout": None, - "isPreemptible": False, - "metricsURL": "aws-testing.paperspace.io", - "customMetrics": None, - "experimentId": "ecgrgm7ok8chv", - "tags": None, - }, - "user": { - "firstName": "first", - "lastName": "last", - "email": "bartosz@paperspace.com", - "handle": "ukgvw4i8", - }, -} - -GET_JOB_METRICS_RESPONSE = { - "handle": "jstkd2lapucirs", - "object_type": "mljob", - "charts": { - "cpuPercentage": { - "mljob-ecgrgm7ok8chv-0-worker": [ - {"time_stamp": 1588155157, "value": "0"}, - {"time_stamp": 1588155187, "value": "0"}, - {"time_stamp": 1588155217, "value": "0"}, - ], - }, - "memoryUsage": { - "mljob-ecgrgm7ok8chv-0-worker": [ - {"time_stamp": 1588155097, "value": "0"}, - {"time_stamp": 1588155127, "value": "5881856"}, - {"time_stamp": 1588155157, "value": "5881856"}, - {"time_stamp": 1588155187, "value": "5881856"}, - ], - }, - }, -} - -JOBS_METRICS_GET_RESPONSE_WHEN_NO_DATA_WAS_FOUND = { - "handle": "jsy5wnxbtqw1e9", - "object_type": "mljob", - "charts": { - "cpuPercentage": None, - "memoryUsage": None, - }, -} - NOTEBOOKS_METRICS_GET_RESPONSE = { "handle": "npmnnm6e", "object_type": "notebook", diff --git a/tests/functional/test_models.py b/tests/functional/test_models.py index cad9e3b6..4232530e 100644 --- a/tests/functional/test_models.py +++ b/tests/functional/test_models.py @@ -21,34 +21,31 @@ class TestModelsList(object): URL = "https://api.paperspace.io/mlModels/getModelList/" COMMAND = ["models", "list"] - COMMAND_WITH_FILTERING_BY_EXPERIMENT_ID = [ - "models", "list", - "--experimentId", "some_experiment_id", - "--projectId", "some_project_id", - "--tag", "some_tag", - "--tag", "some_other_tag", - ] - EXPECTED_REQUEST_JSON_WITH_FILTERING = {"filter": {"where": {"and": [{"projectId": "some_project_id", - "experimentId": "some_experiment_id"}]}}, + EXPECTED_REQUEST_JSON_WITH_FILTERING = {"filter": {"where": {"and": [{"projectId": "some_project_id"}]}}, "tagFilter": ("some_tag", "some_other_tag")} - COMMAND_WITH_API_KEY_PARAMETER_USED = ["models", "list", "--apiKey", "some_key"] - COMMAND_WITH_OPTIONS_FILE = ["models", "list", "--optionsFile", ] # path added in test - - EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND = {"modelList": [], "total": 1, "displayTotal": 0} - - EXPECTED_STDOUT = """+------+-----------------+------------+------------+---------------+ -| Name | ID | Model Type | Project ID | Experiment ID | -+------+-----------------+------------+------------+---------------+ -| None | mosu30xm7q8vb0p | Tensorflow | prmr22ve0 | ehla1kvbwzaco | -+------+-----------------+------------+------------+---------------+ + COMMAND_WITH_API_KEY_PARAMETER_USED = [ + "models", "list", "--apiKey", "some_key"] + COMMAND_WITH_OPTIONS_FILE = ["models", "list", + "--optionsFile", ] # path added in test + + EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND = { + "modelList": [], "total": 1, "displayTotal": 0} + + EXPECTED_STDOUT = """+------+-----------------+------------+------------+ +| Name | ID | Model Type | Project ID | ++------+-----------------+------------+------------+ +| None | mosu30xm7q8vb0p | Tensorflow | prmr22ve0 | ++------+-----------------+------------+------------+ """ - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"status": 401, "message": "No such API token"} + EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = { + "status": 401, "message": "No such API token"} @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_list_of_experiments(self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_MODELS_RESPONSE_JSON) + def test_should_send_get_request_and_print_list_of_models(self, get_patched): + get_patched.return_value = MockResponse( + example_responses.LIST_MODELS_RESPONSE_JSON) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -63,10 +60,12 @@ def test_should_send_get_request_and_print_list_of_experiments(self, get_patched @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_replate_api_key_in_headers_when_api_key_parameter_was_used(self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_MODELS_RESPONSE_JSON) + get_patched.return_value = MockResponse( + example_responses.LIST_MODELS_RESPONSE_JSON) runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) + result = runner.invoke( + cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) get_patched.assert_called_once_with(self.URL, headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, @@ -78,7 +77,8 @@ def test_should_replate_api_key_in_headers_when_api_key_parameter_was_used(self, @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_read_options_from_yaml_file(self, get_patched, models_list_config_path): - get_patched.return_value = MockResponse(example_responses.LIST_MODELS_RESPONSE_JSON) + get_patched.return_value = MockResponse( + example_responses.LIST_MODELS_RESPONSE_JSON) command = self.COMMAND_WITH_OPTIONS_FILE[:] + [models_list_config_path] runner = CliRunner() @@ -92,24 +92,11 @@ def test_should_read_options_from_yaml_file(self, get_patched, models_list_confi assert result.output == self.EXPECTED_STDOUT assert EXPECTED_HEADERS["X-API-Key"] != "some_key" - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_list_of_models_filtered_experiment_id(self, get_patched): - get_patched.return_value = MockResponse(example_responses.LIST_MODELS_RESPONSE_JSON) - - runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_FILTERING_BY_EXPERIMENT_ID) - - get_patched.assert_called_once_with(self.URL, - headers=EXPECTED_HEADERS, - json=self.EXPECTED_REQUEST_JSON_WITH_FILTERING, - params={"limit": -1}) - - assert result.output == self.EXPECTED_STDOUT - @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_send_get_request_and_print_proper_message_when_no_models_were_found( self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND) + get_patched.return_value = MockResponse( + self.EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -123,7 +110,8 @@ def test_should_send_get_request_and_print_proper_message_when_no_models_were_fo @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_print_proper_message_when_wrong_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) + get_patched.return_value = MockResponse( + self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -141,12 +129,15 @@ class TestDeleteModel(object): COMMAND = ["models", "delete", "--id", "some_id"] EXPECTED_REQUEST_JSON = {"id": "some_id"} - COMMAND_WITH_API_KEY_PARAMETER_USED = ["models", "delete", "--id", "some_id", "--apiKey", "some_key"] - COMMAND_WITH_OPTIONS_FILE = ["models", "delete", "--id", "some_id", "--optionsFile", ] # path added in test + COMMAND_WITH_API_KEY_PARAMETER_USED = [ + "models", "delete", "--id", "some_id", "--apiKey", "some_key"] + COMMAND_WITH_OPTIONS_FILE = [ + "models", "delete", "--id", "some_id", "--optionsFile", ] # path added in test EXPECTED_STDOUT = "Model deleted\n" - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"status": 400, "message": "Invalid API token"} + EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = { + "status": 400, "message": "Invalid API token"} @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_send_post_request_when_models_delete_command_was_executed(self, post_patched): @@ -170,7 +161,8 @@ def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, post_patched.return_value = MockResponse(status_code=204) runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) + result = runner.invoke( + cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) post_patched.assert_called_once_with(self.URL, headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, @@ -185,7 +177,8 @@ def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_read_options_from_yaml_file(self, post_patched, models_delete_config_path): post_patched.return_value = MockResponse(status_code=204) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [models_delete_config_path] + command = self.COMMAND_WITH_OPTIONS_FILE[:] + \ + [models_delete_config_path] runner = CliRunner() result = runner.invoke(cli.cli, command) @@ -203,7 +196,8 @@ def test_should_read_options_from_yaml_file(self, post_patched, models_delete_co @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_send_post_request_and_print_proper_message_when_model_with_given_id_was_not_found( self, post_patched): - post_patched.return_value = MockResponse(example_responses.DELETE_MODEL_404_RESPONSE_JSON, status_code=404) + post_patched.return_value = MockResponse( + example_responses.DELETE_MODEL_404_RESPONSE_JSON, status_code=404) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -219,7 +213,8 @@ def test_should_send_post_request_and_print_proper_message_when_model_with_given @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_print_proper_message_when_wrong_api_key_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) + post_patched.return_value = MockResponse( + self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -233,6 +228,7 @@ def test_should_print_proper_message_when_wrong_api_key_was_used(self, post_patc assert result.output == "Failed to delete resource: Invalid API token\n" + class TestModelCreate(object): URL = "https://api.paperspace.io/mlModels/createModelV2" BASE_COMMAND = [ @@ -264,7 +260,6 @@ class TestModelCreate(object): "datasetRef": "dsr8k5qzn401lb5:latest", } - EXPECTED_STDOUT = "Model created with ID: some_model_id\n" CREATE_MODEL_V2_REPONSE = example_responses.MODEL_CREATE_RESPONSE_JSON_V2 @@ -280,11 +275,11 @@ def test_should_send_post_request_when_models_create_command_was_used_with_basic assert result.output == self.EXPECTED_STDOUT, result.exc_info post_patched.assert_has_calls([ mock.call(self.URL, - headers=EXPECTED_HEADERS, - json=None, - files=None, - data=None, - params=self.BASE_PARAMS), + headers=EXPECTED_HEADERS, + json=None, + files=None, + data=None, + params=self.BASE_PARAMS), ]) @mock.patch("gradient.api_sdk.clients.http_client.requests.post") @@ -298,11 +293,11 @@ def test_should_send_post_request_when_models_update_command_was_used_with_all_o assert result.output == self.EXPECTED_STDOUT, result.exc_info post_patched.assert_has_calls([ mock.call(self.URL, - headers=EXPECTED_HEADERS, - json=None, - files=None, - data=None, - params=self.ALL_OPTIONS_PARAMS), + headers=EXPECTED_HEADERS, + json=None, + files=None, + data=None, + params=self.ALL_OPTIONS_PARAMS), ]) @@ -363,18 +358,22 @@ class TestModelUpload(object): "--clusterId", "some_cluster_id", "--apiKey", "some_key", ] - COMMAND_WITH_OPTIONS_FILE = ["models", "upload", "--optionsFile", ] # path added in test + COMMAND_WITH_OPTIONS_FILE = ["models", "upload", + "--optionsFile", ] # path added in test EXPECTED_STDOUT = "Model uploaded with ID: some_model_id\n" GET_PRESIGNED_URL = "https://api.paperspace.io/mlModels/getPresignedModelUrl" - GET_PRESIGNED_URL_PARAMS = {"fileName": "saved_model.pb", "modelHandle": "some_model_id", "contentType": "", "clusterId": "some_cluster_id"} - GET_PRESIGNED_URL_PARAMS_BASIC = {"fileName": "saved_model.pb", "modelHandle": "some_model_id", "contentType": ""} + GET_PRESIGNED_URL_PARAMS = {"fileName": "saved_model.pb", + "modelHandle": "some_model_id", "contentType": "", "clusterId": "some_cluster_id"} + GET_PRESIGNED_URL_PARAMS_BASIC = { + "fileName": "saved_model.pb", "modelHandle": "some_model_id", "contentType": ""} GET_PRESIGNED_URL_RESPONSE = example_responses.MODEL_UPLOAD_GET_PRESIGNED_URL_RESPONSE CREATE_MODEL_V2_REPONSE = example_responses.MODEL_CREATE_RESPONSE_JSON_V2 - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"status": 400, "message": "Invalid API token"} + EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = { + "status": 400, "message": "Invalid API token"} UPDATE_TAGS_RESPONSE_JSON_200 = example_responses.UPDATE_TAGS_RESPONSE @mock.patch("gradient.api_sdk.clients.http_client.requests.get") @@ -384,7 +383,8 @@ def test_should_send_post_request_when_models_update_command_was_used_with_basic self, post_patched, put_patched, get_patched): post_patched.return_value = MockResponse(self.CREATE_MODEL_V2_REPONSE) put_patched.return_value = MockResponse() - get_patched.return_value = MockResponse(self.GET_PRESIGNED_URL_RESPONSE) + get_patched.return_value = MockResponse( + self.GET_PRESIGNED_URL_RESPONSE) runner = CliRunner() with runner.isolated_filesystem(): @@ -425,7 +425,8 @@ def test_should_send_post_request_when_models_update_command_was_used_with_all_o self, post_patched, put_patched, get_patched): post_patched.return_value = MockResponse(self.CREATE_MODEL_V2_REPONSE) put_patched.return_value = MockResponse() - get_patched.return_value = MockResponse(self.GET_PRESIGNED_URL_RESPONSE) + get_patched.return_value = MockResponse( + self.GET_PRESIGNED_URL_RESPONSE) runner = CliRunner() with runner.isolated_filesystem(): @@ -465,14 +466,16 @@ def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used( self, post_patched, put_patched, get_patched): post_patched.return_value = MockResponse(self.CREATE_MODEL_V2_REPONSE) put_patched.return_value = MockResponse() - get_patched.return_value = MockResponse(self.GET_PRESIGNED_URL_RESPONSE) + get_patched.return_value = MockResponse( + self.GET_PRESIGNED_URL_RESPONSE) runner = CliRunner() with runner.isolated_filesystem(): with open(self.MODEL_FILE, "w") as h: h.write("I'm a model!") - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) + result = runner.invoke( + cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) assert result.output == self.EXPECTED_STDOUT, result.exc_info post_patched.assert_has_calls([ @@ -507,8 +510,10 @@ def test_should_read_options_from_yaml_file( self, post_patched, put_patched, get_patched, models_upload_config_path): post_patched.return_value = MockResponse(self.CREATE_MODEL_V2_REPONSE) put_patched.return_value = MockResponse() - get_patched.return_value = MockResponse(self.GET_PRESIGNED_URL_RESPONSE) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [models_upload_config_path] + get_patched.return_value = MockResponse( + self.GET_PRESIGNED_URL_RESPONSE) + command = self.COMMAND_WITH_OPTIONS_FILE[:] + \ + [models_upload_config_path] runner = CliRunner() with runner.isolated_filesystem(): @@ -544,7 +549,8 @@ def test_should_read_options_from_yaml_file( @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_print_proper_message_when_wrong_api_key_was_used(self, post_patched): - post_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) + post_patched.return_value = MockResponse( + self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) runner = CliRunner() with runner.isolated_filesystem(): @@ -628,17 +634,19 @@ class TestModelDetails(object): EXPECTED_REQUEST_JSON = {"filter": {"where": {"and": [{"id": "some_id"}]}}} - COMMAND_WITH_API_KEY_PARAMETER_USED = ["models", "details", "--id", "some_id", "--apiKey", "some_key"] - COMMAND_WITH_OPTIONS_FILE = ["models", "details", "--optionsFile", ] # path added in test + COMMAND_WITH_API_KEY_PARAMETER_USED = [ + "models", "details", "--id", "some_id", "--apiKey", "some_key"] + COMMAND_WITH_OPTIONS_FILE = [ + "models", "details", "--optionsFile", ] # path added in test - EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND = {"modelList": [], "total": 1, "displayTotal": 0} + EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND = { + "modelList": [], "total": 1, "displayTotal": 0} EXPECTED_STDOUT = """+------------------+----------------------------------------------------------------------------+ | ID | some_id | +------------------+----------------------------------------------------------------------------+ | Name | some_name | | Project ID | some_project_id | -| Experiment ID | some_experiment_id | | Model Type | Tensorflow | | URL | s3://ps-projects-development/asdf/some_project_id/some_experiment_id/model | | Deployment State | Stopped | @@ -651,7 +659,6 @@ class TestModelDetails(object): +------------------+----------------------------------------------------------------------------+ | Name | some_name | | Project ID | some_project_id | -| Experiment ID | some_experiment_id | | Model Type | Tensorflow | | URL | s3://ps-projects-development/asdf/some_project_id/some_experiment_id/model | | Deployment State | Stopped | @@ -659,11 +666,13 @@ class TestModelDetails(object): +------------------+----------------------------------------------------------------------------+ """ - EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = {"status": 400, "message": "Invalid API token"} + EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED = { + "status": 400, "message": "Invalid API token"} @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_details_of_experiment(self, get_patched): - get_patched.return_value = MockResponse(example_responses.MODEL_DETAILS_RESPONSE_JSON) + def test_should_send_get_request_and_print_details_of_model(self, get_patched): + get_patched.return_value = MockResponse( + example_responses.MODEL_DETAILS_RESPONSE_JSON) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -677,8 +686,9 @@ def test_should_send_get_request_and_print_details_of_experiment(self, get_patch assert EXPECTED_HEADERS["X-API-Key"] != "some_key" @mock.patch("gradient.api_sdk.clients.http_client.requests.get") - def test_should_send_get_request_and_print_details_of_experiment_that_has_some_tags(self, get_patched): - get_patched.return_value = MockResponse(example_responses.MODEL_DETAILS_RESPONSE_JSON_WITH_TAGS) + def test_should_send_get_request_and_print_details_of_model_that_has_some_tags(self, get_patched): + get_patched.return_value = MockResponse( + example_responses.MODEL_DETAILS_RESPONSE_JSON_WITH_TAGS) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -693,10 +703,12 @@ def test_should_send_get_request_and_print_details_of_experiment_that_has_some_t @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, get_patched): - get_patched.return_value = MockResponse(example_responses.MODEL_DETAILS_RESPONSE_JSON) + get_patched.return_value = MockResponse( + example_responses.MODEL_DETAILS_RESPONSE_JSON) runner = CliRunner() - result = runner.invoke(cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) + result = runner.invoke( + cli.cli, self.COMMAND_WITH_API_KEY_PARAMETER_USED) get_patched.assert_called_once_with(self.URL, headers=EXPECTED_HEADERS_WITH_CHANGED_API_KEY, @@ -708,8 +720,10 @@ def test_should_replace_api_key_in_headers_when_api_key_parameter_was_used(self, @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_read_options_from_yaml_file(self, get_patched, models_details_config_path): - get_patched.return_value = MockResponse(example_responses.MODEL_DETAILS_RESPONSE_JSON) - command = self.COMMAND_WITH_OPTIONS_FILE[:] + [models_details_config_path] + get_patched.return_value = MockResponse( + example_responses.MODEL_DETAILS_RESPONSE_JSON) + command = self.COMMAND_WITH_OPTIONS_FILE[:] + \ + [models_details_config_path] runner = CliRunner() result = runner.invoke(cli.cli, command) @@ -725,7 +739,8 @@ def test_should_read_options_from_yaml_file(self, get_patched, models_details_co @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_send_get_request_and_print_proper_message_when_no_models_were_found( self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND) + get_patched.return_value = MockResponse( + self.EXPECTED_RESPONSE_JSON_WHEN_NO_MODELS_WERE_FOUND) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -739,7 +754,8 @@ def test_should_send_get_request_and_print_proper_message_when_no_models_were_fo @mock.patch("gradient.api_sdk.clients.http_client.requests.get") def test_should_print_proper_message_when_wrong_api_key_was_used(self, get_patched): - get_patched.return_value = MockResponse(self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) + get_patched.return_value = MockResponse( + self.EXPECTED_RESPONSE_WHEN_WRONG_API_KEY_WAS_USED, 401) runner = CliRunner() result = runner.invoke(cli.cli, self.COMMAND) @@ -758,7 +774,8 @@ class TestDownloadModelFiles(object): DESTINATION_DIR_NAME = "dest" DESTINATION_DIR_PATH = os.path.join(tempfile.gettempdir(), "dest") - COMMAND = ["models", "download", "--id", "some_model_id", "--destinationDir", DESTINATION_DIR_PATH] + COMMAND = ["models", "download", "--id", "some_model_id", + "--destinationDir", DESTINATION_DIR_PATH] @classmethod def teardown_method(cls): @@ -818,7 +835,8 @@ def test_should_get_a_list_of_files_and_download_them_to_defined_directory_when_ with open(hello2_txt_path) as h: assert h.read() == "\"Hello Paperspace 2\n\"" - elo_txt_path = os.path.join(self.DESTINATION_DIR_PATH, "keton", "elo.txt") + elo_txt_path = os.path.join( + self.DESTINATION_DIR_PATH, "keton", "elo.txt") assert os.path.exists(elo_txt_path) assert not os.path.isdir(elo_txt_path) with open(elo_txt_path) as h: From 2d9ec6b739dc30ef2fa3f139912a00cbde16979f Mon Sep 17 00:00:00 2001 From: Ben Batha Date: Tue, 12 Apr 2022 13:32:01 -0400 Subject: [PATCH 3/3] fixup! fixup! feat: retire gradient v1 and v2 --- gradient/api_sdk/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gradient/api_sdk/config.py b/gradient/api_sdk/config.py index d5f99cc3..4b4f1f4b 100644 --- a/gradient/api_sdk/config.py +++ b/gradient/api_sdk/config.py @@ -24,7 +24,6 @@ def get_api_key(config_dir_path, config_file_name): _DEFAULT_HELP_HEADERS_COLOR = "yellow" _DEFAULT_HELP_OPTIONS_COLOR = "green" _DEFAULT_USE_CONSOLE_COLORS = True -_DEFAULT_USE_LEGACY_DEPLOYMENTS = False def get_help_colors_dict(use_colors, help_headers_color, help_options_color): @@ -43,8 +42,6 @@ class config(object): WEB_URL = os.environ.get("PAPERSPACE_WEB_URL", _DEFAULT_WEB_URL) API_HOST = os.environ.get("PAPERSPACE_API_HOST", _DEFAULT_API_HOST) - USE_LEGACY_DEPLOYMENTS = os.environ.get( - "USE_LEGACY_DEPLOYMENTS", _DEFAULT_USE_LEGACY_DEPLOYMENTS) CONFIG_HOST = os.environ.get( "PAPERSPACE_CONFIG_HOST", _DEFAULT_CONFIG_HOST) CONFIG_LOG_HOST = os.environ.get(