From 3a7dce2a95f595d821d54f1071c00a1f993c5fb5 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Mon, 13 Jul 2020 15:19:14 -0400 Subject: [PATCH 1/9] add ai platform guide --- .../cloud/example_dags/example_mlengine.py | 30 +++ docs/build | 1 - docs/howto/operator/gcp/mlengine.rst | 217 ++++++++++++++++++ docs/operators-and-hooks-ref.rst | 6 + 4 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 docs/howto/operator/gcp/mlengine.rst diff --git a/airflow/providers/google/cloud/example_dags/example_mlengine.py b/airflow/providers/google/cloud/example_dags/example_mlengine.py index b3ed17249e55e..f805340814f3d 100644 --- a/airflow/providers/google/cloud/example_dags/example_mlengine.py +++ b/airflow/providers/google/cloud/example_dags/example_mlengine.py @@ -62,6 +62,7 @@ schedule_interval=None, # Override to match your needs tags=['example'], ) as dag: + # [START howto_operator_gcp_mlengine_training] training = MLEngineStartTrainingJobOperator( task_id="training", project_id=PROJECT_ID, @@ -74,7 +75,9 @@ training_python_module=TRAINER_PY_MODULE, training_args=[], ) + # [END howto_operator_gcp_mlengine_training] + # [START howto_operator_gcp_mlengine_create_model] create_model = MLEngineCreateModelOperator( task_id="create-model", project_id=PROJECT_ID, @@ -82,18 +85,24 @@ "name": MODEL_NAME, }, ) + # [END howto_operator_gcp_mlengine_create_model] + # [START howto_operator_gcp_mlengine_get_model] get_model = MLEngineGetModelOperator( task_id="get-model", project_id=PROJECT_ID, model_name=MODEL_NAME, ) + # [END howto_operator_gcp_mlengine_get_model] + # [START howto_operator_gcp_mlengine_print_model] get_model_result = BashOperator( bash_command="echo \"{{ task_instance.xcom_pull('get-model') }}\"", task_id="get-model-result", ) + # [END howto_operator_gcp_mlengine_print_model] + # [START howto_operator_gcp_mlengine_create_version1] create_version = MLEngineCreateVersionOperator( task_id="create-version", project_id=PROJECT_ID, @@ -108,7 +117,9 @@ "pythonVersion": "3.7" } ) + # [END howto_operator_gcp_mlengine_create_version1] + # [START howto_operator_gcp_mlengine_create_version2] create_version_2 = MLEngineCreateVersionOperator( task_id="create-version-2", project_id=PROJECT_ID, @@ -123,25 +134,33 @@ "pythonVersion": "3.7" } ) + # [END howto_operator_gcp_mlengine_create_version2] + # [START howto_operator_gcp_mlengine_default_version] set_defaults_version = MLEngineSetDefaultVersionOperator( task_id="set-default-version", project_id=PROJECT_ID, model_name=MODEL_NAME, version_name="v2", ) + # [END howto_operator_gcp_mlengine_default_version] + # [START howto_operator_gcp_mlengine_list_versions] list_version = MLEngineListVersionsOperator( task_id="list-version", project_id=PROJECT_ID, model_name=MODEL_NAME, ) + # [END howto_operator_gcp_mlengine_list_versions] + # [START howto_operator_gcp_mlengine_print_versions] list_version_result = BashOperator( bash_command="echo \"{{ task_instance.xcom_pull('list-version') }}\"", task_id="list-version-result", ) + # [END howto_operator_gcp_mlengine_print_versions] + # [START howto_operator_gcp_mlengine_get_prediction] prediction = MLEngineStartBatchPredictionJobOperator( task_id="prediction", project_id=PROJECT_ID, @@ -152,20 +171,25 @@ input_paths=[PREDICTION_INPUT], output_path=PREDICTION_OUTPUT, ) + # [END howto_operator_gcp_mlengine_get_prediction] + # [START howto_operator_gcp_mlengine_delete_version] delete_version = MLEngineDeleteVersionOperator( task_id="delete-version", project_id=PROJECT_ID, model_name=MODEL_NAME, version_name="v1" ) + # [END howto_operator_gcp_mlengine_delete_version] + # [START howto_operator_gcp_mlengine_delete_model] delete_model = MLEngineDeleteModelOperator( task_id="delete-model", project_id=PROJECT_ID, model_name=MODEL_NAME, delete_contents=True ) + # [END howto_operator_gcp_mlengine_delete_model] training >> create_version training >> create_version_2 @@ -178,6 +202,7 @@ list_version >> delete_version delete_version >> delete_model + # [START howto_operator_gcp_mlengine_get_metric] def get_metric_fn_and_keys(): """ Gets metric function and keys used to generate summary @@ -186,7 +211,9 @@ def normalize_value(inst: Dict): val = float(inst['dense_4'][0]) return tuple([val]) # returns a tuple. return normalize_value, ['val'] # key order must match. + # [END howto_operator_gcp_mlengine_get_metric] + # [START howto_operator_gcp_mlengine_validate_error] def validate_err_and_count(summary: Dict) -> Dict: """ Validate summary result @@ -198,7 +225,9 @@ def validate_err_and_count(summary: Dict) -> Dict: if summary['count'] != 20: raise ValueError('Invalid value val != 20; summary={}'.format(summary)) return summary + # [END howto_operator_gcp_mlengine_validate_error] + # [START howto_operator_gcp_mlengine_evaluate] evaluate_prediction, evaluate_summary, evaluate_validation = mlengine_operator_utils.create_evaluate_ops( task_prefix="evaluate-ops", data_format="TEXT", @@ -218,6 +247,7 @@ def validate_err_and_count(summary: Dict) -> Dict: version_name="v1", py_interpreter="python3", ) + # [END howto_operator_gcp_mlengine_evaluate] create_model >> create_version >> evaluate_prediction evaluate_validation >> delete_version diff --git a/docs/build b/docs/build index a30b6a1a9215b..d8a93da286acd 100755 --- a/docs/build +++ b/docs/build @@ -358,7 +358,6 @@ MISSING_GOOGLLE_DOC_GUIDES = { 'datastore', 'dlp', 'gcs_to_bigquery', - 'mlengine', 'mssql_to_gcs', 'mysql_to_gcs', 'postgres_to_gcs', diff --git a/docs/howto/operator/gcp/mlengine.rst b/docs/howto/operator/gcp/mlengine.rst new file mode 100644 index 0000000000000..9a9a8b9ff2d9d --- /dev/null +++ b/docs/howto/operator/gcp/mlengine.rst @@ -0,0 +1,217 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Google Cloud AI Platform Operators +================================== + +`Google Cloud AI Platform `__ (formerly known +as ML Engine) can be used to train machine learning models at scale, host trained models +in the cloud, and use models to make predictions for new data. AI Platform is a collection +of tools for training, evaluating, and tuning machine learning models. AI Platform can also +be used to deploy a trained model, make predictions, and manage various model versions. + +.. contents:: + :depth: 1 + :local: + +Prerequisite tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: _partials/prerequisite_tasks.rst + +Launching a Job +^^^^^^^^^^^^^^^ +To start a machine learning operation with AI Platform, you must launch a training job. +This creates a virtual machine that can run code specified in the trainer file, which +contains the main application code. A job can be initiated with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartTrainingJobOperator`. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_training] + :end-before: [END howto_operator_gcp_mlengine_training] + +Creating a model +^^^^^^^^^^^^^^^^ +A model is a container that can hold multiple model versions. A new model can be created through the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateModelOperator`. +The ``model`` field should be defined with a dictionary containing the information about the model. +``name`` is a required field in this dictionary. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_create_model] + :end-before: [END howto_operator_gcp_mlengine_create_model] + +Getting a model +^^^^^^^^^^^^^^^ +The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineGetModelOperator` +can be used to obtain a model previously created. To obtain the correct model, ``model_name`` +must be defined in the operator. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_get_model] + :end-before: [END howto_operator_gcp_mlengine_get_model] + +You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` +fields to dynamically determine their values. The result are saved to :ref:`XCom `, +allowing them to be used by other operators. In this case, the +:class:`~airflow.operators.bash.BashOperator` is used to print the model information. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_print_model] + :end-before: [END howto_operator_gcp_mlengine_print_model] + +Creating model versions +^^^^^^^^^^^^^^^^^^^^^^^ +A model version is a subset of the model container where the code runs. A new version of the model can be created +through the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator`. +The model must be specified by ``model_name``, and the ``version`` parameter should contain a dictionary of +all the information about the version. Within the ``version`` parameter’s dictionary, the ``name`` field is +required. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_create_version1] + :end-before: [END howto_operator_gcp_mlengine_create_version1] + +The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator` +can also be used to create more versions with varying parameters. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_create_version2] + :end-before: [END howto_operator_gcp_mlengine_create_version2] + +Managing model versions +^^^^^^^^^^^^^^^^^^^^^^^ +By default, the model code will run using the default model version. You can set the model version through the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineSetDefaultVersionOperator` +by specifying the ``model_name`` and ``version_name`` parameters. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_default_version] + :end-before: [END howto_operator_gcp_mlengine_default_version] + +To list the model versions available, use the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineListVersionsOperator` +while specifying the ``model_name`` parameter. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_list_versions] + :end-before: [END howto_operator_gcp_mlengine_list_versions] + +You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` +fields to dynamically determine their values. The result are saved to :ref:`XCom `, +allowing them to be used by other operators. In this case, the +:class:`~airflow.operators.bash.BashOperator` is used to print the version information. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_print_versions] + :end-before: [END howto_operator_gcp_mlengine_print_versions] + +Making predictions +^^^^^^^^^^^^^^^^^^ +A Google Cloud AI Platform prediction job can be started with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartBatchPredictionJobOperator`. +For specifying the model origin, you need to provide either the ``model_name``, ``uri``, or ``model_name`` and +``version_name``. If you do not provide the ``version_name``, the operator will use the default model version. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_get_prediction] + :end-before: [END howto_operator_gcp_mlengine_get_prediction] + +Cleaning up +^^^^^^^^^^^ +A model version can be deleted with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteVersionOperator` by +the ``version_name`` and ``model_name`` parameters. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_delete_version] + :end-before: [END howto_operator_gcp_mlengine_delete_version] + +You can also delete a model with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteModelOperator` +by providing the ``model_name`` parameter. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_delete_model] + :end-before: [END howto_operator_gcp_mlengine_delete_model] + +Evaluating a model +^^^^^^^^^^^^^^^^^^ +To evaluate a prediction and model, specify a metric function to generate a summary and customize +the evaluation of the model. This function receives a dictionary derived from a json in the batch +prediction result, then returns a tuple of metrics. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_get_metric] + :end-before: [END howto_operator_gcp_mlengine_get_metric] + +To evaluate a prediction and model, it’s useful to have a function to validate the summary result. +This function receives a dictionary of the averaged metrics the function above generated. It then +raises an exception if a task fails or should not proceed. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_validate_error] + :end-before: [END howto_operator_gcp_mlengine_validate_error] + +Prediction results and a model summary can be generated through a function such as +:class:`~airflow.providers.google.cloud.utils.mlengine_operator_utils.create_evaluate_ops`. +It makes predictions using the specified inputs and then summarizes and validates the result. The +functions created above should be passed in through the ``metric_fn_and_keys`` and ``validate_fn`` fields. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_evaluate] + :end-before: [END howto_operator_gcp_mlengine_evaluate] + +Reference +^^^^^^^^^ + +For further information, look at: + +* `Client Library Documentation `__ +* `Product Documentation `__ diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index 43b02390f9b45..d6b8b1c700ba4 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -764,6 +764,12 @@ These integrations allow you to perform various operations within the Google Clo - :mod:`airflow.providers.google.cloud.operators.cloud_memorystore` - + * - `ML Engine `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.mlengine` + - :mod:`airflow.providers.google.cloud.operators.mlengine` + - + * - `Natural Language `__ - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.natural_language` From 7b55ec7f5fb730b4def1b8bb48692d2ab1e64581 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Tue, 14 Jul 2020 10:47:36 -0400 Subject: [PATCH 2/9] change paths to example DAG --- docs/howto/operator/gcp/mlengine.rst | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/howto/operator/gcp/mlengine.rst b/docs/howto/operator/gcp/mlengine.rst index 9a9a8b9ff2d9d..b0367cc9eff9a 100644 --- a/docs/howto/operator/gcp/mlengine.rst +++ b/docs/howto/operator/gcp/mlengine.rst @@ -42,7 +42,7 @@ This creates a virtual machine that can run code specified in the trainer file, contains the main application code. A job can be initiated with the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartTrainingJobOperator`. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_training] @@ -55,7 +55,7 @@ A model is a container that can hold multiple model versions. A new model can be The ``model`` field should be defined with a dictionary containing the information about the model. ``name`` is a required field in this dictionary. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_create_model] @@ -67,7 +67,7 @@ The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineGetModelO can be used to obtain a model previously created. To obtain the correct model, ``model_name`` must be defined in the operator. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_get_model] @@ -78,7 +78,7 @@ fields to dynamically determine their values. The result are saved to :ref:`XCom allowing them to be used by other operators. In this case, the :class:`~airflow.operators.bash.BashOperator` is used to print the model information. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_print_model] @@ -92,7 +92,7 @@ The model must be specified by ``model_name``, and the ``version`` parameter sho all the information about the version. Within the ``version`` parameter’s dictionary, the ``name`` field is required. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_create_version1] @@ -101,7 +101,7 @@ required. The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator` can also be used to create more versions with varying parameters. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_create_version2] @@ -113,7 +113,7 @@ By default, the model code will run using the default model version. You can set :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineSetDefaultVersionOperator` by specifying the ``model_name`` and ``version_name`` parameters. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_default_version] @@ -123,7 +123,7 @@ To list the model versions available, use the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineListVersionsOperator` while specifying the ``model_name`` parameter. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_list_versions] @@ -134,7 +134,7 @@ fields to dynamically determine their values. The result are saved to :ref:`XCom allowing them to be used by other operators. In this case, the :class:`~airflow.operators.bash.BashOperator` is used to print the version information. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_print_versions] @@ -147,7 +147,7 @@ A Google Cloud AI Platform prediction job can be started with the For specifying the model origin, you need to provide either the ``model_name``, ``uri``, or ``model_name`` and ``version_name``. If you do not provide the ``version_name``, the operator will use the default model version. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_get_prediction] @@ -159,7 +159,7 @@ A model version can be deleted with the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteVersionOperator` by the ``version_name`` and ``model_name`` parameters. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_delete_version] @@ -169,7 +169,7 @@ You can also delete a model with the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteModelOperator` by providing the ``model_name`` parameter. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_delete_model] @@ -181,7 +181,7 @@ To evaluate a prediction and model, specify a metric function to generate a summ the evaluation of the model. This function receives a dictionary derived from a json in the batch prediction result, then returns a tuple of metrics. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_get_metric] @@ -191,7 +191,7 @@ To evaluate a prediction and model, it’s useful to have a function to validate This function receives a dictionary of the averaged metrics the function above generated. It then raises an exception if a task fails or should not proceed. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_validate_error] @@ -202,7 +202,7 @@ Prediction results and a model summary can be generated through a function such It makes predictions using the specified inputs and then summarizes and validates the result. The functions created above should be passed in through the ``metric_fn_and_keys`` and ``validate_fn`` fields. -.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_mlengine.py +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_evaluate] From e0518e1d11372b331b960728fe41808180bcaca4 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Tue, 14 Jul 2020 10:52:11 -0400 Subject: [PATCH 3/9] update ML Engine block in operators and hooks --- docs/operators-and-hooks-ref.rst | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index d6b8b1c700ba4..dbfe63b3ea6a8 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -752,8 +752,8 @@ These integrations allow you to perform various operations within the Google Clo - :mod:`airflow.providers.google.cloud.operators.kubernetes_engine` - - * - `Machine Learning Engine `__ - - + * - `Machine Learning Engine `__ + - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.mlengine` - :mod:`airflow.providers.google.cloud.operators.mlengine` - @@ -764,12 +764,6 @@ These integrations allow you to perform various operations within the Google Clo - :mod:`airflow.providers.google.cloud.operators.cloud_memorystore` - - * - `ML Engine `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.mlengine` - - :mod:`airflow.providers.google.cloud.operators.mlengine` - - - * - `Natural Language `__ - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.natural_language` From 38db1a368b021216752d6aab7c111e9e0cef5167 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Tue, 14 Jul 2020 11:38:41 -0400 Subject: [PATCH 4/9] fix include error with prereq tasks --- docs/howto/operator/gcp/mlengine.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/operator/gcp/mlengine.rst b/docs/howto/operator/gcp/mlengine.rst index b0367cc9eff9a..6d43e8e0136a1 100644 --- a/docs/howto/operator/gcp/mlengine.rst +++ b/docs/howto/operator/gcp/mlengine.rst @@ -33,7 +33,7 @@ be used to deploy a trained model, make predictions, and manage various model ve Prerequisite tasks ^^^^^^^^^^^^^^^^^^ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Launching a Job ^^^^^^^^^^^^^^^ From 309b013aa0fda2449222bfa970989047a5d3a4ec Mon Sep 17 00:00:00 2001 From: vuppalli Date: Mon, 13 Jul 2020 15:19:14 -0400 Subject: [PATCH 5/9] add ai platform guide --- docs/operators-and-hooks-ref.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index dbfe63b3ea6a8..61a4fc8a8c7a9 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -764,6 +764,12 @@ These integrations allow you to perform various operations within the Google Clo - :mod:`airflow.providers.google.cloud.operators.cloud_memorystore` - + * - `ML Engine `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.mlengine` + - :mod:`airflow.providers.google.cloud.operators.mlengine` + - + * - `Natural Language `__ - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.natural_language` From 6a033c731b128ac257e5071d17ba86f66f816865 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Tue, 14 Jul 2020 10:52:11 -0400 Subject: [PATCH 6/9] update ML Engine block in operators and hooks --- docs/operators-and-hooks-ref.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index 61a4fc8a8c7a9..dbfe63b3ea6a8 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -764,12 +764,6 @@ These integrations allow you to perform various operations within the Google Clo - :mod:`airflow.providers.google.cloud.operators.cloud_memorystore` - - * - `ML Engine `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.mlengine` - - :mod:`airflow.providers.google.cloud.operators.mlengine` - - - * - `Natural Language `__ - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.natural_language` From f21eec4fc36aeda7e0005a6f3b7e37698a2b8525 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Tue, 14 Jul 2020 14:07:03 -0400 Subject: [PATCH 7/9] remove guide in incorrect folder --- docs/howto/operator/gcp/mlengine.rst | 217 --------------------------- 1 file changed, 217 deletions(-) delete mode 100644 docs/howto/operator/gcp/mlengine.rst diff --git a/docs/howto/operator/gcp/mlengine.rst b/docs/howto/operator/gcp/mlengine.rst deleted file mode 100644 index 6d43e8e0136a1..0000000000000 --- a/docs/howto/operator/gcp/mlengine.rst +++ /dev/null @@ -1,217 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - - -Google Cloud AI Platform Operators -================================== - -`Google Cloud AI Platform `__ (formerly known -as ML Engine) can be used to train machine learning models at scale, host trained models -in the cloud, and use models to make predictions for new data. AI Platform is a collection -of tools for training, evaluating, and tuning machine learning models. AI Platform can also -be used to deploy a trained model, make predictions, and manage various model versions. - -.. contents:: - :depth: 1 - :local: - -Prerequisite tasks -^^^^^^^^^^^^^^^^^^ - -.. include:: ../_partials/prerequisite_tasks.rst - -Launching a Job -^^^^^^^^^^^^^^^ -To start a machine learning operation with AI Platform, you must launch a training job. -This creates a virtual machine that can run code specified in the trainer file, which -contains the main application code. A job can be initiated with the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartTrainingJobOperator`. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_training] - :end-before: [END howto_operator_gcp_mlengine_training] - -Creating a model -^^^^^^^^^^^^^^^^ -A model is a container that can hold multiple model versions. A new model can be created through the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateModelOperator`. -The ``model`` field should be defined with a dictionary containing the information about the model. -``name`` is a required field in this dictionary. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_create_model] - :end-before: [END howto_operator_gcp_mlengine_create_model] - -Getting a model -^^^^^^^^^^^^^^^ -The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineGetModelOperator` -can be used to obtain a model previously created. To obtain the correct model, ``model_name`` -must be defined in the operator. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_get_model] - :end-before: [END howto_operator_gcp_mlengine_get_model] - -You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` -fields to dynamically determine their values. The result are saved to :ref:`XCom `, -allowing them to be used by other operators. In this case, the -:class:`~airflow.operators.bash.BashOperator` is used to print the model information. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_print_model] - :end-before: [END howto_operator_gcp_mlengine_print_model] - -Creating model versions -^^^^^^^^^^^^^^^^^^^^^^^ -A model version is a subset of the model container where the code runs. A new version of the model can be created -through the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator`. -The model must be specified by ``model_name``, and the ``version`` parameter should contain a dictionary of -all the information about the version. Within the ``version`` parameter’s dictionary, the ``name`` field is -required. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_create_version1] - :end-before: [END howto_operator_gcp_mlengine_create_version1] - -The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator` -can also be used to create more versions with varying parameters. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_create_version2] - :end-before: [END howto_operator_gcp_mlengine_create_version2] - -Managing model versions -^^^^^^^^^^^^^^^^^^^^^^^ -By default, the model code will run using the default model version. You can set the model version through the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineSetDefaultVersionOperator` -by specifying the ``model_name`` and ``version_name`` parameters. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_default_version] - :end-before: [END howto_operator_gcp_mlengine_default_version] - -To list the model versions available, use the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineListVersionsOperator` -while specifying the ``model_name`` parameter. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_list_versions] - :end-before: [END howto_operator_gcp_mlengine_list_versions] - -You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` -fields to dynamically determine their values. The result are saved to :ref:`XCom `, -allowing them to be used by other operators. In this case, the -:class:`~airflow.operators.bash.BashOperator` is used to print the version information. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_print_versions] - :end-before: [END howto_operator_gcp_mlengine_print_versions] - -Making predictions -^^^^^^^^^^^^^^^^^^ -A Google Cloud AI Platform prediction job can be started with the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartBatchPredictionJobOperator`. -For specifying the model origin, you need to provide either the ``model_name``, ``uri``, or ``model_name`` and -``version_name``. If you do not provide the ``version_name``, the operator will use the default model version. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_get_prediction] - :end-before: [END howto_operator_gcp_mlengine_get_prediction] - -Cleaning up -^^^^^^^^^^^ -A model version can be deleted with the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteVersionOperator` by -the ``version_name`` and ``model_name`` parameters. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_delete_version] - :end-before: [END howto_operator_gcp_mlengine_delete_version] - -You can also delete a model with the -:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteModelOperator` -by providing the ``model_name`` parameter. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_delete_model] - :end-before: [END howto_operator_gcp_mlengine_delete_model] - -Evaluating a model -^^^^^^^^^^^^^^^^^^ -To evaluate a prediction and model, specify a metric function to generate a summary and customize -the evaluation of the model. This function receives a dictionary derived from a json in the batch -prediction result, then returns a tuple of metrics. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_get_metric] - :end-before: [END howto_operator_gcp_mlengine_get_metric] - -To evaluate a prediction and model, it’s useful to have a function to validate the summary result. -This function receives a dictionary of the averaged metrics the function above generated. It then -raises an exception if a task fails or should not proceed. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_validate_error] - :end-before: [END howto_operator_gcp_mlengine_validate_error] - -Prediction results and a model summary can be generated through a function such as -:class:`~airflow.providers.google.cloud.utils.mlengine_operator_utils.create_evaluate_ops`. -It makes predictions using the specified inputs and then summarizes and validates the result. The -functions created above should be passed in through the ``metric_fn_and_keys`` and ``validate_fn`` fields. - -.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py - :language: python - :dedent: 4 - :start-after: [START howto_operator_gcp_mlengine_evaluate] - :end-before: [END howto_operator_gcp_mlengine_evaluate] - -Reference -^^^^^^^^^ - -For further information, look at: - -* `Client Library Documentation `__ -* `Product Documentation `__ From 20a16879cf70f8b2910ae79d6f9efc8c88b7d0c5 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Tue, 14 Jul 2020 16:03:48 -0400 Subject: [PATCH 8/9] add ai platform guide --- docs/howto/operator/google/cloud/mlengine.rst | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 docs/howto/operator/google/cloud/mlengine.rst diff --git a/docs/howto/operator/google/cloud/mlengine.rst b/docs/howto/operator/google/cloud/mlengine.rst new file mode 100644 index 0000000000000..6d43e8e0136a1 --- /dev/null +++ b/docs/howto/operator/google/cloud/mlengine.rst @@ -0,0 +1,217 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Google Cloud AI Platform Operators +================================== + +`Google Cloud AI Platform `__ (formerly known +as ML Engine) can be used to train machine learning models at scale, host trained models +in the cloud, and use models to make predictions for new data. AI Platform is a collection +of tools for training, evaluating, and tuning machine learning models. AI Platform can also +be used to deploy a trained model, make predictions, and manage various model versions. + +.. contents:: + :depth: 1 + :local: + +Prerequisite tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: ../_partials/prerequisite_tasks.rst + +Launching a Job +^^^^^^^^^^^^^^^ +To start a machine learning operation with AI Platform, you must launch a training job. +This creates a virtual machine that can run code specified in the trainer file, which +contains the main application code. A job can be initiated with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartTrainingJobOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_training] + :end-before: [END howto_operator_gcp_mlengine_training] + +Creating a model +^^^^^^^^^^^^^^^^ +A model is a container that can hold multiple model versions. A new model can be created through the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateModelOperator`. +The ``model`` field should be defined with a dictionary containing the information about the model. +``name`` is a required field in this dictionary. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_create_model] + :end-before: [END howto_operator_gcp_mlengine_create_model] + +Getting a model +^^^^^^^^^^^^^^^ +The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineGetModelOperator` +can be used to obtain a model previously created. To obtain the correct model, ``model_name`` +must be defined in the operator. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_get_model] + :end-before: [END howto_operator_gcp_mlengine_get_model] + +You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` +fields to dynamically determine their values. The result are saved to :ref:`XCom `, +allowing them to be used by other operators. In this case, the +:class:`~airflow.operators.bash.BashOperator` is used to print the model information. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_print_model] + :end-before: [END howto_operator_gcp_mlengine_print_model] + +Creating model versions +^^^^^^^^^^^^^^^^^^^^^^^ +A model version is a subset of the model container where the code runs. A new version of the model can be created +through the :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator`. +The model must be specified by ``model_name``, and the ``version`` parameter should contain a dictionary of +all the information about the version. Within the ``version`` parameter’s dictionary, the ``name`` field is +required. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_create_version1] + :end-before: [END howto_operator_gcp_mlengine_create_version1] + +The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineCreateVersionOperator` +can also be used to create more versions with varying parameters. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_create_version2] + :end-before: [END howto_operator_gcp_mlengine_create_version2] + +Managing model versions +^^^^^^^^^^^^^^^^^^^^^^^ +By default, the model code will run using the default model version. You can set the model version through the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineSetDefaultVersionOperator` +by specifying the ``model_name`` and ``version_name`` parameters. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_default_version] + :end-before: [END howto_operator_gcp_mlengine_default_version] + +To list the model versions available, use the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineListVersionsOperator` +while specifying the ``model_name`` parameter. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_list_versions] + :end-before: [END howto_operator_gcp_mlengine_list_versions] + +You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` +fields to dynamically determine their values. The result are saved to :ref:`XCom `, +allowing them to be used by other operators. In this case, the +:class:`~airflow.operators.bash.BashOperator` is used to print the version information. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_print_versions] + :end-before: [END howto_operator_gcp_mlengine_print_versions] + +Making predictions +^^^^^^^^^^^^^^^^^^ +A Google Cloud AI Platform prediction job can be started with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineStartBatchPredictionJobOperator`. +For specifying the model origin, you need to provide either the ``model_name``, ``uri``, or ``model_name`` and +``version_name``. If you do not provide the ``version_name``, the operator will use the default model version. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_get_prediction] + :end-before: [END howto_operator_gcp_mlengine_get_prediction] + +Cleaning up +^^^^^^^^^^^ +A model version can be deleted with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteVersionOperator` by +the ``version_name`` and ``model_name`` parameters. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_delete_version] + :end-before: [END howto_operator_gcp_mlengine_delete_version] + +You can also delete a model with the +:class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteModelOperator` +by providing the ``model_name`` parameter. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_delete_model] + :end-before: [END howto_operator_gcp_mlengine_delete_model] + +Evaluating a model +^^^^^^^^^^^^^^^^^^ +To evaluate a prediction and model, specify a metric function to generate a summary and customize +the evaluation of the model. This function receives a dictionary derived from a json in the batch +prediction result, then returns a tuple of metrics. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_get_metric] + :end-before: [END howto_operator_gcp_mlengine_get_metric] + +To evaluate a prediction and model, it’s useful to have a function to validate the summary result. +This function receives a dictionary of the averaged metrics the function above generated. It then +raises an exception if a task fails or should not proceed. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_validate_error] + :end-before: [END howto_operator_gcp_mlengine_validate_error] + +Prediction results and a model summary can be generated through a function such as +:class:`~airflow.providers.google.cloud.utils.mlengine_operator_utils.create_evaluate_ops`. +It makes predictions using the specified inputs and then summarizes and validates the result. The +functions created above should be passed in through the ``metric_fn_and_keys`` and ``validate_fn`` fields. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_mlengine.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcp_mlengine_evaluate] + :end-before: [END howto_operator_gcp_mlengine_evaluate] + +Reference +^^^^^^^^^ + +For further information, look at: + +* `Client Library Documentation `__ +* `Product Documentation `__ From e8809bcbae25ba805e3931c8d366310979c61525 Mon Sep 17 00:00:00 2001 From: vuppalli Date: Wed, 15 Jul 2020 11:41:02 -0400 Subject: [PATCH 9/9] add references to guide in class definitions --- .../google/cloud/operators/mlengine.py | 36 +++++++++++++++++++ docs/howto/operator/google/cloud/mlengine.rst | 16 +++++++++ 2 files changed, 52 insertions(+) diff --git a/airflow/providers/google/cloud/operators/mlengine.py b/airflow/providers/google/cloud/operators/mlengine.py index c402c478f38cd..623c14d540c2c 100644 --- a/airflow/providers/google/cloud/operators/mlengine.py +++ b/airflow/providers/google/cloud/operators/mlengine.py @@ -71,6 +71,10 @@ class MLEngineStartBatchPredictionJobOperator(BaseOperator): """ Start a Google Cloud ML Engine prediction job. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineStartBatchPredictionJobOperator` + NOTE: For model origin, users should consider exactly one from the three options below: @@ -351,6 +355,10 @@ class MLEngineCreateModelOperator(BaseOperator): """ Creates a new model. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineCreateModelOperator` + The model should be provided by the `model` parameter. :param model: A dictionary containing the information about the model. @@ -395,6 +403,10 @@ class MLEngineGetModelOperator(BaseOperator): """ Gets a particular model + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineGetModelOperator` + The name of model shold be specified in `model_name`. :param model_name: The name of the model. @@ -438,6 +450,10 @@ class MLEngineDeleteModelOperator(BaseOperator): """ Deletes a model. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineDeleteModelOperator` + The model should be provided by the `model_name` parameter. :param model_name: The name of the model. @@ -615,6 +631,10 @@ class MLEngineCreateVersionOperator(BaseOperator): """ Creates a new version in the model + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineCreateVersionOperator` + Model should be specified by `model_name`, in which case the `version` parameter should contain all the information to create that version @@ -678,6 +698,10 @@ class MLEngineSetDefaultVersionOperator(BaseOperator): """ Sets a version in the model. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineSetDefaultVersionOperator` + The model should be specified by `model_name` to be the default. The name of the version should be specified in the `version_name` parameter. @@ -741,6 +765,10 @@ class MLEngineListVersionsOperator(BaseOperator): """ Lists all available versions of the model + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineListVersionsOperator` + The model should be specified by `model_name`. :param model_name: The name of the Google Cloud ML Engine model that the version @@ -794,6 +822,10 @@ class MLEngineDeleteVersionOperator(BaseOperator): """ Deletes the version from the model. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineDeleteVersionOperator` + The name of the version should be specified in `version_name` parameter from the model specified by `model_name`. @@ -874,6 +906,10 @@ class MLEngineStartTrainingJobOperator(BaseOperator): """ Operator for launching a MLEngine training job. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:MLEngineStartTrainingJobOperator` + :param job_id: A unique templated id for the submitted Google MLEngine training job. (templated) :type job_id: str diff --git a/docs/howto/operator/google/cloud/mlengine.rst b/docs/howto/operator/google/cloud/mlengine.rst index 6d43e8e0136a1..0b3dd66f816ed 100644 --- a/docs/howto/operator/google/cloud/mlengine.rst +++ b/docs/howto/operator/google/cloud/mlengine.rst @@ -35,6 +35,8 @@ Prerequisite tasks .. include:: ../_partials/prerequisite_tasks.rst +.. _howto/operator:MLEngineStartTrainingJobOperator: + Launching a Job ^^^^^^^^^^^^^^^ To start a machine learning operation with AI Platform, you must launch a training job. @@ -48,6 +50,8 @@ contains the main application code. A job can be initiated with the :start-after: [START howto_operator_gcp_mlengine_training] :end-before: [END howto_operator_gcp_mlengine_training] +.. _howto/operator:MLEngineCreateModelOperator: + Creating a model ^^^^^^^^^^^^^^^^ A model is a container that can hold multiple model versions. A new model can be created through the @@ -61,6 +65,8 @@ The ``model`` field should be defined with a dictionary containing the informati :start-after: [START howto_operator_gcp_mlengine_create_model] :end-before: [END howto_operator_gcp_mlengine_create_model] +.. _howto/operator:MLEngineGetModelOperator: + Getting a model ^^^^^^^^^^^^^^^ The :class:`~airflow.providers.google.cloud.operators.mlengine.MLEngineGetModelOperator` @@ -84,6 +90,8 @@ allowing them to be used by other operators. In this case, the :start-after: [START howto_operator_gcp_mlengine_print_model] :end-before: [END howto_operator_gcp_mlengine_print_model] +.. _howto/operator:MLEngineCreateVersionOperator: + Creating model versions ^^^^^^^^^^^^^^^^^^^^^^^ A model version is a subset of the model container where the code runs. A new version of the model can be created @@ -107,6 +115,9 @@ can also be used to create more versions with varying parameters. :start-after: [START howto_operator_gcp_mlengine_create_version2] :end-before: [END howto_operator_gcp_mlengine_create_version2] +.. _howto/operator:MLEngineSetDefaultVersionOperator: +.. _howto/operator:MLEngineListVersionsOperator: + Managing model versions ^^^^^^^^^^^^^^^^^^^^^^^ By default, the model code will run using the default model version. You can set the model version through the @@ -140,6 +151,8 @@ allowing them to be used by other operators. In this case, the :start-after: [START howto_operator_gcp_mlengine_print_versions] :end-before: [END howto_operator_gcp_mlengine_print_versions] +.. _howto/operator:MLEngineStartBatchPredictionJobOperator: + Making predictions ^^^^^^^^^^^^^^^^^^ A Google Cloud AI Platform prediction job can be started with the @@ -153,6 +166,9 @@ For specifying the model origin, you need to provide either the ``model_name``, :start-after: [START howto_operator_gcp_mlengine_get_prediction] :end-before: [END howto_operator_gcp_mlengine_get_prediction] +.. _howto/operator:MLEngineDeleteVersionOperator: +.. _howto/operator:MLEngineDeleteModelOperator: + Cleaning up ^^^^^^^^^^^ A model version can be deleted with the