[ML]Remove empty attributes when node -> RestObj and component -> component dict

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_builders/base_node.py

@@ -408,9 +408,7 @@ def _to_rest_object(self, **kwargs) -> dict:  # pylint: disable=unused-argument
         """Convert self to a rest object for remote call."""
         base_dict, rest_obj = self._to_dict(), {}
         for key in self._picked_fields_from_dict_to_rest_object():
-            if key not in base_dict:
-                rest_obj[key] = None
-            else:
+            if key in base_dict:
                 rest_obj[key] = base_dict.get(key)
 
         rest_obj.update(

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py

@@ -468,6 +468,9 @@ def _to_rest_object(self) -> ComponentVersionData:
             component["type"] = NodeType.COMMAND
             component["inputs"] = component.pop("source")
             component["outputs"] = dict({"output": component.pop("output")})
+            # method _to_dict() will remove empty keys
+            if "tags" not in component:
+                component["tags"] = {}
             component["tags"]["component_type_overwrite"] = NodeType.IMPORT
             component["command"] = NodeType.IMPORT
 

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_util.py

@@ -5,7 +5,6 @@
 import json
 import os
 import shutil
-from collections import OrderedDict
 from typing import Any, Dict, List, Optional, Union
 from unittest import mock
 
@@ -191,21 +190,34 @@ def validate_attribute_type(attrs_to_check: dict, attr_type_map: dict):
                 error_type=ValidationErrorType.INVALID_VALUE,
             )
 
+def is_empty_target(obj):
+    """Determines if it's empty target"""
+    return (obj is None
+            # some objs have overloaded "==" and will cause error. e.g CommandComponent obj
+            or (isinstance(obj, dict) and len(obj) == 0)
+        )
 
-def convert_ordered_dict_to_dict(target_object: Union[Dict, List]) -> Union[Dict, List]:
-    """Convert ordered dict to dict.
+def convert_ordered_dict_to_dict(target_object: Union[Dict, List], remove_empty=True) -> Union[Dict, List]:
+    """Convert ordered dict to dict. Remove keys with None value.
 
     This is a workaround for rest request must be in dict instead of
     ordered dict.
     """
     # OrderedDict can appear nested in a list
     if isinstance(target_object, list):
-        target_object = [convert_ordered_dict_to_dict(obj) for obj in target_object]
+        new_list = []
+        for item in target_object:
+            item = convert_ordered_dict_to_dict(item)
+            if not is_empty_target(item) or not remove_empty:
+                new_list.append(item)
+        return new_list
     if isinstance(target_object, dict):
-        for key, dict_candidate in target_object.items():
-            target_object[key] = convert_ordered_dict_to_dict(dict_candidate)
-    if isinstance(target_object, OrderedDict):
-        return dict(**target_object)
+        new_dict = {}
+        for key, value in target_object.items():
+            value = convert_ordered_dict_to_dict(value)
+            if not is_empty_target(value) or not remove_empty:
+                new_dict[key] = value
+        return new_dict
     return target_object
 
 

sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py

@@ -182,7 +182,6 @@ def test_parallel_component(self, client: MLClient, randstr: Callable[[str], str
             "mini_batch_size": "10240",
             "outputs": {"scored_result": {"type": "mltable"}, "scoring_summary": {"type": "uri_file"}},
             "retry_settings": {"max_retries": 10, "timeout": 3},
-            "tags": {},
             "type": "parallel",
             "version": "1.0.0",
         }
@@ -198,7 +197,6 @@ def test_parallel_component(self, client: MLClient, randstr: Callable[[str], str
     def test_automl_component(self, client: MLClient, registry_client: MLClient, randstr: Callable[[str], str]) -> None:
         expected_component_dict = {
             "description": "Component that executes an AutoML Classification task model training in a pipeline.",
-            "tags": {},
             "version": "1.0",
             "$schema": "http://azureml/sdk-2-0/AutoMLComponent.json",
             "display_name": "AutoML Classification",
@@ -787,7 +785,6 @@ def test_simple_pipeline_component_create(self, client: MLClient, randstr: Calla
                 # The azureml: prefix has been resolve and removed by service
                 "node_compute": {"type": "string", "default": "cpu-cluster"},
             },
-            "outputs": {},
             "type": "pipeline",
         }
         assert component_dict == expected_dict

sdk/ml/azure-ai-ml/tests/component/unittests/test_command_component_entity.py

@@ -52,7 +52,6 @@ def test_command_component_to_dict(self):
         assert command_component._other_parameter.get("mock_option_param") == yaml_dict["mock_option_param"]
 
         yaml_dict["version"] = str(yaml_dict["version"])
-        yaml_dict["inputs"] = {}
         component_dict = command_component._to_dict()
         component_dict.pop("is_deterministic")
         assert yaml_dict == component_dict
@@ -211,17 +210,6 @@ def test_command_component_code_git_path(self):
     def test_command_component_version_as_a_function(self):
         expected_rest_component = {
             "componentId": "fake_component",
-            "computeId": None,
-            "display_name": None,
-            "distribution": None,
-            "environment_variables": {},
-            "inputs": {},
-            "properties": {},
-            "limits": None,
-            "name": None,
-            "outputs": {},
-            "resources": None,
-            "tags": {},
             "type": "command",
             "_source": "YAML.COMPONENT",
         }
@@ -250,20 +238,10 @@ def test_command_component_version_as_a_function(self):
     def test_command_component_version_as_a_function_with_inputs(self):
         expected_rest_component = {
             "componentId": "fake_component",
-            "computeId": None,
-            "display_name": None,
-            "distribution": None,
-            "environment_variables": {},
             "inputs": {
                 "component_in_number": {"job_input_type": "literal", "value": "10"},
                 "component_in_path": {"job_input_type": "literal", "value": "${{parent.inputs.pipeline_input}}"},
             },
-            "limits": None,
-            "name": None,
-            "outputs": {},
-            "resources": None,
-            "tags": {},
-            "properties": {},
             "type": "command",
             "_source": "YAML.COMPONENT",
         }
@@ -404,7 +382,6 @@ def test_primitive_output(self):
             "description": "This is the basic command component",
             "display_name": "CommandComponentBasic",
             "environment": "azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1",
-            "inputs": {},
             "is_deterministic": True,
             "name": "sample_command_component_basic",
             "outputs": {

sdk/ml/azure-ai-ml/tests/component/unittests/test_parallel_component_entity.py

@@ -80,8 +80,6 @@ def test_parallel_component_version_as_a_function_with_inputs(self):
         expected_rest_component = {
             "componentId": "fake_component",
             "_source": "YAML.COMPONENT",
-            "computeId": None,
-            "display_name": None,
             "input_data": "${{inputs.component_in_path}}",
             "inputs": {
                 "component_in_number": {"job_input_type": "literal", "value": "10"},
@@ -90,21 +88,9 @@ def test_parallel_component_version_as_a_function_with_inputs(self):
                     "value": "${{parent.inputs.pipeline_input}}",
                 },
             },
-            "name": None,
-            "outputs": {},
-            "tags": {},
-            "properties": {},
             "input_data": "${{inputs.component_in_path}}",
             "type": "parallel",
-            "error_threshold": None,
-            "logging_level": None,
-            "max_concurrency_per_instance": None,
-            "partition_keys": None,
-            "mini_batch_error_threshold": None,
             "mini_batch_size": 10485760,
-            "retry_settings": None,
-            "resources": None,
-            "environment_variables": {},
             "task": {
                 "append_row_to": "${{outputs.scoring_summary}}",
                 "program_arguments": "--label ${{inputs.label}} --model ${{inputs.model}} "

sdk/ml/azure-ai-ml/tests/component/unittests/test_pipeline_component_entity.py

@@ -42,26 +42,19 @@ def test_inline_helloworld_pipeline_component(self) -> None:
                 "component_in_path": {"type": "uri_folder", "description": "A path"},
                 "node_compute": {"type": "string", "default": "azureml:cpu-cluster"},
             },
-            "outputs": {},
             "type": "pipeline",
             "jobs": {
                 "component_a_job": {
-                    "properties": {},
                     "component": {
                         "command": 'echo "hello" && echo ' '"world" > ' "${{outputs.world_output}}/world.txt",
                         "environment": "azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu@latest",
-                        "inputs": {},
                         "is_deterministic": True,
                         "name": "azureml_anonymous",
                         "outputs": {"world_output": {"type": "uri_folder"}},
-                        "tags": {},
                         "type": "command",
                         "version": "1",
                     },
                     "compute": "${{parent.inputs.node_compute}}",
-                    "environment_variables": {},
-                    "inputs": {},
-                    "outputs": {},
                     "type": "command",
                 },
             },
@@ -89,7 +82,6 @@ def test_helloworld_pipeline_component(self) -> None:
             },
             "jobs": {
                 "component_a_job": {
-                    "properties": {},
                     "component": {
                         "$schema": "https://azuremlschemas.azureedge.net/development/commandComponent.schema.json",
                         "command": "echo Hello World & "
@@ -120,7 +112,6 @@ def test_helloworld_pipeline_component(self) -> None:
                         "type": "command",
                         "version": "1",
                     },
-                    "environment_variables": {},
                     "inputs": {
                         "component_in_number": {"path": "${{parent.inputs.component_in_number}}"},
                         "component_in_path": {"path": "${{parent.inputs.component_in_path}}"},
@@ -159,10 +150,8 @@ def test_helloworld_nested_pipeline_component(self) -> None:
                 "pipeline_component": {
                     "component": {
                         "$schema": "https://azuremlschemas.azureedge.net/development/pipelineComponent.schema.json",
-                        "creation_context": None,
                         "description": "This is the " "basic pipeline " "component",
                         "display_name": "Hello World " "Pipeline " "Component",
-                        "id": None,
                         "inputs": {
                             "component_in_number": {
                                 "default": "10.99",
@@ -172,7 +161,6 @@ def test_helloworld_nested_pipeline_component(self) -> None:
                             },
                             "component_in_path": {"description": "A " "path", "type": "uri_folder"},
                         },
-                        "is_deterministic": None,
                         "jobs": {
                             "component_a_job": {
                                 "component": {
@@ -210,26 +198,21 @@ def test_helloworld_nested_pipeline_component(self) -> None:
                                     "type": "command",
                                     "version": "1",
                                 },
-                                "environment_variables": {},
                                 "inputs": {
                                     "component_in_number": {"path": "${{parent.inputs.component_in_number}}"},
                                     "component_in_path": {"path": "${{parent.inputs.component_in_path}}"},
                                 },
                                 "outputs": {"component_out_path": "${{parent.outputs.output_path}}"},
-                                "properties": {},
                                 "type": "command",
                             }
                         },
-                        "latest_version": None,
                         "name": "azureml_anonymous",
                         "outputs": {"output_path": {"type": "uri_folder"}},
                         "tags": {"owner": "sdkteam", "tag": "tagvalue"},
                         "type": "pipeline",
                         "version": "1",
                     },
-                    "properties": {},
                     "inputs": {"component_in_path": {"path": "${{parent.inputs.component_in_path}}"}},
-                    "outputs": {},
                     "type": "pipeline",
                 }
             },
@@ -255,33 +238,25 @@ def test_pipeline_job_to_component(self):
             },
             "jobs": {
                 "hello_world_component": {
-                    "properties": {},
                     "component": "azureml:microsoftsamplesCommandComponentBasic_second:1",
                     "compute": "azureml:cpu-cluster",
-                    "environment_variables": {},
                     "inputs": {
                         "component_in_number": {"path": "${{parent.inputs.job_in_number}}"},
                         "component_in_path": {"path": "${{parent.inputs.job_in_path}}"},
                     },
-                    "outputs": {},
                     "type": "command",
                 },
                 "hello_world_component_2": {
-                    "properties": {},
                     "component": "azureml:microsoftsamplesCommandComponentBasic_second:1",
                     "compute": "azureml:cpu-cluster",
-                    "environment_variables": {},
                     "inputs": {
                         "component_in_number": {"path": "${{parent.inputs.job_in_other_number}}"},
                         "component_in_path": {"path": "${{parent.inputs.job_in_path}}"},
                     },
-                    "outputs": {},
                     "type": "command",
                 },
             },
             "name": "azureml_anonymous",
-            "outputs": {},
-            "tags": {},
             "type": "pipeline",
             "version": "1",
         }

sdk/ml/azure-ai-ml/tests/component/unittests/test_spark_component_entity.py

@@ -75,13 +75,10 @@ def test_spark_component_entity(self):
     def test_spark_component_version_as_a_function_with_inputs(self):
         expected_rest_component = {
             "type": "spark",
-            "properties": {},
             "resources": {"instance_type": "Standard_E8S_V3", "runtime_version": "3.1.0"},
             "entry": {"file": "add_greeting_column.py", "spark_job_entry_type": "SparkJobPythonEntry"},
             "py_files": ["utils.zip"],
-            "jars": None,
             "files": ["my_files.txt"],
-            "archives": None,
             "identity": {"identity_type": "UserIdentity"},
             "conf": {
                 "spark.driver.cores": 2,
@@ -91,14 +88,9 @@ def test_spark_component_version_as_a_function_with_inputs(self):
                 "spark.executor.memory": "1g",
             },
             "args": "--file_input ${{inputs.file_input}}",
-            "name": None,
-            "display_name": None,
-            "tags": {},
-            "computeId": None,
             "inputs": {
                 "file_input": {"job_input_type": "literal", "value": "${{parent.inputs.pipeline_input}}"},
             },
-            "outputs": {},
             "_source": "YAML.COMPONENT",
             "componentId": "fake_component",
         }