diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_component_builder.py b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_component_builder.py index f5f71f823e60..3fb72fcdfe60 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_component_builder.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_component_builder.py @@ -181,12 +181,14 @@ def add_node(self, node: Union[BaseNode, AutoMLJob]): """ self.nodes.append(node) - def build(self, *, user_provided_kwargs=None, non_pipeline_params_dict=None) -> PipelineComponent: + def build(self, *, user_provided_kwargs=None, + non_pipeline_inputs_dict=None, non_pipeline_inputs=None) -> PipelineComponent: """ Build a pipeline component from current pipeline builder. :param user_provided_kwargs: The kwargs user provided to dsl pipeline function. None if not provided. - :param non_pipeline_params_dict: Non-pipeline parameters to provided value. None if not exist. + :param non_pipeline_inputs_dict: The non-pipeline input provided key-value. None if not exist. + :param non_pipeline_inputs: List of non-pipeline input name. None if not exist. """ if user_provided_kwargs is None: user_provided_kwargs = {} @@ -198,8 +200,9 @@ def build(self, *, user_provided_kwargs=None, non_pipeline_params_dict=None) -> user_provided_kwargs=user_provided_kwargs, # TODO: support result() for pipeline input inside parameter group group_default_kwargs=self._get_group_parameter_defaults(), - non_pipeline_parameter_dict=non_pipeline_params_dict + non_pipeline_inputs=non_pipeline_inputs ) + kwargs.update(non_pipeline_inputs_dict or {}) # We use this stack to store the dsl pipeline definition hierarchy _definition_builder_stack.push(self) @@ -401,7 +404,7 @@ def _get_name_or_component_name(node: Union[BaseNode, AutoMLJob]): def _build_pipeline_parameter( - func, *, user_provided_kwargs, group_default_kwargs=None, non_pipeline_parameter_dict=None): + func, *, user_provided_kwargs, group_default_kwargs=None, non_pipeline_inputs=None): # Pass group defaults into kwargs to support group.item can be used even if no default on function. # example: # @parameter_group @@ -413,14 +416,14 @@ def _build_pipeline_parameter( # component_func(input=param.key) <--- param.key should be val. # transform kwargs - transformed_kwargs = non_pipeline_parameter_dict or {} + transformed_kwargs, non_pipeline_inputs = {}, non_pipeline_inputs or [] if group_default_kwargs: transformed_kwargs.update( { key: _wrap_pipeline_parameter( key, default_value=value, actual_value=value ) for key, value in group_default_kwargs.items() - if key not in non_pipeline_parameter_dict + if key not in non_pipeline_inputs } ) @@ -434,7 +437,7 @@ def all_params(parameters): parameters = all_params(signature(func).parameters) # transform default values for left_args in parameters: - if left_args.name not in transformed_kwargs.keys(): + if left_args.name not in transformed_kwargs.keys() and left_args.name not in non_pipeline_inputs: default_value = left_args.default if left_args.default is not Parameter.empty else None actual_value = user_provided_kwargs.get(left_args.name) transformed_kwargs[left_args.name] = _wrap_pipeline_parameter( diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_decorator.py b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_decorator.py index 9f7b12792cb1..dd6c21d3a9f4 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_decorator.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_pipeline_decorator.py @@ -165,12 +165,13 @@ def wrapper(*args, **kwargs) -> PipelineJob: provided_positional_args = _validate_args(func, args, kwargs, non_pipeline_inputs) # Convert args to kwargs kwargs.update(provided_positional_args) - non_pipeline_params_dict = {k: v for k, v in kwargs.items() if k in non_pipeline_inputs} + non_pipeline_inputs_dict = {k: v for k, v in kwargs.items() if k in non_pipeline_inputs} # TODO: cache built pipeline component pipeline_component = pipeline_builder.build( user_provided_kwargs=kwargs, - non_pipeline_params_dict=non_pipeline_params_dict + non_pipeline_inputs_dict=non_pipeline_inputs_dict, + non_pipeline_inputs=non_pipeline_inputs ) finally: # use `finally` to ensure pop operation from the stack diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/utils.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/utils.py index 1d51a818f513..f2c01a43087c 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/utils.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/utils.py @@ -211,6 +211,7 @@ def _split(_fields): inherited_fields = _get_inherited_fields() # From annotations get field with type annotations = getattr(cls_or_func, "__annotations__", {}) + annotations = {k: v for k, v in annotations.items() if k not in skip_params} annotations = _update_io_from_mldesigner(annotations) annotation_fields = _get_fields(annotations) # Update fields use class field with defaults from class dict or signature(func).paramters diff --git a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py index 73f22811c9a2..5a4d6886cb8f 100644 --- a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py +++ b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py @@ -1,6 +1,7 @@ import os from io import StringIO from pathlib import Path +from typing import Dict from unittest import mock from unittest.mock import patch @@ -1930,20 +1931,24 @@ def test_pipeline_with_non_pipeline_inputs(self): component_func1 = load_component(source=component_yaml, params_override=[{"name": "component_name_1"}]) component_func2 = load_component(source=component_yaml, params_override=[{"name": "component_name_2"}]) - @dsl.pipeline(non_pipeline_inputs=["other_params", "is_add_component"]) - def pipeline_func(job_in_number, job_in_path, other_params, is_add_component): + @dsl.pipeline(non_pipeline_inputs=["other_params", "is_add_component", + "param_with_annotation", "param_with_default"]) + def pipeline_func(job_in_number, job_in_path, other_params, is_add_component, + param_with_annotation: Dict[str, str], param_with_default: int = 1): + assert param_with_default == 1 + assert param_with_annotation == {"mock": "dict"} component_func1(component_in_number=job_in_number, component_in_path=job_in_path) component_func2(component_in_number=other_params, component_in_path=job_in_path) if is_add_component: component_func2(component_in_number=other_params, component_in_path=job_in_path) - pipeline = pipeline_func(10, Input(path="/a/path/on/ds"), 15, False) + pipeline = pipeline_func(10, Input(path="/a/path/on/ds"), 15, False, {"mock": "dict"}) assert len(pipeline.jobs) == 2 assert "other_params" not in pipeline.inputs assert isinstance(pipeline.jobs[component_func1.name].inputs["component_in_number"]._data, PipelineInput) assert pipeline.jobs[component_func2.name].inputs["component_in_number"]._data == 15 - pipeline = pipeline_func(10, Input(path="/a/path/on/ds"), 15, True) + pipeline = pipeline_func(10, Input(path="/a/path/on/ds"), 15, True, {"mock": "dict"}) assert len(pipeline.jobs) == 3 @dsl.pipeline(non_pipeline_parameters=["other_params", "is_add_component"])