diff --git a/gapic/schema/api.py b/gapic/schema/api.py index 084349d488..d3aa9f07ca 100644 --- a/gapic/schema/api.py +++ b/gapic/schema/api.py @@ -23,10 +23,12 @@ import keyword import os import sys -from typing import Callable, Container, Dict, FrozenSet, Mapping, Optional, Sequence, Set, Tuple from types import MappingProxyType +from typing import Callable, Container, Dict, FrozenSet, Mapping, Optional, Sequence, Set, Tuple +import yaml from google.api_core import exceptions +from google.api import client_pb2 # type: ignore from google.api import http_pb2 # type: ignore from google.api import resource_pb2 # type: ignore from google.api import service_pb2 # type: ignore @@ -58,6 +60,14 @@ TRANSPORT_REST = "rest" +class MethodSettingsError(ValueError): + """ + Raised when `google.api.client_pb2.MethodSettings` contains + an invalid value. + """ + pass + + @dataclasses.dataclass(frozen=True) class Proto: """A representation of a particular proto file within an API.""" @@ -560,6 +570,133 @@ def mixin_http_options(self): res[s] = [rule for rule in opt_gen if rule] return res + @cached_property + def all_methods(self) -> Mapping[str, MethodDescriptorProto]: + """Return a map of all methods for the API. + + Return: + Mapping[str, MethodDescriptorProto]: A mapping of MethodDescriptorProto + values for the API. + """ + return { + f"{service_key}.{method_key}": method_value + for service_key, service_value in self.services.items() + for method_key, method_value in service_value.methods.items() + } + + def enforce_valid_method_settings( + self, service_method_settings: Sequence[client_pb2.MethodSettings] + ) -> None: + """ + Checks each `google.api.client.MethodSettings` provided for validity and + raises an exception if invalid values are found. If + `google.api.client.MethodSettings.auto_populated_fields` + is set, verify each field against the criteria of AIP-4235 + (https://google.aip.dev/client-libraries/4235). All of the conditions + below must be true: + + - The field must be of type string + - The field must be at the top-level of the request message + - The RPC must be a unary RPC (i.e. streaming RPCs are not supported) + - The field must not be annotated with google.api.field_behavior = REQUIRED. + - The field must be annotated with google.api.field_info.format = UUID4. + + Note that the field presence requirements in AIP-4235 should be checked at run + time. + + Args: + service_method_settings (Sequence[client_pb2.MethodSettings]): Method + settings to be used when generating API methods. + Return: + None + Raises: + MethodSettingsError: if fields in `method_settings.auto_populated_fields` + cannot be automatically populated. + """ + + all_errors: dict = {} + selectors_seen = [] + for method_settings in service_method_settings: + # Check if this selector is defind more than once + if method_settings.selector in selectors_seen: + all_errors[method_settings.selector] = ["Duplicate selector"] + continue + selectors_seen.append(method_settings.selector) + + method_descriptor = self.all_methods.get(method_settings.selector) + # Check if this selector can be mapped to a method in the API. + if not method_descriptor: + all_errors[method_settings.selector] = [ + "Method was not found." + ] + continue + + if method_settings.auto_populated_fields: + # Check if the selector maps to a streaming method + if ( + method_descriptor.client_streaming + or method_descriptor.server_streaming + ): + all_errors[method_settings.selector] = [ + "Method is not a unary method." + ] + continue + top_level_request_message = self.messages[ + method_descriptor.input_type.lstrip(".") + ] + selector_errors = [] + for field_str in method_settings.auto_populated_fields: + if field_str not in top_level_request_message.fields: + selector_errors.append( + f"Field `{field_str}` was not found" + ) + else: + field = top_level_request_message.fields[field_str] + if field.type != wrappers.PrimitiveType.build(str): + selector_errors.append( + f"Field `{field_str}` is not of type string." + ) + if field.required: + selector_errors.append( + f"Field `{field_str}` is a required field." + ) + if not field.uuid4: + selector_errors.append( + f"Field `{field_str}` is not annotated with " + "`google.api.field_info.format = \"UUID4\"." + ) + if selector_errors: + all_errors[method_settings.selector] = selector_errors + if all_errors: + raise MethodSettingsError(yaml.dump(all_errors)) + + @cached_property + def all_method_settings(self) -> Mapping[str, Sequence[client_pb2.MethodSettings]]: + """Return a map of all `google.api.client.MethodSettings` to be used + when generating methods. + https://github.com/googleapis/googleapis/blob/7dab3de7ec79098bb367b6b2ac3815512a49dd56/google/api/client.proto#L325 + + Return: + Mapping[str, Sequence[client_pb2.MethodSettings]]: A mapping of all method + settings read from the service YAML. + + Raises: + gapic.schema.api.MethodSettingsError: if the method settings do not + meet the requirements of https://google.aip.dev/client-libraries/4235. + """ + self.enforce_valid_method_settings( + self.service_yaml_config.publishing.method_settings + ) + + return { + method_setting.selector: client_pb2.MethodSettings( + selector=method_setting.selector, + long_running=method_setting.long_running, + auto_populated_fields=method_setting.auto_populated_fields, + ) + for method_setting in self.service_yaml_config.publishing.method_settings + } + @cached_property def has_location_mixin(self) -> bool: return len(list(filter(lambda api: api.name == "google.cloud.location.Locations", self.service_yaml_config.apis))) > 0 diff --git a/tests/unit/schema/test_api.py b/tests/unit/schema/test_api.py index e493e68d0c..b242fd9e81 100644 --- a/tests/unit/schema/test_api.py +++ b/tests/unit/schema/test_api.py @@ -13,13 +13,17 @@ # limitations under the License. import collections - +import re +from typing import Sequence from unittest import mock +import yaml import pytest from google.api import annotations_pb2 # type: ignore from google.api import client_pb2 +from google.api import field_behavior_pb2 +from google.api import field_info_pb2 from google.api import resource_pb2 from google.api_core import exceptions from google.cloud import extended_operations_pb2 as ex_ops_pb2 @@ -39,10 +43,10 @@ from test_utils.test_utils import ( make_enum_pb2, + make_field, make_field_pb2, make_file_pb2, make_message_pb2, - make_method, make_naming, make_oneof_pb2, ) @@ -2602,3 +2606,436 @@ def test_has_iam_mixin(): }) api_schema = api.API.build(fd, 'google.example.v1', opts=opts) assert api_schema.has_iam_mixin + + +def get_file_descriptor_proto_for_method_settings_tests( + fields: Sequence[descriptor_pb2.FieldDescriptorProto] = None, + client_streaming: bool = False, + server_streaming: bool = False, +) -> descriptor_pb2.FileDescriptorProto: + """ + Args: + fields (Sequence[descriptor_pb2.FieldDescriptorProto]): Fields to include + in messages in the return object `descriptor_pb2.FileDescriptorProto`. + client_streaming (bool): Whether the methods in the return object + `descriptor_pb2.FileDescriptorProto` should use client streaming. + server_streaming (bool): Whether the methods in the return object + `descriptor_pb2.FileDescriptorProto` should use server streaming. + Return: + descriptor_pb2.FileDescriptorProto: Returns an object describing the API. + """ + + field_options = descriptor_pb2.FieldOptions() + field_options.Extensions[ + field_info_pb2.field_info + ].format = field_info_pb2.FieldInfo.Format.Value("UUID4") + + fd = ( + make_file_pb2( + name="someexample.proto", + package="google.example.v1beta1", + messages=( + make_message_pb2(name="ExampleRequest", fields=fields), + make_message_pb2(name="ExampleResponse", fields=()), + make_message_pb2( + name='NestedMessage', + fields=( + make_field_pb2( + name="squid", + options=field_options, + type="TYPE_STRING", + number=1 + ), + ), + options=descriptor_pb2.MessageOptions(map_entry=True), + ) + ), + services=( + descriptor_pb2.ServiceDescriptorProto( + name="ServiceOne", + method=( + descriptor_pb2.MethodDescriptorProto( + name="Example1", + input_type="google.example.v1beta1.ExampleRequest", + output_type="google.example.v1beta1.ExampleResponse", + client_streaming=client_streaming, + server_streaming=server_streaming, + ), + ), + ), + descriptor_pb2.ServiceDescriptorProto( + name="ServiceTwo", + method=( + descriptor_pb2.MethodDescriptorProto( + name="Example1", + input_type="google.example.v1beta1.ExampleRequest", + output_type="google.example.v1beta1.ExampleResponse", + client_streaming=client_streaming, + server_streaming=server_streaming, + ), + ), + ), + ), + ), + ) + return fd + + +def test_api_all_methods(): + """ + Tests the `all_methods` method of `gapic.schema.api` method which returns a map of + all methods for the API. + """ + fd = get_file_descriptor_proto_for_method_settings_tests() + api_schema = api.API.build(fd, "google.example.v1beta1") + assert len(api_schema.all_methods) == 2 + assert list(api_schema.all_methods.keys()) == [ + "google.example.v1beta1.ServiceOne.Example1", + "google.example.v1beta1.ServiceTwo.Example1", + ] + + +def test_read_method_settings_from_service_yaml(): + """ + Tests the `gapic.schema.api.all_method_settings` method which reads + `MethodSettings` from the service config YAML. + https://github.com/googleapis/googleapis/blob/7dab3de7ec79098bb367b6b2ac3815512a49dd56/google/api/client.proto#L325 + """ + service_yaml_config = { + "apis": [ + {"name": "google.example.v1beta1.ServiceOne.Example1"}, + ], + "publishing": { + "method_settings": [ + { + "selector": "google.example.v1beta1.ServiceOne.Example1", + "auto_populated_fields": [ + "squid", + "mollusc", + ], + }, + ] + }, + } + cli_options = Options(service_yaml_config=service_yaml_config) + field_options = descriptor_pb2.FieldOptions() + field_options.Extensions[ + field_info_pb2.field_info + ].format = field_info_pb2.FieldInfo.Format.Value("UUID4") + + squid = make_field_pb2( + name="squid", type="TYPE_STRING", options=field_options, number=1 + ) + mollusc = make_field_pb2( + name="mollusc", type="TYPE_STRING", options=field_options, number=2 + ) + fields = [squid, mollusc] + fd = get_file_descriptor_proto_for_method_settings_tests(fields=fields) + api_schema = api.API.build(fd, "google.example.v1beta1", opts=cli_options) + assert api_schema.all_method_settings == { + "google.example.v1beta1.ServiceOne.Example1": client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["squid", "mollusc"], + long_running=client_pb2.MethodSettings.LongRunning(), + ) + } + + +def test_method_settings_duplicate_selector_raises_error(): + """ + Test that `MethodSettingsError` is raised when there are duplicate selectors in + `client_pb2.MethodSettings`. + """ + fd = get_file_descriptor_proto_for_method_settings_tests() + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + ), + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + ), + ] + with pytest.raises( + api.MethodSettingsError, match="(?i)duplicate selector" + ): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_invalid_selector_raises_error(): + """ + Test that `MethodSettingsError` when `client_pb2.MethodSettings.selector` + cannot be mapped to a method in the API. + """ + method_example1 = "google.example.v1beta1.DoesNotExist.Example1" + method_example2 = "google.example.v1beta1.ServiceOne.DoesNotExist" + + fd = get_file_descriptor_proto_for_method_settings_tests() + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector=method_example1, + ), + client_pb2.MethodSettings( + selector=method_example2, + ), + ] + + with pytest.raises(api.MethodSettingsError) as ex: + api_schema.enforce_valid_method_settings(methodsettings) + + error_yaml = yaml.safe_load(ex.value.args[0]) + + assert re.match( + ".*not found.*", + error_yaml[method_example1][0].lower() + ) + assert re.match( + ".*not found.*", + error_yaml[method_example2][0].lower() + ) + + +def test_method_settings_unsupported_auto_populated_field_type_raises_error(): + """ + Test that `MethodSettingsError` is raised when a field in + `client_pb2.MethodSettings.auto_populated_fields` is not of type string. + """ + squid = make_field_pb2(name="squid", type="TYPE_INT32", number=1) + fd = get_file_descriptor_proto_for_method_settings_tests(fields=[squid]) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises(api.MethodSettingsError, match="(?i)type string"): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_auto_populated_field_not_found_raises_error(): + """ + Test that `MethodSettingsError` is raised when a field in + `client_pb2.MethodSettings.auto_populated_fields` is not found in the top-level + request message of the selector. + """ + fd = get_file_descriptor_proto_for_method_settings_tests() + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["whelk"], + ), + ] + with pytest.raises(api.MethodSettingsError, match="(?i)not found"): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_auto_populated_nested_field_raises_error(): + """ + Test that `MethodSettingsError` is raised when a field in + `client_pb2.MethodSettings.auto_populated_fields` is not found in the top-level + request message of the selector. Instead, the field exists in a nested message. + """ + + octopus = make_field( + name='octopus', + type_name='google.example.v1beta1.NestedMessage', + label=3, + type='TYPE_MESSAGE', + ) + + fd = get_file_descriptor_proto_for_method_settings_tests( + fields=[octopus.field_pb] + ) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises(api.MethodSettingsError, match="(?i)not found"): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_auto_populated_field_client_streaming_rpc_raises_error(): + """ + Test that `MethodSettingsError` is raised when the selector in + `client_pb2.MethodSettings.selector` maps to a method which uses client streaming. + """ + fd = get_file_descriptor_proto_for_method_settings_tests( + client_streaming=True + ) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises( + api.MethodSettingsError, match="(?i)not a unary method" + ): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_auto_populated_field_server_streaming_rpc_raises_error(): + """ + Test that `MethodSettingsError` is raised when the selector in + `client_pb2.MethodSettings.selector` maps to a method which uses server streaming. + """ + fd = get_file_descriptor_proto_for_method_settings_tests( + server_streaming=True + ) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises( + api.MethodSettingsError, match="(?i)not a unary method" + ): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_unsupported_auto_populated_field_behavior_raises_error(): + """ + Test that `MethodSettingsError` is raised when a field in + `client_pb2.MethodSettings.auto_populated_fields` is a required field. + """ + field_options = descriptor_pb2.FieldOptions() + field_options.Extensions[field_behavior_pb2.field_behavior].append( + field_behavior_pb2.FieldBehavior.Value("REQUIRED") + ) + squid = make_field_pb2( + name="squid", type="TYPE_STRING", options=field_options, number=1 + ) + fd = get_file_descriptor_proto_for_method_settings_tests(fields=[squid]) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.ServiceOne.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises( + api.MethodSettingsError, match="(?i)required field" + ): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_auto_populated_field_field_info_format_not_specified_raises_error(): + """ + Test that `MethodSettingsError` is raised when a field in + `client_pb2.MethodSettings.auto_populated_fields` is not annotated with + `google.api.field_info.format = UUID4`. For this test case, + the format of the field is not specified. + """ + squid = make_field_pb2(name="squid", type="TYPE_STRING", number=1) + fd = get_file_descriptor_proto_for_method_settings_tests(fields=[squid]) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.SomeExample.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises(api.MethodSettingsError): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_unsupported_auto_populated_field_field_info_format_raises_error(): + """ + Test that `MethodSettingsError` is raised when a field in + `client_pb2.MethodSettings.auto_populated_fields` is not annotated with + `google.api.field_info.format = UUID4`.For this test case, + the format of the field is `IPV4`. + """ + field_options = descriptor_pb2.FieldOptions() + field_options.Extensions[ + field_info_pb2.field_info + ].format = field_info_pb2.FieldInfo.Format.Value("IPV4") + squid = make_field_pb2( + name="squid", type="TYPE_STRING", options=field_options, number=1 + ) + fd = get_file_descriptor_proto_for_method_settings_tests(fields=[squid]) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector="google.example.v1beta1.SomeExample.Example1", + auto_populated_fields=["squid"], + ), + ] + with pytest.raises(api.MethodSettingsError): + api_schema.enforce_valid_method_settings(methodsettings) + + +def test_method_settings_invalid_multiple_issues(): + """ + A kitchen sink type of test to ensure `MethodSettingsError` is raised and the contents + of the exception includes sufficient detail describing each issue. + """ + method_example1 = "google.example.v1beta1.ServiceTwo.Example1" + method_example2 = "google.example.v1beta1.ServiceThree.Example2" + field_options = descriptor_pb2.FieldOptions() + + # Field Squid Errors + # - Not annotated with google.api.field_info.format = UUID4 + # - Not of type string + # - Required field + field_options.Extensions[ + field_info_pb2.field_info + ].format = field_info_pb2.FieldInfo.Format.Value("IPV4") + squid = make_field_pb2( + name="squid", type="TYPE_INT32", options=field_options, number=1 + ) + field_options = descriptor_pb2.FieldOptions() + field_options.Extensions[field_behavior_pb2.field_behavior].append( + field_behavior_pb2.FieldBehavior.Value("REQUIRED") + ) + + # Field Octopus Errors + # - Not annotated with google.api.field_info.format = UUID4 + octopus = make_field_pb2(name="octopus", type="TYPE_STRING", number=1) + fd = get_file_descriptor_proto_for_method_settings_tests( + fields=[squid, octopus] + ) + api_schema = api.API.build(fd, "google.example.v1beta1") + methodsettings = [ + client_pb2.MethodSettings( + selector=method_example1, + auto_populated_fields=[ + "squid", + "octopus", + ], + ), + client_pb2.MethodSettings( + selector=method_example2, + auto_populated_fields=["squid", "octopus"], + ), + ] + with pytest.raises(api.MethodSettingsError) as ex: + api_schema.enforce_valid_method_settings(methodsettings) + + error_yaml = yaml.safe_load(ex.value.args[0]) + + assert re.match( + ".*squid.*not.*string.*", + error_yaml[method_example1][0].lower() + ) + assert re.match( + ".*squid.*not.*uuid4.*", + error_yaml[method_example1][1].lower() + ) + assert re.match( + ".*octopus.*not.*uuid4.*", + error_yaml[method_example1][2].lower() + ) + assert re.match( + ".*method.*not found.*", + error_yaml[method_example2][0].lower() + )