From e62825fa24480c1ec89ba88ffb4846eb0e71e596 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 15 Jul 2019 15:23:21 -0700 Subject: [PATCH 01/11] Shared connection (sync) draft --- .../azure-eventhubs/azure/eventhub/client.py | 28 +- .../azure/eventhub/connection_manager.py | 50 +++ .../azure/eventhub/consumer.py | 341 +++++------------- .../azure/eventhub/producer.py | 275 ++++++-------- 4 files changed, 286 insertions(+), 408 deletions(-) create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 308aa2000a6d..9770f30dc114 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -27,6 +27,7 @@ from azure.eventhub.error import ConnectError from .client_abstract import EventHubClientAbstract from .common import EventHubSASTokenCredential, EventHubSharedKeyCredential +from .connection_manager import _ConnectionManager log = logging.getLogger(__name__) @@ -47,6 +48,23 @@ class EventHubClient(EventHubClientAbstract): """ + def __init__(self, host, event_hub_path, credential, **kwargs): + super(EventHubClient, self).__init__(host, event_hub_path, credential, **kwargs) + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password": self._auth_config.get("iot_password") + } + self._conn_manager = _ConnectionManager() + + def __del__(self): + self._conn_manager.close_connection() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + def _create_auth(self, username=None, password=None): """ Create an ~uamqp.authentication.SASTokenAuth instance to authenticate @@ -100,15 +118,18 @@ def _management_request(self, mgmt_msg, op_type): while True: connect_count += 1 mgmt_auth = self._create_auth(**alt_creds) - mgmt_client = uamqp.AMQPClient(self.mgmt_target, auth=mgmt_auth, debug=self.config.network_tracing) + mgmt_client = uamqp.AMQPClient(self.mgmt_target) try: - mgmt_client.open() + conn = self._conn_manager.get_connection() + mgmt_client.open(connection=self._conn_manager.get_connection()) + print(conn._state) response = mgmt_client.mgmt_request( mgmt_msg, constants.READ_OPERATION, op_type=op_type, status_code_field=b'status-code', description_fields=b'status-description') + print(conn._state) return response except (errors.AMQPConnectionError, errors.TokenAuthFailure, compat.TimeoutException) as failure: if connect_count >= self.config.max_retries: @@ -268,3 +289,6 @@ def create_producer(self, partition_id=None, operation=None, send_timeout=None): handler = EventHubProducer( self, target, partition=partition_id, send_timeout=send_timeout) return handler + + def close(self): + self._conn_manager.close_connection() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py new file mode 100644 index 000000000000..026348c3a215 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py @@ -0,0 +1,50 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import threading +from uamqp import Connection, TransportType + + +class _ConnectionManager(object): + def __init__(self, **kwargs): + self._lock = threading.Lock() + self._conn = None + + self._container_id = kwargs.get("container_id") + self._debug = kwargs.get("debug") + self._error_policy = kwargs.get("error_policy") + self._properties = kwargs.get("properties") + self._encoding = kwargs.get("encoding") or "UTF-8" + self._transport_type = kwargs.get('transport_type') or TransportType.Amqp + self._http_proxy = kwargs.get('http_proxy') + self._max_frame_size = kwargs.get("max_frame_size") + self._channel_max = kwargs.get("channel_max") + self._idle_timeout = kwargs.get("idle_timeout") + self._remote_idle_timeout_empty_frame_send_ratio = kwargs.get("remote_idle_timeout_empty_frame_send_ratio") + + def get_connection(self, host, auth): + # type: (...) -> Connection + with self._lock: + if self._conn is None: + self._conn = Connection( + host, + auth, + container_id=self._container_id, + max_frame_size=self._max_frame_size, + channel_max=self._channel_max, + idle_timeout=self._idle_timeout, + properties=self._properties, + remote_idle_timeout_empty_frame_send_ratio=self._remote_idle_timeout_empty_frame_send_ratio, + error_policy=self._error_policy, + debug=self._debug, + encoding=self._encoding) + return self._conn + + def close_connection(self): + with self._lock: + if self._conn: + self._conn.destroy() + self._conn = None + diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index 856c77d6fb65..2909d0fe5f0c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -75,17 +75,7 @@ def __init__(self, client, source, event_position=None, prefetch=300, owner_leve source.set_filter(self.offset._selector()) # pylint: disable=protected-access if owner_level: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} - self._handler = ReceiveClient( - source, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access + self._handler = None def __enter__(self): return self @@ -97,84 +87,52 @@ def __iter__(self): return self def __next__(self): - self._open() max_retries = self.client.config.max_retries - connecting_count = 0 + retry_count = 0 while True: - connecting_count += 1 try: + self._open() if not self.messages_iter: self.messages_iter = self._handler.receive_messages_iter() message = next(self.messages_iter) event_data = EventData(message=message) self.offset = EventPosition(event_data.offset, inclusive=False) return event_data - except errors.AuthenticationException as auth_error: - if connecting_count < max_retries: - log.info("EventHubConsumer disconnected due to token error. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer authentication failed. Shutting down.") - error = AuthenticationError(str(auth_error), auth_error) - self.close(auth_error) - raise error - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("EventHubConsumer detached. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer detached. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(error) - raise error - except errors.AMQPConnectionError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer connection lost. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer connection lost. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(error) - raise error - except compat.TimeoutException as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer timed out receiving event data. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer timed out. Shutting down.") - self.close(shutdown) - raise ConnectionLostError(str(shutdown), shutdown) - except StopIteration: - raise - except KeyboardInterrupt: - log.info("EventHubConsumer stops due to keyboard interrupt") - self.close() - raise - except Exception as e: - log.error("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receive failed: {}".format(e), e) - self.close(exception=error) - raise error + except Exception as exception: + self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 def _check_closed(self): if self.error: raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.", self.error) + def _create_handler(self): + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset._selector()) + self._handler = ReceiveClient( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.config.network_tracing, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties( + self.client.config.user_agent)) # pylint: disable=protected-access + self.messages_iter = None + def _redirect(self, redirect): self.redirected = redirect self.running = False self.messages_iter = None - self._open() + self._close_connection() def _open(self): """ @@ -184,129 +142,79 @@ def _open(self): """ # pylint: disable=protected-access - self._check_closed() - if self.redirected: - self.client._process_redirect_uri(self.redirected) - self.source = self.redirected.address - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) - - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} - self._handler = ReceiveClient( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access if not self.running: - self._connect() + if self.redirected: + self.client._process_redirect_uri(self.redirected) + self.source = self.redirected.address + self._create_handler() + self._handler.open(connection=self.client._conn_manager.get_connection( + self.client.address.hostname, + self.client.get_auth() + )) + while not self._handler.client_ready(): + time.sleep(0.05) self.running = True - def _connect(self): - connected = self._build_connection() - if not connected: - time.sleep(self.reconnect_backoff) - while not self._build_connection(is_reconnect=True): - time.sleep(self.reconnect_backoff) - - def _build_connection(self, is_reconnect=False): - """ + def _close_handler(self): + self._handler.close() # close the link (sharing connection) or connection (not sharing) + self.running = False - :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. - False - the 1st time to connect - :return: True - connected. False - not connected - """ - # pylint: disable=protected-access - if is_reconnect: - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} - self._handler.close() - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) - self._handler = ReceiveClient( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties( - self.client.config.user_agent)) # pylint: disable=protected-access - self.messages_iter = None - try: - self._handler.open() - while not self._handler.client_ready(): - time.sleep(0.05) - return True - except errors.AuthenticationException as shutdown: - if is_reconnect: - log.info("EventHubConsumer couldn't authenticate. Shutting down. (%r)", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error - else: - log.info("EventHubConsumer couldn't authenticate. Attempting reconnect.") - return False - except errors.LinkRedirect as redirect: - self._redirect(redirect) - return True - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: - log.info("EventHubConsumer detached. Attempting reconnect.") - return False - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if is_reconnect: + def _close_connection(self): + self._close_handler() + self.client._conn_manager.close_connection() # close the shared connection. + + def _handle_exception(self, exception, retry_count, max_retries): + if isinstance(exception, KeyboardInterrupt): + log.info("EventHubConsumer stops due to keyboard interrupt") + self.close() + raise + elif retry_count >= max_retries: + log.info("EventHubConsumer has an error and has exhausted retrying. (%r)", exception) + if isinstance(exception, errors.AuthenticationException): + log.info("EventHubConsumer authentication failed. Shutting down.") + error = AuthenticationError(str(exception), exception) + elif isinstance(exception, errors.LinkDetach): + log.info("EventHubConsumer link detached. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.ConnectionClose): + log.info("EventHubConsumer connection closed. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.MessageHandlerError): log.info("EventHubConsumer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.AMQPConnectionError): + log.info("EventHubConsumer connection lost. Shutting down.") + error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ + else ConnectError + error = error_type(str(exception), exception) + elif isinstance(exception, compat.TimeoutException): + log.info("EventHubConsumer timed out. Shutting down.") + error = ConnectionLostError(str(exception), exception) else: - log.info("EventHubConsumer detached. Attempting reconnect.") - return False - except errors.AMQPConnectionError as shutdown: - if is_reconnect: - log.info("EventHubConsumer connection error (%r). Shutting down.", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error - else: - log.info("EventHubConsumer couldn't authenticate. Attempting reconnect.") - return False - except compat.TimeoutException as shutdown: - if is_reconnect: - log.info("EventHubConsumer authentication timed out. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error - else: - log.info("EventHubConsumer authentication timed out. Attempting reconnect.") - return False - except Exception as e: - log.error("Unexpected error occurred when building connection (%r). Shutting down.", e) - error = EventHubError("Unexpected error occurred when building connection", e) + log.error("Unexpected error occurred (%r). Shutting down.", exception) + error = EventHubError("Receive failed: {}".format(exception), exception) self.close(exception=error) raise error - - def _reconnect(self): - return self._build_connection(is_reconnect=True) + else: + log.info("EventHubConsumer has an exception (%r). Retrying...", exception) + if isinstance(exception, errors.AuthenticationException): + self._close_connection() + elif isinstance(exception, errors.LinkRedirect): + log.info("EventHubConsumer link redirected. Redirecting...") + redirect = exception + self._redirect(redirect) + elif isinstance(exception, errors.LinkDetach): + self._close_handler() + elif isinstance(exception, errors.ConnectionClose): + self._close_connection() + elif isinstance(exception, errors.MessageHandlerError): + self._close_handler() + elif isinstance(exception, errors.AMQPConnectionError): + self._close_connection() + elif isinstance(exception, compat.TimeoutException): + pass # Timeout doesn't need to recreate link or exception + else: + self._close_connection() @property def queue_size(self): @@ -348,17 +256,16 @@ def receive(self, max_batch_size=None, timeout=None): """ self._check_closed() - self._open() max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size timeout = self.client.config.receive_timeout if timeout is None else timeout data_batch = [] # type: List[EventData] max_retries = self.client.config.max_retries - connecting_count = 0 + retry_count = 0 while True: - connecting_count += 1 try: + self._open() timeout_ms = 1000 * timeout if timeout else 0 message_batch = self._handler.receive_message_batch( max_batch_size=max_batch_size - (len(data_batch) if data_batch else 0), @@ -368,59 +275,9 @@ def receive(self, max_batch_size=None, timeout=None): self.offset = EventPosition(event_data.offset) data_batch.append(event_data) return data_batch - except errors.AuthenticationException as auth_error: - if connecting_count < max_retries: - log.info("EventHubConsumer disconnected due to token error. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer authentication failed. Shutting down.") - error = AuthenticationError(str(auth_error), auth_error) - self.close(auth_error) - raise error - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("EventHubConsumer detached. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer detached. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(error) - raise error - except errors.AMQPConnectionError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer connection lost. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer connection lost. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(error) - raise error - except compat.TimeoutException as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer timed out receiving event data. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubConsumer timed out. Shutting down.") - self.close(shutdown) - raise ConnectionLostError(str(shutdown), shutdown) - except KeyboardInterrupt: - log.info("EventHubConsumer stops due to keyboard interrupt") - self.close() - raise - except Exception as e: - log.error("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receive failed: {}".format(e), e) - self.close(exception=error) - raise error + except Exception as exception: + self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 def close(self, exception=None): # type:(Exception) -> None @@ -456,6 +313,6 @@ def close(self, exception=None): self.error = EventHubError(str(exception)) else: self.error = EventHubError("This receive handler is now closed.") - self._handler.close() + self._handler.close() # this will close link if sharing connection. Otherwise close connection next = __next__ # for python2.7 diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index 3f95b7be08c3..dcfec2b22ee6 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -7,7 +7,7 @@ import uuid import logging import time -from typing import Iterator, Generator, List, Union +from typing import Iterable, Union from uamqp import constants, errors from uamqp import compat @@ -85,6 +85,23 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close(exc_val) + def _create_handler(self): + self._handler = SendClient( + self.target, + auth=self.client.get_auth(), + debug=self.client.config.network_tracing, + msg_timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access + + def _redirect(self, redirect): + self.redirected = redirect + self.running = False + self.messages_iter = None + self._close_connection() + def _open(self): """ Open the EventHubProducer using the supplied connection. @@ -93,182 +110,112 @@ def _open(self): """ # pylint: disable=protected-access - self._check_closed() - if self.redirected: - self.target = self.redirected.address - self._handler = SendClient( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent)) if not self.running: - self._connect() + if self.redirected: + self.target = self.redirected.address + self._create_handler() + self._handler.open(connection=self.client._conn_manager.get_connection( + self.client.address.hostname, + self.client.get_auth() + )) + while not self._handler.client_ready(): + time.sleep(0.05) self.running = True - def _connect(self): - connected = self._build_connection() - if not connected: - time.sleep(self.reconnect_backoff) - while not self._build_connection(is_reconnect=True): - time.sleep(self.reconnect_backoff) - - def _build_connection(self, is_reconnect=False): - """ + def _close_handler(self): + self._handler.close() # close the link (sharing connection) or connection (not sharing) + self.running = False - :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. - False - the 1st time to connect - :return: True - connected. False - not connected - """ - # pylint: disable=protected-access - if is_reconnect: - self._handler.close() - self._handler = SendClient( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent)) - try: - self._handler.open() - while not self._handler.client_ready(): - time.sleep(0.05) - return True - except errors.AuthenticationException as shutdown: - if is_reconnect: - log.info("EventHubProducer couldn't authenticate. Shutting down. (%r)", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error - else: - log.info("EventHubProducer couldn't authenticate. Attempting reconnect.") - return False - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: - log.info("EventHubProducer detached. Attempting reconnect.") - return False - else: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if is_reconnect: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error - else: - log.info("EventHubProducer detached. Attempting reconnect.") - return False - except errors.AMQPConnectionError as shutdown: - if is_reconnect: - log.info("EventHubProducer connection error (%r). Shutting down.", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error - else: - log.info("EventHubProducer couldn't authenticate. Attempting reconnect.") - return False - except compat.TimeoutException as shutdown: - if is_reconnect: - log.info("EventHubProducer authentication timed out. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error + def _close_connection(self): + self._close_handler() + self.client._conn_manager.close_connection() # close the shared connection. + + def _handle_exception(self, exception, retry_count, max_retries): + if isinstance(exception, KeyboardInterrupt): + log.info("EventHubConsumer stops due to keyboard interrupt") + self.close() + raise + elif isinstance(exception, ( + errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) + ): + log.error("Event data error (%r)", exception) + error = EventDataError(str(exception), exception) + self.close(exception) + raise error + elif isinstance(exception, errors.MessageException): + log.error("Event data send error (%r)", exception) + error = EventDataSendError(str(exception), exception) + self.close(exception) + raise error + elif retry_count >= max_retries: + log.info("EventHubConsumer has an error and has exhausted retrying. (%r)", exception) + if isinstance(exception, errors.AuthenticationException): + log.info("EventHubConsumer authentication failed. Shutting down.") + error = AuthenticationError(str(exception), exception) + elif isinstance(exception, errors.LinkDetach): + log.info("EventHubConsumer link detached. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.ConnectionClose): + log.info("EventHubConsumer connection closed. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.MessageHandlerError): + log.info("EventHubConsumer detached. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.AMQPConnectionError): + log.info("EventHubConsumer connection lost. Shutting down.") + error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ + else ConnectError + error = error_type(str(exception), exception) + elif isinstance(exception, compat.TimeoutException): + log.info("EventHubConsumer timed out. Shutting down.") + error = ConnectionLostError(str(exception), exception) else: - log.info("EventHubProducer authentication timed out. Attempting reconnect.") - return False - except Exception as e: - log.info("Unexpected error occurred when building connection (%r). Shutting down.", e) - error = EventHubError("Unexpected error occurred when building connection", e) + log.error("Unexpected error occurred (%r). Shutting down.", exception) + error = EventHubError("Receive failed: {}".format(exception), exception) self.close(exception=error) raise error - - def _reconnect(self): - return self._build_connection(is_reconnect=True) + else: + log.info("EventHubConsumer has an exception (%r). Retrying...", exception) + if isinstance(exception, errors.AuthenticationException): + self._close_connection() + elif isinstance(exception, errors.LinkRedirect): + log.info("EventHubConsumer link redirected. Redirecting...") + redirect = exception + self._redirect(redirect) + elif isinstance(exception, errors.LinkDetach): + self._close_handler() + elif isinstance(exception, errors.ConnectionClose): + self._close_connection() + elif isinstance(exception, errors.MessageHandlerError): + self._close_handler() + elif isinstance(exception, errors.AMQPConnectionError): + self._close_connection() + elif isinstance(exception, compat.TimeoutException): + pass # Timeout doesn't need to recreate link or exception + else: + self._close_connection() def _send_event_data(self): self._open() max_retries = self.client.config.max_retries - connecting_count = 0 + retry_count = 0 while True: - connecting_count += 1 try: if self.unsent_events: self._handler.queue_message(*self.unsent_events) self._handler.wait() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: - EventHubProducer._error(self._outcome, self._condition) + _error(self._outcome, self._condition) return - except (errors.MessageAccepted, - errors.MessageAlreadySettled, - errors.MessageModified, - errors.MessageRejected, - errors.MessageReleased, - errors.MessageContentTooLarge) as msg_error: - raise EventDataError(str(msg_error), msg_error) - except errors.MessageException as failed: - log.error("Send event data error (%r)", failed) - error = EventDataSendError(str(failed), failed) - self.close(exception=error) - raise error - except errors.AuthenticationException as auth_error: - if connecting_count < max_retries: - log.info("EventHubProducer disconnected due to token error. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubProducer authentication failed. Shutting down.") - error = AuthenticationError(str(auth_error), auth_error) - self.close(auth_error) - raise error - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: - log.info("EventHubProducer detached. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if connecting_count < max_retries: - log.info("EventHubProducer detached. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(error) - raise error - except errors.AMQPConnectionError as shutdown: - if connecting_count < max_retries: - log.info("EventHubProducer connection lost. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubProducer connection lost. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - self.close(error) - raise error - except compat.TimeoutException as shutdown: - if connecting_count < max_retries: - log.info("EventHubProducer timed out sending event data. Attempting reconnect.") - self._reconnect() - else: - log.info("EventHubProducer timed out. Shutting down.") - self.close(shutdown) - raise ConnectionLostError(str(shutdown), shutdown) - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Send failed: {}".format(e), e) - self.close(exception=error) - raise error + except Exception as exception: + self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 def _check_closed(self): if self.error: @@ -293,13 +240,8 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - @staticmethod - def _error(outcome, condition): - if outcome != constants.MessageSendResult.Ok: - raise condition - def send(self, event_data, partition_key=None): - # type:(Union[EventData, Union[List[EventData], Iterator[EventData], Generator[EventData]]], Union[str, bytes]) -> None + # type:(Union[EventData, Iterable[EventData]], Union[str, bytes]) -> None """ Sends an event data and blocks until acknowledgement is received or operation times out. @@ -370,3 +312,8 @@ def close(self, exception=None): else: self.error = EventHubError("This send handler is now closed.") self._handler.close() + + +def _error(outcome, condition): + if outcome != constants.MessageSendResult.Ok: + raise condition From 84ae397327a0f0cbb9b56fe7dd2e4a61555eb96e Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 16 Jul 2019 11:52:43 -0700 Subject: [PATCH 02/11] Shared connection (sync) draft 2 --- ...tion_manager.py => _connection_manager.py} | 18 ++++- .../azure-eventhubs/azure/eventhub/client.py | 29 ++++--- .../azure/eventhub/consumer.py | 65 +++------------- .../azure-eventhubs/azure/eventhub/error.py | 76 ++++++++++++++++++- .../azure/eventhub/producer.py | 73 +----------------- 5 files changed, 117 insertions(+), 144 deletions(-) rename sdk/eventhub/azure-eventhubs/azure/eventhub/{connection_manager.py => _connection_manager.py} (86%) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py similarity index 86% rename from sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py rename to sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py index 026348c3a215..3e1a17a1051d 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/connection_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py @@ -7,7 +7,7 @@ from uamqp import Connection, TransportType -class _ConnectionManager(object): +class _SharedConnectionManager(object): def __init__(self, **kwargs): self._lock = threading.Lock() self._conn = None @@ -42,9 +42,23 @@ def get_connection(self, host, auth): encoding=self._encoding) return self._conn - def close_connection(self): + def close_connection(self, conn=None): with self._lock: if self._conn: self._conn.destroy() self._conn = None + +class _SeparateConnectionManager(object): + def __init__(self, **kwargs): + pass + + def get_connection(self, host, auth): + return None + + def close_connection(self): + pass + + +def get_connection_manager(**kwargs): + return _SeparateConnectionManager(**kwargs) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 9770f30dc114..a88beebdc275 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -24,10 +24,10 @@ from azure.eventhub.producer import EventHubProducer from azure.eventhub.consumer import EventHubConsumer from azure.eventhub.common import parse_sas_token, EventPosition -from azure.eventhub.error import ConnectError +from azure.eventhub.error import ConnectError, EventHubError from .client_abstract import EventHubClientAbstract from .common import EventHubSASTokenCredential, EventHubSharedKeyCredential -from .connection_manager import _ConnectionManager +from ._connection_manager import get_connection_manager log = logging.getLogger(__name__) @@ -54,7 +54,7 @@ def __init__(self, host, event_hub_path, credential, **kwargs): "username": self._auth_config.get("iot_username"), "password": self._auth_config.get("iot_password") } - self._conn_manager = _ConnectionManager() + self._conn_manager = get_connection_manager(**kwargs) def __del__(self): self._conn_manager.close_connection() @@ -111,28 +111,23 @@ def _create_auth(self, username=None, password=None): transport_type=transport_type) def _management_request(self, mgmt_msg, op_type): - alt_creds = { - "username": self._auth_config.get("iot_username"), - "password": self._auth_config.get("iot_password")} - connect_count = 0 - while True: - connect_count += 1 - mgmt_auth = self._create_auth(**alt_creds) + retry_count = 0 + while retry_count <= self.config.max_retries: + retry_count += 1 + mgmt_auth = self._create_auth() mgmt_client = uamqp.AMQPClient(self.mgmt_target) try: - conn = self._conn_manager.get_connection() - mgmt_client.open(connection=self._conn_manager.get_connection()) - print(conn._state) + conn = self._conn_manager.get_connection(self.host, mgmt_auth) + mgmt_client.open(connection=conn) response = mgmt_client.mgmt_request( mgmt_msg, constants.READ_OPERATION, op_type=op_type, status_code_field=b'status-code', description_fields=b'status-description') - print(conn._state) return response except (errors.AMQPConnectionError, errors.TokenAuthFailure, compat.TimeoutException) as failure: - if connect_count >= self.config.max_retries: + if retry_count >= self.config.max_retries: err = ConnectError( "Can not connect to EventHubs or get management info from the service. " "Please make sure the connection string or token is correct and retry. " @@ -140,6 +135,10 @@ def _management_request(self, mgmt_msg, op_type): failure ) raise err + except Exception as failure: + if retry_count >= self.config.max_retries: + err = EventHubError("Unexpected error happened during management request", failure) + raise err finally: mgmt_client.close() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index 2909d0fe5f0c..b8fc6ff4a2ea 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -14,7 +14,8 @@ from uamqp import ReceiveClient, Source from azure.eventhub.common import EventData, EventPosition -from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, \ + _error_handler, _handle_exception log = logging.getLogger(__name__) @@ -70,9 +71,6 @@ def __init__(self, client, source, event_position=None, prefetch=300, owner_leve self.error = None partition = self.source.split('/')[-1] self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) # pylint: disable=protected-access if owner_level: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} self._handler = None @@ -146,10 +144,15 @@ def _open(self): if self.redirected: self.client._process_redirect_uri(self.redirected) self.source = self.redirected.address + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + else: + alt_creds = {} self._create_handler() self._handler.open(connection=self.client._conn_manager.get_connection( self.client.address.hostname, - self.client.get_auth() + self.client.get_auth(**alt_creds) )) while not self._handler.client_ready(): time.sleep(0.05) @@ -164,57 +167,7 @@ def _close_connection(self): self.client._conn_manager.close_connection() # close the shared connection. def _handle_exception(self, exception, retry_count, max_retries): - if isinstance(exception, KeyboardInterrupt): - log.info("EventHubConsumer stops due to keyboard interrupt") - self.close() - raise - elif retry_count >= max_retries: - log.info("EventHubConsumer has an error and has exhausted retrying. (%r)", exception) - if isinstance(exception, errors.AuthenticationException): - log.info("EventHubConsumer authentication failed. Shutting down.") - error = AuthenticationError(str(exception), exception) - elif isinstance(exception, errors.LinkDetach): - log.info("EventHubConsumer link detached. Shutting down.") - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.ConnectionClose): - log.info("EventHubConsumer connection closed. Shutting down.") - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.MessageHandlerError): - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.AMQPConnectionError): - log.info("EventHubConsumer connection lost. Shutting down.") - error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ - else ConnectError - error = error_type(str(exception), exception) - elif isinstance(exception, compat.TimeoutException): - log.info("EventHubConsumer timed out. Shutting down.") - error = ConnectionLostError(str(exception), exception) - else: - log.error("Unexpected error occurred (%r). Shutting down.", exception) - error = EventHubError("Receive failed: {}".format(exception), exception) - self.close(exception=error) - raise error - else: - log.info("EventHubConsumer has an exception (%r). Retrying...", exception) - if isinstance(exception, errors.AuthenticationException): - self._close_connection() - elif isinstance(exception, errors.LinkRedirect): - log.info("EventHubConsumer link redirected. Redirecting...") - redirect = exception - self._redirect(redirect) - elif isinstance(exception, errors.LinkDetach): - self._close_handler() - elif isinstance(exception, errors.ConnectionClose): - self._close_connection() - elif isinstance(exception, errors.MessageHandlerError): - self._close_handler() - elif isinstance(exception, errors.AMQPConnectionError): - self._close_connection() - elif isinstance(exception, compat.TimeoutException): - pass # Timeout doesn't need to recreate link or exception - else: - self._close_connection() + _handle_exception(exception, retry_count, max_retries, self, log) @property def queue_size(self): diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index 6932daa7cc0f..d15dabf0c051 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -4,7 +4,7 @@ # -------------------------------------------------------------------------------------------- import six -from uamqp import constants, errors +from uamqp import constants, errors, compat _NO_RETRY_ERRORS = ( @@ -128,3 +128,77 @@ class EventDataSendError(EventHubError): """ pass + + +def _handle_exception(exception, retry_count, max_retries, closable, log): + if isinstance(exception, KeyboardInterrupt): + log.info("EventHubConsumer stops due to keyboard interrupt") + closable.close() + raise + elif isinstance(exception, ( + errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) + ): + log.error("Event data error (%r)", exception) + error = EventDataError(str(exception), exception) + closable.close(exception) + raise error + elif isinstance(exception, errors.MessageException): + log.error("Event data send error (%r)", exception) + error = EventDataSendError(str(exception), exception) + closable.close(exception) + raise error + elif retry_count >= max_retries: + log.info("EventHubConsumer has an error and has exhausted retrying. (%r)", exception) + if isinstance(exception, errors.AuthenticationException): + log.info("EventHubConsumer authentication failed. Shutting down.") + error = AuthenticationError(str(exception), exception) + elif isinstance(exception, errors.VendorLinkDetach): + log.info("EventHubConsumer link detached. Shutting down.") + error = ConnectError(str(exception), exception) + elif isinstance(exception, errors.LinkDetach): + log.info("EventHubConsumer link detached. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.ConnectionClose): + log.info("EventHubConsumer connection closed. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.MessageHandlerError): + log.info("EventHubConsumer detached. Shutting down.") + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.AMQPConnectionError): + log.info("EventHubConsumer connection lost. Shutting down.") + error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ + else ConnectError + error = error_type(str(exception), exception) + elif isinstance(exception, compat.TimeoutException): + log.info("EventHubConsumer timed out. Shutting down.") + error = ConnectionLostError(str(exception), exception) + else: + log.error("Unexpected error occurred (%r). Shutting down.", exception) + error = EventHubError("Receive failed: {}".format(exception), exception) + closable.close(exception=error) + raise error + else: + log.info("EventHubConsumer has an exception (%r). Retrying...", exception) + if isinstance(exception, errors.AuthenticationException): + closable._close_connection() + elif isinstance(exception, errors.LinkRedirect): + log.info("EventHubConsumer link redirected. Redirecting...") + redirect = exception + closable._redirect(redirect) + elif isinstance(exception, errors.LinkDetach): + closable._close_handler() + elif isinstance(exception, errors.ConnectionClose): + closable._close_connection() + elif isinstance(exception, errors.MessageHandlerError): + closable._close_handler() + elif isinstance(exception, errors.AMQPConnectionError): + closable._close_connection() + elif isinstance(exception, compat.TimeoutException): + pass # Timeout doesn't need to recreate link or exception + else: + closable._close_connection() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index dcfec2b22ee6..cfc32c7157cd 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -15,7 +15,7 @@ from azure.eventhub.common import EventData, _BatchSendEventData from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler + AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler, _handle_exception log = logging.getLogger(__name__) @@ -131,82 +131,15 @@ def _close_connection(self): self.client._conn_manager.close_connection() # close the shared connection. def _handle_exception(self, exception, retry_count, max_retries): - if isinstance(exception, KeyboardInterrupt): - log.info("EventHubConsumer stops due to keyboard interrupt") - self.close() - raise - elif isinstance(exception, ( - errors.MessageAccepted, - errors.MessageAlreadySettled, - errors.MessageModified, - errors.MessageRejected, - errors.MessageReleased, - errors.MessageContentTooLarge) - ): - log.error("Event data error (%r)", exception) - error = EventDataError(str(exception), exception) - self.close(exception) - raise error - elif isinstance(exception, errors.MessageException): - log.error("Event data send error (%r)", exception) - error = EventDataSendError(str(exception), exception) - self.close(exception) - raise error - elif retry_count >= max_retries: - log.info("EventHubConsumer has an error and has exhausted retrying. (%r)", exception) - if isinstance(exception, errors.AuthenticationException): - log.info("EventHubConsumer authentication failed. Shutting down.") - error = AuthenticationError(str(exception), exception) - elif isinstance(exception, errors.LinkDetach): - log.info("EventHubConsumer link detached. Shutting down.") - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.ConnectionClose): - log.info("EventHubConsumer connection closed. Shutting down.") - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.MessageHandlerError): - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.AMQPConnectionError): - log.info("EventHubConsumer connection lost. Shutting down.") - error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ - else ConnectError - error = error_type(str(exception), exception) - elif isinstance(exception, compat.TimeoutException): - log.info("EventHubConsumer timed out. Shutting down.") - error = ConnectionLostError(str(exception), exception) - else: - log.error("Unexpected error occurred (%r). Shutting down.", exception) - error = EventHubError("Receive failed: {}".format(exception), exception) - self.close(exception=error) - raise error - else: - log.info("EventHubConsumer has an exception (%r). Retrying...", exception) - if isinstance(exception, errors.AuthenticationException): - self._close_connection() - elif isinstance(exception, errors.LinkRedirect): - log.info("EventHubConsumer link redirected. Redirecting...") - redirect = exception - self._redirect(redirect) - elif isinstance(exception, errors.LinkDetach): - self._close_handler() - elif isinstance(exception, errors.ConnectionClose): - self._close_connection() - elif isinstance(exception, errors.MessageHandlerError): - self._close_handler() - elif isinstance(exception, errors.AMQPConnectionError): - self._close_connection() - elif isinstance(exception, compat.TimeoutException): - pass # Timeout doesn't need to recreate link or exception - else: - self._close_connection() + _handle_exception(exception, retry_count, max_retries, self, log) def _send_event_data(self): - self._open() max_retries = self.client.config.max_retries retry_count = 0 while True: try: if self.unsent_events: + self._open() self._handler.queue_message(*self.unsent_events) self._handler.wait() self.unsent_events = self._handler.pending_messages From 6e6c82798bf6c3032a0b149a28714fdd85b4aa90 Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 16 Jul 2019 11:53:06 -0700 Subject: [PATCH 03/11] Shared connection (sync) test update --- .../azure-eventhubs/tests/test_negative.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/tests/test_negative.py b/sdk/eventhub/azure-eventhubs/tests/test_negative.py index 1bf9855c80eb..ac19a01f76c9 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_negative.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_negative.py @@ -26,7 +26,7 @@ def test_send_with_invalid_hostname(invalid_hostname, connstr_receivers): client = EventHubClient.from_connection_string(invalid_hostname, network_tracing=False) sender = client.create_producer() with pytest.raises(AuthenticationError): - sender._open() + sender.send(EventData("test data")) @pytest.mark.liveTest @@ -34,7 +34,7 @@ def test_receive_with_invalid_hostname_sync(invalid_hostname): client = EventHubClient.from_connection_string(invalid_hostname, network_tracing=False) receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1")) with pytest.raises(AuthenticationError): - receiver._open() + receiver.receive(timeout=3) @pytest.mark.liveTest @@ -43,7 +43,7 @@ def test_send_with_invalid_key(invalid_key, connstr_receivers): client = EventHubClient.from_connection_string(invalid_key, network_tracing=False) sender = client.create_producer() with pytest.raises(AuthenticationError): - sender._open() + sender.send(EventData("test data")) @pytest.mark.liveTest @@ -51,7 +51,7 @@ def test_receive_with_invalid_key_sync(invalid_key): client = EventHubClient.from_connection_string(invalid_key, network_tracing=False) receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1")) with pytest.raises(AuthenticationError): - receiver._open() + receiver.receive(timeout=3) @pytest.mark.liveTest @@ -60,7 +60,7 @@ def test_send_with_invalid_policy(invalid_policy, connstr_receivers): client = EventHubClient.from_connection_string(invalid_policy, network_tracing=False) sender = client.create_producer() with pytest.raises(AuthenticationError): - sender._open() + sender.send(EventData("test data")) @pytest.mark.liveTest @@ -68,7 +68,7 @@ def test_receive_with_invalid_policy_sync(invalid_policy): client = EventHubClient.from_connection_string(invalid_policy, network_tracing=False) receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1")) with pytest.raises(AuthenticationError): - receiver._open() + receiver.receive(timeout=3) @pytest.mark.liveTest @@ -90,7 +90,7 @@ def test_non_existing_entity_sender(connection_str): client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) sender = client.create_producer(partition_id="1") with pytest.raises(AuthenticationError): - sender._open() + sender.send(EventData("test data")) @pytest.mark.liveTest @@ -98,7 +98,7 @@ def test_non_existing_entity_receiver(connection_str): client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1")) with pytest.raises(AuthenticationError): - receiver._open() + receiver.receive(timeout=3) @pytest.mark.liveTest @@ -122,7 +122,7 @@ def test_send_to_invalid_partitions(connection_str): sender = client.create_producer(partition_id=p) try: with pytest.raises(ConnectError): - sender._open() + sender.send(EventData("test data")) finally: sender.close() From a86146e776d455795f53fe9f577188d1380e9a6b Mon Sep 17 00:00:00 2001 From: yijxie Date: Wed, 17 Jul 2019 12:14:56 -0700 Subject: [PATCH 04/11] Shared connection --- .../azure/eventhub/_connection_manager.py | 25 +- .../eventhub/aio/_connection_manager_async.py | 77 +++++ .../azure/eventhub/aio/client_async.py | 45 ++- .../azure/eventhub/aio/consumer_async.py | 300 ++++-------------- .../azure/eventhub/aio/error_async.py | 79 +++++ .../azure/eventhub/aio/producer_async.py | 232 +++----------- .../azure-eventhubs/azure/eventhub/client.py | 30 +- .../azure/eventhub/consumer.py | 5 +- .../azure-eventhubs/azure/eventhub/error.py | 45 +-- .../azure/eventhub/producer.py | 31 +- 10 files changed, 365 insertions(+), 504 deletions(-) create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py index 3e1a17a1051d..9600226df2fb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py @@ -3,14 +3,14 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import threading -from uamqp import Connection, TransportType +from threading import RLock +from uamqp import Connection, TransportType, c_uamqp class _SharedConnectionManager(object): def __init__(self, **kwargs): - self._lock = threading.Lock() - self._conn = None + self._lock = RLock() + self._conn = None # type: Connection self._container_id = kwargs.get("container_id") self._debug = kwargs.get("debug") @@ -42,12 +42,22 @@ def get_connection(self, host, auth): encoding=self._encoding) return self._conn - def close_connection(self, conn=None): + def close_connection(self): with self._lock: if self._conn: self._conn.destroy() self._conn = None + def reset_connection_if_broken(self): + with self._lock: + if self._conn and self._conn._state in ( + c_uamqp.ConnectionState.CLOSE_RCVD, + c_uamqp.ConnectionState.CLOSE_SENT, + c_uamqp.ConnectionState.DISCARDING, + c_uamqp.ConnectionState.END, + ): + self._conn = None + class _SeparateConnectionManager(object): def __init__(self, **kwargs): @@ -59,6 +69,9 @@ def get_connection(self, host, auth): def close_connection(self): pass + def reset_connection_if_broken(self): + pass + def get_connection_manager(**kwargs): - return _SeparateConnectionManager(**kwargs) + return _SharedConnectionManager(**kwargs) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py new file mode 100644 index 000000000000..bd54a2bc4e3a --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py @@ -0,0 +1,77 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from asyncio import Lock +from uamqp import TransportType, c_uamqp +from uamqp.async_ops import ConnectionAsync + + +class _SharedConnectionManager(object): + def __init__(self, **kwargs): + self._lock = Lock() + self._conn = None + + self._container_id = kwargs.get("container_id") + self._debug = kwargs.get("debug") + self._error_policy = kwargs.get("error_policy") + self._properties = kwargs.get("properties") + self._encoding = kwargs.get("encoding") or "UTF-8" + self._transport_type = kwargs.get('transport_type') or TransportType.Amqp + self._http_proxy = kwargs.get('http_proxy') + self._max_frame_size = kwargs.get("max_frame_size") + self._channel_max = kwargs.get("channel_max") + self._idle_timeout = kwargs.get("idle_timeout") + self._remote_idle_timeout_empty_frame_send_ratio = kwargs.get("remote_idle_timeout_empty_frame_send_ratio") + + async def get_connection(self, host, auth): + # type: (...) -> ConnectionAsync + async with self._lock: + if self._conn is None: + self._conn = ConnectionAsync( + host, + auth, + container_id=self._container_id, + max_frame_size=self._max_frame_size, + channel_max=self._channel_max, + idle_timeout=self._idle_timeout, + properties=self._properties, + remote_idle_timeout_empty_frame_send_ratio=self._remote_idle_timeout_empty_frame_send_ratio, + error_policy=self._error_policy, + debug=self._debug, + encoding=self._encoding) + return self._conn + + async def close_connection(self): + async with self._lock: + if self._conn: + await self._conn.destroy_async() + self._conn = None + + def reset_connection_if_broken(self): + with self._lock: + if self._conn and self._conn._state in ( + c_uamqp.ConnectionState.CLOSE_RCVD, + c_uamqp.ConnectionState.CLOSE_SENT, + c_uamqp.ConnectionState.DISCARDING, + c_uamqp.ConnectionState.END, + ): + self._conn = None + +class _SeparateConnectionManager(object): + def __init__(self, **kwargs): + pass + + async def get_connection(self, host, auth): + pass # return None + + async def close_connection(self): + pass + + def reset_connection_if_broken(self): + pass + + +def get_connection_manager(**kwargs): + return _SharedConnectionManager(**kwargs) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py index f552cb0a167b..0141923f9fe3 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py @@ -22,6 +22,8 @@ from .producer_async import EventHubProducer from .consumer_async import EventHubConsumer +from ._connection_manager_async import get_connection_manager +from .error_async import _handle_exception log = logging.getLogger(__name__) @@ -42,6 +44,16 @@ class EventHubClient(EventHubClientAbstract): """ + def __init__(self, host, event_hub_path, credential, **kwargs): + super(EventHubClient, self).__init__(host, event_hub_path, credential, **kwargs) + self._conn_manager = get_connection_manager(**kwargs) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + def _create_auth(self, username=None, password=None): """ Create an ~uamqp.authentication.cbs_auth_async.SASTokenAuthAsync instance to authenticate @@ -85,17 +97,21 @@ def _create_auth(self, username=None, password=None): get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) + async def _handle_exception(self, exception, retry_count, max_retries): + await _handle_exception(exception, retry_count, max_retries, self, log) + + async def _close_connection(self): + self._conn_manager.reset_connection_if_broken() + async def _management_request(self, mgmt_msg, op_type): - alt_creds = { - "username": self._auth_config.get("iot_username"), - "password": self._auth_config.get("iot_password")} - connect_count = 0 + max_retries = self.config.max_retries + retry_count = 0 while True: - connect_count += 1 - mgmt_auth = self._create_auth(**alt_creds) + mgmt_auth = self._create_auth() mgmt_client = AMQPClientAsync(self.mgmt_target, auth=mgmt_auth, debug=self.config.network_tracing) try: - await mgmt_client.open_async() + conn = await self._conn_manager.get_connection(self.host, mgmt_auth) + await mgmt_client.open_async(connection=conn) response = await mgmt_client.mgmt_request_async( mgmt_msg, constants.READ_OPERATION, @@ -103,15 +119,9 @@ async def _management_request(self, mgmt_msg, op_type): status_code_field=b'status-code', description_fields=b'status-description') return response - except (errors.AMQPConnectionError, errors.TokenAuthFailure, compat.TimeoutException) as failure: - if connect_count >= self.config.max_retries: - err = ConnectError( - "Can not connect to EventHubs or get management info from the service. " - "Please make sure the connection string or token is correct and retry. " - "Besides, this method doesn't work if you use an IoT connection string.", - failure - ) - raise err + except Exception as exception: + await self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 finally: await mgmt_client.close_async() @@ -263,3 +273,6 @@ def create_producer( handler = EventHubProducer( self, target, partition=partition_id, send_timeout=send_timeout, loop=loop) return handler + + async def close(self): + await self._conn_manager.close_connection() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py index 6cf020176d96..203cdd9882bd 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py @@ -12,6 +12,7 @@ from azure.eventhub import EventData, EventPosition from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler +from ..aio.error_async import _handle_exception log = logging.getLogger(__name__) @@ -71,23 +72,9 @@ def __init__( # pylint: disable=super-init-not-called self.properties = None partition = self.source.split('/')[-1] self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) # pylint: disable=protected-access if owner_level: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} - self._handler = ReceiveClientAsync( - source, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access - loop=self.loop) + self._handler = None async def __aenter__(self): return self @@ -99,74 +86,52 @@ def __aiter__(self): return self async def __anext__(self): - await self._open() max_retries = self.client.config.max_retries - connecting_count = 0 + retry_count = 0 while True: - connecting_count += 1 try: + await self._open() if not self.messages_iter: self.messages_iter = self._handler.receive_messages_iter_async() message = await self.messages_iter.__anext__() event_data = EventData(message=message) self.offset = EventPosition(event_data.offset, inclusive=False) return event_data - except errors.AuthenticationException as auth_error: - if connecting_count < max_retries: - log.info("EventHubConsumer disconnected due to token error. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer authentication failed. Shutting down.") - error = AuthenticationError(str(auth_error), auth_error) - await self.close(auth_error) - raise error - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("EventHubConsumer detached. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer detached. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(error) - raise error - except errors.AMQPConnectionError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer connection lost. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer connection lost. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(error) - raise error - except compat.TimeoutException as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer timed out receiving event data. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer timed out. Shutting down.") - await self.close(shutdown) - raise ConnectionLostError(str(shutdown), shutdown) - except StopAsyncIteration: - raise - except Exception as e: - log.error("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receive failed: {}".format(e), e) - await self.close(exception=error) - raise error + except Exception as exception: + await self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 def _check_closed(self): if self.error: - raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.", - self.error) + raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.") + + def _create_handler(self): + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset._selector()) + self._handler = ReceiveClientAsync( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.config.network_tracing, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties( + self.client.config.user_agent), # pylint: disable=protected-access + loop=self.loop) + self.messages_iter = None + + async def _redirect(self, redirect): + self.redirected = redirect + self.running = False + self.messages_iter = None + await self._close_connection() async def _open(self): """ @@ -176,121 +141,34 @@ async def _open(self): """ # pylint: disable=protected-access - self._check_closed() - if self.redirected: - self.source = self.redirected.address - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) # pylint: disable=protected-access - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} - self._handler = ReceiveClientAsync( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access - loop=self.loop) if not self.running: - await self._connect() + if self.redirected: + self.client._process_redirect_uri(self.redirected) + self.source = self.redirected.address + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + else: + alt_creds = {} + self._create_handler() + await self._handler.open_async(connection=await self.client._conn_manager.get_connection( + self.client.address.hostname, + self.client.get_auth(**alt_creds) + )) + while not await self._handler.client_ready_async(): + await asyncio.sleep(0.05) self.running = True - async def _connect(self): - connected = await self._build_connection() - if not connected: - await asyncio.sleep(self.reconnect_backoff) - while not await self._build_connection(is_reconnect=True): - await asyncio.sleep(self.reconnect_backoff) + async def _close_handler(self): + await self._handler.close_async() # close the link (sharing connection) or connection (not sharing) + self.running = False - async def _build_connection(self, is_reconnect=False): # pylint: disable=too-many-statements - # pylint: disable=protected-access - if is_reconnect: - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} - await self._handler.close_async() - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) # pylint: disable=protected-access - self._handler = ReceiveClientAsync( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access - loop=self.loop) - self.messages_iter = None - try: - await self._handler.open_async() - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - return True - except errors.AuthenticationException as shutdown: - if is_reconnect: - log.info("EventHubConsumer couldn't authenticate. Shutting down. (%r)", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubConsumer couldn't authenticate. Attempting reconnect.") - return False - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: - log.info("EventHubConsumer detached. Attempting reconnect.") - return False - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if is_reconnect: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubConsumer detached. Attempting reconnect.") - return False - except errors.AMQPConnectionError as shutdown: - if is_reconnect: - log.info("EventHubConsumer connection error (%r). Shutting down.", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubConsumer couldn't authenticate. Attempting reconnect.") - return False - except compat.TimeoutException as shutdown: - if is_reconnect: - log.info("EventHubConsumer authentication timed out. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubConsumer authentication timed out. Attempting reconnect.") - return False - except Exception as e: - log.error("Unexpected error occurred when building connection (%r). Shutting down.", e) - error = EventHubError("Unexpected error occurred when building connection", e) - await self.close(exception=error) - raise error - - async def _reconnect(self): - """If the EventHubConsumer was disconnected from the service with - a retryable error - attempt to reconnect.""" - return await self._build_connection(is_reconnect=True) + async def _close_connection(self): + await self._close_handler() + self.client._conn_manager.reset_connection_if_broken() + + async def _handle_exception(self, exception, retry_count, max_retries): + await _handle_exception(exception, retry_count, max_retries, self, log) @property def queue_size(self): @@ -333,17 +211,15 @@ async def receive(self, max_batch_size=None, timeout=None): """ self._check_closed() - await self._open() - max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size timeout = self.client.config.receive_timeout if timeout is None else timeout data_batch = [] max_retries = self.client.config.max_retries - connecting_count = 0 + retry_count = 0 while True: - connecting_count += 1 try: + await self._open() timeout_ms = 1000 * timeout if timeout else 0 message_batch = await self._handler.receive_message_batch_async( max_batch_size=max_batch_size, @@ -353,55 +229,9 @@ async def receive(self, max_batch_size=None, timeout=None): self.offset = EventPosition(event_data.offset) data_batch.append(event_data) return data_batch - except errors.AuthenticationException as auth_error: - if connecting_count < max_retries: - log.info("EventHubConsumer disconnected due to token error. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer authentication failed. Shutting down.") - error = AuthenticationError(str(auth_error), auth_error) - await self.close(auth_error) - raise error - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("EventHubConsumer detached. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer detached. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(error) - raise error - except errors.AMQPConnectionError as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer connection lost. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer connection lost. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(error) - raise error - except compat.TimeoutException as shutdown: - if connecting_count < max_retries: - log.info("EventHubConsumer timed out receiving event data. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubConsumer timed out. Shutting down.") - await self.close(shutdown) - raise ConnectionLostError(str(shutdown), shutdown) - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receive failed: {}".format(e), e) - await self.close(exception=error) - raise error + except Exception as exception: + await self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 async def close(self, exception=None): # type: (Exception) -> None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py new file mode 100644 index 000000000000..500dde08e8a7 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py @@ -0,0 +1,79 @@ +from uamqp import errors, compat +from ..error import EventHubError, EventDataSendError, \ + EventDataError, ConnectError, ConnectionLostError, AuthenticationError + + +async def _handle_exception(exception, retry_count, max_retries, closable, log): + type_name = type(closable).__name__ + if isinstance(exception, KeyboardInterrupt): + log.info("{} stops due to keyboard interrupt".format(type_name)) + await closable.close() + raise + + elif isinstance(exception, ( + errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) + ): + log.error("Event data error (%r)", exception) + error = EventDataError(str(exception), exception) + await closable.close(exception) + raise error + elif isinstance(exception, errors.MessageException): + log.error("Event data send error (%r)", exception) + error = EventDataSendError(str(exception), exception) + await closable.close(exception) + raise error + elif retry_count >= max_retries: + log.info("{} has an error and has exhausted retrying. (%r)".format(type_name), exception) + if isinstance(exception, errors.AuthenticationException): + log.info("{} authentication failed. Shutting down.".format(type_name)) + error = AuthenticationError(str(exception), exception) + elif isinstance(exception, errors.VendorLinkDetach): + log.info("{} link detached. Shutting down.".format(type_name)) + error = ConnectError(str(exception), exception) + elif isinstance(exception, errors.LinkDetach): + log.info("{} link detached. Shutting down.".format(type_name)) + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.ConnectionClose): + log.info("{} connection closed. Shutting down.".format(type_name)) + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.MessageHandlerError): + log.info("{} detached. Shutting down.".format(type_name)) + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.AMQPConnectionError): + log.info("{} connection lost. Shutting down.".format(type_name)) + error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ + else ConnectError + error = error_type(str(exception), exception) + elif isinstance(exception, compat.TimeoutException): + log.info("{} timed out. Shutting down.".format(type_name)) + error = ConnectionLostError(str(exception), exception) + else: + log.error("Unexpected error occurred (%r). Shutting down.", exception) + error = EventHubError("Receive failed: {}".format(exception), exception) + await closable.close() + raise error + else: + log.info("{} has an exception (%r). Retrying...".format(type_name), exception) + if isinstance(exception, errors.AuthenticationException): + await closable._close_connection() + elif isinstance(exception, errors.LinkRedirect): + log.info("{} link redirected. Redirecting...".format(type_name)) + redirect = exception + await closable._redirect(redirect) + elif isinstance(exception, errors.LinkDetach): + await closable._close_handler() + elif isinstance(exception, errors.ConnectionClose): + await closable._close_connection() + elif isinstance(exception, errors.MessageHandlerError): + await closable._close_handler() + elif isinstance(exception, errors.AMQPConnectionError): + await closable._close_connection() + elif isinstance(exception, compat.TimeoutException): + pass # Timeout doesn't need to recreate link or connection to retry + else: + await closable._close_connection() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index aef8dc50ff02..07237efe5936 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -13,6 +13,8 @@ from azure.eventhub.common import EventData, _BatchSendEventData from azure.eventhub.error import EventHubError, ConnectError, \ AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler +from .error_async import _handle_exception +from ..producer import _error, _set_partition_key log = logging.getLogger(__name__) @@ -68,6 +70,17 @@ def __init__( # pylint: disable=super-init-not-called if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) + self._handler = None + self._outcome = None + self._condition = None + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close(exc_val) + + def _create_handler(self): self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), @@ -76,16 +89,14 @@ def __init__( # pylint: disable=super-init-not-called error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access + properties=self.client._create_properties( + self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) - self._outcome = None - self._condition = None - - async def __aenter__(self): - return self - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close(exc_val) + async def _redirect(self, redirect): + self.redirected = redirect + self.running = False + await self._close_connection() async def _open(self): """ @@ -94,188 +105,49 @@ async def _open(self): context will be used to create a new handler before opening it. """ - if self.redirected: - self.target = self.redirected.address - self._handler = SendClientAsync( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access - loop=self.loop) if not self.running: - await self._connect() + if self.redirected: + self.target = self.redirected.address + self._create_handler() + await self._handler.open_async(connection=await self.client._conn_manager.get_connection( + self.client.address.hostname, + self.client.get_auth() + )) + while not await self._handler.client_ready_async(): + await asyncio.sleep(0.05) self.running = True - async def _connect(self): - connected = await self._build_connection() - if not connected: - await asyncio.sleep(self.reconnect_backoff) - while not await self._build_connection(is_reconnect=True): - await asyncio.sleep(self.reconnect_backoff) - - async def _build_connection(self, is_reconnect=False): - """ + async def _close_handler(self): + await self._handler.close_async() # close the link (sharing connection) or connection (not sharing) + self.running = False - :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. - False - the 1st time to connect - :return: True - connected. False - not connected - """ - # pylint: disable=protected-access - if is_reconnect: - await self._handler.close_async() - self._handler = SendClientAsync( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent), - loop=self.loop) - try: - await self._handler.open_async() - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - return True - except errors.AuthenticationException as shutdown: - if is_reconnect: - log.info("EventHubProducer couldn't authenticate. Shutting down. (%r)", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubProducer couldn't authenticate. Attempting reconnect.") - return False - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: - log.info("EventHubProducer detached. Attempting reconnect.") - return False - else: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if is_reconnect: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubProducer detached. Attempting reconnect.") - return False - except errors.AMQPConnectionError as shutdown: - if is_reconnect: - log.info("EventHubProducer connection error (%r). Shutting down.", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubProducer couldn't authenticate. Attempting reconnect.") - return False - except compat.TimeoutException as shutdown: - if is_reconnect: - log.info("EventHubProducer authentication timed out. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - else: - log.info("EventHubProducer authentication timed out. Attempting reconnect.") - return False - except Exception as e: - log.info("Unexpected error occurred when building connection (%r). Shutting down.", e) - error = EventHubError("Unexpected error occurred when building connection", e) - await self.close(exception=error) - raise error + async def _close_connection(self): + await self._close_handler() + await self.client._conn_manager.close_connection() # close the shared connection. - async def _reconnect(self): - return await self._build_connection(is_reconnect=True) + async def _handle_exception(self, exception, retry_count, max_retries): + await _handle_exception(exception, retry_count, max_retries, self, log) async def _send_event_data(self): - await self._open() max_retries = self.client.config.max_retries - connecting_count = 0 + retry_count = 0 while True: - connecting_count += 1 try: if self.unsent_events: + await self._open() self._handler.queue_message(*self.unsent_events) await self._handler.wait_async() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: - EventHubProducer._error(self._outcome, self._condition) + _error(self._outcome, self._condition) return - except (errors.MessageAccepted, - errors.MessageAlreadySettled, - errors.MessageModified, - errors.MessageRejected, - errors.MessageReleased, - errors.MessageContentTooLarge) as msg_error: - raise EventDataError(str(msg_error), msg_error) - except errors.MessageException as failed: - log.error("Send event data error (%r)", failed) - error = EventDataSendError(str(failed), failed) - await self.close(exception=error) - raise error - except errors.AuthenticationException as auth_error: - if connecting_count < max_retries: - log.info("EventHubProducer disconnected due to token error. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubProducer authentication failed. Shutting down.") - error = AuthenticationError(str(auth_error), auth_error) - await self.close(auth_error) - raise error - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: - log.info("EventHubProducer detached. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if connecting_count < max_retries: - log.info("EventHubProducer detached. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubProducer detached. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(error) - raise error - except errors.AMQPConnectionError as shutdown: - if connecting_count < max_retries: - log.info("EventHubProducer connection lost. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubProducer connection lost. Shutting down.") - error = ConnectionLostError(str(shutdown), shutdown) - await self.close(error) - raise error - except compat.TimeoutException as shutdown: - if connecting_count < max_retries: - log.info("EventHubProducer timed out sending event data. Attempting reconnect.") - await self._reconnect() - else: - log.info("EventHubProducer timed out. Shutting down.") - await self.close(shutdown) - raise ConnectionLostError(str(shutdown), shutdown) - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Send failed: {}".format(e), e) - await self.close(exception=error) - raise error + except Exception as exception: + await self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 def _check_closed(self): if self.error: - raise EventHubError("This producer has been closed. Please create a new producer to send event data.", - self.error) + raise EventHubError("This producer has been closed. Please create a new producer to send event data.") def _on_outcome(self, outcome, condition): """ @@ -289,20 +161,8 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - @staticmethod - def _error(outcome, condition): - if outcome != constants.MessageSendResult.Ok: - raise condition - - @staticmethod - def _set_partition_key(event_datas, partition_key): - ed_iter = iter(event_datas) - for ed in ed_iter: - ed._set_partition_key(partition_key) - yield ed - async def send(self, event_data, partition_key=None): - # type:(Union[EventData, Union[List[EventData], Iterator[EventData], Generator[EventData]]], Union[str, bytes]) -> None + # type:(Union[EventData, Iterable[EventData]], Union[str, bytes]) -> None """ Sends an event data and blocks until acknowledgement is received or operation times out. @@ -332,7 +192,7 @@ async def send(self, event_data, partition_key=None): event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: - event_data_with_pk = self._set_partition_key(event_data, partition_key) + event_data_with_pk = _set_partition_key(event_data, partition_key) wrapper_event_data = _BatchSendEventData( event_data_with_pk, partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) @@ -373,4 +233,4 @@ async def close(self, exception=None): self.error = EventHubError(str(exception)) else: self.error = EventHubError("This send handler is now closed.") - await self._handler.close_async() \ No newline at end of file + await self._handler.close_async() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index a88beebdc275..65f22b19662a 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -28,7 +28,7 @@ from .client_abstract import EventHubClientAbstract from .common import EventHubSASTokenCredential, EventHubSharedKeyCredential from ._connection_manager import get_connection_manager - +from .error import _handle_exception log = logging.getLogger(__name__) @@ -50,10 +50,6 @@ class EventHubClient(EventHubClientAbstract): def __init__(self, host, event_hub_path, credential, **kwargs): super(EventHubClient, self).__init__(host, event_hub_path, credential, **kwargs) - alt_creds = { - "username": self._auth_config.get("iot_username"), - "password": self._auth_config.get("iot_password") - } self._conn_manager = get_connection_manager(**kwargs) def __del__(self): @@ -110,10 +106,16 @@ def _create_auth(self, username=None, password=None): get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) + def _handle_exception(self, exception, retry_count, max_retries): + _handle_exception(exception, retry_count, max_retries, self, log) + + def _close_connection(self): + self._conn_manager.reset_connection_if_broken() + def _management_request(self, mgmt_msg, op_type): + max_retries = self.config.max_retries retry_count = 0 while retry_count <= self.config.max_retries: - retry_count += 1 mgmt_auth = self._create_auth() mgmt_client = uamqp.AMQPClient(self.mgmt_target) try: @@ -126,19 +128,9 @@ def _management_request(self, mgmt_msg, op_type): status_code_field=b'status-code', description_fields=b'status-description') return response - except (errors.AMQPConnectionError, errors.TokenAuthFailure, compat.TimeoutException) as failure: - if retry_count >= self.config.max_retries: - err = ConnectError( - "Can not connect to EventHubs or get management info from the service. " - "Please make sure the connection string or token is correct and retry. " - "Besides, this method doesn't work if you use an IoT connection string.", - failure - ) - raise err - except Exception as failure: - if retry_count >= self.config.max_retries: - err = EventHubError("Unexpected error happened during management request", failure) - raise err + except Exception as exception: + self._handle_exception(exception, retry_count, max_retries) + retry_count += 1 finally: mgmt_client.close() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index b8fc6ff4a2ea..dde3a456311b 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -102,8 +102,7 @@ def __next__(self): def _check_closed(self): if self.error: - raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.", - self.error) + raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.") def _create_handler(self): alt_creds = { @@ -164,7 +163,7 @@ def _close_handler(self): def _close_connection(self): self._close_handler() - self.client._conn_manager.close_connection() # close the shared connection. + self.client._conn_manager.reset_connection_if_broken() def _handle_exception(self, exception, retry_count, max_retries): _handle_exception(exception, retry_count, max_retries, self, log) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index d15dabf0c051..db8ae4794e1e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -131,8 +131,9 @@ class EventDataSendError(EventHubError): def _handle_exception(exception, retry_count, max_retries, closable, log): + type_name = type(closable).__name__ if isinstance(exception, KeyboardInterrupt): - log.info("EventHubConsumer stops due to keyboard interrupt") + log.info("{} stops due to keyboard interrupt".format(type_name)) closable.close() raise elif isinstance(exception, ( @@ -153,52 +154,58 @@ def _handle_exception(exception, retry_count, max_retries, closable, log): closable.close(exception) raise error elif retry_count >= max_retries: - log.info("EventHubConsumer has an error and has exhausted retrying. (%r)", exception) + log.info("{} has an error and has exhausted retrying. (%r)".format(type_name), exception) if isinstance(exception, errors.AuthenticationException): - log.info("EventHubConsumer authentication failed. Shutting down.") + log.info("{} authentication failed. Shutting down.".format(type_name)) error = AuthenticationError(str(exception), exception) elif isinstance(exception, errors.VendorLinkDetach): - log.info("EventHubConsumer link detached. Shutting down.") + log.info("{} link detached. Shutting down.".format(type_name)) error = ConnectError(str(exception), exception) elif isinstance(exception, errors.LinkDetach): - log.info("EventHubConsumer link detached. Shutting down.") + log.info("{} link detached. Shutting down.".format(type_name)) error = ConnectionLostError(str(exception), exception) elif isinstance(exception, errors.ConnectionClose): - log.info("EventHubConsumer connection closed. Shutting down.") + log.info("{} connection closed. Shutting down.".format(type_name)) error = ConnectionLostError(str(exception), exception) elif isinstance(exception, errors.MessageHandlerError): - log.info("EventHubConsumer detached. Shutting down.") + log.info("{} detached. Shutting down.".format(type_name)) error = ConnectionLostError(str(exception), exception) elif isinstance(exception, errors.AMQPConnectionError): - log.info("EventHubConsumer connection lost. Shutting down.") + log.info("{} connection lost. Shutting down.".format(type_name)) error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ else ConnectError error = error_type(str(exception), exception) elif isinstance(exception, compat.TimeoutException): - log.info("EventHubConsumer timed out. Shutting down.") + log.info("{} timed out. Shutting down.".format(type_name)) error = ConnectionLostError(str(exception), exception) else: log.error("Unexpected error occurred (%r). Shutting down.", exception) error = EventHubError("Receive failed: {}".format(exception), exception) - closable.close(exception=error) + closable.close() raise error else: - log.info("EventHubConsumer has an exception (%r). Retrying...", exception) + log.info("{} has an exception (%r). Retrying...".format(type_name), exception) if isinstance(exception, errors.AuthenticationException): closable._close_connection() elif isinstance(exception, errors.LinkRedirect): - log.info("EventHubConsumer link redirected. Redirecting...") + log.info("{} link redirected. Redirecting...".format(type_name)) redirect = exception - closable._redirect(redirect) + if hasattr(closable, "_redirect"): + closable._redirect(redirect) elif isinstance(exception, errors.LinkDetach): - closable._close_handler() + if hasattr(closable, "_close_handler"): + closable._close_handler() elif isinstance(exception, errors.ConnectionClose): - closable._close_connection() + if hasattr(closable, "_close_connection"): + closable._close_connection() elif isinstance(exception, errors.MessageHandlerError): - closable._close_handler() + if hasattr(closable, "_close_handler"): + closable._close_handler() elif isinstance(exception, errors.AMQPConnectionError): - closable._close_connection() + if hasattr(closable, "_close_connection"): + closable._close_connection() elif isinstance(exception, compat.TimeoutException): - pass # Timeout doesn't need to recreate link or exception + pass # Timeout doesn't need to recreate link or connection to retry else: - closable._close_connection() + if hasattr(closable, "_close_connection"): + closable._close_connection() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index cfc32c7157cd..3f3ff4c9492e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -67,15 +67,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) - self._handler = SendClient( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access + self._handler = None self._outcome = None self._condition = None @@ -99,7 +91,6 @@ def _create_handler(self): def _redirect(self, redirect): self.redirected = redirect self.running = False - self.messages_iter = None self._close_connection() def _open(self): @@ -128,7 +119,7 @@ def _close_handler(self): def _close_connection(self): self._close_handler() - self.client._conn_manager.close_connection() # close the shared connection. + self.client._conn_manager.reset_connection_if_broken() def _handle_exception(self, exception, retry_count, max_retries): _handle_exception(exception, retry_count, max_retries, self, log) @@ -152,14 +143,7 @@ def _send_event_data(self): def _check_closed(self): if self.error: - raise EventHubError("This producer has been closed. Please create a new producer to send event data.", self.error) - - @staticmethod - def _set_partition_key(event_datas, partition_key): - ed_iter = iter(event_datas) - for ed in ed_iter: - ed._set_partition_key(partition_key) - yield ed + raise EventHubError("This producer has been closed. Please create a new producer to send event data.") def _on_outcome(self, outcome, condition): """ @@ -205,7 +189,7 @@ def send(self, event_data, partition_key=None): event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: - event_data_with_pk = self._set_partition_key(event_data, partition_key) + event_data_with_pk = _set_partition_key(event_data, partition_key) wrapper_event_data = _BatchSendEventData( event_data_with_pk, partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) @@ -250,3 +234,10 @@ def close(self, exception=None): def _error(outcome, condition): if outcome != constants.MessageSendResult.Ok: raise condition + + +def _set_partition_key(event_datas, partition_key): + ed_iter = iter(event_datas) + for ed in ed_iter: + ed._set_partition_key(partition_key) + yield ed From c5a23c8953644ec7f50d083cd7ccfa4510b71dce Mon Sep 17 00:00:00 2001 From: yijxie Date: Wed, 17 Jul 2019 12:24:29 -0700 Subject: [PATCH 05/11] Fix an issue --- .../azure/eventhub/aio/error_async.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py index 500dde08e8a7..3afea84c3904 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py @@ -60,20 +60,27 @@ async def _handle_exception(exception, retry_count, max_retries, closable, log): else: log.info("{} has an exception (%r). Retrying...".format(type_name), exception) if isinstance(exception, errors.AuthenticationException): - await closable._close_connection() + if hasattr(closable, "_close_connection"): + await closable._close_connection() elif isinstance(exception, errors.LinkRedirect): log.info("{} link redirected. Redirecting...".format(type_name)) redirect = exception - await closable._redirect(redirect) + if hasattr(closable, "_redirect"): + await closable._redirect(redirect) elif isinstance(exception, errors.LinkDetach): - await closable._close_handler() + if hasattr(closable, "_close_handler"): + await closable._close_handler() elif isinstance(exception, errors.ConnectionClose): - await closable._close_connection() + if hasattr(closable, "_close_connection"): + await closable._close_connection() elif isinstance(exception, errors.MessageHandlerError): - await closable._close_handler() + if hasattr(closable, "_close_handler"): + await closable._close_handler() elif isinstance(exception, errors.AMQPConnectionError): - await closable._close_connection() + if hasattr(closable, "_close_connection"): + await closable._close_connection() elif isinstance(exception, compat.TimeoutException): pass # Timeout doesn't need to recreate link or connection to retry else: - await closable._close_connection() \ No newline at end of file + if hasattr(closable, "_close_connection"): + await closable._close_connection() From b83264d16408c3bdaed215a5f4ae2c307705d3bf Mon Sep 17 00:00:00 2001 From: yijxie Date: Sun, 21 Jul 2019 23:25:19 -0700 Subject: [PATCH 06/11] add retry exponential delay and timeout to exception handling --- .../eventhub/_consumer_producer_mixin.py | 109 +++++++++++++++++ .../eventhub/aio/_connection_manager_async.py | 5 +- .../aio/_consumer_producer_mixin_async.py | 112 ++++++++++++++++++ .../azure/eventhub/aio/client_async.py | 2 +- .../azure/eventhub/aio/consumer_async.py | 75 +++++------- .../azure/eventhub/aio/error_async.py | 100 ++++++++++------ .../azure/eventhub/aio/producer_async.py | 98 ++++++--------- .../azure-eventhubs/azure/eventhub/client.py | 3 - .../azure-eventhubs/azure/eventhub/common.py | 1 + .../azure/eventhub/configuration.py | 6 +- .../azure/eventhub/consumer.py | 93 +++++---------- .../azure-eventhubs/azure/eventhub/error.py | 103 ++++++++++------ .../azure/eventhub/producer.py | 95 ++++++--------- 13 files changed, 481 insertions(+), 321 deletions(-) create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py new file mode 100644 index 000000000000..9ac6fb468945 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py @@ -0,0 +1,109 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from __future__ import unicode_literals + +import logging +import time + +from uamqp import errors +from azure.eventhub.error import EventHubError, _handle_exception + +log = logging.getLogger(__name__) + + +class ConsumerProducerMixin(object): + def __init__(self): + self.client = None + self._handler = None + self.name = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close(exc_val) + + def _check_closed(self): + if self.error: + raise EventHubError("{} has been closed. Please create a new consumer to receive event data.".format(self.name)) + + def _create_handler(self): + pass + + def _redirect(self, redirect): + self.redirected = redirect + self.running = False + self._close_connection() + + def _open(self, timeout_time=None): + """ + Open the EventHubConsumer using the supplied connection. + If the handler has previously been redirected, the redirect + context will be used to create a new handler before opening it. + + """ + # pylint: disable=protected-access + if not self.running: + if self.redirected: + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + else: + alt_creds = {} + self._create_handler() + self._handler.open(connection=self.client._conn_manager.get_connection( + self.client.address.hostname, + self.client.get_auth(**alt_creds) + )) + while not self._handler.client_ready(): + if timeout_time and time.time() >= timeout_time: + return + time.sleep(0.05) + self.running = True + + def _close_handler(self): + self._handler.close() # close the link (sharing connection) or connection (not sharing) + self.running = False + + def _close_connection(self): + self._close_handler() + self.client._conn_manager.reset_connection_if_broken() + + def _handle_exception(self, exception, retry_count, max_retries, timeout_time): + _handle_exception(exception, retry_count, max_retries, self, timeout_time) + + def close(self, exception=None): + # type:(Exception) -> None + """ + Close down the handler. If the handler has already closed, + this will be a no op. An optional exception can be passed in to + indicate that the handler was shutdown due to error. + + :param exception: An optional exception if the handler is closing + due to an error. + :type exception: Exception + + Example: + .. literalinclude:: ../examples/test_examples_eventhub.py + :start-after: [START eventhub_client_receiver_close] + :end-before: [END eventhub_client_receiver_close] + :language: python + :dedent: 4 + :caption: Close down the handler. + + """ + self.running = False + if self.error: + return + if isinstance(exception, errors.LinkRedirect): + self.redirected = exception + elif isinstance(exception, EventHubError): + self.error = exception + elif exception: + self.error = EventHubError(str(exception)) + else: + self.error = EventHubError("{} handler is closed.".format(self.name)) + if self._handler: + self._handler.close() # this will close link if sharing connection. Otherwise close connection diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py index bd54a2bc4e3a..3178e1fb72a7 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_connection_manager_async.py @@ -49,8 +49,8 @@ async def close_connection(self): await self._conn.destroy_async() self._conn = None - def reset_connection_if_broken(self): - with self._lock: + async def reset_connection_if_broken(self): + async with self._lock: if self._conn and self._conn._state in ( c_uamqp.ConnectionState.CLOSE_RCVD, c_uamqp.ConnectionState.CLOSE_SENT, @@ -59,6 +59,7 @@ def reset_connection_if_broken(self): ): self._conn = None + class _SeparateConnectionManager(object): def __init__(self, **kwargs): pass diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py new file mode 100644 index 000000000000..5a0f0d9eaa4d --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py @@ -0,0 +1,112 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import asyncio +import logging +import time + +from uamqp import errors +from azure.eventhub.error import EventHubError, ConnectError +from ..aio.error_async import _handle_exception + +log = logging.getLogger(__name__) + + +class ConsumerProducerMixin(object): + + def __init__(self): + self.client = None + self._handler = None + self.name = None + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close(exc_val) + + def _check_closed(self): + if self.error: + raise EventHubError("{} has been closed. Please create a new consumer to receive event data.".format(self.name)) + + def _create_handler(self): + pass + + async def _redirect(self, redirect): + self.redirected = redirect + self.running = False + await self._close_connection() + + async def _open(self, timeout_time=None): + """ + Open the EventHubConsumer using the supplied connection. + If the handler has previously been redirected, the redirect + context will be used to create a new handler before opening it. + + """ + # pylint: disable=protected-access + if not self.running: + if self.redirected: + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + else: + alt_creds = {} + self._create_handler() + await self._handler.open_async(connection=await self.client._conn_manager.get_connection( + self.client.address.hostname, + self.client.get_auth(**alt_creds) + )) + while not await self._handler.client_ready_async(): + if timeout_time and time.time() >= timeout_time: + return + await asyncio.sleep(0.05) + self.running = True + + async def _close_handler(self): + await self._handler.close_async() # close the link (sharing connection) or connection (not sharing) + self.running = False + + async def _close_connection(self): + await self._close_handler() + await self.client._conn_manager.reset_connection_if_broken() + + async def _handle_exception(self, exception, retry_count, max_retries, timeout_time): + await _handle_exception(exception, retry_count, max_retries, self, timeout_time) + + async def close(self, exception=None): + # type: (Exception) -> None + """ + Close down the handler. If the handler has already closed, + this will be a no op. An optional exception can be passed in to + indicate that the handler was shutdown due to error. + + :param exception: An optional exception if the handler is closing + due to an error. + :type exception: Exception + + Example: + .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py + :start-after: [START eventhub_client_async_receiver_close] + :end-before: [END eventhub_client_async_receiver_close] + :language: python + :dedent: 4 + :caption: Close down the handler. + + """ + self.running = False + if self.error: + return + if isinstance(exception, errors.LinkRedirect): + self.redirected = exception + elif isinstance(exception, EventHubError): + self.error = exception + elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): + self.error = ConnectError(str(exception), exception) + elif exception: + self.error = EventHubError(str(exception)) + else: + self.error = EventHubError("This receive handler is now closed.") + if self._handler: + await self._handler.close_async() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py index 0141923f9fe3..513e13e4f75e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py @@ -101,7 +101,7 @@ async def _handle_exception(self, exception, retry_count, max_retries): await _handle_exception(exception, retry_count, max_retries, self, log) async def _close_connection(self): - self._conn_manager.reset_connection_if_broken() + await self._conn_manager.reset_connection_if_broken() async def _management_request(self, mgmt_msg, op_type): max_retries = self.config.max_retries diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py index 203cdd9882bd..acd20181845f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py @@ -6,6 +6,7 @@ import uuid import logging from typing import List +import time from uamqp import errors, types, compat from uamqp import ReceiveClientAsync, Source @@ -13,11 +14,12 @@ from azure.eventhub import EventData, EventPosition from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler from ..aio.error_async import _handle_exception +from ._consumer_producer_mixin_async import ConsumerProducerMixin log = logging.getLogger(__name__) -class EventHubConsumer(object): +class EventHubConsumer(ConsumerProducerMixin): """ A consumer responsible for reading EventData from a specific Event Hub partition and as a member of a specific consumer group. @@ -55,6 +57,7 @@ def __init__( # pylint: disable=super-init-not-called :type owner_level: int :param loop: An event loop. """ + super(EventHubConsumer, self).__init__() self.loop = loop or asyncio.get_event_loop() self.running = False self.client = client @@ -76,12 +79,6 @@ def __init__( # pylint: disable=super-init-not-called self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} self._handler = None - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close(exc_val) - def __aiter__(self): return self @@ -101,10 +98,6 @@ async def __anext__(self): await self._handle_exception(exception, retry_count, max_retries) retry_count += 1 - def _check_closed(self): - if self.error: - raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.") - def _create_handler(self): alt_creds = { "username": self.client._auth_config.get("iot_username"), @@ -128,12 +121,10 @@ def _create_handler(self): self.messages_iter = None async def _redirect(self, redirect): - self.redirected = redirect - self.running = False self.messages_iter = None - await self._close_connection() + await super(EventHubConsumer, self)._redirect(redirect) - async def _open(self): + async def _open(self, timeout_time=None): """ Open the EventHubConsumer using the supplied connection. If the handler has previously been redirected, the redirect @@ -141,34 +132,10 @@ async def _open(self): """ # pylint: disable=protected-access - if not self.running: - if self.redirected: - self.client._process_redirect_uri(self.redirected) - self.source = self.redirected.address - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} - else: - alt_creds = {} - self._create_handler() - await self._handler.open_async(connection=await self.client._conn_manager.get_connection( - self.client.address.hostname, - self.client.get_auth(**alt_creds) - )) - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - self.running = True - - async def _close_handler(self): - await self._handler.close_async() # close the link (sharing connection) or connection (not sharing) - self.running = False - - async def _close_connection(self): - await self._close_handler() - self.client._conn_manager.reset_connection_if_broken() - - async def _handle_exception(self, exception, retry_count, max_retries): - await _handle_exception(exception, retry_count, max_retries, self, log) + if not self.running and self.redirected: + self.client._process_redirect_uri(self.redirected) + self.source = self.redirected.address + await super(EventHubConsumer, self)._open(timeout_time) @property def queue_size(self): @@ -213,24 +180,38 @@ async def receive(self, max_batch_size=None, timeout=None): self._check_closed() max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size timeout = self.client.config.receive_timeout if timeout is None else timeout + if not timeout: + timeout = 100_000 # timeout None or 0 mean no timeout. 100000 seconds is equivalent to no timeout data_batch = [] + start_time = time.time() + timeout_time = start_time + timeout max_retries = self.client.config.max_retries retry_count = 0 + last_exception = None while True: try: - await self._open() - timeout_ms = 1000 * timeout if timeout else 0 + await self._open(timeout_time) + remaining_time = timeout_time - time.time() + if remaining_time <= 0.0: + if last_exception: + log.info("%r receive operation timed out. (%r)", self.name, last_exception) + raise last_exception + return data_batch + + remaining_time_ms = 1000 * remaining_time message_batch = await self._handler.receive_message_batch_async( max_batch_size=max_batch_size, - timeout=timeout_ms) + timeout=remaining_time_ms) for message in message_batch: event_data = EventData(message=message) self.offset = EventPosition(event_data.offset) data_batch.append(event_data) return data_batch + except EventHubError: + raise except Exception as exception: - await self._handle_exception(exception, retry_count, max_retries) + last_exception = await self._handle_exception(exception, retry_count, max_retries, timeout_time) retry_count += 1 async def close(self, exception=None): diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py index 3afea84c3904..d78233c1f896 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py @@ -1,15 +1,49 @@ +import asyncio +import time +import logging + from uamqp import errors, compat from ..error import EventHubError, EventDataSendError, \ EventDataError, ConnectError, ConnectionLostError, AuthenticationError -async def _handle_exception(exception, retry_count, max_retries, closable, log): - type_name = type(closable).__name__ +log = logging.getLogger(__name__) + + +def _create_eventhub_exception(exception): + if isinstance(exception, errors.AuthenticationException): + error = AuthenticationError(str(exception), exception) + elif isinstance(exception, errors.VendorLinkDetach): + error = ConnectError(str(exception), exception) + elif isinstance(exception, errors.LinkDetach): + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.ConnectionClose): + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.MessageHandlerError): + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.AMQPConnectionError): + error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ + else ConnectError + error = error_type(str(exception), exception) + elif isinstance(exception, compat.TimeoutException): + error = ConnectionLostError(str(exception), exception) + else: + error = EventHubError(str(exception), exception) + return error + + +async def _handle_exception(exception, retry_count, max_retries, closable, timeout_time): + try: + name = closable.name + except AttributeError: + name = closable.container_id if isinstance(exception, KeyboardInterrupt): - log.info("{} stops due to keyboard interrupt".format(type_name)) - await closable.close() + log.info("%r stops due to keyboard interrupt", name) + closable.close() + raise + elif isinstance(exception, EventHubError): + closable.close() raise - elif isinstance(exception, ( errors.MessageAccepted, errors.MessageAlreadySettled, @@ -18,52 +52,23 @@ async def _handle_exception(exception, retry_count, max_retries, closable, log): errors.MessageReleased, errors.MessageContentTooLarge) ): - log.error("Event data error (%r)", exception) + log.info("%r Event data error (%r)", name, exception) error = EventDataError(str(exception), exception) - await closable.close(exception) raise error elif isinstance(exception, errors.MessageException): - log.error("Event data send error (%r)", exception) + log.info("%r Event data send error (%r)", name, exception) error = EventDataSendError(str(exception), exception) - await closable.close(exception) raise error elif retry_count >= max_retries: - log.info("{} has an error and has exhausted retrying. (%r)".format(type_name), exception) - if isinstance(exception, errors.AuthenticationException): - log.info("{} authentication failed. Shutting down.".format(type_name)) - error = AuthenticationError(str(exception), exception) - elif isinstance(exception, errors.VendorLinkDetach): - log.info("{} link detached. Shutting down.".format(type_name)) - error = ConnectError(str(exception), exception) - elif isinstance(exception, errors.LinkDetach): - log.info("{} link detached. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.ConnectionClose): - log.info("{} connection closed. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.MessageHandlerError): - log.info("{} detached. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.AMQPConnectionError): - log.info("{} connection lost. Shutting down.".format(type_name)) - error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ - else ConnectError - error = error_type(str(exception), exception) - elif isinstance(exception, compat.TimeoutException): - log.info("{} timed out. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - else: - log.error("Unexpected error occurred (%r). Shutting down.", exception) - error = EventHubError("Receive failed: {}".format(exception), exception) - await closable.close() + error = _create_eventhub_exception(exception) + log.info("%r has exhausted retry. Exception still occurs (%r)", name, exception) raise error else: - log.info("{} has an exception (%r). Retrying...".format(type_name), exception) if isinstance(exception, errors.AuthenticationException): if hasattr(closable, "_close_connection"): await closable._close_connection() elif isinstance(exception, errors.LinkRedirect): - log.info("{} link redirected. Redirecting...".format(type_name)) + log.info("%r link redirect received. Redirecting...", name) redirect = exception if hasattr(closable, "_redirect"): await closable._redirect(redirect) @@ -84,3 +89,20 @@ async def _handle_exception(exception, retry_count, max_retries, closable, log): else: if hasattr(closable, "_close_connection"): await closable._close_connection() + # start processing retry delay + try: + backoff_factor = closable.client.config.backoff_factor + backoff_max = closable.client.config.backoff_max + except AttributeError: + backoff_factor = closable.config.backoff_factor + backoff_max = closable.config.backoff_max + backoff = backoff_factor * 2 ** retry_count + if backoff <= backoff_max and time.time() + backoff <= timeout_time: + await asyncio.sleep(backoff) + log.info("%r has an exception (%r). Retrying...", format(name), exception) + return _create_eventhub_exception(exception) + else: + error = _create_eventhub_exception(exception) + log.info("%r operation has timed out. Last exception before timeout is (%r)", name, error) + raise error + # end of processing retry delay \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index 07237efe5936..96d60ffc8943 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -5,21 +5,22 @@ import uuid import asyncio import logging -from typing import Iterator, Generator, List, Union +from typing import Iterable, Union +import time -from uamqp import constants, errors, compat +from uamqp import constants, errors from uamqp import SendClientAsync from azure.eventhub.common import EventData, _BatchSendEventData -from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler -from .error_async import _handle_exception +from azure.eventhub.error import _error_handler, OperationTimeoutError from ..producer import _error, _set_partition_key +from ._consumer_producer_mixin_async import ConsumerProducerMixin + log = logging.getLogger(__name__) -class EventHubProducer(object): +class EventHubProducer(ConsumerProducerMixin): """ A producer responsible for transmitting EventData to a specific Event Hub, grouped together in batches. Depending on the options specified at creation, the producer may @@ -53,6 +54,7 @@ def __init__( # pylint: disable=super-init-not-called :type auto_reconnect: bool :param loop: An event loop. If not specified the default event loop will be used. """ + super(EventHubProducer, self).__init__() self.loop = loop or asyncio.get_event_loop() self.running = False self.client = client @@ -74,12 +76,6 @@ def __init__( # pylint: disable=super-init-not-called self._outcome = None self._condition = None - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close(exc_val) - def _create_handler(self): self._handler = SendClientAsync( self.target, @@ -93,48 +89,39 @@ def _create_handler(self): self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) - async def _redirect(self, redirect): - self.redirected = redirect - self.running = False - await self._close_connection() - - async def _open(self): + async def _open(self, timeout_time=None): """ Open the EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. """ - if not self.running: - if self.redirected: - self.target = self.redirected.address - self._create_handler() - await self._handler.open_async(connection=await self.client._conn_manager.get_connection( - self.client.address.hostname, - self.client.get_auth() - )) - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - self.running = True - - async def _close_handler(self): - await self._handler.close_async() # close the link (sharing connection) or connection (not sharing) - self.running = False - - async def _close_connection(self): - await self._close_handler() - await self.client._conn_manager.close_connection() # close the shared connection. - - async def _handle_exception(self, exception, retry_count, max_retries): - await _handle_exception(exception, retry_count, max_retries, self, log) - - async def _send_event_data(self): + if not self.running and self.redirected: + self.client._process_redirect_uri(self.redirected) + self.target = self.redirected.address + await super(EventHubProducer, self)._open(timeout_time) + + async def _send_event_data(self, timeout=None): + timeout = self.client.config.send_timeout if timeout is None else timeout + if not timeout: + timeout = 100_000 # timeout None or 0 mean no timeout. 100000 seconds is equivalent to no timeout + start_time = time.time() + timeout_time = start_time + timeout max_retries = self.client.config.max_retries retry_count = 0 + last_exception = None while True: try: if self.unsent_events: - await self._open() + await self._open(timeout_time) + remaining_time = timeout_time - time.time() + if remaining_time < 0.0: + if last_exception: + error = last_exception + else: + error = OperationTimeoutError("send operation timed out") + log.info("%r send operation timed out. (%r)", self.name, error) + raise error self._handler.queue_message(*self.unsent_events) await self._handler.wait_async() self.unsent_events = self._handler.pending_messages @@ -142,13 +129,9 @@ async def _send_event_data(self): _error(self._outcome, self._condition) return except Exception as exception: - await self._handle_exception(exception, retry_count, max_retries) + last_exception = await self._handle_exception(exception, retry_count, max_retries, timeout_time) retry_count += 1 - def _check_closed(self): - if self.error: - raise EventHubError("This producer has been closed. Please create a new producer to send event data.") - def _on_outcome(self, outcome, condition): """ Called when the outcome is received for a delivery. @@ -161,7 +144,7 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - async def send(self, event_data, partition_key=None): + async def send(self, event_data, partition_key=None, timeout=None): # type:(Union[EventData, Iterable[EventData]], Union[str, bytes]) -> None """ Sends an event data and blocks until acknowledgement is @@ -198,7 +181,7 @@ async def send(self, event_data, partition_key=None): partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome self.unsent_events = [wrapper_event_data.message] - await self._send_event_data() + await self._send_event_data(timeout) async def close(self, exception=None): # type: (Exception) -> None @@ -220,17 +203,4 @@ async def close(self, exception=None): :caption: Close down the handler. """ - self.running = False - if self.error: - return - if isinstance(exception, errors.LinkRedirect): - self.redirected = exception - elif isinstance(exception, EventHubError): - self.error = exception - elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): - self.error = ConnectError(str(exception), exception) - elif exception: - self.error = EventHubError(str(exception)) - else: - self.error = EventHubError("This send handler is now closed.") - await self._handler.close_async() + await super(EventHubProducer, self).close(exception) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 65f22b19662a..5de8f8531093 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -52,9 +52,6 @@ def __init__(self, host, event_hub_path, credential, **kwargs): super(EventHubClient, self).__init__(host, event_hub_path, credential, **kwargs) self._conn_manager = get_connection_manager(**kwargs) - def __del__(self): - self._conn_manager.close_connection() - def __enter__(self): return self diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 5a6702a60324..5ac6258eeb0a 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -8,6 +8,7 @@ import calendar import json import six +from enum import Enum from uamqp import BatchMessage, Message, types from uamqp.message import MessageHeader, MessageProperties diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py index 27eb649628ec..58563bdba0e0 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py @@ -8,7 +8,11 @@ class _Configuration(object): def __init__(self, **kwargs): self.user_agent = kwargs.get("user_agent") - self.max_retries = kwargs.get("max_retries", 3) + self.retry_total = kwargs.pop('retry_total', 3) + self.max_retries = self.retry_total or kwargs.get("max_retries", 3) + self.backoff_factor = kwargs.pop('retry_backoff_factor', 0.8) + self.backoff_max = kwargs.pop('retry_backoff_max', 120) + self.network_tracing = kwargs.get("network_tracing", False) self.http_proxy = kwargs.get("http_proxy") self.transport_type = TransportType.AmqpOverWebsocket if self.http_proxy \ diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index dde3a456311b..95cdeedd43aa 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -10,18 +10,16 @@ from typing import List from uamqp import types, errors -from uamqp import compat from uamqp import ReceiveClient, Source from azure.eventhub.common import EventData, EventPosition -from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, \ - _error_handler, _handle_exception - +from azure.eventhub.error import _error_handler, EventHubError +from ._consumer_producer_mixin import ConsumerProducerMixin log = logging.getLogger(__name__) -class EventHubConsumer(object): +class EventHubConsumer(ConsumerProducerMixin): """ A consumer responsible for reading EventData from a specific Event Hub partition and as a member of a specific consumer group. @@ -55,6 +53,7 @@ def __init__(self, client, source, event_position=None, prefetch=300, owner_leve consumer if owner_level is set. :type owner_level: int """ + super(EventHubConsumer, self).__init__() self.running = False self.client = client self.source = source @@ -70,17 +69,11 @@ def __init__(self, client, source, event_position=None, prefetch=300, owner_leve self.redirected = None self.error = None partition = self.source.split('/')[-1] - self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) + self.name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition) if owner_level: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} self._handler = None - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close(exc_val) - def __iter__(self): return self @@ -100,10 +93,6 @@ def __next__(self): self._handle_exception(exception, retry_count, max_retries) retry_count += 1 - def _check_closed(self): - if self.error: - raise EventHubError("This consumer has been closed. Please create a new consumer to receive event data.") - def _create_handler(self): alt_creds = { "username": self.client._auth_config.get("iot_username"), @@ -126,12 +115,10 @@ def _create_handler(self): self.messages_iter = None def _redirect(self, redirect): - self.redirected = redirect - self.running = False self.messages_iter = None - self._close_connection() + super(EventHubConsumer, self)._redirect(redirect) - def _open(self): + def _open(self, timeout_time=None): """ Open the EventHubConsumer using the supplied connection. If the handler has previously been redirected, the redirect @@ -139,34 +126,10 @@ def _open(self): """ # pylint: disable=protected-access - if not self.running: - if self.redirected: - self.client._process_redirect_uri(self.redirected) - self.source = self.redirected.address - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} - else: - alt_creds = {} - self._create_handler() - self._handler.open(connection=self.client._conn_manager.get_connection( - self.client.address.hostname, - self.client.get_auth(**alt_creds) - )) - while not self._handler.client_ready(): - time.sleep(0.05) - self.running = True - - def _close_handler(self): - self._handler.close() # close the link (sharing connection) or connection (not sharing) - self.running = False - - def _close_connection(self): - self._close_handler() - self.client._conn_manager.reset_connection_if_broken() - - def _handle_exception(self, exception, retry_count, max_retries): - _handle_exception(exception, retry_count, max_retries, self, log) + if not self.running and self.redirected: + self.client._process_redirect_uri(self.redirected) + self.source = self.redirected.address + super(EventHubConsumer, self)._open(timeout_time) @property def queue_size(self): @@ -211,24 +174,37 @@ def receive(self, max_batch_size=None, timeout=None): max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size timeout = self.client.config.receive_timeout if timeout is None else timeout + if not timeout: + timeout = 100_000 # timeout None or 0 mean no timeout. 100000 seconds is equivalent to no timeout data_batch = [] # type: List[EventData] + start_time = time.time() + timeout_time = start_time + timeout max_retries = self.client.config.max_retries retry_count = 0 + last_exception = None while True: try: - self._open() - timeout_ms = 1000 * timeout if timeout else 0 + self._open(timeout_time) + remaining_time = timeout_time - time.time() + if remaining_time <= 0.0: + if last_exception: + log.info("%r receive operation timed out. (%r)", self.name, last_exception) + raise last_exception + return data_batch + remaining_time_ms = 1000 * remaining_time message_batch = self._handler.receive_message_batch( max_batch_size=max_batch_size - (len(data_batch) if data_batch else 0), - timeout=timeout_ms) + timeout=remaining_time_ms) for message in message_batch: event_data = EventData(message=message) self.offset = EventPosition(event_data.offset) data_batch.append(event_data) return data_batch + except EventHubError: + raise except Exception as exception: - self._handle_exception(exception, retry_count, max_retries) + last_exception = self._handle_exception(exception, retry_count, max_retries, timeout_time) retry_count += 1 def close(self, exception=None): @@ -254,17 +230,6 @@ def close(self, exception=None): if self.messages_iter: self.messages_iter.close() self.messages_iter = None - self.running = False - if self.error: - return - if isinstance(exception, errors.LinkRedirect): - self.redirected = exception - elif isinstance(exception, EventHubError): - self.error = exception - elif exception: - self.error = EventHubError(str(exception)) - else: - self.error = EventHubError("This receive handler is now closed.") - self._handler.close() # this will close link if sharing connection. Otherwise close connection + super(EventHubConsumer, self).close(exception) next = __next__ # for python2.7 diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index db8ae4794e1e..650f6bacfec6 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import six +import time +import logging from uamqp import constants, errors, compat @@ -15,6 +17,7 @@ b"com.microsoft:argument-error" ) +log = logging.getLogger(__name__) def _error_handler(error): """ @@ -130,10 +133,45 @@ class EventDataSendError(EventHubError): pass -def _handle_exception(exception, retry_count, max_retries, closable, log): - type_name = type(closable).__name__ +class OperationTimeoutError(EventHubError): + """Operation times out + + """ + pass + + +def _create_eventhub_exception(exception): + if isinstance(exception, errors.AuthenticationException): + error = AuthenticationError(str(exception), exception) + elif isinstance(exception, errors.VendorLinkDetach): + error = ConnectError(str(exception), exception) + elif isinstance(exception, errors.LinkDetach): + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.ConnectionClose): + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.MessageHandlerError): + error = ConnectionLostError(str(exception), exception) + elif isinstance(exception, errors.AMQPConnectionError): + error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ + else ConnectError + error = error_type(str(exception), exception) + elif isinstance(exception, compat.TimeoutException): + error = ConnectionLostError(str(exception), exception) + else: + error = EventHubError(str(exception), exception) + return error + + +def _handle_exception(exception, retry_count, max_retries, closable, timeout_time): + try: + name = closable.name + except AttributeError: + name = closable.container_id if isinstance(exception, KeyboardInterrupt): - log.info("{} stops due to keyboard interrupt".format(type_name)) + log.info("%r stops due to keyboard interrupt", name) + closable.close() + raise + elif isinstance(exception, EventHubError): closable.close() raise elif isinstance(exception, ( @@ -144,51 +182,23 @@ def _handle_exception(exception, retry_count, max_retries, closable, log): errors.MessageReleased, errors.MessageContentTooLarge) ): - log.error("Event data error (%r)", exception) + log.info("%r Event data error (%r)", name, exception) error = EventDataError(str(exception), exception) - closable.close(exception) raise error elif isinstance(exception, errors.MessageException): - log.error("Event data send error (%r)", exception) + log.info("%r Event data send error (%r)", name, exception) error = EventDataSendError(str(exception), exception) - closable.close(exception) raise error elif retry_count >= max_retries: - log.info("{} has an error and has exhausted retrying. (%r)".format(type_name), exception) - if isinstance(exception, errors.AuthenticationException): - log.info("{} authentication failed. Shutting down.".format(type_name)) - error = AuthenticationError(str(exception), exception) - elif isinstance(exception, errors.VendorLinkDetach): - log.info("{} link detached. Shutting down.".format(type_name)) - error = ConnectError(str(exception), exception) - elif isinstance(exception, errors.LinkDetach): - log.info("{} link detached. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.ConnectionClose): - log.info("{} connection closed. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.MessageHandlerError): - log.info("{} detached. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - elif isinstance(exception, errors.AMQPConnectionError): - log.info("{} connection lost. Shutting down.".format(type_name)) - error_type = AuthenticationError if str(exception).startswith("Unable to open authentication session") \ - else ConnectError - error = error_type(str(exception), exception) - elif isinstance(exception, compat.TimeoutException): - log.info("{} timed out. Shutting down.".format(type_name)) - error = ConnectionLostError(str(exception), exception) - else: - log.error("Unexpected error occurred (%r). Shutting down.", exception) - error = EventHubError("Receive failed: {}".format(exception), exception) - closable.close() + error = _create_eventhub_exception(exception) + log.info("%r has exhausted retry. Exception still occurs (%r)", name, exception) raise error else: - log.info("{} has an exception (%r). Retrying...".format(type_name), exception) if isinstance(exception, errors.AuthenticationException): - closable._close_connection() + if hasattr(closable, "_close_connection"): + closable._close_connection() elif isinstance(exception, errors.LinkRedirect): - log.info("{} link redirected. Redirecting...".format(type_name)) + log.info("%r link redirect received. Redirecting...", name) redirect = exception if hasattr(closable, "_redirect"): closable._redirect(redirect) @@ -209,3 +219,20 @@ def _handle_exception(exception, retry_count, max_retries, closable, log): else: if hasattr(closable, "_close_connection"): closable._close_connection() + # start processing retry delay + try: + backoff_factor = closable.client.config.backoff_factor + backoff_max = closable.client.config.backoff_max + except AttributeError: + backoff_factor = closable.config.backoff_factor + backoff_max = closable.config.backoff_max + backoff = backoff_factor * 2 ** retry_count + if backoff <= backoff_max and time.time() + backoff <= timeout_time: + time.sleep(backoff) + log.info("%r has an exception (%r). Retrying...", format(name), exception) + return _create_eventhub_exception(exception) + else: + error = _create_eventhub_exception(exception) + log.info("%r operation has timed out. Last exception before timeout is (%r)", name, error) + raise error + # end of processing retry delay diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index 3f3ff4c9492e..956da690801c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -14,13 +14,13 @@ from uamqp import SendClient from azure.eventhub.common import EventData, _BatchSendEventData -from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler, _handle_exception +from azure.eventhub.error import OperationTimeoutError, _error_handler +from ._consumer_producer_mixin import ConsumerProducerMixin log = logging.getLogger(__name__) -class EventHubProducer(object): +class EventHubProducer(ConsumerProducerMixin): """ A producer responsible for transmitting EventData to a specific Event Hub, grouped together in batches. Depending on the options specified at creation, the producer may @@ -51,6 +51,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N Default value is `True`. :type auto_reconnect: bool """ + super(EventHubProducer, self).__init__() self.running = False self.client = client self.target = target @@ -71,12 +72,6 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N self._outcome = None self._condition = None - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close(exc_val) - def _create_handler(self): self._handler = SendClient( self.target, @@ -88,12 +83,7 @@ def _create_handler(self): client_name=self.name, properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access - def _redirect(self, redirect): - self.redirected = redirect - self.running = False - self._close_connection() - - def _open(self): + def _open(self, timeout_time=None): """ Open the EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect @@ -101,50 +91,42 @@ def _open(self): """ # pylint: disable=protected-access - if not self.running: - if self.redirected: - self.target = self.redirected.address - self._create_handler() - self._handler.open(connection=self.client._conn_manager.get_connection( - self.client.address.hostname, - self.client.get_auth() - )) - while not self._handler.client_ready(): - time.sleep(0.05) - self.running = True - - def _close_handler(self): - self._handler.close() # close the link (sharing connection) or connection (not sharing) - self.running = False - - def _close_connection(self): - self._close_handler() - self.client._conn_manager.reset_connection_if_broken() - - def _handle_exception(self, exception, retry_count, max_retries): - _handle_exception(exception, retry_count, max_retries, self, log) - - def _send_event_data(self): + if not self.running and self.redirected: + self.client._process_redirect_uri(self.redirected) + self.target = self.redirected.address + super(EventHubProducer, self)._open() + + def _send_event_data(self, timeout=None): + timeout = self.client.config.send_timeout if timeout is None else timeout + if not timeout: + timeout = 100_000 # timeout None or 0 mean no timeout. 100000 seconds is equivalent to no timeout + start_time = time.time() + timeout_time = start_time + timeout max_retries = self.client.config.max_retries retry_count = 0 + last_exception = None while True: try: if self.unsent_events: - self._open() + self._open(timeout_time) + remaining_time = timeout_time - time.time() + if remaining_time <= 0.0: + if last_exception: + error = last_exception + else: + error = OperationTimeoutError("send operation timed out") + log.info("%r send operation timed out. (%r)", self.name, error) + raise error self._handler.queue_message(*self.unsent_events) self._handler.wait() self.unsent_events = self._handler.pending_messages - if self._outcome != constants.MessageSendResult.Ok: - _error(self._outcome, self._condition) + if self._outcome != constants.MessageSendResult.Ok: + _error(self._outcome, self._condition) return except Exception as exception: - self._handle_exception(exception, retry_count, max_retries) + last_exception = self._handle_exception(exception, retry_count, max_retries, timeout_time) retry_count += 1 - def _check_closed(self): - if self.error: - raise EventHubError("This producer has been closed. Please create a new producer to send event data.") - def _on_outcome(self, outcome, condition): """ Called when the outcome is received for a delivery. @@ -157,8 +139,8 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - def send(self, event_data, partition_key=None): - # type:(Union[EventData, Iterable[EventData]], Union[str, bytes]) -> None + def send(self, event_data, partition_key=None, timeout=None): + # type:(Union[EventData, Iterable[EventData]], Union[str, bytes], float) -> None """ Sends an event data and blocks until acknowledgement is received or operation times out. @@ -195,7 +177,7 @@ def send(self, event_data, partition_key=None): partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome self.unsent_events = [wrapper_event_data.message] - self._send_event_data() + self._send_event_data(timeout=timeout) def close(self, exception=None): # type:(Exception) -> None @@ -217,18 +199,7 @@ def close(self, exception=None): :caption: Close down the handler. """ - self.running = False - if self.error: - return - if isinstance(exception, errors.LinkRedirect): - self.redirected = exception - elif isinstance(exception, EventHubError): - self.error = exception - elif exception: - self.error = EventHubError(str(exception)) - else: - self.error = EventHubError("This send handler is now closed.") - self._handler.close() + super(EventHubProducer, self).close(exception) def _error(outcome, condition): From 4f1778105765a1603a3b5bae006f7537389d5113 Mon Sep 17 00:00:00 2001 From: yijxie Date: Sun, 21 Jul 2019 23:32:07 -0700 Subject: [PATCH 07/11] put module method before class def --- .../azure/eventhub/producer.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index 956da690801c..a6746ee1aff5 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -20,6 +20,18 @@ log = logging.getLogger(__name__) +def _error(outcome, condition): + if outcome != constants.MessageSendResult.Ok: + raise condition + + +def _set_partition_key(event_datas, partition_key): + ed_iter = iter(event_datas) + for ed in ed_iter: + ed._set_partition_key(partition_key) + yield ed + + class EventHubProducer(ConsumerProducerMixin): """ A producer responsible for transmitting EventData to a specific Event Hub, @@ -200,15 +212,3 @@ def close(self, exception=None): """ super(EventHubProducer, self).close(exception) - - -def _error(outcome, condition): - if outcome != constants.MessageSendResult.Ok: - raise condition - - -def _set_partition_key(event_datas, partition_key): - ed_iter = iter(event_datas) - for ed in ed_iter: - ed._set_partition_key(partition_key) - yield ed From f0d98d1eb5f9b6580a0c4f48f471bb87eddfc3d1 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 22 Jul 2019 11:38:49 -0700 Subject: [PATCH 08/11] fixed Client.get_properties error --- .../azure-eventhubs/azure/eventhub/aio/client_async.py | 2 +- .../azure-eventhubs/azure/eventhub/aio/error_async.py | 4 ++-- sdk/eventhub/azure-eventhubs/azure/eventhub/client.py | 2 +- sdk/eventhub/azure-eventhubs/azure/eventhub/error.py | 4 ++-- sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py index 513e13e4f75e..20462a7753f4 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py @@ -98,7 +98,7 @@ def _create_auth(self, username=None, password=None): transport_type=transport_type) async def _handle_exception(self, exception, retry_count, max_retries): - await _handle_exception(exception, retry_count, max_retries, self, log) + await _handle_exception(exception, retry_count, max_retries, self) async def _close_connection(self): await self._conn_manager.reset_connection_if_broken() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py index d78233c1f896..957a3005662e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py @@ -32,7 +32,7 @@ def _create_eventhub_exception(exception): return error -async def _handle_exception(exception, retry_count, max_retries, closable, timeout_time): +async def _handle_exception(exception, retry_count, max_retries, closable, timeout_time=None): try: name = closable.name except AttributeError: @@ -97,7 +97,7 @@ async def _handle_exception(exception, retry_count, max_retries, closable, timeo backoff_factor = closable.config.backoff_factor backoff_max = closable.config.backoff_max backoff = backoff_factor * 2 ** retry_count - if backoff <= backoff_max and time.time() + backoff <= timeout_time: + if backoff <= backoff_max and (timeout_time is None or time.time() + backoff <= timeout_time): await asyncio.sleep(backoff) log.info("%r has an exception (%r). Retrying...", format(name), exception) return _create_eventhub_exception(exception) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 5de8f8531093..b7fad40779c9 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -104,7 +104,7 @@ def _create_auth(self, username=None, password=None): transport_type=transport_type) def _handle_exception(self, exception, retry_count, max_retries): - _handle_exception(exception, retry_count, max_retries, self, log) + _handle_exception(exception, retry_count, max_retries, self) def _close_connection(self): self._conn_manager.reset_connection_if_broken() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index 650f6bacfec6..31f456f84eb8 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -162,7 +162,7 @@ def _create_eventhub_exception(exception): return error -def _handle_exception(exception, retry_count, max_retries, closable, timeout_time): +def _handle_exception(exception, retry_count, max_retries, closable, timeout_time=None): try: name = closable.name except AttributeError: @@ -227,7 +227,7 @@ def _handle_exception(exception, retry_count, max_retries, closable, timeout_tim backoff_factor = closable.config.backoff_factor backoff_max = closable.config.backoff_max backoff = backoff_factor * 2 ** retry_count - if backoff <= backoff_max and time.time() + backoff <= timeout_time: + if backoff <= backoff_max and (timeout_time is None or time.time() + backoff <= timeout_time): time.sleep(backoff) log.info("%r has an exception (%r). Retrying...", format(name), exception) return _create_eventhub_exception(exception) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index a6746ee1aff5..0a839463e4fb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -106,7 +106,7 @@ def _open(self, timeout_time=None): if not self.running and self.redirected: self.client._process_redirect_uri(self.redirected) self.target = self.redirected.address - super(EventHubProducer, self)._open() + super(EventHubProducer, self)._open(timeout_time) def _send_event_data(self, timeout=None): timeout = self.client.config.send_timeout if timeout is None else timeout From df1a4909f2b26d80a9c3cd2ad00eb80658d18841 Mon Sep 17 00:00:00 2001 From: Yunhao Ling <47871814+yunhaoling@users.noreply.github.com> Date: Wed, 24 Jul 2019 16:02:05 -0700 Subject: [PATCH 09/11] Improve send timeout (#6481) --- .../azure/eventhub/aio/producer_async.py | 10 ++++++++-- .../azure-eventhubs/azure/eventhub/producer.py | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index 96d60ffc8943..9541fcc0efa2 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -122,11 +122,14 @@ async def _send_event_data(self, timeout=None): error = OperationTimeoutError("send operation timed out") log.info("%r send operation timed out. (%r)", self.name, error) raise error + self._handler._msg_timeout = remaining_time # pylint: disable=protected-access self._handler.queue_message(*self.unsent_events) await self._handler.wait_async() self.unsent_events = self._handler.pending_messages - if self._outcome != constants.MessageSendResult.Ok: - _error(self._outcome, self._condition) + if self._outcome != constants.MessageSendResult.Ok: + if self._outcome == constants.MessageSendResult.Timeout: + self._condition = OperationTimeoutError("send operation timed out") + _error(self._outcome, self._condition) return except Exception as exception: last_exception = await self._handle_exception(exception, retry_count, max_retries, timeout_time) @@ -155,6 +158,9 @@ async def send(self, event_data, partition_key=None, timeout=None): :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. :type partition_key: str + :param timeout: The maximum wait time to send the event data. + If not specified, the default wait time specified when the producer was created will be used. + :type timeout:float :raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError, ~azure.eventhub.EventDataError, ~azure.eventhub.EventDataSendError, ~azure.eventhub.EventHubError :return: None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index 0a839463e4fb..9ea370bc6aef 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -129,10 +129,13 @@ def _send_event_data(self, timeout=None): error = OperationTimeoutError("send operation timed out") log.info("%r send operation timed out. (%r)", self.name, error) raise error + self._handler._msg_timeout = remaining_time # pylint: disable=protected-access self._handler.queue_message(*self.unsent_events) self._handler.wait() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: + if self._outcome == constants.MessageSendResult.Timeout: + self._condition = OperationTimeoutError("send operation timed out") _error(self._outcome, self._condition) return except Exception as exception: @@ -162,6 +165,9 @@ def send(self, event_data, partition_key=None, timeout=None): :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. :type partition_key: str + :param timeout: The maximum wait time to send the event data. + If not specified, the default wait time specified when the producer was created will be used. + :type timeout:float :raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError, ~azure.eventhub.EventDataError, ~azure.eventhub.EventDataSendError, ~azure.eventhub.EventHubError From 365f1d0abbea520f6662b9b7ff09db8f2b1657c7 Mon Sep 17 00:00:00 2001 From: Yunhao Ling <47871814+yunhaoling@users.noreply.github.com> Date: Wed, 24 Jul 2019 20:46:39 -0700 Subject: [PATCH 10/11] Add timeout information to the link prop during link creation process (#6485) --- .../azure-eventhubs/azure/eventhub/aio/consumer_async.py | 9 ++++++--- .../azure-eventhubs/azure/eventhub/aio/producer_async.py | 5 ++++- sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py | 9 ++++++--- sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py | 5 ++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py index acd20181845f..5cc8df6941a0 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py @@ -35,6 +35,7 @@ class EventHubConsumer(ConsumerProducerMixin): """ timeout = 0 _epoch = b'com.microsoft:epoch' + _timeout = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called self, client, source, event_position=None, prefetch=300, owner_level=None, @@ -72,11 +73,13 @@ def __init__( # pylint: disable=super-init-not-called self.reconnect_backoff = 1 self.redirected = None self.error = None - self.properties = None + self._link_properties = {} partition = self.source.split('/')[-1] self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) if owner_level: - self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} + self._link_properties[types.AMQPSymbol(self._epoch)] = types.AMQPLong(int(owner_level)) + link_property_timeout_ms = (self.client.config.receive_timeout or self.timeout) * 1000 + self._link_properties[types.AMQPSymbol(self._timeout)] = types.AMQPLong(int(link_property_timeout_ms)) self._handler = None def __aiter__(self): @@ -110,7 +113,7 @@ def _create_handler(self): auth=self.client.get_auth(**alt_creds), debug=self.client.config.network_tracing, prefetch=self.prefetch, - link_properties=self.properties, + link_properties=self._link_properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index 9541fcc0efa2..16ddb97a36b9 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -8,7 +8,7 @@ from typing import Iterable, Union import time -from uamqp import constants, errors +from uamqp import types, constants, errors from uamqp import SendClientAsync from azure.eventhub.common import EventData, _BatchSendEventData @@ -28,6 +28,7 @@ class EventHubProducer(ConsumerProducerMixin): to a partition. """ + _timeout = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called self, client, target, partition=None, send_timeout=60, @@ -75,6 +76,7 @@ def __init__( # pylint: disable=super-init-not-called self._handler = None self._outcome = None self._condition = None + self._link_properties = {types.AMQPSymbol(self._timeout): types.AMQPLong(int(self.timeout * 1000))} def _create_handler(self): self._handler = SendClientAsync( @@ -85,6 +87,7 @@ def _create_handler(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, + link_properties=self._link_properties, properties=self.client._create_properties( self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index 95cdeedd43aa..53368dc292b0 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -35,6 +35,7 @@ class EventHubConsumer(ConsumerProducerMixin): """ timeout = 0 _epoch = b'com.microsoft:epoch' + _timeout = b'com.microsoft:timeout' def __init__(self, client, source, event_position=None, prefetch=300, owner_level=None, keep_alive=None, auto_reconnect=True): @@ -65,13 +66,15 @@ def __init__(self, client, source, event_position=None, prefetch=300, owner_leve self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 - self.properties = None + self._link_properties = {} self.redirected = None self.error = None partition = self.source.split('/')[-1] self.name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition) if owner_level: - self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))} + self._link_properties[types.AMQPSymbol(self._epoch)] = types.AMQPLong(int(owner_level)) + link_property_timeout_ms = (self.client.config.receive_timeout or self.timeout) * 1000 + self._link_properties[types.AMQPSymbol(self._timeout)] = types.AMQPLong(int(link_property_timeout_ms)) self._handler = None def __iter__(self): @@ -105,7 +108,7 @@ def _create_handler(self): auth=self.client.get_auth(**alt_creds), debug=self.client.config.network_tracing, prefetch=self.prefetch, - link_properties=self.properties, + link_properties=self._link_properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index 9ea370bc6aef..d85a965381ad 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -9,7 +9,7 @@ import time from typing import Iterable, Union -from uamqp import constants, errors +from uamqp import types, constants, errors from uamqp import compat from uamqp import SendClient @@ -40,6 +40,7 @@ class EventHubProducer(ConsumerProducerMixin): to a partition. """ + _timeout = b'com.microsoft:timeout' def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True): """ @@ -83,6 +84,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N self._handler = None self._outcome = None self._condition = None + self._link_properties = {types.AMQPSymbol(self._timeout): types.AMQPLong(int(self.timeout * 1000))} def _create_handler(self): self._handler = SendClient( @@ -93,6 +95,7 @@ def _create_handler(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, + link_properties=self._link_properties, properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access def _open(self, timeout_time=None): From 90dd7c1698a4e30d7b82ddb8d4c1023ce2e88ffd Mon Sep 17 00:00:00 2001 From: Yunhao Ling <47871814+yunhaoling@users.noreply.github.com> Date: Fri, 26 Jul 2019 16:28:00 -0700 Subject: [PATCH 11/11] Update optional parameters in public api into kwargs and update comments (#6510) * Update optional parameters in public api into kwargs and update some comments * Update more optional parameters into kwargs paramter --- .../azure/eventhub/_connection_manager.py | 2 +- .../azure/eventhub/aio/client_async.py | 16 +++++++--- .../azure/eventhub/aio/consumer_async.py | 30 ++++++++++++------- .../azure/eventhub/aio/producer_async.py | 23 +++++++++----- .../azure-eventhubs/azure/eventhub/client.py | 14 ++++++--- .../azure/eventhub/client_abstract.py | 9 +++--- .../azure-eventhubs/azure/eventhub/common.py | 15 +++++++--- .../azure/eventhub/consumer.py | 28 ++++++++++------- .../azure-eventhubs/azure/eventhub/error.py | 4 ++- .../azure/eventhub/producer.py | 21 +++++++++---- 10 files changed, 110 insertions(+), 52 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py index 9600226df2fb..166703d698e7 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/_connection_manager.py @@ -74,4 +74,4 @@ def reset_connection_if_broken(self): def get_connection_manager(**kwargs): - return _SharedConnectionManager(**kwargs) + return _SeparateConnectionManager(**kwargs) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py index 20462a7753f4..17479119ccad 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py @@ -194,8 +194,7 @@ async def get_partition_properties(self, partition): return output def create_consumer( - self, consumer_group, partition_id, event_position, owner_level=None, - operation=None, prefetch=None, loop=None): + self, consumer_group, partition_id, event_position, **kwargs): # type: (str, str, EventPosition, int, str, int, asyncio.AbstractEventLoop) -> EventHubConsumer """ Create an async consumer to the client for a particular consumer group and partition. @@ -227,8 +226,12 @@ def create_consumer( :caption: Add an async consumer to the client for a particular consumer group and partition. """ - prefetch = self.config.prefetch if prefetch is None else prefetch + owner_level = kwargs.get("owner_level", None) + operation = kwargs.get("operation", None) + prefetch = kwargs.get("prefetch", None) + loop = kwargs.get("loop", None) + prefetch = prefetch or self.config.prefetch path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) @@ -238,7 +241,7 @@ def create_consumer( return handler def create_producer( - self, partition_id=None, operation=None, send_timeout=None, loop=None): + self, **kwargs): # type: (str, str, float, asyncio.AbstractEventLoop) -> EventHubProducer """ Create an async producer to send EventData object to an EventHub. @@ -265,6 +268,11 @@ def create_producer( :caption: Add an async producer to the client to send EventData. """ + partition_id = kwargs.get("partition_id", None) + operation = kwargs.get("operation", None) + send_timeout = kwargs.get("send_timeout", None) + loop = kwargs.get("loop", None) + target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py index 5cc8df6941a0..d4d4143810af 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py @@ -22,15 +22,15 @@ class EventHubConsumer(ConsumerProducerMixin): """ A consumer responsible for reading EventData from a specific Event Hub - partition and as a member of a specific consumer group. + partition and as a member of a specific consumer group. A consumer may be exclusive, which asserts ownership over the partition for the consumer - group to ensure that only one consumer from that group is reading the from the partition. - These exclusive consumers are sometimes referred to as "Epoch Consumers." + group to ensure that only one consumer from that group is reading the from the partition. + These exclusive consumers are sometimes referred to as "Epoch Consumers." A consumer may also be non-exclusive, allowing multiple consumers from the same consumer - group to be actively reading events from the partition. These non-exclusive consumers are - sometimes referred to as "Non-Epoch Consumers." + group to be actively reading events from the partition. These non-exclusive consumers are + sometimes referred to as "Non-Epoch Consumers." """ timeout = 0 @@ -38,8 +38,7 @@ class EventHubConsumer(ConsumerProducerMixin): _timeout = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called - self, client, source, event_position=None, prefetch=300, owner_level=None, - keep_alive=None, auto_reconnect=True, loop=None): + self, client, source, **kwargs): """ Instantiate an async consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method in EventHubClient. @@ -58,6 +57,13 @@ def __init__( # pylint: disable=super-init-not-called :type owner_level: int :param loop: An event loop. """ + event_position = kwargs.get("event_position", None) + prefetch = kwargs.get("prefetch", 300) + owner_level = kwargs.get("owner_level", None) + keep_alive = kwargs.get("keep_alive", None) + auto_reconnect = kwargs.get("auto_reconnect", True) + loop = kwargs.get("loop", None) + super(EventHubConsumer, self).__init__() self.loop = loop or asyncio.get_event_loop() self.running = False @@ -153,7 +159,7 @@ def queue_size(self): return self._handler._received_messages.qsize() return 0 - async def receive(self, max_batch_size=None, timeout=None): + async def receive(self, **kwargs): # type: (int, float) -> List[EventData] """ Receive events asynchronously from the EventHub. @@ -180,6 +186,9 @@ async def receive(self, max_batch_size=None, timeout=None): :caption: Receives events asynchronously """ + max_batch_size = kwargs.get("max_batch_size", None) + timeout = kwargs.get("timeout", None) + self._check_closed() max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size timeout = self.client.config.receive_timeout if timeout is None else timeout @@ -217,7 +226,7 @@ async def receive(self, max_batch_size=None, timeout=None): last_exception = await self._handle_exception(exception, retry_count, max_retries, timeout_time) retry_count += 1 - async def close(self, exception=None): + async def close(self, **kwargs): # type: (Exception) -> None """ Close down the handler. If the handler has already closed, @@ -237,6 +246,7 @@ async def close(self, exception=None): :caption: Close down the handler. """ + exception = kwargs.get("exception", None) self.running = False if self.error: return @@ -250,4 +260,4 @@ async def close(self, exception=None): self.error = EventHubError(str(exception)) else: self.error = EventHubError("This receive handler is now closed.") - await self._handler.close_async() \ No newline at end of file + await self._handler.close_async() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index 16ddb97a36b9..9612b4156327 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -23,16 +23,15 @@ class EventHubProducer(ConsumerProducerMixin): """ A producer responsible for transmitting EventData to a specific Event Hub, - grouped together in batches. Depending on the options specified at creation, the producer may - be created to allow event data to be automatically routed to an available partition or specific - to a partition. + grouped together in batches. Depending on the options specified at creation, the producer may + be created to allow event data to be automatically routed to an available partition or specific + to a partition. """ _timeout = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called - self, client, target, partition=None, send_timeout=60, - keep_alive=None, auto_reconnect=True, loop=None): + self, client, target, **kwargs): """ Instantiate an async EventHubProducer. EventHubProducer should be instantiated by calling the `create_producer` method in EventHubClient. @@ -55,6 +54,12 @@ def __init__( # pylint: disable=super-init-not-called :type auto_reconnect: bool :param loop: An event loop. If not specified the default event loop will be used. """ + partition = kwargs.get("partition", None) + send_timeout = kwargs.get("send_timeout", 60) + keep_alive = kwargs.get("keep_alive", None) + auto_reconnect = kwargs.get("auto_reconnect", True) + loop = kwargs.get("loop", None) + super(EventHubProducer, self).__init__() self.loop = loop or asyncio.get_event_loop() self.running = False @@ -150,7 +155,7 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - async def send(self, event_data, partition_key=None, timeout=None): + async def send(self, event_data, **kwargs): # type:(Union[EventData, Iterable[EventData]], Union[str, bytes]) -> None """ Sends an event data and blocks until acknowledgement is @@ -178,6 +183,9 @@ async def send(self, event_data, partition_key=None, timeout=None): :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ + partition_key = kwargs.get("partition_key", None) + timeout = kwargs.get("timeout", None) + self._check_closed() if isinstance(event_data, EventData): if partition_key: @@ -192,7 +200,7 @@ async def send(self, event_data, partition_key=None, timeout=None): self.unsent_events = [wrapper_event_data.message] await self._send_event_data(timeout) - async def close(self, exception=None): + async def close(self, **kwargs): # type: (Exception) -> None """ Close down the handler. If the handler has already closed, @@ -212,4 +220,5 @@ async def close(self, exception=None): :caption: Close down the handler. """ + exception = kwargs.get("exception", None) await super(EventHubProducer, self).close(exception) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index b7fad40779c9..deda0ddc01fb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -200,8 +200,7 @@ def get_partition_properties(self, partition): return output def create_consumer( - self, consumer_group, partition_id, event_position, - owner_level=None, operation=None, prefetch=None, + self, consumer_group, partition_id, event_position, **kwargs ): # type: (str, str, EventPosition, int, str, int) -> EventHubConsumer """ @@ -233,8 +232,11 @@ def create_consumer( :caption: Add a consumer to the client for a particular consumer group and partition. """ - prefetch = self.config.prefetch if prefetch is None else prefetch + owner_level = kwargs.get("owner_level", None) + operation = kwargs.get("operation", None) + prefetch = kwargs.get("prefetch", None) + prefetch = prefetch or self.config.prefetch path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) @@ -243,7 +245,7 @@ def create_consumer( prefetch=prefetch) return handler - def create_producer(self, partition_id=None, operation=None, send_timeout=None): + def create_producer(self, **kwargs): # type: (str, str, float) -> EventHubProducer """ Create an producer to send EventData object to an EventHub. @@ -269,6 +271,10 @@ def create_producer(self, partition_id=None, operation=None, send_timeout=None): :caption: Add a producer to the client to send EventData. """ + partition_id = kwargs.get("partition_id", None) + operation = kwargs.get("operation", None) + send_timeout = kwargs.get("send_timeout", None) + target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 38e2afde2615..7f6afb51c7fc 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -219,7 +219,7 @@ def _process_redirect_uri(self, redirect): self.mgmt_target = redirect_uri @classmethod - def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): + def from_connection_string(cls, conn_str, **kwargs): """Create an EventHubClient from an EventHub/IotHub connection string. :param conn_str: The connection string of an eventhub or IoT hub @@ -266,6 +266,7 @@ def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): :caption: Create an EventHubClient from a connection string. """ + event_hub_path = kwargs.get("event_hub_path", None) is_iot_conn_str = conn_str.lstrip().lower().startswith("hostname") if not is_iot_conn_str: address, policy, key, entity = _parse_conn_str(conn_str) @@ -281,12 +282,10 @@ def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): @abstractmethod def create_consumer( - self, consumer_group, partition_id, event_position, owner_level=None, - operation=None, - prefetch=None, + self, consumer_group, partition_id, event_position, **kwargs ): pass @abstractmethod - def create_producer(self, partition_id=None, operation=None, send_timeout=None): + def create_producer(self, **kwargs): pass diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 5ac6258eeb0a..8faded746a74 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -51,7 +51,7 @@ class EventData(object): PROP_TIMESTAMP = b"x-opt-enqueued-time" PROP_DEVICE_ID = b"iothub-connection-device-id" - def __init__(self, body=None, to_device=None, message=None): + def __init__(self, **kwargs): """ Initialize EventData. @@ -64,6 +64,10 @@ def __init__(self, body=None, to_device=None, message=None): :param message: The received message. :type message: ~uamqp.message.Message """ + body = kwargs.get("body", None) + to_device = kwargs.get("to_device", None) + message = kwargs.get("message", None) + self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) self._annotations = {} self._app_properties = {} @@ -206,7 +210,7 @@ def body(self): except TypeError: raise ValueError("Message data empty.") - def body_as_str(self, encoding='UTF-8'): + def body_as_str(self, **kwargs): """ The body of the event data as a string if the data is of a compatible type. @@ -215,6 +219,7 @@ def body_as_str(self, encoding='UTF-8'): Default is 'UTF-8' :rtype: str or unicode """ + encoding = kwargs.get("encoding", 'UTF-8') data = self.body try: return "".join(b.decode(encoding) for b in data) @@ -227,7 +232,7 @@ def body_as_str(self, encoding='UTF-8'): except Exception as e: raise TypeError("Message data is not compatible with string type: {}".format(e)) - def body_as_json(self, encoding='UTF-8'): + def body_as_json(self, **kwargs): """ The body of the event loaded as a JSON object is the data is compatible. @@ -235,6 +240,7 @@ def body_as_json(self, encoding='UTF-8'): Default is 'UTF-8' :rtype: dict """ + encoding = kwargs.get("encoding", 'UTF-8') data_str = self.body_as_str(encoding=encoding) try: return json.loads(data_str) @@ -280,7 +286,7 @@ class EventPosition(object): >>> event_pos = EventPosition(1506968696002) """ - def __init__(self, value, inclusive=False): + def __init__(self, value, **kwargs): """ Initialize EventPosition. @@ -289,6 +295,7 @@ def __init__(self, value, inclusive=False): :param inclusive: Whether to include the supplied value as the start point. :type inclusive: bool """ + inclusive = kwargs.get("inclusive", False) self.value = value if value is not None else "-1" self.inclusive = inclusive diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index 53368dc292b0..855e6cf97d53 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -22,23 +22,22 @@ class EventHubConsumer(ConsumerProducerMixin): """ A consumer responsible for reading EventData from a specific Event Hub - partition and as a member of a specific consumer group. + partition and as a member of a specific consumer group. A consumer may be exclusive, which asserts ownership over the partition for the consumer - group to ensure that only one consumer from that group is reading the from the partition. - These exclusive consumers are sometimes referred to as "Epoch Consumers." + group to ensure that only one consumer from that group is reading the from the partition. + These exclusive consumers are sometimes referred to as "Epoch Consumers." A consumer may also be non-exclusive, allowing multiple consumers from the same consumer - group to be actively reading events from the partition. These non-exclusive consumers are - sometimes referred to as "Non-Epoch Consumers." + group to be actively reading events from the partition. These non-exclusive consumers are + sometimes referred to as "Non-Epoch Consumers." """ timeout = 0 _epoch = b'com.microsoft:epoch' _timeout = b'com.microsoft:timeout' - def __init__(self, client, source, event_position=None, prefetch=300, owner_level=None, - keep_alive=None, auto_reconnect=True): + def __init__(self, client, source, **kwargs): """ Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method in EventHubClient. @@ -54,6 +53,12 @@ def __init__(self, client, source, event_position=None, prefetch=300, owner_leve consumer if owner_level is set. :type owner_level: int """ + event_position = kwargs.get("event_position", None) + prefetch = kwargs.get("prefetch", 300) + owner_level = kwargs.get("owner_level", None) + keep_alive = kwargs.get("keep_alive", None) + auto_reconnect = kwargs.get("auto_reconnect", True) + super(EventHubConsumer, self).__init__() self.running = False self.client = client @@ -147,7 +152,7 @@ def queue_size(self): return self._handler._received_messages.qsize() return 0 - def receive(self, max_batch_size=None, timeout=None): + def receive(self, **kwargs): # type:(int, float) -> List[EventData] """ Receive events from the EventHub. @@ -173,8 +178,10 @@ def receive(self, max_batch_size=None, timeout=None): :caption: Receive events from the EventHub. """ - self._check_closed() + max_batch_size = kwargs.get("max_batch_size", None) + timeout = kwargs.get("timeout", None) + self._check_closed() max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size timeout = self.client.config.receive_timeout if timeout is None else timeout if not timeout: @@ -210,7 +217,7 @@ def receive(self, max_batch_size=None, timeout=None): last_exception = self._handle_exception(exception, retry_count, max_retries, timeout_time) retry_count += 1 - def close(self, exception=None): + def close(self, **kwargs): # type:(Exception) -> None """ Close down the handler. If the handler has already closed, @@ -230,6 +237,7 @@ def close(self, exception=None): :caption: Close down the handler. """ + exception = kwargs.get("exception", None) if self.messages_iter: self.messages_iter.close() self.messages_iter = None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index 31f456f84eb8..0fb6933e3015 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -19,6 +19,7 @@ log = logging.getLogger(__name__) + def _error_handler(error): """ Called internally when an event has failed to send so we @@ -56,7 +57,8 @@ class EventHubError(Exception): :vartype details: dict[str, str] """ - def __init__(self, message, details=None): + def __init__(self, message, **kwargs): + details = kwargs.get("details", None) self.error = None self.message = message self.details = details diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index d85a965381ad..465fbf45d9ef 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -35,14 +35,14 @@ def _set_partition_key(event_datas, partition_key): class EventHubProducer(ConsumerProducerMixin): """ A producer responsible for transmitting EventData to a specific Event Hub, - grouped together in batches. Depending on the options specified at creation, the producer may - be created to allow event data to be automatically routed to an available partition or specific - to a partition. + grouped together in batches. Depending on the options specified at creation, the producer may + be created to allow event data to be automatically routed to an available partition or specific + to a partition. """ _timeout = b'com.microsoft:timeout' - def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True): + def __init__(self, client, target, **kwargs): """ Instantiate an EventHubProducer. EventHubProducer should be instantiated by calling the `create_producer` method in EventHubClient. @@ -64,6 +64,11 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N Default value is `True`. :type auto_reconnect: bool """ + partition = kwargs.get("partition", None) + send_timeout = kwargs.get("send_timeout", 60) + keep_alive = kwargs.get("keep_alive", None) + auto_reconnect = kwargs.get("auto_reconnect", True) + super(EventHubProducer, self).__init__() self.running = False self.client = client @@ -157,7 +162,7 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - def send(self, event_data, partition_key=None, timeout=None): + def send(self, event_data, **kwargs): # type:(Union[EventData, Iterable[EventData]], Union[str, bytes], float) -> None """ Sends an event data and blocks until acknowledgement is @@ -186,6 +191,9 @@ def send(self, event_data, partition_key=None, timeout=None): :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ + partition_key = kwargs.get("partition_key", None) + timeout = kwargs.get("timeout", None) + self._check_closed() if isinstance(event_data, EventData): if partition_key: @@ -200,7 +208,7 @@ def send(self, event_data, partition_key=None, timeout=None): self.unsent_events = [wrapper_event_data.message] self._send_event_data(timeout=timeout) - def close(self, exception=None): + def close(self, **kwargs): # type:(Exception) -> None """ Close down the handler. If the handler has already closed, @@ -220,4 +228,5 @@ def close(self, exception=None): :caption: Close down the handler. """ + exception = kwargs.get("exception", None) super(EventHubProducer, self).close(exception)