From 520c4c10a564c24ecada99d31517545d919e50d0 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Sat, 7 Sep 2019 20:18:04 -0700 Subject: [PATCH 1/4] Small fix --- sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py index e737ee6889d7..93e7e85b4287 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py @@ -61,12 +61,12 @@ def send(sender, args): total += 1 except ValueError: sender.send(batch, timeout=0) - print("Sent total {} of partition {}".format(total, sender.partition)) + print("Sent total {} of partition {}".format(total, sender._partition)) batch = sender.create_batch() except Exception as err: - print("Partition {} send failed {}".format(sender.partition, err)) + print("Partition {} send failed {}".format(sender._partition, err)) raise - print("Sent total {} of partition {}".format(total, sender.partition)) + print("Sent total {} of partition {}".format(total, sender._partition)) @pytest.mark.liveTest From 6dd1c1c7edce4e469ef41b8b81aa406d9adc40b0 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Mon, 9 Sep 2019 15:51:28 -0700 Subject: [PATCH 2/4] Update history and readme --- sdk/eventhub/azure-eventhubs/HISTORY.md | 21 +++++++++++++-------- sdk/eventhub/azure-eventhubs/README.md | 6 +----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/sdk/eventhub/azure-eventhubs/HISTORY.md b/sdk/eventhub/azure-eventhubs/HISTORY.md index a2dcdb1e5b55..4bdb2c8ab671 100644 --- a/sdk/eventhub/azure-eventhubs/HISTORY.md +++ b/sdk/eventhub/azure-eventhubs/HISTORY.md @@ -2,17 +2,22 @@ ## 5.0.0b3 (2019-09-10) **New features** -- `EventProcessor` has a load balancer that balances load among multiple EventProcessors automatically -- A new `PartitionManager` implementation that uses Azure Blob Storage is added -to centrally store the checkpoint data for event processors. It's packaged separately as a plug-in to this package. -Refer to [Azure Blob Storage Partition Manager](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio) for details. + +- Added support for automatic load balancing among multiple `EventProcessor`. +- Added `BlobPartitionManager` which implements `PartitionManager`. + - Azure Blob Storage is applied for storing data used by `EventProcessor`. + - Packaged separately as a plug-in to `EventProcessor`. + - For details, please refer to [Azure Blob Storage Partition Manager](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio). +- Added property `system_properties` on `EventData`. **Breaking changes** -- `PartitionProcessor` constructor removed argument "checkpoint_manager". Its methods (initialize, process_events, -process_error, close) added argument "partition_context", which has method update_checkpoint. -- `CheckpointManager` was replaced by `PartitionContext` -- Renamed `Sqlite3PartitionManager` to `SamplePartitionManager` +- Removed constructor method of `PartitionProcessor`. For initialization please implement the method `initialize` . +- Replaced `CheckpointManager` by `PartitionContext`. + - `PartitionContext` has partition context information and method `update_checkpoint`. +- Updated all methods of `PartitionProcessor` to include `PartitionContext` as part of the arguments. +- Updated accessibility of class members in `EventHub/EventHubConsumer/EventHubProducer`to be private. + ## 5.0.0b2 (2019-08-06) diff --git a/sdk/eventhub/azure-eventhubs/README.md b/sdk/eventhub/azure-eventhubs/README.md index 94280c2df912..f3f29afd49fe 100644 --- a/sdk/eventhub/azure-eventhubs/README.md +++ b/sdk/eventhub/azure-eventhubs/README.md @@ -9,7 +9,7 @@ The Azure Event Hubs client library allows for publishing and consuming of Azure - Observe interesting operations and interactions happening within your business or other ecosystem, allowing loosely coupled systems to interact without the need to bind them together. - Receive events from one or more publishers, transform them to better meet the needs of your ecosystem, then publish the transformed events to a new stream for consumers to observe. -[Source code](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs) | [Package (PyPi)](https://pypi.org/project/azure-eventhub/) | [API reference documentation](https://azure.github.io/azure-sdk-for-python/ref/azure.eventhub) | [Product documentation](https://docs.microsoft.com/en-ca/azure/event-hubs/) +[Source code](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs) | [Package (PyPi)](https://pypi.org/project/azure-eventhub/) | [API reference documentation](https://azure.github.io/azure-sdk-for-python/ref/azure.eventhub) | [Product documentation](https://docs.microsoft.com/en-us/azure/event-hubs/) ## Getting started @@ -20,10 +20,6 @@ Install the Azure Event Hubs client library for Python with pip: ``` $ pip install --pre azure-eventhub ``` -For Python2.7, please install package "typing". This is a workaround for [issue 6767](https://github.com/Azure/azure-sdk-for-python/issues/6767). -``` -$ pip install typing -``` **Prerequisites** From 13fcd035a550f51f9c7da8b3b4b30eb0db34e961 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Mon, 9 Sep 2019 16:23:15 -0700 Subject: [PATCH 3/4] Update docstring of eventhub --- .../azure-eventhubs/azure/eventhub/aio/consumer_async.py | 1 + .../azure/eventhub/aio/eventprocessor/event_processor.py | 2 +- .../azure-eventhubs/azure/eventhub/aio/producer_async.py | 1 + sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py | 1 + sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py index 3aa1a7d6bbe5..61da41316cb3 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py @@ -31,6 +31,7 @@ class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instan group to be actively reading events from the partition. These non-exclusive consumers are sometimes referred to as "Non-Epoch Consumers." + Please use the method `create_consumer` on `EventHubClient` for creating `EventHubConsumer`. """ _timeout = 0 _epoch_symbol = b'com.microsoft:epoch' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py index 4356a7c1b74b..0b8ca10f20ba 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py @@ -23,7 +23,7 @@ class EventProcessor(object): # pylint:disable=too-many-instance-attributes """ - An EventProcessor constantly receives events from all partitions of the Event Hub in the context of a given + An EventProcessor constantly receives events from multiple partitions of the Event Hub in the context of a given consumer group. The received data will be sent to PartitionProcessor to be processed. It provides the user a convenient way to receive events from multiple partitions and save checkpoints. diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index 999bdc09c787..ec4e39c87116 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -26,6 +26,7 @@ class EventHubProducer(ConsumerProducerMixin): # pylint: disable=too-many-insta be created to allow event data to be automatically routed to an available partition or specific to a partition. + Please use the method `create_producer` on `EventHubClient` for creating `EventHubProducer`. """ _timeout_symbol = b'com.microsoft:timeout' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index 499c3ba5429e..3ba14020aaf7 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -33,6 +33,7 @@ class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instan group to be actively reading events from the partition. These non-exclusive consumers are sometimes referred to as "Non-Epoch Consumers." + Please use the method `create_consumer` on `EventHubClient` for creating `EventHubConsumer`. """ _timeout = 0 _epoch_symbol = b'com.microsoft:epoch' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index 8008fac7ecd0..cab9638f2acc 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -39,6 +39,7 @@ class EventHubProducer(ConsumerProducerMixin): # pylint:disable=too-many-instan be created to allow event data to be automatically routed to an available partition or specific to a partition. + Please use the method `create_producer` on `EventHubClient` for creating `EventHubProducer`. """ _timeout_symbol = b'com.microsoft:timeout' From 49bb76b1be21fbf8cfdca98752e7c73689ff21d1 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Mon, 9 Sep 2019 16:26:40 -0700 Subject: [PATCH 4/4] Small fix --- sdk/eventhub/azure-eventhubs/HISTORY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/eventhub/azure-eventhubs/HISTORY.md b/sdk/eventhub/azure-eventhubs/HISTORY.md index 4bdb2c8ab671..456b0fb18e98 100644 --- a/sdk/eventhub/azure-eventhubs/HISTORY.md +++ b/sdk/eventhub/azure-eventhubs/HISTORY.md @@ -1,4 +1,5 @@ # Release History + ## 5.0.0b3 (2019-09-10) **New features** @@ -12,7 +13,7 @@ **Breaking changes** -- Removed constructor method of `PartitionProcessor`. For initialization please implement the method `initialize` . +- Removed constructor method of `PartitionProcessor`. For initialization please implement the method `initialize`. - Replaced `CheckpointManager` by `PartitionContext`. - `PartitionContext` has partition context information and method `update_checkpoint`. - Updated all methods of `PartitionProcessor` to include `PartitionContext` as part of the arguments.